chore: retry failing tests in CI (#17681)

This PR introduces failing test retries in CI for e2e tests, Go tests
with the in-memory database, Go tests with Postgres, and the CLI tests.
Retries are not enabled for race tests.

The goal is to reduce how often flakes disrupt developers' workflows.

.github/workflows/ci.yaml

@@ -382,8 +382,8 @@ jobs:
             touch ~/.bash_profile && echo "export BASH_SILENCE_DEPRECATION_WARNING=1" >> ~/.bash_profile
           fi
           export TS_DEBUG_DISCO=true
-          gotestsum --junitfile="gotests.xml" --jsonfile="gotests.json" \
-            --packages="./..." -- $PARALLEL_FLAG -short -failfast
+          gotestsum --junitfile="gotests.xml" --jsonfile="gotests.json" --rerun-fails=2 \
+            --packages="./..." -- $PARALLEL_FLAG -short
 
       - name: Upload Test Cache
         uses: ./.github/actions/test-cache/upload
@@ -436,6 +436,7 @@ jobs:
           TS_DEBUG_DISCO: "true"
           LC_CTYPE: "en_US.UTF-8"
           LC_ALL: "en_US.UTF-8"
+          TEST_RETRIES: 2
         shell: bash
         run: |
           # By default Go will use the number of logical CPUs, which
@@ -499,6 +500,7 @@ jobs:
           TS_DEBUG_DISCO: "true"
           LC_CTYPE: "en_US.UTF-8"
           LC_ALL: "en_US.UTF-8"
+          TEST_RETRIES: 2
         shell: bash
         run: |
           # By default Go will use the number of logical CPUs, which
@@ -560,6 +562,7 @@ jobs:
         env:
           POSTGRES_VERSION: "16"
           TS_DEBUG_DISCO: "true"
+          TEST_RETRIES: 2
         run: |
           make test-postgres
 
@@ -784,6 +787,7 @@ jobs:
         if: ${{ !matrix.variant.premium }}
         env:
           DEBUG: pw:api
+          CODER_E2E_TEST_RETRIES: 2
         working-directory: site
 
       # Run all of the tests with a premium license
@@ -793,6 +797,7 @@ jobs:
           DEBUG: pw:api
           CODER_E2E_LICENSE: ${{ secrets.CODER_E2E_LICENSE }}
           CODER_E2E_REQUIRE_PREMIUM_TESTS: "1"
+          CODER_E2E_TEST_RETRIES: 2
         working-directory: site
 
       - name: Upload Playwright Failed Tests

Makefile

@@ -875,12 +875,19 @@ provisioner/terraform/testdata/version:
 	fi
 .PHONY: provisioner/terraform/testdata/version
 
+# Set the retry flags if TEST_RETRIES is set
+ifdef TEST_RETRIES
+GOTESTSUM_RETRY_FLAGS := --rerun-fails=$(TEST_RETRIES)
+else
+GOTESTSUM_RETRY_FLAGS :=
+endif
+
 test:
-	$(GIT_FLAGS) gotestsum --format standard-quiet -- -v -short -count=1 ./... $(if $(RUN),-run $(RUN))
+	$(GIT_FLAGS) gotestsum --format standard-quiet $(GOTESTSUM_RETRY_FLAGS) --packages="./..." -- -v -short -count=1 $(if $(RUN),-run $(RUN))
 .PHONY: test
 
 test-cli:
-	$(GIT_FLAGS) gotestsum --format standard-quiet -- -v -short -count=1 ./cli/...
+	$(GIT_FLAGS) gotestsum --format standard-quiet $(GOTESTSUM_RETRY_FLAGS) --packages="./cli/..." -- -v -short -count=1
 .PHONY: test-cli
 
 # sqlc-cloud-is-setup will fail if no SQLc auth token is set. Use this as a
@@ -919,9 +926,9 @@ test-postgres: test-postgres-docker
 	$(GIT_FLAGS)  DB=ci gotestsum \
 		--junitfile="gotests.xml" \
 		--jsonfile="gotests.json" \
+		$(GOTESTSUM_RETRY_FLAGS) \
 		--packages="./..." -- \
 		-timeout=20m \
-		-failfast \
 		-count=1
 .PHONY: test-postgres
 

site/e2e/playwright.config.ts

@@ -10,12 +10,30 @@ import {
 } from "./constants";
 
 export const wsEndpoint = process.env.CODER_E2E_WS_ENDPOINT;
+export const retries = (() => {
+	if (process.env.CODER_E2E_TEST_RETRIES === undefined) {
+		return undefined;
+	}
+	const count = Number.parseInt(process.env.CODER_E2E_TEST_RETRIES, 10);
+	if (Number.isNaN(count)) {
+		throw new Error(
+			`CODER_E2E_TEST_RETRIES is not a number: ${process.env.CODER_E2E_TEST_RETRIES}`,
+		);
+	}
+	if (count < 0) {
+		throw new Error(
+			`CODER_E2E_TEST_RETRIES is less than 0: ${process.env.CODER_E2E_TEST_RETRIES}`,
+		);
+	}
+	return count;
+})();
 
 const localURL = (port: number, path: string): string => {
 	return `http://localhost:${port}${path}`;
 };
 
 export default defineConfig({
+	retries,
 	globalSetup: require.resolve("./setup/preflight"),
 	projects: [
 		{