fix: limit test parallelism in make test (#19465)

In order to get `make test` to reliably pass again on our dogfood
workspaces, we're having to resort to setting parallelism.

It also reworks our CI to call the `make test` target, instead of
rolling a different command.

Behavior changes:
 * sets 8 packages x 8 tests in parallel by default on `make test`
* by default, removes the `-short` flag. In my testing it makes only a
few seconds difference on ~200s, or 1-2%
* by default, removes the `-count=1` flag that busts Go's test cache.
With a fresh cache and no code changes, `make test` executes in ~15
seconds.

Signed-off-by: Spike Curtis <spike@coder.com>

.github/workflows/ci.yaml

@@ -465,30 +465,28 @@ jobs:
             # running in parallel, and dbtestutil.NewDB starts to take more than
             # 10s to complete sometimes causing test timeouts. With 16x8=128 tests
             # Postgres tends not to choke.
-            NUM_PARALLEL_PACKAGES=8
-            NUM_PARALLEL_TESTS=16
+            export TEST_NUM_PARALLEL_PACKAGES=8
+            export TEST_NUM_PARALLEL_TESTS=16
             # Only the CLI and Agent are officially supported on Windows and the rest are too flaky
-            PACKAGES="./cli/... ./enterprise/cli/... ./agent/..."
+            export TEST_PACKAGES="./cli/... ./enterprise/cli/... ./agent/..."
           elif [ "${RUNNER_OS}" == "macOS" ]; then
             # Our macOS runners have 8 cores. We set NUM_PARALLEL_TESTS to 16
             # because the tests complete faster and Postgres doesn't choke. It seems
             # that macOS's tmpfs is faster than the one on Windows.
-            NUM_PARALLEL_PACKAGES=8
-            NUM_PARALLEL_TESTS=16
+            export TEST_NUM_PARALLEL_PACKAGES=8
+            export TEST_NUM_PARALLEL_TESTS=16
             # Only the CLI and Agent are officially supported on macOS and the rest are too flaky
-            PACKAGES="./cli/... ./enterprise/cli/... ./agent/..."
+            export TEST_PACKAGES="./cli/... ./enterprise/cli/... ./agent/..."
           elif [ "${RUNNER_OS}" == "Linux" ]; then
             # Our Linux runners have 8 cores.
-            NUM_PARALLEL_PACKAGES=8
-            NUM_PARALLEL_TESTS=8
-            PACKAGES="./..."
+            export TEST_NUM_PARALLEL_PACKAGES=8
+            export TEST_NUM_PARALLEL_TESTS=8
           fi
 
           # by default, run tests with cache
-          TESTCOUNT=""
           if [ "${GITHUB_REF}" == "refs/heads/main" ]; then
             # on main, run tests without cache
-            TESTCOUNT="-count=1"
+            export TEST_COUNT="1"
           fi
 
           mkdir -p "$RUNNER_TEMP/sym"
@@ -498,8 +496,7 @@ jobs:
           # invalidated. See scripts/normalize_path.sh for more details.
           normalize_path_with_symlinks "$RUNNER_TEMP/sym" "$(dirname "$(which terraform)")"
 
-          gotestsum --format standard-quiet --packages "$PACKAGES" \
-            -- -timeout=20m -v -p $NUM_PARALLEL_PACKAGES -parallel=$NUM_PARALLEL_TESTS $TESTCOUNT
+          make test
 
       - name: Upload failed test db dumps
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2

Makefile

@@ -958,12 +958,31 @@ else
 GOTESTSUM_RETRY_FLAGS :=
 endif
 
+# default to 8x8 parallelism to avoid overwhelming our workspaces. Hopefully we can remove these defaults
+# when we get our test suite's resource utilization under control.
+GOTEST_FLAGS := -v -p $(or $(TEST_NUM_PARALLEL_PACKAGES),"8") -parallel=$(or $(TEST_NUM_PARALLEL_TESTS),"8")
+
+# The most common use is to set TEST_COUNT=1 to avoid Go's test cache.
+ifdef TEST_COUNT
+GOTEST_FLAGS += -count=$(TEST_COUNT)
+endif
+
+ifdef TEST_SHORT
+GOTEST_FLAGS += -short
+endif
+
+ifdef RUN
+GOTEST_FLAGS += -run $(RUN)
+endif
+
+TEST_PACKAGES ?= ./...
+
 test:
-	$(GIT_FLAGS) gotestsum --format standard-quiet $(GOTESTSUM_RETRY_FLAGS) --packages="./..." -- -v -short -count=1 $(if $(RUN),-run $(RUN))
+	$(GIT_FLAGS) gotestsum --format standard-quiet $(GOTESTSUM_RETRY_FLAGS) --packages="$(TEST_PACKAGES)" -- $(GOTEST_FLAGS)
 .PHONY: test
 
 test-cli:
-	$(GIT_FLAGS) gotestsum --format standard-quiet $(GOTESTSUM_RETRY_FLAGS) --packages="./cli/..." -- -v -short -count=1
+	$(MAKE) test TEST_PACKAGES="./cli..."
 .PHONY: test-cli
 
 # sqlc-cloud-is-setup will fail if no SQLc auth token is set. Use this as a