From f543a87b7871cbea2b9f23e1ac6a8a807608dafb Mon Sep 17 00:00:00 2001 From: Kacper Sawicki Date: Wed, 12 Nov 2025 09:43:22 +0100 Subject: [PATCH] chore: cache terraform providers for workspaces terraform tests (#20603) Fixes flaky `TestWorkspaceTagsTerraform` and `TestWorkspaceTemplateParamsChange` tests that were failing with `connection reset by peer` errors when downloading the coder/coder provider. This applies the same caching solution which was done in https://github.com/coder/coder/pull/17373 1. Extracts provider caching logic into `testutil/terraform_cache.go` 2. Updates TestProvision to use the shared caching helpers 3. Updates enterprise workspace tests to use the shared caching helpers The cache is persisted at `~/.cache/coderv2-test/` and automatically cached between CI runs via existing GitHub Actions cache setup. Closes https://github.com/coder/internal/issues/607 --- enterprise/coderd/workspaces_test.go | 52 ++----- provisioner/terraform/provision_test.go | 170 +--------------------- testutil/terraform_cache.go | 185 ++++++++++++++++++++++++ 3 files changed, 196 insertions(+), 211 deletions(-) create mode 100644 testutil/terraform_cache.go diff --git a/enterprise/coderd/workspaces_test.go b/enterprise/coderd/workspaces_test.go index 5201e613f7..44244f238f 100644 --- a/enterprise/coderd/workspaces_test.go +++ b/enterprise/coderd/workspaces_test.go @@ -7,8 +7,6 @@ import ( "encoding/json" "fmt" "net/http" - "os" - "os/exec" "path/filepath" "strings" "sync/atomic" @@ -3390,51 +3388,19 @@ func workspaceTagsTerraform(t *testing.T, tc testWorkspaceTagsTerraformCase, dyn } } -// downloadProviders is a test helper that creates a temporary file and writes a -// terraform CLI config file with a provider_installation stanza for coder/coder -// using dev_overrides. It also fetches the latest provider release from GitHub -// and extracts the binary to the temporary dir. It is the responsibility of the -// caller to set TF_CLI_CONFIG_FILE. +// downloadProviders is a test helper that caches Terraform providers and returns +// the path to a Terraform CLI config file that uses the cached providers. +// This uses the shared testutil caching infrastructure to avoid re-downloading +// providers on every test run. It is the responsibility of the caller to set +// TF_CLI_CONFIG_FILE. func downloadProviders(t *testing.T, providersTf string) string { t.Helper() - // We firstly write a Terraform CLI config file to a temporary directory: - var ( - tempDir = t.TempDir() - cacheDir = filepath.Join(tempDir, ".cache") - providersTfPath = filepath.Join(tempDir, "providers.tf") - cliConfigPath = filepath.Join(tempDir, "local.tfrc") - ) - // Write files to disk - require.NoError(t, os.MkdirAll(cacheDir, os.ModePerm|os.ModeDir)) - require.NoError(t, os.WriteFile(providersTfPath, []byte(providersTf), os.ModePerm)) // nolint:gosec - cliConfigTemplate := ` - provider_installation { - filesystem_mirror { - path = %q - include = ["*/*/*"] - } - direct { - exclude = ["*/*/*"] - } - }` - err := os.WriteFile(cliConfigPath, []byte(fmt.Sprintf(cliConfigTemplate, cacheDir)), os.ModePerm) // nolint:gosec - require.NoError(t, err, "failed to write %s", cliConfigPath) - - ctx := testutil.Context(t, testutil.WaitLong) - - // Run terraform providers mirror to mirror required providers to cacheDir - cmd := exec.CommandContext(ctx, "terraform", "providers", "mirror", cacheDir) - cmd.Env = os.Environ() // without this terraform may complain about path - cmd.Env = append(cmd.Env, "TF_CLI_CONFIG_FILE="+cliConfigPath) - cmd.Dir = tempDir - out, err := cmd.CombinedOutput() - if !assert.NoError(t, err) { - t.Log("failed to download providers:") - t.Log(string(out)) - t.FailNow() - } + cacheRootDir := filepath.Join(testutil.PersistentCacheDir(t), "terraform_workspace_tags_test") + templateFiles := map[string]string{"providers.tf": providersTf} + testName := "TestWorkspaceTagsTerraform" + cliConfigPath := testutil.CacheTFProviders(t, cacheRootDir, testName, templateFiles) t.Logf("Set TF_CLI_CONFIG_FILE=%s", cliConfigPath) return cliConfigPath } diff --git a/provisioner/terraform/provision_test.go b/provisioner/terraform/provision_test.go index 450dd04b06..9a8a49c29b 100644 --- a/provisioner/terraform/provision_test.go +++ b/provisioner/terraform/provision_test.go @@ -3,17 +3,13 @@ package terraform_test import ( - "bytes" "context" - "crypto/sha256" - "encoding/hex" "encoding/json" "errors" "fmt" "net" "net/http" "os" - "os/exec" "path/filepath" "sort" "strings" @@ -94,168 +90,6 @@ func configure(ctx context.Context, t *testing.T, client proto.DRPCProvisionerCl return sess } -func hashTemplateFilesAndTestName(t *testing.T, testName string, templateFiles map[string]string) string { - t.Helper() - - sortedFileNames := make([]string, 0, len(templateFiles)) - for fileName := range templateFiles { - sortedFileNames = append(sortedFileNames, fileName) - } - sort.Strings(sortedFileNames) - - // Inserting a delimiter between the file name and the file content - // ensures that a file named `ab` with content `cd` - // will not hash to the same value as a file named `abc` with content `d`. - // This can still happen if the file name or content include the delimiter, - // but hopefully they won't. - delimiter := []byte("🎉 🌱 🌷") - - hasher := sha256.New() - for _, fileName := range sortedFileNames { - file := templateFiles[fileName] - _, err := hasher.Write([]byte(fileName)) - require.NoError(t, err) - _, err = hasher.Write(delimiter) - require.NoError(t, err) - _, err = hasher.Write([]byte(file)) - require.NoError(t, err) - } - _, err := hasher.Write(delimiter) - require.NoError(t, err) - _, err = hasher.Write([]byte(testName)) - require.NoError(t, err) - - return hex.EncodeToString(hasher.Sum(nil)) -} - -const ( - terraformConfigFileName = "terraform.rc" - cacheProvidersDirName = "providers" - cacheTemplateFilesDirName = "files" -) - -// Writes a Terraform CLI config file (`terraform.rc`) in `dir` to enforce using the local provider mirror. -// This blocks network access for providers, forcing Terraform to use only what's cached in `dir`. -// Returns the path to the generated config file. -func writeCliConfig(t *testing.T, dir string) string { - t.Helper() - - cliConfigPath := filepath.Join(dir, terraformConfigFileName) - require.NoError(t, os.MkdirAll(filepath.Dir(cliConfigPath), 0o700)) - - content := fmt.Sprintf(` - provider_installation { - filesystem_mirror { - path = "%s" - include = ["*/*"] - } - direct { - exclude = ["*/*"] - } - } - `, filepath.Join(dir, cacheProvidersDirName)) - require.NoError(t, os.WriteFile(cliConfigPath, []byte(content), 0o600)) - return cliConfigPath -} - -func runCmd(t *testing.T, dir string, args ...string) { - t.Helper() - - stdout, stderr := bytes.NewBuffer(nil), bytes.NewBuffer(nil) - cmd := exec.Command(args[0], args[1:]...) //#nosec - cmd.Dir = dir - cmd.Stdout = stdout - cmd.Stderr = stderr - if err := cmd.Run(); err != nil { - t.Fatalf("failed to run %s: %s\nstdout: %s\nstderr: %s", strings.Join(args, " "), err, stdout.String(), stderr.String()) - } -} - -// Each test gets a unique cache dir based on its name and template files. -// This ensures that tests can download providers in parallel and that they -// will redownload providers if the template files change. -func getTestCacheDir(t *testing.T, rootDir string, testName string, templateFiles map[string]string) string { - t.Helper() - - hash := hashTemplateFilesAndTestName(t, testName, templateFiles) - dir := filepath.Join(rootDir, hash[:12]) - return dir -} - -// Ensures Terraform providers are downloaded and cached locally in a unique directory for the test. -// Uses `terraform init` then `mirror` to populate the cache if needed. -// Returns the cache directory path. -func downloadProviders(t *testing.T, rootDir string, testName string, templateFiles map[string]string) string { - t.Helper() - - dir := getTestCacheDir(t, rootDir, testName, templateFiles) - if _, err := os.Stat(dir); err == nil { - t.Logf("%s: using cached terraform providers", testName) - return dir - } - filesDir := filepath.Join(dir, cacheTemplateFilesDirName) - defer func() { - // The files dir will contain a copy of terraform providers generated - // by the terraform init command. We don't want to persist them since - // we already have a registry mirror in the providers dir. - if err := os.RemoveAll(filesDir); err != nil { - t.Logf("failed to remove files dir %s: %s", filesDir, err) - } - if !t.Failed() { - return - } - // If `downloadProviders` function failed, clean up the cache dir. - // We don't want to leave it around because it may be incomplete or corrupted. - if err := os.RemoveAll(dir); err != nil { - t.Logf("failed to remove dir %s: %s", dir, err) - } - }() - - require.NoError(t, os.MkdirAll(filesDir, 0o700)) - - for fileName, file := range templateFiles { - filePath := filepath.Join(filesDir, fileName) - require.NoError(t, os.MkdirAll(filepath.Dir(filePath), 0o700)) - require.NoError(t, os.WriteFile(filePath, []byte(file), 0o600)) - } - - providersDir := filepath.Join(dir, cacheProvidersDirName) - require.NoError(t, os.MkdirAll(providersDir, 0o700)) - - // We need to run init because if a test uses modules in its template, - // the mirror command will fail without it. - runCmd(t, filesDir, "terraform", "init") - // Now, mirror the providers into `providersDir`. We use this explicit mirror - // instead of relying only on the standard Terraform plugin cache. - // - // Why? Because this mirror, when used with the CLI config from `writeCliConfig`, - // prevents Terraform from hitting the network registry during `plan`. This cuts - // down on network calls, making CI tests less flaky. - // - // In contrast, the standard cache *still* contacts the registry for metadata - // during `init`, even if the plugins are already cached locally - see link below. - // - // Ref: https://developer.hashicorp.com/terraform/cli/config/config-file#provider-plugin-cache - // > When a plugin cache directory is enabled, the terraform init command will - // > still use the configured or implied installation methods to obtain metadata - // > about which plugins are available - runCmd(t, filesDir, "terraform", "providers", "mirror", providersDir) - - return dir -} - -// Caches providers locally and generates a Terraform CLI config to use *only* that cache. -// This setup prevents network access for providers during `terraform init`, improving reliability -// in subsequent test runs. -// Returns the path to the generated CLI config file. -func cacheProviders(t *testing.T, rootDir string, testName string, templateFiles map[string]string) string { - t.Helper() - - providersParentDir := downloadProviders(t, rootDir, testName, templateFiles) - cliConfigPath := writeCliConfig(t, providersParentDir) - return cliConfigPath -} - func readProvisionLog(t *testing.T, response proto.DRPCProvisioner_SessionClient) string { var logBuf strings.Builder for { @@ -1177,7 +1011,7 @@ func TestProvision(t *testing.T) { cacheRootDir := filepath.Join(testutil.PersistentCacheDir(t), "terraform_provision_test") expectedCacheDirs := make(map[string]bool) for _, testCase := range testCases { - cacheDir := getTestCacheDir(t, cacheRootDir, testCase.Name, testCase.Files) + cacheDir := testutil.GetTestTFCacheDir(t, cacheRootDir, testCase.Name, testCase.Files) expectedCacheDirs[cacheDir] = true } currentCacheDirs, err := filepath.Glob(filepath.Join(cacheRootDir, "*")) @@ -1199,7 +1033,7 @@ func TestProvision(t *testing.T) { cliConfigPath := "" if !testCase.SkipCacheProviders { - cliConfigPath = cacheProviders( + cliConfigPath = testutil.CacheTFProviders( t, cacheRootDir, testCase.Name, diff --git a/testutil/terraform_cache.go b/testutil/terraform_cache.go new file mode 100644 index 0000000000..f1e21e23c0 --- /dev/null +++ b/testutil/terraform_cache.go @@ -0,0 +1,185 @@ +//go:build linux || darwin + +package testutil + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "fmt" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +const ( + // terraformConfigFileName is the name of the Terraform CLI config file. + terraformConfigFileName = "terraform.rc" + // cacheProvidersDirName is the subdirectory name for the provider mirror. + cacheProvidersDirName = "providers" + // cacheTemplateFilesDirName is the subdirectory name for template files. + cacheTemplateFilesDirName = "files" +) + +// hashTemplateFilesAndTestName generates a unique hash based on test name and template files. +func hashTemplateFilesAndTestName(t *testing.T, testName string, templateFiles map[string]string) string { + t.Helper() + + sortedFileNames := make([]string, 0, len(templateFiles)) + for fileName := range templateFiles { + sortedFileNames = append(sortedFileNames, fileName) + } + sort.Strings(sortedFileNames) + + // Inserting a delimiter between the file name and the file content + // ensures that a file named `ab` with content `cd` + // will not hash to the same value as a file named `abc` with content `d`. + // This can still happen if the file name or content include the delimiter, + // but hopefully they won't. + delimiter := []byte("🎉 🌱 🌷") + + hasher := sha256.New() + for _, fileName := range sortedFileNames { + file := templateFiles[fileName] + _, err := hasher.Write([]byte(fileName)) + require.NoError(t, err) + _, err = hasher.Write(delimiter) + require.NoError(t, err) + _, err = hasher.Write([]byte(file)) + require.NoError(t, err) + } + _, err := hasher.Write(delimiter) + require.NoError(t, err) + _, err = hasher.Write([]byte(testName)) + require.NoError(t, err) + + return hex.EncodeToString(hasher.Sum(nil)) +} + +// WriteTFCliConfig writes a Terraform CLI config file (`terraform.rc`) in `dir` to enforce using the local provider mirror. +// This blocks network access for providers, forcing Terraform to use only what's cached in `dir`. +// Returns the path to the generated config file. +func WriteTFCliConfig(t *testing.T, dir string) string { + t.Helper() + + cliConfigPath := filepath.Join(dir, terraformConfigFileName) + require.NoError(t, os.MkdirAll(filepath.Dir(cliConfigPath), 0o700)) + + content := fmt.Sprintf(` + provider_installation { + filesystem_mirror { + path = "%s" + include = ["*/*"] + } + direct { + exclude = ["*/*"] + } + } + `, filepath.Join(dir, cacheProvidersDirName)) + require.NoError(t, os.WriteFile(cliConfigPath, []byte(content), 0o600)) + return cliConfigPath +} + +func runCmd(t *testing.T, dir string, args ...string) { + t.Helper() + + stdout, stderr := bytes.NewBuffer(nil), bytes.NewBuffer(nil) + cmd := exec.Command(args[0], args[1:]...) //#nosec + cmd.Dir = dir + cmd.Stdout = stdout + cmd.Stderr = stderr + if err := cmd.Run(); err != nil { + t.Fatalf("failed to run %s: %s\nstdout: %s\nstderr: %s", strings.Join(args, " "), err, stdout.String(), stderr.String()) + } +} + +// GetTestTFCacheDir returns a unique cache directory path based on the test name and template files. +// Each test gets a unique cache dir based on its name and template files. +// This ensures that tests can download providers in parallel and that they +// will redownload providers if the template files change. +func GetTestTFCacheDir(t *testing.T, rootDir string, testName string, templateFiles map[string]string) string { + t.Helper() + + hash := hashTemplateFilesAndTestName(t, testName, templateFiles) + dir := filepath.Join(rootDir, hash[:12]) + return dir +} + +// DownloadTFProviders ensures Terraform providers are downloaded and cached locally in a unique directory for the test. +// Uses `terraform init` then `mirror` to populate the cache if needed. +// Returns the cache directory path. +func DownloadTFProviders(t *testing.T, rootDir string, testName string, templateFiles map[string]string) string { + t.Helper() + + dir := GetTestTFCacheDir(t, rootDir, testName, templateFiles) + if _, err := os.Stat(dir); err == nil { + t.Logf("%s: using cached terraform providers", testName) + return dir + } + filesDir := filepath.Join(dir, cacheTemplateFilesDirName) + defer func() { + // The files dir will contain a copy of terraform providers generated + // by the terraform init command. We don't want to persist them since + // we already have a registry mirror in the providers dir. + if err := os.RemoveAll(filesDir); err != nil { + t.Logf("failed to remove files dir %s: %s", filesDir, err) + } + if !t.Failed() { + return + } + // If `DownloadTFProviders` function failed, clean up the cache dir. + // We don't want to leave it around because it may be incomplete or corrupted. + if err := os.RemoveAll(dir); err != nil { + t.Logf("failed to remove dir %s: %s", dir, err) + } + }() + + require.NoError(t, os.MkdirAll(filesDir, 0o700)) + + for fileName, file := range templateFiles { + filePath := filepath.Join(filesDir, fileName) + require.NoError(t, os.MkdirAll(filepath.Dir(filePath), 0o700)) + require.NoError(t, os.WriteFile(filePath, []byte(file), 0o600)) + } + + providersDir := filepath.Join(dir, cacheProvidersDirName) + require.NoError(t, os.MkdirAll(providersDir, 0o700)) + + // We need to run init because if a test uses modules in its template, + // the mirror command will fail without it. + runCmd(t, filesDir, "terraform", "init") + // Now, mirror the providers into `providersDir`. We use this explicit mirror + // instead of relying only on the standard Terraform plugin cache. + // + // Why? Because this mirror, when used with the CLI config from `WriteCliConfig`, + // prevents Terraform from hitting the network registry during `plan`. This cuts + // down on network calls, making CI tests less flaky. + // + // In contrast, the standard cache *still* contacts the registry for metadata + // during `init`, even if the plugins are already cached locally - see link below. + // + // Ref: https://developer.hashicorp.com/terraform/cli/config/config-file#provider-plugin-cache + // > When a plugin cache directory is enabled, the terraform init command will + // > still use the configured or implied installation methods to obtain metadata + // > about which plugins are available + runCmd(t, filesDir, "terraform", "providers", "mirror", providersDir) + + return dir +} + +// CacheTFProviders caches providers locally and generates a Terraform CLI config to use *only* that cache. +// This setup prevents network access for providers during `terraform init`, improving reliability +// in subsequent test runs. +// Returns the path to the generated CLI config file. +func CacheTFProviders(t *testing.T, rootDir string, testName string, templateFiles map[string]string) string { + t.Helper() + + providersParentDir := DownloadTFProviders(t, rootDir, testName, templateFiles) + cliConfigPath := WriteTFCliConfig(t, providersParentDir) + return cliConfigPath +}