package clitest import ( "bytes" "context" "flag" "fmt" "os" "path/filepath" "regexp" "strings" "sync" "testing" "github.com/google/go-cmp/cmp" "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/coder/coder/v2/cli/config" "github.com/coder/coder/v2/coderd/coderdtest" "github.com/coder/coder/v2/coderd/database/dbtestutil" "github.com/coder/coder/v2/codersdk" "github.com/coder/coder/v2/testutil" "github.com/coder/serpent" ) // UpdateGoldenFiles indicates golden files should be updated. // To update the golden files: // make gen/golden-files var UpdateGoldenFiles = flag.Bool("update", false, "update .golden files") var timestampRegex = regexp.MustCompile(`(?i)\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(.\d+)?(Z|[+-]\d+:\d+)`) type CommandHelpCase struct { Name string Cmd []string } func DefaultCases() []CommandHelpCase { return []CommandHelpCase{ { Name: "coder --help", Cmd: []string{"--help"}, }, { Name: "coder server --help", Cmd: []string{"server", "--help"}, }, } } // TestCommandHelp will test the help output of the given commands // using golden files. func TestCommandHelp(t *testing.T, getRoot func(t *testing.T) *serpent.Command, cases []CommandHelpCase) { t.Parallel() rootClient, replacements := prepareTestData(t) root := getRoot(t) ExtractCommandPathsLoop: for _, cp := range extractVisibleCommandPaths(nil, root.Children) { name := fmt.Sprintf("coder %s --help", strings.Join(cp, " ")) //nolint:gocritic cmd := append(cp, "--help") for _, tt := range cases { if tt.Name == name { continue ExtractCommandPathsLoop } } cases = append(cases, CommandHelpCase{Name: name, Cmd: cmd}) } for _, tt := range cases { t.Run(tt.Name, func(t *testing.T) { t.Parallel() ctx := testutil.Context(t, testutil.WaitLong) var outBuf bytes.Buffer caseCmd := getRoot(t) inv, cfg := NewWithCommand(t, caseCmd, tt.Cmd...) inv.Stderr = &outBuf inv.Stdout = &outBuf inv.Environ.Set("CODER_URL", rootClient.URL.String()) inv.Environ.Set("CODER_SESSION_TOKEN", rootClient.SessionToken()) inv.Environ.Set("CODER_CACHE_DIRECTORY", "~/.cache") SetupConfig(t, rootClient, cfg) StartWithWaiter(t, inv.WithContext(ctx)).RequireSuccess() TestGoldenFile(t, tt.Name, outBuf.Bytes(), replacements) }) } } // Output captures stdout and stderr from an invocation and formats them with // prefixes for golden file testing, preserving their interleaved order. type Output struct { mu sync.Mutex stdout bytes.Buffer stderr bytes.Buffer combined bytes.Buffer } // prefixWriter wraps a buffer and prefixes each line with a given prefix. type prefixWriter struct { mu *sync.Mutex prefix string raw *bytes.Buffer combined *bytes.Buffer line bytes.Buffer // buffer for incomplete lines } // Write implements io.Writer, adding a prefix to each complete line. func (w *prefixWriter) Write(p []byte) (n int, err error) { w.mu.Lock() defer w.mu.Unlock() // Write unprefixed to raw buffer. _, _ = w.raw.Write(p) // Append to line buffer. _, _ = w.line.Write(p) // Split on newlines. lines := bytes.Split(w.line.Bytes(), []byte{'\n'}) // Write all complete lines (all but the last, which may be incomplete). for i := 0; i < len(lines)-1; i++ { _, _ = w.combined.WriteString(w.prefix) _, _ = w.combined.Write(lines[i]) _ = w.combined.WriteByte('\n') } // Keep the last line (incomplete) in the buffer. w.line.Reset() _, _ = w.line.Write(lines[len(lines)-1]) return len(p), nil } // Capture sets up stdout and stderr writers on the invocation that prefix each // line with "out: " or "err: " while preserving their order. func Capture(inv *serpent.Invocation) *Output { output := &Output{} inv.Stdout = &prefixWriter{mu: &output.mu, prefix: "out: ", raw: &output.stdout, combined: &output.combined} inv.Stderr = &prefixWriter{mu: &output.mu, prefix: "err: ", raw: &output.stderr, combined: &output.combined} return output } // Golden returns the formatted output with lines prefixed by "err: " or "out: ". func (o *Output) Golden() []byte { return o.combined.Bytes() } // Stdout returns the unprefixed stdout content for parsing (e.g., JSON). func (o *Output) Stdout() string { return o.stdout.String() } // Stderr returns the unprefixed stderr content. func (o *Output) Stderr() string { return o.stderr.String() } // TestGoldenFile will test the given bytes slice input against the // golden file with the given file name, optionally using the given replacements. func TestGoldenFile(t *testing.T, fileName string, actual []byte, replacements map[string]string) { if len(actual) == 0 { t.Fatal("no output") } for k, v := range replacements { actual = bytes.ReplaceAll(actual, []byte(k), []byte(v)) } actual = normalizeGoldenFile(t, actual) goldenPath := filepath.Join("testdata", strings.ReplaceAll(fileName, " ", "_")+".golden") if *UpdateGoldenFiles { t.Logf("update golden file for: %q: %s", fileName, goldenPath) err := os.WriteFile(goldenPath, actual, 0o600) require.NoError(t, err, "update golden file") } expected, err := os.ReadFile(goldenPath) require.NoError(t, err, "read golden file, run \"make gen/golden-files\" and commit the changes") expected = normalizeGoldenFile(t, expected) assert.Empty(t, cmp.Diff(string(expected), string(actual)), "golden file mismatch (-want +got): %s, run \"make gen/golden-files\", verify and commit the changes", goldenPath) } // normalizeGoldenFile replaces any strings that are system or timing dependent // with a placeholder so that the golden files can be compared with a simple // equality check. func normalizeGoldenFile(t *testing.T, byt []byte) []byte { // Replace any timestamps with a placeholder. byt = timestampRegex.ReplaceAll(byt, []byte(pad("[timestamp]", 20))) homeDir, err := os.UserHomeDir() require.NoError(t, err) configDir := config.DefaultDir() byt = bytes.ReplaceAll(byt, []byte(configDir), []byte("~/.config/coderv2")) byt = bytes.ReplaceAll(byt, []byte(codersdk.DefaultCacheDir()), []byte("[cache dir]")) // The home directory changes depending on the test environment. byt = bytes.ReplaceAll(byt, []byte(homeDir), []byte("~")) // Normalize the temp directory. os.TempDir() may include a trailing slash // (macOS) or not (Linux/Windows), and the temp directory may be followed by // more filepath elements with an OS-specific separator. We handle all cases // by replacing tempdir+separator first, then tempdir alone. tempDir := filepath.Clean(os.TempDir()) byt = bytes.ReplaceAll(byt, []byte(tempDir+string(filepath.Separator)), []byte("/tmp/")) byt = bytes.ReplaceAll(byt, []byte(tempDir), []byte("/tmp")) // Clean up trailing slash when temp dir is used standalone (e.g., "/tmp/)" -> "/tmp)"). byt = bytes.ReplaceAll(byt, []byte("/tmp/)"), []byte("/tmp)")) for _, r := range []struct { old string new string }{ {"\r\n", "\n"}, {`~\.cache\coder`, "~/.cache/coder"}, {`C:\Users\RUNNER~1\AppData\Local\Temp`, "/tmp"}, } { byt = bytes.ReplaceAll(byt, []byte(r.old), []byte(r.new)) } return byt } func extractVisibleCommandPaths(cmdPath []string, cmds []*serpent.Command) [][]string { var cmdPaths [][]string for _, c := range cmds { if c.Hidden { continue } cmdPath := append(cmdPath, c.Name()) cmdPaths = append(cmdPaths, cmdPath) cmdPaths = append(cmdPaths, extractVisibleCommandPaths(cmdPath, c.Children)...) } return cmdPaths } func prepareTestData(t *testing.T) (*codersdk.Client, map[string]string) { t.Helper() ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong) defer cancel() // This needs to be a fixed timezone because timezones increase the length // of timestamp strings. The increased length can pad table formatting's // and differ the table header spacings. //nolint:gocritic db, pubsub := dbtestutil.NewDB(t, dbtestutil.WithTimezone("UTC")) rootClient := coderdtest.New(t, &coderdtest.Options{ Database: db, Pubsub: pubsub, IncludeProvisionerDaemon: true, }) firstUser := coderdtest.CreateFirstUser(t, rootClient) secondUser, err := rootClient.CreateUserWithOrgs(ctx, codersdk.CreateUserRequestWithOrgs{ Email: "testuser2@coder.com", Username: "testuser2", Password: coderdtest.FirstUserParams.Password, OrganizationIDs: []uuid.UUID{firstUser.OrganizationID}, }) require.NoError(t, err) version := coderdtest.CreateTemplateVersion(t, rootClient, firstUser.OrganizationID, nil) version = coderdtest.AwaitTemplateVersionJobCompleted(t, rootClient, version.ID) template := coderdtest.CreateTemplate(t, rootClient, firstUser.OrganizationID, version.ID, func(req *codersdk.CreateTemplateRequest) { req.Name = "test-template" }) workspace := coderdtest.CreateWorkspace(t, rootClient, template.ID, func(req *codersdk.CreateWorkspaceRequest) { req.Name = "test-workspace" }) workspaceBuild := coderdtest.AwaitWorkspaceBuildJobCompleted(t, rootClient, workspace.LatestBuild.ID) replacements := map[string]string{ firstUser.UserID.String(): pad("[first user ID]", 36), secondUser.ID.String(): pad("[second user ID]", 36), firstUser.OrganizationID.String(): pad("[first org ID]", 36), version.ID.String(): pad("[version ID]", 36), version.Name: pad("[version name]", 36), version.Job.ID.String(): pad("[version job ID]", 36), version.Job.FileID.String(): pad("[version file ID]", 36), version.Job.WorkerID.String(): pad("[version worker ID]", 36), template.ID.String(): pad("[template ID]", 36), workspace.ID.String(): pad("[workspace ID]", 36), workspaceBuild.ID.String(): pad("[workspace build ID]", 36), workspaceBuild.Job.ID.String(): pad("[workspace build job ID]", 36), workspaceBuild.Job.FileID.String(): pad("[workspace build file ID]", 36), workspaceBuild.Job.WorkerID.String(): pad("[workspace build worker ID]", 36), } return rootClient, replacements } func pad(s string, n int) string { if len(s) >= n { return s } n -= len(s) pre := n / 2 post := n - pre return strings.Repeat("=", pre) + s + strings.Repeat("=", post) }