diff --git a/.claude/docs/AGENT_FAILURES.md b/.claude/docs/AGENT_FAILURES.md index 62e651f2a8..7cd1eeaa31 100644 --- a/.claude/docs/AGENT_FAILURES.md +++ b/.claude/docs/AGENT_FAILURES.md @@ -63,6 +63,22 @@ shown below when adding new failures. video, browser console output, and command output before retrying or cleaning the workspace. +## Symptom: Go test failure without preserved diagnostics + +- Likely cause: The failing CI job summary or compact failures artifact was + discarded before reporting or retrying the failure. +- How to reproduce: Let a Go test job fail in CI, then report the failure using + only the final job status instead of the job summary and artifacts. +- How to diagnose: Open the failed Go test job summary for the inline failure + table and per-test details. Download `go-test-failures-*.ndjson` for deeper + inspection of the compact failures-only records. +- Existing docs or tools: `.github/workflows/ci.yaml` Go test jobs and + `scripts/gotestsummary`. +- Missing harness piece: Agents need a central reminder to preserve the small + Go test diagnostics artifact instead of the old raw test log. +- Proposed prevention: Attach or summarize the inline job summary and preserve + `go-test-failures-*.ndjson` when reporting CI Go test failures. + ## Symptom: Port collision across worktrees - Likely cause: Multiple worktrees use the same default develop ports. diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2407459cb3..6afc09a5b4 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -560,14 +560,21 @@ jobs: - name: Publish Go test failure summary if: failure() && github.actor != 'dependabot[bot]' && runner.os == 'Linux' && !github.event.pull_request.head.repo.fork - run: bash scripts/go-test-failure-summary.sh "${RUNNER_TEMP}/go-test.json" >> "$GITHUB_STEP_SUMMARY" + run: | + go run ./scripts/gotestsummary \ + --jsonfile "${RUNNER_TEMP}/go-test.json" \ + --markdown-out - \ + --failures-out "${RUNNER_TEMP}/go-test-failures.ndjson" \ + --max-output-bytes 16384 \ + --max-failures 50 \ + >> "$GITHUB_STEP_SUMMARY" - - name: Upload Go test JSON + - name: Upload Go test failures if: failure() && github.actor != 'dependabot[bot]' && runner.os == 'Linux' && !github.event.pull_request.head.repo.fork uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: go-test-json-${{ github.job }}-${{ github.sha }} - path: ${{ runner.temp }}/go-test.json + name: go-test-failures-${{ github.job }}-${{ github.sha }} + path: ${{ runner.temp }}/go-test-failures.ndjson retention-days: 7 - name: Upload failed test db dumps @@ -671,14 +678,21 @@ jobs: - name: Publish Go test failure summary if: failure() && github.actor != 'dependabot[bot]' && runner.os == 'Linux' && !github.event.pull_request.head.repo.fork - run: bash scripts/go-test-failure-summary.sh "${RUNNER_TEMP}/go-test.json" >> "$GITHUB_STEP_SUMMARY" + run: | + go run ./scripts/gotestsummary \ + --jsonfile "${RUNNER_TEMP}/go-test.json" \ + --markdown-out - \ + --failures-out "${RUNNER_TEMP}/go-test-failures.ndjson" \ + --max-output-bytes 16384 \ + --max-failures 50 \ + >> "$GITHUB_STEP_SUMMARY" - - name: Upload Go test JSON + - name: Upload Go test failures if: failure() && github.actor != 'dependabot[bot]' && runner.os == 'Linux' && !github.event.pull_request.head.repo.fork uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: go-test-json-${{ github.job }}-${{ github.sha }} - path: ${{ runner.temp }}/go-test.json + name: go-test-failures-${{ github.job }}-${{ github.sha }} + path: ${{ runner.temp }}/go-test-failures.ndjson retention-days: 7 - name: Upload Test Cache @@ -766,14 +780,21 @@ jobs: - name: Publish Go test failure summary if: failure() && github.actor != 'dependabot[bot]' && runner.os == 'Linux' && !github.event.pull_request.head.repo.fork - run: bash scripts/go-test-failure-summary.sh "${RUNNER_TEMP}/go-test.json" >> "$GITHUB_STEP_SUMMARY" + run: | + go run ./scripts/gotestsummary \ + --jsonfile "${RUNNER_TEMP}/go-test.json" \ + --markdown-out - \ + --failures-out "${RUNNER_TEMP}/go-test-failures.ndjson" \ + --max-output-bytes 16384 \ + --max-failures 50 \ + >> "$GITHUB_STEP_SUMMARY" - - name: Upload Go test JSON + - name: Upload Go test failures if: failure() && github.actor != 'dependabot[bot]' && runner.os == 'Linux' && !github.event.pull_request.head.repo.fork uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: go-test-json-${{ github.job }}-${{ github.sha }} - path: ${{ runner.temp }}/go-test.json + name: go-test-failures-${{ github.job }}-${{ github.sha }} + path: ${{ runner.temp }}/go-test-failures.ndjson retention-days: 7 - name: Upload Test Cache diff --git a/scripts/go-test-failure-summary.sh b/scripts/go-test-failure-summary.sh deleted file mode 100755 index b3bfac4898..0000000000 --- a/scripts/go-test-failure-summary.sh +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env bash - -# Summarize failed Go tests from go test JSON output. - -set -euo pipefail -# shellcheck source=scripts/lib.sh -# shellcheck disable=SC1091 -source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -cdroot - -if [[ $# -ne 1 ]]; then - error "Usage: go-test-failure-summary.sh " -fi - -results_file=$1 -if [[ ! -s "$results_file" ]]; then - exit 0 -fi - -if ! command -v jq >/dev/null; then - error "jq is required to summarize Go test failures." -fi - -jq -sr ' - def clean_block: - tostring - | gsub("\u001b\\[[0-9;?]*[ -/]*[@-~]"; "") - | gsub("```"; "``"); - def clean_inline: - tostring | gsub("`"; "") | gsub("[\r\n]"; " "); - def truncate($max): - if length > $max then .[0:$max] + "..." else . end; - def terminal_action: - .Action == "pass" or .Action == "fail" or .Action == "skip"; - def test_key: - (.Package // "") + "\u0000" + (.Test // ""); - def output_for($events; $package; $test): - [ - $events[] - | select(.Action == "output") - | select((.Package // "") == $package) - | select((.Test // "") == $test) - | .Output // "" - ] - | join("") - | clean_block - | if . == "" then "No output recorded." else . end - | truncate(600); - - map(select(type == "object")) as $events - | [ - $events - | to_entries[] - | .value + {idx: .key} - | select((.Test // "") != "") - | select(terminal_action) - ] as $terminal_tests - | [ - $terminal_tests - | group_by(test_key) - | .[] - | max_by(.idx) - | select(.Action == "fail") - | { - package: ((.Package // "unknown") | clean_inline), - test: ((.Test // "unknown") | clean_inline), - elapsed: (.Elapsed // 0), - output: output_for($events; (.Package // ""); (.Test // "")) - } - ] as $failures - | if ($failures | length) == 0 then - empty - else - ($failures | length) as $failed - | ($failures | map(.package) | unique | length) as $packages - | ([ - $events[] - | select((.Test // "") == "") - | select(.Action == "pass" or .Action == "fail") - | .Elapsed // 0 - ] | add // 0) as $duration - | ([ - $events[] - | select((.Test // "") == "") - | select(.Action == "fail") - | .Package // empty - ] | unique | length) as $package_failures - | [ - "## Go test failures (\($failed) in \($packages))", - "- Duration: \($duration)s", - "- Package failures: \($package_failures)", - "", - ($failures[] - | "### \(.package) :: \(.test)\n" - + "- Elapsed: \(.elapsed)s\n\n" - + "```\n\(.output)\n```\n") - ] - | join("\n") - end -' "$results_file" diff --git a/scripts/gotestsummary/main.go b/scripts/gotestsummary/main.go new file mode 100644 index 0000000000..713fdb121a --- /dev/null +++ b/scripts/gotestsummary/main.go @@ -0,0 +1,482 @@ +package main + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "html" + "io" + "os" + "regexp" + "slices" + "sort" + "strings" + + "golang.org/x/xerrors" +) + +const defaultFailuresCapBytes = 4 * 1024 * 1024 + +var ansiEscapePattern = regexp.MustCompile(`\x1b\[[0-9;?]*[\x20-\x2f]*[\x40-\x7e]`) + +type config struct { + JSONFile string + MarkdownOut string + FailuresOut string + MaxOutputBytes int + MaxFailures int + FailuresCapBytes int +} + +type testEvent struct { + Action string `json:"Action"` + Package string `json:"Package"` + Test string `json:"Test"` + Elapsed float64 `json:"Elapsed"` + Output string `json:"Output"` +} + +type testKey struct { + pkg string + test string +} + +type failure struct { + Package string + Test string + Elapsed float64 + Output string +} + +type summary struct { + Failures []failure + DurationSeconds float64 + PackageFailureCount int + MalformedLineWarning int +} + +type tailBuffer struct { + maxBytes int + value string +} + +func main() { + cfg := config{MarkdownOut: "-", MaxOutputBytes: 8192, FailuresCapBytes: defaultFailuresCapBytes} + flags := flag.NewFlagSet(os.Args[0], flag.ExitOnError) + flags.StringVar(&cfg.JSONFile, "jsonfile", cfg.JSONFile, "path to go test JSON output") + flags.StringVar(&cfg.MarkdownOut, "markdown-out", cfg.MarkdownOut, "path for Markdown output, or - for stdout") + flags.StringVar(&cfg.FailuresOut, "failures-out", cfg.FailuresOut, "path for failures NDJSON output") + flags.IntVar(&cfg.MaxOutputBytes, "max-output-bytes", cfg.MaxOutputBytes, "maximum output bytes captured per failure") + flags.IntVar(&cfg.MaxFailures, "max-failures", cfg.MaxFailures, "maximum failures to render in Markdown, or 0 for all") + if err := flags.Parse(os.Args[1:]); err != nil { + _, _ = fmt.Fprintln(os.Stderr, err) + os.Exit(2) + } + if err := run(context.Background(), cfg, os.Stdout, os.Stderr, os.Getenv); err != nil { + _, _ = fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func run(ctx context.Context, cfg config, stdout, stderr io.Writer, getenv func(string) string) error { + if cfg.JSONFile == "" { + return xerrors.New("--jsonfile is required") + } + if cfg.MarkdownOut == "" { + cfg.MarkdownOut = "-" + } + if cfg.MaxOutputBytes < 0 { + return xerrors.New("--max-output-bytes must be non-negative") + } + if cfg.MaxFailures < 0 { + return xerrors.New("--max-failures must be non-negative") + } + if cfg.FailuresCapBytes <= 0 { + cfg.FailuresCapBytes = defaultFailuresCapBytes + } + + stat, err := os.Stat(cfg.JSONFile) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return writeEmptyOutputs(cfg) + } + return xerrors.Errorf("stat json file: %w", err) + } + if stat.Size() == 0 { + return writeEmptyOutputs(cfg) + } + + file, err := os.Open(cfg.JSONFile) + if err != nil { + return xerrors.Errorf("open json file: %w", err) + } + defer file.Close() + + result, err := summarize(ctx, file, cfg.MaxOutputBytes, stderr) + if err != nil { + return err + } + if cfg.FailuresOut != "" { + if err := writeFailuresNDJSON(cfg.FailuresOut, result.Failures, cfg.FailuresCapBytes); err != nil { + return err + } + } + if len(result.Failures) == 0 { + if cfg.MarkdownOut != "-" { + return os.WriteFile(cfg.MarkdownOut, nil, 0o600) + } + return nil + } + markdown := renderMarkdown(result, cfg.MaxFailures, cfg.FailuresOut, getenv("GITHUB_JOB")) + if cfg.MarkdownOut == "-" { + _, err = io.WriteString(stdout, markdown) + return err + } + return os.WriteFile(cfg.MarkdownOut, []byte(markdown), 0o600) +} + +func writeEmptyOutputs(cfg config) error { + if cfg.FailuresOut != "" { + if err := os.WriteFile(cfg.FailuresOut, nil, 0o600); err != nil { + return err + } + } + if cfg.MarkdownOut != "" && cfg.MarkdownOut != "-" { + return os.WriteFile(cfg.MarkdownOut, nil, 0o600) + } + return nil +} + +func summarize(ctx context.Context, r io.Reader, maxOutputBytes int, stderr io.Writer) (summary, error) { + reader := bufio.NewReader(r) + buffers := map[testKey]*tailBuffer{} + failures := map[testKey]failure{} + packageFailures := map[string]struct{}{} + var durationSeconds float64 + var malformedWarnings int + + for lineNumber := 1; ; lineNumber++ { + if err := ctx.Err(); err != nil { + return summary{}, err + } + line, err := reader.ReadString('\n') + if errors.Is(err, io.EOF) && line == "" { + break + } + if err != nil && !errors.Is(err, io.EOF) { + return summary{}, xerrors.Errorf("read json line: %w", err) + } + line = strings.TrimSpace(line) + if line == "" { + if errors.Is(err, io.EOF) { + break + } + continue + } + + var raw map[string]json.RawMessage + if err := json.Unmarshal([]byte(line), &raw); err != nil { + malformedWarnings++ + writef(stderr, "warning: skipping malformed go test JSON line %d: %v\n", lineNumber, err) + continue + } + if raw == nil { + malformedWarnings++ + writef(stderr, "warning: skipping non-object go test JSON line %d\n", lineNumber) + continue + } + var event testEvent + if err := json.Unmarshal([]byte(line), &event); err != nil { + malformedWarnings++ + writef(stderr, "warning: skipping malformed go test JSON line %d: %v\n", lineNumber, err) + continue + } + + key := testKey{pkg: event.Package, test: event.Test} + switch event.Action { + case "output": + bufferFor(buffers, key, maxOutputBytes).Append(stripANSI(event.Output)) + case "pass", "skip": + delete(buffers, key) + delete(failures, key) + if event.Test == "" { + delete(packageFailures, event.Package) + if event.Action == "pass" { + durationSeconds += event.Elapsed + } + } + case "fail": + if event.Test == "" { + durationSeconds += event.Elapsed + if event.Package != "" { + packageFailures[event.Package] = struct{}{} + } + } + output := bufferFor(buffers, key, maxOutputBytes).String() + if output == "" && event.Test != "" { + output = bufferFor(buffers, testKey{pkg: event.Package}, maxOutputBytes).String() + } + failures[key] = failure{ + Package: cmpString(event.Package, "unknown"), + Test: displayTestName(event.Test), + Elapsed: event.Elapsed, + Output: strings.ToValidUTF8(output, ""), + } + } + + if errors.Is(err, io.EOF) { + break + } + } + + failureList := make([]failure, 0, len(failures)) + for _, item := range failures { + failureList = append(failureList, item) + } + sort.Slice(failureList, func(i, j int) bool { + if failureList[i].Package != failureList[j].Package { + return failureList[i].Package < failureList[j].Package + } + return failureList[i].Test < failureList[j].Test + }) + + return summary{ + Failures: failureList, + DurationSeconds: durationSeconds, + PackageFailureCount: len(packageFailures), + MalformedLineWarning: malformedWarnings, + }, nil +} + +func bufferFor(buffers map[testKey]*tailBuffer, key testKey, maxOutputBytes int) *tailBuffer { + buffer := buffers[key] + if buffer == nil { + buffer = &tailBuffer{maxBytes: maxOutputBytes} + buffers[key] = buffer + } + return buffer +} + +func (b *tailBuffer) Append(output string) { + if b.maxBytes == 0 || output == "" { + return + } + b.value += output + if len(b.value) > b.maxBytes { + b.value = b.value[len(b.value)-b.maxBytes:] + } +} + +func (b *tailBuffer) String() string { + return strings.ToValidUTF8(b.value, "") +} + +func renderMarkdown(result summary, maxFailures int, failuresOut string, githubJob string) string { + failures := result.Failures + visibleFailures := failures + if maxFailures > 0 && len(failures) > maxFailures { + visibleFailures = failures[:maxFailures] + } + packageNames := map[string]struct{}{} + for _, item := range failures { + packageNames[item.Package] = struct{}{} + } + + var builder strings.Builder + writeBuilderf(&builder, "## Go test failures (%d in %d packages)\n\n", len(failures), len(packageNames)) + writeBuilderf(&builder, "Duration: %s · Packages with failures: %d", formatSeconds(result.DurationSeconds), result.PackageFailureCount) + if githubJob != "" { + writeBuilderf(&builder, " · Job: %s", escapeMarkdownLine(githubJob)) + } + writeBuilderString(&builder, "\n\n") + writeBuilderString(&builder, "| Package | Test | Elapsed |\n") + writeBuilderString(&builder, "|---|---|---|\n") + for _, item := range visibleFailures { + writeBuilderf(&builder, "| %s | %s | %s |\n", + escapeTableCell(item.Package), + escapeTableCell(item.Test), + formatSeconds(item.Elapsed), + ) + } + writeBuilderString(&builder, "\n") + + for _, item := range visibleFailures { + output := item.Output + if output == "" { + output = "No output recorded." + } + output = strings.ReplaceAll(strings.ToValidUTF8(output, ""), "```", "``") + writeBuilderf(&builder, "
\n%s :: %s (%s)\n\n", + html.EscapeString(item.Package), + html.EscapeString(item.Test), + formatSeconds(item.Elapsed), + ) + writeBuilderString(&builder, "```text\n") + writeBuilderString(&builder, output) + if !strings.HasSuffix(output, "\n") { + writeBuilderString(&builder, "\n") + } + writeBuilderString(&builder, "```\n\n
\n\n") + } + + if omitted := len(failures) - len(visibleFailures); omitted > 0 { + writeBuilderf(&builder, "_... and %d more failed tests omitted.", omitted) + if failuresOut != "" { + writeBuilderString(&builder, " Download the failures-only artifact for the full list.") + } + writeBuilderString(&builder, "_\n") + } + return builder.String() +} + +func writeBuilderf(builder *strings.Builder, format string, args ...any) { + _, _ = fmt.Fprintf(builder, format, args...) +} + +func writef(writer io.Writer, format string, args ...any) { + _, _ = fmt.Fprintf(writer, format, args...) +} + +func writeBuilderString(builder *strings.Builder, value string) { + _, _ = builder.WriteString(value) +} + +func writeFailuresNDJSON(path string, failures []failure, capBytes int) error { + var output bytes.Buffer + for index, item := range failures { + recordLine, err := marshalRecord(failureRecord{ + Package: item.Package, + Test: item.Test, + ElapsedS: item.Elapsed, + Output: strings.ToValidUTF8(item.Output, ""), + }) + if err != nil { + return err + } + if output.Len()+len(recordLine) <= capBytes { + _, _ = output.Write(recordLine) + continue + } + + remainingAfterCurrent := len(failures) - index - 1 + summaryLine, err := marshalRecord(truncationRecord{Truncated: true, RemainingFailures: remainingAfterCurrent}) + if err != nil { + return err + } + availableForRecord := capBytes - output.Len() + if remainingAfterCurrent > 0 { + availableForRecord -= len(summaryLine) + } + truncatedLine, ok, err := truncateFailureRecord(item, availableForRecord) + if err != nil { + return err + } + if ok { + _, _ = output.Write(truncatedLine) + if remainingAfterCurrent > 0 && output.Len()+len(summaryLine) <= capBytes { + _, _ = output.Write(summaryLine) + } + break + } + + summaryLine, err = marshalRecord(truncationRecord{Truncated: true, RemainingFailures: len(failures) - index}) + if err != nil { + return err + } + if output.Len()+len(summaryLine) <= capBytes { + _, _ = output.Write(summaryLine) + } + break + } + return os.WriteFile(path, output.Bytes(), 0o600) +} + +type failureRecord struct { + Package string `json:"package"` + Test string `json:"test"` + ElapsedS float64 `json:"elapsed_s"` + Output string `json:"output"` + OutputTruncated bool `json:"output_truncated,omitempty"` +} + +type truncationRecord struct { + Truncated bool `json:"truncated"` + RemainingFailures int `json:"remaining_failures"` +} + +func truncateFailureRecord(item failure, capBytes int) ([]byte, bool, error) { + if capBytes <= 0 { + return nil, false, nil + } + output := []byte(item.Output) + low, high := 0, len(output) + var best []byte + for low <= high { + mid := low + (high-low)/2 + recordLine, err := marshalRecord(failureRecord{ + Package: item.Package, + Test: item.Test, + ElapsedS: item.Elapsed, + Output: strings.ToValidUTF8(string(output[:mid]), ""), + OutputTruncated: true, + }) + if err != nil { + return nil, false, err + } + if len(recordLine) <= capBytes { + best = slices.Clone(recordLine) + low = mid + 1 + continue + } + high = mid - 1 + } + if best == nil { + return nil, false, nil + } + return best, true, nil +} + +func marshalRecord(record any) ([]byte, error) { + line, err := json.Marshal(record) + if err != nil { + return nil, err + } + line = append(line, '\n') + return line, nil +} + +func stripANSI(output string) string { + return ansiEscapePattern.ReplaceAllString(output, "") +} + +func displayTestName(name string) string { + if name == "" { + return "(package)" + } + return name +} + +func formatSeconds(seconds float64) string { + return fmt.Sprintf("%.2fs", seconds) +} + +func escapeTableCell(value string) string { + value = strings.ReplaceAll(value, "|", `\|`) + value = strings.NewReplacer("\r", " ", "\n", " ", "`", "`").Replace(value) + return html.EscapeString(value) +} + +func escapeMarkdownLine(value string) string { + return strings.NewReplacer("\r", " ", "\n", " ").Replace(value) +} + +func cmpString(value, fallback string) string { + if value == "" { + return fallback + } + return value +} diff --git a/scripts/gotestsummary/main_test.go b/scripts/gotestsummary/main_test.go new file mode 100644 index 0000000000..1e0fbb9b5c --- /dev/null +++ b/scripts/gotestsummary/main_test.go @@ -0,0 +1,235 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestRunEmptyInputWritesNoMarkdownAndEmptyFailures(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + jsonFile := filepath.Join(dir, "go-test.json") + failuresFile := filepath.Join(dir, "failures.ndjson") + require.NoError(t, os.WriteFile(jsonFile, nil, 0o600)) + + var stdout bytes.Buffer + err := run(context.Background(), config{ + JSONFile: jsonFile, + MarkdownOut: "-", + FailuresOut: failuresFile, + MaxOutputBytes: 8192, + }, &stdout, ioDiscard{}, emptyEnv) + require.NoError(t, err) + require.Empty(t, stdout.String()) + assertFileContent(t, failuresFile, "") +} + +func TestRunPassingInputWritesNoMarkdown(t *testing.T) { + t.Parallel() + + jsonFile := writeEvents(t, + testEvent{Action: "output", Package: "example.com/pkg", Test: "TestOK", Output: "ok\n"}, + testEvent{Action: "pass", Package: "example.com/pkg", Test: "TestOK", Elapsed: 0.01}, + testEvent{Action: "pass", Package: "example.com/pkg", Elapsed: 0.02}, + ) + failuresFile := filepath.Join(t.TempDir(), "failures.ndjson") + + var stdout bytes.Buffer + err := run(context.Background(), config{ + JSONFile: jsonFile, + MarkdownOut: "-", + FailuresOut: failuresFile, + MaxOutputBytes: 8192, + }, &stdout, ioDiscard{}, emptyEnv) + require.NoError(t, err) + require.Empty(t, stdout.String()) + assertFileContent(t, failuresFile, "") +} + +func TestRunSingleFailureRendersBoundedOutput(t *testing.T) { + t.Parallel() + + jsonFile := writeEvents(t, + testEvent{Action: "output", Package: "example.com/pkg", Test: "TestFail", Output: "prefix-" + strings.Repeat("x", 20)}, + testEvent{Action: "fail", Package: "example.com/pkg", Test: "TestFail", Elapsed: 1.25}, + testEvent{Action: "fail", Package: "example.com/pkg", Elapsed: 1.50}, + ) + + markdown := runMarkdown(t, jsonFile, config{MaxOutputBytes: 10}) + require.Contains(t, markdown, "## Go test failures (2 in 1 packages)") + require.Contains(t, markdown, "| example.com/pkg | TestFail | 1.25s |") + require.NotContains(t, markdown, "prefix") + require.Contains(t, markdown, strings.Repeat("x", 10)) +} + +func TestRunSubtestFailureCapturesSlashName(t *testing.T) { + t.Parallel() + + jsonFile := writeEvents(t, + testEvent{Action: "output", Package: "example.com/pkg", Test: "TestParent/subcase", Output: "subtest failed\n"}, + testEvent{Action: "fail", Package: "example.com/pkg", Test: "TestParent/subcase", Elapsed: 0.20}, + ) + + markdown := runMarkdown(t, jsonFile, config{MaxOutputBytes: 8192}) + require.Contains(t, markdown, "TestParent/subcase") + require.Contains(t, markdown, "subtest failed") +} + +func TestRunRerunPassRemovesPriorFailure(t *testing.T) { + t.Parallel() + + jsonFile := writeEvents(t, + testEvent{Action: "output", Package: "example.com/pkg", Test: "TestFlake", Output: "first run failed\n"}, + testEvent{Action: "fail", Package: "example.com/pkg", Test: "TestFlake", Elapsed: 0.10}, + testEvent{Action: "output", Package: "example.com/pkg", Test: "TestFlake", Output: "retry passed\n"}, + testEvent{Action: "pass", Package: "example.com/pkg", Test: "TestFlake", Elapsed: 0.05}, + ) + + markdown := runMarkdown(t, jsonFile, config{MaxOutputBytes: 8192}) + require.Empty(t, markdown) +} + +func TestRunStripsANSIOutput(t *testing.T) { + t.Parallel() + + jsonFile := writeEvents(t, + testEvent{Action: "output", Package: "example.com/pkg", Test: "TestFail", Output: "\x1b[31mred\x1b[0m\n"}, + testEvent{Action: "fail", Package: "example.com/pkg", Test: "TestFail", Elapsed: 0.10}, + ) + + markdown := runMarkdown(t, jsonFile, config{MaxOutputBytes: 8192}) + require.Contains(t, markdown, "red") + require.NotContains(t, markdown, "\x1b") +} + +func TestRunEscapesTripleBackticksInOutput(t *testing.T) { + t.Parallel() + + jsonFile := writeEvents(t, + testEvent{Action: "output", Package: "example.com/pkg", Test: "TestFail", Output: "before ``` after\n"}, + testEvent{Action: "fail", Package: "example.com/pkg", Test: "TestFail", Elapsed: 0.10}, + ) + + markdown := runMarkdown(t, jsonFile, config{MaxOutputBytes: 8192}) + require.Contains(t, markdown, "before `` after") + require.Equal(t, 2, strings.Count(markdown, "```")) +} + +func TestRunMaxFailuresAddsOmittedLine(t *testing.T) { + t.Parallel() + + jsonFile := writeEvents(t, + testEvent{Action: "fail", Package: "example.com/pkg", Test: "TestA", Elapsed: 0.10}, + testEvent{Action: "fail", Package: "example.com/pkg", Test: "TestB", Elapsed: 0.20}, + ) + + markdown := runMarkdown(t, jsonFile, config{ + MaxOutputBytes: 8192, + MaxFailures: 1, + FailuresOut: filepath.Join(t.TempDir(), "failures.ndjson"), + }) + require.Contains(t, markdown, "TestA") + require.NotContains(t, markdown, "TestB") + require.Contains(t, markdown, "_... and 1 more failed tests omitted. Download the failures-only artifact for the full list._") +} + +func TestWriteFailuresNDJSONAppliesCap(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + path := filepath.Join(dir, "failures.ndjson") + failures := []failure{ + {Package: "example.com/pkg", Test: "TestA", Elapsed: 0.10, Output: strings.Repeat("a", 1000)}, + {Package: "example.com/pkg", Test: "TestB", Elapsed: 0.20, Output: "second"}, + } + summaryLine, err := marshalRecord(truncationRecord{Truncated: true, RemainingFailures: 1}) + require.NoError(t, err) + minimumLine, err := marshalRecord(failureRecord{ + Package: failures[0].Package, + Test: failures[0].Test, + ElapsedS: failures[0].Elapsed, + Output: "", + OutputTruncated: true, + }) + require.NoError(t, err) + capBytes := len(summaryLine) + len(minimumLine) + 20 + + require.NoError(t, writeFailuresNDJSON(path, failures, capBytes)) + content, err := os.ReadFile(path) + require.NoError(t, err) + require.LessOrEqual(t, len(content), capBytes) + + lines := strings.Split(strings.TrimSpace(string(content)), "\n") + require.Len(t, lines, 2) + var first map[string]any + require.NoError(t, json.Unmarshal([]byte(lines[0]), &first)) + require.Equal(t, true, first["output_truncated"]) + require.Equal(t, "TestA", first["test"]) + require.Less(t, len(first["output"].(string)), 1000) + var second map[string]any + require.NoError(t, json.Unmarshal([]byte(lines[1]), &second)) + require.Equal(t, true, second["truncated"]) + require.Equal(t, float64(1), second["remaining_failures"]) +} + +func TestRunPackageLevelFailure(t *testing.T) { + t.Parallel() + + jsonFile := writeEvents(t, + testEvent{Action: "output", Package: "example.com/pkg", Output: "setup failed\n"}, + testEvent{Action: "fail", Package: "example.com/pkg", Elapsed: 0.30}, + ) + + markdown := runMarkdown(t, jsonFile, config{MaxOutputBytes: 8192}) + require.Contains(t, markdown, "(package)") + require.Contains(t, markdown, "setup failed") +} + +func runMarkdown(t *testing.T, jsonFile string, cfg config) string { + t.Helper() + cfg.JSONFile = jsonFile + cfg.MarkdownOut = "-" + if cfg.MaxOutputBytes == 0 { + cfg.MaxOutputBytes = 8192 + } + var stdout bytes.Buffer + err := run(context.Background(), cfg, &stdout, ioDiscard{}, emptyEnv) + require.NoError(t, err) + return stdout.String() +} + +func writeEvents(t *testing.T, events ...testEvent) string { + t.Helper() + dir := t.TempDir() + path := filepath.Join(dir, "go-test.json") + var content strings.Builder + for _, event := range events { + line, err := json.Marshal(event) + require.NoError(t, err) + _, _ = content.Write(line) + _ = content.WriteByte('\n') + } + require.NoError(t, os.WriteFile(path, []byte(content.String()), 0o600)) + return path +} + +func assertFileContent(t *testing.T, path string, expected string) { + t.Helper() + content, err := os.ReadFile(path) + require.NoError(t, err) + require.Equal(t, expected, string(content)) +} + +func emptyEnv(string) string { return "" } + +type ioDiscard struct{} + +func (ioDiscard) Write(p []byte) (int, error) { return len(p), nil }