mirror of
https://github.com/coder/coder.git
synced 2026-06-02 20:48:20 +00:00
feat: stream go test failure summary and drop raw json artifact (#25146)
This follows up on https://github.com/coder/coder/actions/runs/25684936801/job/75406131184?pr=25139 by replacing the large raw Go test JSON artifact with inline structured summaries and a compact failures-only artifact. ## What changed - Added `scripts/gotestsummary`, a streaming Go tool that reads gotestsum JSON and renders failed tests as Markdown. - Updated the three Go test jobs to publish per-test `<details>` sections in the job summary. - Removed upload of the raw `go-test.json` artifact. - Added upload of `go-test-failures-*.ndjson` with compact failure records for deeper inspection. - Deleted the old bash and `jq` summary script. ## Why - The previous raw artifact was about 35 MB compressed and 445 MB raw in the linked run. - Passing-test output made the artifact noisy and slow to inspect. - The old summary truncated output to 600 characters. - The new path keeps streaming, bounded output and writes structured diagnostics for only final failed tests. ## Validation - `gofmt -w scripts/gotestsummary` - `gofmt -l scripts/gotestsummary` - `go test ./scripts/gotestsummary/...` - `go vet ./scripts/gotestsummary/...` - `grep -rn 'go-test-failure-summary.sh' . || true` - `grep -rn 'go-test-failure-summary.sh\|go-test.json\|go-test-json-' .claude .agents docs AGENTS.md || true` - `make lint/agents` - `make lint/emdash` - `make lint/markdown` - `make lint/shellcheck` - `git diff --check origin/main..HEAD` > This PR was prepared by Mux working on Mike's behalf.
This commit is contained in:
Michael Suchacz
@@ -1,100 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Summarize failed Go tests from go test JSON output.
|
||||
|
||||
set -euo pipefail
|
||||
# shellcheck source=scripts/lib.sh
|
||||
# shellcheck disable=SC1091
|
||||
source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
|
||||
cdroot
|
||||
|
||||
if [[ $# -ne 1 ]]; then
|
||||
error "Usage: go-test-failure-summary.sh <go-test.json>"
|
||||
fi
|
||||
|
||||
results_file=$1
|
||||
if [[ ! -s "$results_file" ]]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if ! command -v jq >/dev/null; then
|
||||
error "jq is required to summarize Go test failures."
|
||||
fi
|
||||
|
||||
jq -sr '
|
||||
def clean_block:
|
||||
tostring
|
||||
| gsub("\u001b\\[[0-9;?]*[ -/]*[@-~]"; "")
|
||||
| gsub("```"; "``");
|
||||
def clean_inline:
|
||||
tostring | gsub("`"; "") | gsub("[\r\n]"; " ");
|
||||
def truncate($max):
|
||||
if length > $max then .[0:$max] + "..." else . end;
|
||||
def terminal_action:
|
||||
.Action == "pass" or .Action == "fail" or .Action == "skip";
|
||||
def test_key:
|
||||
(.Package // "") + "\u0000" + (.Test // "");
|
||||
def output_for($events; $package; $test):
|
||||
[
|
||||
$events[]
|
||||
| select(.Action == "output")
|
||||
| select((.Package // "") == $package)
|
||||
| select((.Test // "") == $test)
|
||||
| .Output // ""
|
||||
]
|
||||
| join("")
|
||||
| clean_block
|
||||
| if . == "" then "No output recorded." else . end
|
||||
| truncate(600);
|
||||
|
||||
map(select(type == "object")) as $events
|
||||
| [
|
||||
$events
|
||||
| to_entries[]
|
||||
| .value + {idx: .key}
|
||||
| select((.Test // "") != "")
|
||||
| select(terminal_action)
|
||||
] as $terminal_tests
|
||||
| [
|
||||
$terminal_tests
|
||||
| group_by(test_key)
|
||||
| .[]
|
||||
| max_by(.idx)
|
||||
| select(.Action == "fail")
|
||||
| {
|
||||
package: ((.Package // "unknown") | clean_inline),
|
||||
test: ((.Test // "unknown") | clean_inline),
|
||||
elapsed: (.Elapsed // 0),
|
||||
output: output_for($events; (.Package // ""); (.Test // ""))
|
||||
}
|
||||
] as $failures
|
||||
| if ($failures | length) == 0 then
|
||||
empty
|
||||
else
|
||||
($failures | length) as $failed
|
||||
| ($failures | map(.package) | unique | length) as $packages
|
||||
| ([
|
||||
$events[]
|
||||
| select((.Test // "") == "")
|
||||
| select(.Action == "pass" or .Action == "fail")
|
||||
| .Elapsed // 0
|
||||
] | add // 0) as $duration
|
||||
| ([
|
||||
$events[]
|
||||
| select((.Test // "") == "")
|
||||
| select(.Action == "fail")
|
||||
| .Package // empty
|
||||
] | unique | length) as $package_failures
|
||||
| [
|
||||
"## Go test failures (\($failed) in \($packages))",
|
||||
"- Duration: \($duration)s",
|
||||
"- Package failures: \($package_failures)",
|
||||
"",
|
||||
($failures[]
|
||||
| "### \(.package) :: \(.test)\n"
|
||||
+ "- Elapsed: \(.elapsed)s\n\n"
|
||||
+ "```\n\(.output)\n```\n")
|
||||
]
|
||||
| join("\n")
|
||||
end
|
||||
' "$results_file"
|
||||
@@ -0,0 +1,482 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"html"
|
||||
"io"
|
||||
"os"
|
||||
"regexp"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/xerrors"
|
||||
)
|
||||
|
||||
const defaultFailuresCapBytes = 4 * 1024 * 1024
|
||||
|
||||
var ansiEscapePattern = regexp.MustCompile(`\x1b\[[0-9;?]*[\x20-\x2f]*[\x40-\x7e]`)
|
||||
|
||||
type config struct {
|
||||
JSONFile string
|
||||
MarkdownOut string
|
||||
FailuresOut string
|
||||
MaxOutputBytes int
|
||||
MaxFailures int
|
||||
FailuresCapBytes int
|
||||
}
|
||||
|
||||
type testEvent struct {
|
||||
Action string `json:"Action"`
|
||||
Package string `json:"Package"`
|
||||
Test string `json:"Test"`
|
||||
Elapsed float64 `json:"Elapsed"`
|
||||
Output string `json:"Output"`
|
||||
}
|
||||
|
||||
type testKey struct {
|
||||
pkg string
|
||||
test string
|
||||
}
|
||||
|
||||
type failure struct {
|
||||
Package string
|
||||
Test string
|
||||
Elapsed float64
|
||||
Output string
|
||||
}
|
||||
|
||||
type summary struct {
|
||||
Failures []failure
|
||||
DurationSeconds float64
|
||||
PackageFailureCount int
|
||||
MalformedLineWarning int
|
||||
}
|
||||
|
||||
type tailBuffer struct {
|
||||
maxBytes int
|
||||
value string
|
||||
}
|
||||
|
||||
func main() {
|
||||
cfg := config{MarkdownOut: "-", MaxOutputBytes: 8192, FailuresCapBytes: defaultFailuresCapBytes}
|
||||
flags := flag.NewFlagSet(os.Args[0], flag.ExitOnError)
|
||||
flags.StringVar(&cfg.JSONFile, "jsonfile", cfg.JSONFile, "path to go test JSON output")
|
||||
flags.StringVar(&cfg.MarkdownOut, "markdown-out", cfg.MarkdownOut, "path for Markdown output, or - for stdout")
|
||||
flags.StringVar(&cfg.FailuresOut, "failures-out", cfg.FailuresOut, "path for failures NDJSON output")
|
||||
flags.IntVar(&cfg.MaxOutputBytes, "max-output-bytes", cfg.MaxOutputBytes, "maximum output bytes captured per failure")
|
||||
flags.IntVar(&cfg.MaxFailures, "max-failures", cfg.MaxFailures, "maximum failures to render in Markdown, or 0 for all")
|
||||
if err := flags.Parse(os.Args[1:]); err != nil {
|
||||
_, _ = fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(2)
|
||||
}
|
||||
if err := run(context.Background(), cfg, os.Stdout, os.Stderr, os.Getenv); err != nil {
|
||||
_, _ = fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run(ctx context.Context, cfg config, stdout, stderr io.Writer, getenv func(string) string) error {
|
||||
if cfg.JSONFile == "" {
|
||||
return xerrors.New("--jsonfile is required")
|
||||
}
|
||||
if cfg.MarkdownOut == "" {
|
||||
cfg.MarkdownOut = "-"
|
||||
}
|
||||
if cfg.MaxOutputBytes < 0 {
|
||||
return xerrors.New("--max-output-bytes must be non-negative")
|
||||
}
|
||||
if cfg.MaxFailures < 0 {
|
||||
return xerrors.New("--max-failures must be non-negative")
|
||||
}
|
||||
if cfg.FailuresCapBytes <= 0 {
|
||||
cfg.FailuresCapBytes = defaultFailuresCapBytes
|
||||
}
|
||||
|
||||
stat, err := os.Stat(cfg.JSONFile)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return writeEmptyOutputs(cfg)
|
||||
}
|
||||
return xerrors.Errorf("stat json file: %w", err)
|
||||
}
|
||||
if stat.Size() == 0 {
|
||||
return writeEmptyOutputs(cfg)
|
||||
}
|
||||
|
||||
file, err := os.Open(cfg.JSONFile)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("open json file: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
result, err := summarize(ctx, file, cfg.MaxOutputBytes, stderr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if cfg.FailuresOut != "" {
|
||||
if err := writeFailuresNDJSON(cfg.FailuresOut, result.Failures, cfg.FailuresCapBytes); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if len(result.Failures) == 0 {
|
||||
if cfg.MarkdownOut != "-" {
|
||||
return os.WriteFile(cfg.MarkdownOut, nil, 0o600)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
markdown := renderMarkdown(result, cfg.MaxFailures, cfg.FailuresOut, getenv("GITHUB_JOB"))
|
||||
if cfg.MarkdownOut == "-" {
|
||||
_, err = io.WriteString(stdout, markdown)
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(cfg.MarkdownOut, []byte(markdown), 0o600)
|
||||
}
|
||||
|
||||
func writeEmptyOutputs(cfg config) error {
|
||||
if cfg.FailuresOut != "" {
|
||||
if err := os.WriteFile(cfg.FailuresOut, nil, 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if cfg.MarkdownOut != "" && cfg.MarkdownOut != "-" {
|
||||
return os.WriteFile(cfg.MarkdownOut, nil, 0o600)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func summarize(ctx context.Context, r io.Reader, maxOutputBytes int, stderr io.Writer) (summary, error) {
|
||||
reader := bufio.NewReader(r)
|
||||
buffers := map[testKey]*tailBuffer{}
|
||||
failures := map[testKey]failure{}
|
||||
packageFailures := map[string]struct{}{}
|
||||
var durationSeconds float64
|
||||
var malformedWarnings int
|
||||
|
||||
for lineNumber := 1; ; lineNumber++ {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return summary{}, err
|
||||
}
|
||||
line, err := reader.ReadString('\n')
|
||||
if errors.Is(err, io.EOF) && line == "" {
|
||||
break
|
||||
}
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
return summary{}, xerrors.Errorf("read json line: %w", err)
|
||||
}
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
var raw map[string]json.RawMessage
|
||||
if err := json.Unmarshal([]byte(line), &raw); err != nil {
|
||||
malformedWarnings++
|
||||
writef(stderr, "warning: skipping malformed go test JSON line %d: %v\n", lineNumber, err)
|
||||
continue
|
||||
}
|
||||
if raw == nil {
|
||||
malformedWarnings++
|
||||
writef(stderr, "warning: skipping non-object go test JSON line %d\n", lineNumber)
|
||||
continue
|
||||
}
|
||||
var event testEvent
|
||||
if err := json.Unmarshal([]byte(line), &event); err != nil {
|
||||
malformedWarnings++
|
||||
writef(stderr, "warning: skipping malformed go test JSON line %d: %v\n", lineNumber, err)
|
||||
continue
|
||||
}
|
||||
|
||||
key := testKey{pkg: event.Package, test: event.Test}
|
||||
switch event.Action {
|
||||
case "output":
|
||||
bufferFor(buffers, key, maxOutputBytes).Append(stripANSI(event.Output))
|
||||
case "pass", "skip":
|
||||
delete(buffers, key)
|
||||
delete(failures, key)
|
||||
if event.Test == "" {
|
||||
delete(packageFailures, event.Package)
|
||||
if event.Action == "pass" {
|
||||
durationSeconds += event.Elapsed
|
||||
}
|
||||
}
|
||||
case "fail":
|
||||
if event.Test == "" {
|
||||
durationSeconds += event.Elapsed
|
||||
if event.Package != "" {
|
||||
packageFailures[event.Package] = struct{}{}
|
||||
}
|
||||
}
|
||||
output := bufferFor(buffers, key, maxOutputBytes).String()
|
||||
if output == "" && event.Test != "" {
|
||||
output = bufferFor(buffers, testKey{pkg: event.Package}, maxOutputBytes).String()
|
||||
}
|
||||
failures[key] = failure{
|
||||
Package: cmpString(event.Package, "unknown"),
|
||||
Test: displayTestName(event.Test),
|
||||
Elapsed: event.Elapsed,
|
||||
Output: strings.ToValidUTF8(output, ""),
|
||||
}
|
||||
}
|
||||
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
failureList := make([]failure, 0, len(failures))
|
||||
for _, item := range failures {
|
||||
failureList = append(failureList, item)
|
||||
}
|
||||
sort.Slice(failureList, func(i, j int) bool {
|
||||
if failureList[i].Package != failureList[j].Package {
|
||||
return failureList[i].Package < failureList[j].Package
|
||||
}
|
||||
return failureList[i].Test < failureList[j].Test
|
||||
})
|
||||
|
||||
return summary{
|
||||
Failures: failureList,
|
||||
DurationSeconds: durationSeconds,
|
||||
PackageFailureCount: len(packageFailures),
|
||||
MalformedLineWarning: malformedWarnings,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func bufferFor(buffers map[testKey]*tailBuffer, key testKey, maxOutputBytes int) *tailBuffer {
|
||||
buffer := buffers[key]
|
||||
if buffer == nil {
|
||||
buffer = &tailBuffer{maxBytes: maxOutputBytes}
|
||||
buffers[key] = buffer
|
||||
}
|
||||
return buffer
|
||||
}
|
||||
|
||||
func (b *tailBuffer) Append(output string) {
|
||||
if b.maxBytes == 0 || output == "" {
|
||||
return
|
||||
}
|
||||
b.value += output
|
||||
if len(b.value) > b.maxBytes {
|
||||
b.value = b.value[len(b.value)-b.maxBytes:]
|
||||
}
|
||||
}
|
||||
|
||||
func (b *tailBuffer) String() string {
|
||||
return strings.ToValidUTF8(b.value, "")
|
||||
}
|
||||
|
||||
func renderMarkdown(result summary, maxFailures int, failuresOut string, githubJob string) string {
|
||||
failures := result.Failures
|
||||
visibleFailures := failures
|
||||
if maxFailures > 0 && len(failures) > maxFailures {
|
||||
visibleFailures = failures[:maxFailures]
|
||||
}
|
||||
packageNames := map[string]struct{}{}
|
||||
for _, item := range failures {
|
||||
packageNames[item.Package] = struct{}{}
|
||||
}
|
||||
|
||||
var builder strings.Builder
|
||||
writeBuilderf(&builder, "## Go test failures (%d in %d packages)\n\n", len(failures), len(packageNames))
|
||||
writeBuilderf(&builder, "Duration: %s · Packages with failures: %d", formatSeconds(result.DurationSeconds), result.PackageFailureCount)
|
||||
if githubJob != "" {
|
||||
writeBuilderf(&builder, " · Job: %s", escapeMarkdownLine(githubJob))
|
||||
}
|
||||
writeBuilderString(&builder, "\n\n")
|
||||
writeBuilderString(&builder, "| Package | Test | Elapsed |\n")
|
||||
writeBuilderString(&builder, "|---|---|---|\n")
|
||||
for _, item := range visibleFailures {
|
||||
writeBuilderf(&builder, "| %s | %s | %s |\n",
|
||||
escapeTableCell(item.Package),
|
||||
escapeTableCell(item.Test),
|
||||
formatSeconds(item.Elapsed),
|
||||
)
|
||||
}
|
||||
writeBuilderString(&builder, "\n")
|
||||
|
||||
for _, item := range visibleFailures {
|
||||
output := item.Output
|
||||
if output == "" {
|
||||
output = "No output recorded."
|
||||
}
|
||||
output = strings.ReplaceAll(strings.ToValidUTF8(output, ""), "```", "``")
|
||||
writeBuilderf(&builder, "<details>\n<summary><code>%s</code> :: <code>%s</code> (%s)</summary>\n\n",
|
||||
html.EscapeString(item.Package),
|
||||
html.EscapeString(item.Test),
|
||||
formatSeconds(item.Elapsed),
|
||||
)
|
||||
writeBuilderString(&builder, "```text\n")
|
||||
writeBuilderString(&builder, output)
|
||||
if !strings.HasSuffix(output, "\n") {
|
||||
writeBuilderString(&builder, "\n")
|
||||
}
|
||||
writeBuilderString(&builder, "```\n\n</details>\n\n")
|
||||
}
|
||||
|
||||
if omitted := len(failures) - len(visibleFailures); omitted > 0 {
|
||||
writeBuilderf(&builder, "_... and %d more failed tests omitted.", omitted)
|
||||
if failuresOut != "" {
|
||||
writeBuilderString(&builder, " Download the failures-only artifact for the full list.")
|
||||
}
|
||||
writeBuilderString(&builder, "_\n")
|
||||
}
|
||||
return builder.String()
|
||||
}
|
||||
|
||||
func writeBuilderf(builder *strings.Builder, format string, args ...any) {
|
||||
_, _ = fmt.Fprintf(builder, format, args...)
|
||||
}
|
||||
|
||||
func writef(writer io.Writer, format string, args ...any) {
|
||||
_, _ = fmt.Fprintf(writer, format, args...)
|
||||
}
|
||||
|
||||
func writeBuilderString(builder *strings.Builder, value string) {
|
||||
_, _ = builder.WriteString(value)
|
||||
}
|
||||
|
||||
func writeFailuresNDJSON(path string, failures []failure, capBytes int) error {
|
||||
var output bytes.Buffer
|
||||
for index, item := range failures {
|
||||
recordLine, err := marshalRecord(failureRecord{
|
||||
Package: item.Package,
|
||||
Test: item.Test,
|
||||
ElapsedS: item.Elapsed,
|
||||
Output: strings.ToValidUTF8(item.Output, ""),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if output.Len()+len(recordLine) <= capBytes {
|
||||
_, _ = output.Write(recordLine)
|
||||
continue
|
||||
}
|
||||
|
||||
remainingAfterCurrent := len(failures) - index - 1
|
||||
summaryLine, err := marshalRecord(truncationRecord{Truncated: true, RemainingFailures: remainingAfterCurrent})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
availableForRecord := capBytes - output.Len()
|
||||
if remainingAfterCurrent > 0 {
|
||||
availableForRecord -= len(summaryLine)
|
||||
}
|
||||
truncatedLine, ok, err := truncateFailureRecord(item, availableForRecord)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if ok {
|
||||
_, _ = output.Write(truncatedLine)
|
||||
if remainingAfterCurrent > 0 && output.Len()+len(summaryLine) <= capBytes {
|
||||
_, _ = output.Write(summaryLine)
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
summaryLine, err = marshalRecord(truncationRecord{Truncated: true, RemainingFailures: len(failures) - index})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if output.Len()+len(summaryLine) <= capBytes {
|
||||
_, _ = output.Write(summaryLine)
|
||||
}
|
||||
break
|
||||
}
|
||||
return os.WriteFile(path, output.Bytes(), 0o600)
|
||||
}
|
||||
|
||||
type failureRecord struct {
|
||||
Package string `json:"package"`
|
||||
Test string `json:"test"`
|
||||
ElapsedS float64 `json:"elapsed_s"`
|
||||
Output string `json:"output"`
|
||||
OutputTruncated bool `json:"output_truncated,omitempty"`
|
||||
}
|
||||
|
||||
type truncationRecord struct {
|
||||
Truncated bool `json:"truncated"`
|
||||
RemainingFailures int `json:"remaining_failures"`
|
||||
}
|
||||
|
||||
func truncateFailureRecord(item failure, capBytes int) ([]byte, bool, error) {
|
||||
if capBytes <= 0 {
|
||||
return nil, false, nil
|
||||
}
|
||||
output := []byte(item.Output)
|
||||
low, high := 0, len(output)
|
||||
var best []byte
|
||||
for low <= high {
|
||||
mid := low + (high-low)/2
|
||||
recordLine, err := marshalRecord(failureRecord{
|
||||
Package: item.Package,
|
||||
Test: item.Test,
|
||||
ElapsedS: item.Elapsed,
|
||||
Output: strings.ToValidUTF8(string(output[:mid]), ""),
|
||||
OutputTruncated: true,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if len(recordLine) <= capBytes {
|
||||
best = slices.Clone(recordLine)
|
||||
low = mid + 1
|
||||
continue
|
||||
}
|
||||
high = mid - 1
|
||||
}
|
||||
if best == nil {
|
||||
return nil, false, nil
|
||||
}
|
||||
return best, true, nil
|
||||
}
|
||||
|
||||
func marshalRecord(record any) ([]byte, error) {
|
||||
line, err := json.Marshal(record)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
line = append(line, '\n')
|
||||
return line, nil
|
||||
}
|
||||
|
||||
func stripANSI(output string) string {
|
||||
return ansiEscapePattern.ReplaceAllString(output, "")
|
||||
}
|
||||
|
||||
func displayTestName(name string) string {
|
||||
if name == "" {
|
||||
return "(package)"
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
func formatSeconds(seconds float64) string {
|
||||
return fmt.Sprintf("%.2fs", seconds)
|
||||
}
|
||||
|
||||
func escapeTableCell(value string) string {
|
||||
value = strings.ReplaceAll(value, "|", `\|`)
|
||||
value = strings.NewReplacer("\r", " ", "\n", " ", "`", "`").Replace(value)
|
||||
return html.EscapeString(value)
|
||||
}
|
||||
|
||||
func escapeMarkdownLine(value string) string {
|
||||
return strings.NewReplacer("\r", " ", "\n", " ").Replace(value)
|
||||
}
|
||||
|
||||
func cmpString(value, fallback string) string {
|
||||
if value == "" {
|
||||
return fallback
|
||||
}
|
||||
return value
|
||||
}
|
||||
@@ -0,0 +1,235 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestRunEmptyInputWritesNoMarkdownAndEmptyFailures(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
jsonFile := filepath.Join(dir, "go-test.json")
|
||||
failuresFile := filepath.Join(dir, "failures.ndjson")
|
||||
require.NoError(t, os.WriteFile(jsonFile, nil, 0o600))
|
||||
|
||||
var stdout bytes.Buffer
|
||||
err := run(context.Background(), config{
|
||||
JSONFile: jsonFile,
|
||||
MarkdownOut: "-",
|
||||
FailuresOut: failuresFile,
|
||||
MaxOutputBytes: 8192,
|
||||
}, &stdout, ioDiscard{}, emptyEnv)
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, stdout.String())
|
||||
assertFileContent(t, failuresFile, "")
|
||||
}
|
||||
|
||||
func TestRunPassingInputWritesNoMarkdown(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
jsonFile := writeEvents(t,
|
||||
testEvent{Action: "output", Package: "example.com/pkg", Test: "TestOK", Output: "ok\n"},
|
||||
testEvent{Action: "pass", Package: "example.com/pkg", Test: "TestOK", Elapsed: 0.01},
|
||||
testEvent{Action: "pass", Package: "example.com/pkg", Elapsed: 0.02},
|
||||
)
|
||||
failuresFile := filepath.Join(t.TempDir(), "failures.ndjson")
|
||||
|
||||
var stdout bytes.Buffer
|
||||
err := run(context.Background(), config{
|
||||
JSONFile: jsonFile,
|
||||
MarkdownOut: "-",
|
||||
FailuresOut: failuresFile,
|
||||
MaxOutputBytes: 8192,
|
||||
}, &stdout, ioDiscard{}, emptyEnv)
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, stdout.String())
|
||||
assertFileContent(t, failuresFile, "")
|
||||
}
|
||||
|
||||
func TestRunSingleFailureRendersBoundedOutput(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
jsonFile := writeEvents(t,
|
||||
testEvent{Action: "output", Package: "example.com/pkg", Test: "TestFail", Output: "prefix-" + strings.Repeat("x", 20)},
|
||||
testEvent{Action: "fail", Package: "example.com/pkg", Test: "TestFail", Elapsed: 1.25},
|
||||
testEvent{Action: "fail", Package: "example.com/pkg", Elapsed: 1.50},
|
||||
)
|
||||
|
||||
markdown := runMarkdown(t, jsonFile, config{MaxOutputBytes: 10})
|
||||
require.Contains(t, markdown, "## Go test failures (2 in 1 packages)")
|
||||
require.Contains(t, markdown, "| example.com/pkg | TestFail | 1.25s |")
|
||||
require.NotContains(t, markdown, "prefix")
|
||||
require.Contains(t, markdown, strings.Repeat("x", 10))
|
||||
}
|
||||
|
||||
func TestRunSubtestFailureCapturesSlashName(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
jsonFile := writeEvents(t,
|
||||
testEvent{Action: "output", Package: "example.com/pkg", Test: "TestParent/subcase", Output: "subtest failed\n"},
|
||||
testEvent{Action: "fail", Package: "example.com/pkg", Test: "TestParent/subcase", Elapsed: 0.20},
|
||||
)
|
||||
|
||||
markdown := runMarkdown(t, jsonFile, config{MaxOutputBytes: 8192})
|
||||
require.Contains(t, markdown, "TestParent/subcase")
|
||||
require.Contains(t, markdown, "subtest failed")
|
||||
}
|
||||
|
||||
func TestRunRerunPassRemovesPriorFailure(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
jsonFile := writeEvents(t,
|
||||
testEvent{Action: "output", Package: "example.com/pkg", Test: "TestFlake", Output: "first run failed\n"},
|
||||
testEvent{Action: "fail", Package: "example.com/pkg", Test: "TestFlake", Elapsed: 0.10},
|
||||
testEvent{Action: "output", Package: "example.com/pkg", Test: "TestFlake", Output: "retry passed\n"},
|
||||
testEvent{Action: "pass", Package: "example.com/pkg", Test: "TestFlake", Elapsed: 0.05},
|
||||
)
|
||||
|
||||
markdown := runMarkdown(t, jsonFile, config{MaxOutputBytes: 8192})
|
||||
require.Empty(t, markdown)
|
||||
}
|
||||
|
||||
func TestRunStripsANSIOutput(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
jsonFile := writeEvents(t,
|
||||
testEvent{Action: "output", Package: "example.com/pkg", Test: "TestFail", Output: "\x1b[31mred\x1b[0m\n"},
|
||||
testEvent{Action: "fail", Package: "example.com/pkg", Test: "TestFail", Elapsed: 0.10},
|
||||
)
|
||||
|
||||
markdown := runMarkdown(t, jsonFile, config{MaxOutputBytes: 8192})
|
||||
require.Contains(t, markdown, "red")
|
||||
require.NotContains(t, markdown, "\x1b")
|
||||
}
|
||||
|
||||
func TestRunEscapesTripleBackticksInOutput(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
jsonFile := writeEvents(t,
|
||||
testEvent{Action: "output", Package: "example.com/pkg", Test: "TestFail", Output: "before ``` after\n"},
|
||||
testEvent{Action: "fail", Package: "example.com/pkg", Test: "TestFail", Elapsed: 0.10},
|
||||
)
|
||||
|
||||
markdown := runMarkdown(t, jsonFile, config{MaxOutputBytes: 8192})
|
||||
require.Contains(t, markdown, "before `` after")
|
||||
require.Equal(t, 2, strings.Count(markdown, "```"))
|
||||
}
|
||||
|
||||
func TestRunMaxFailuresAddsOmittedLine(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
jsonFile := writeEvents(t,
|
||||
testEvent{Action: "fail", Package: "example.com/pkg", Test: "TestA", Elapsed: 0.10},
|
||||
testEvent{Action: "fail", Package: "example.com/pkg", Test: "TestB", Elapsed: 0.20},
|
||||
)
|
||||
|
||||
markdown := runMarkdown(t, jsonFile, config{
|
||||
MaxOutputBytes: 8192,
|
||||
MaxFailures: 1,
|
||||
FailuresOut: filepath.Join(t.TempDir(), "failures.ndjson"),
|
||||
})
|
||||
require.Contains(t, markdown, "TestA")
|
||||
require.NotContains(t, markdown, "<code>TestB</code>")
|
||||
require.Contains(t, markdown, "_... and 1 more failed tests omitted. Download the failures-only artifact for the full list._")
|
||||
}
|
||||
|
||||
func TestWriteFailuresNDJSONAppliesCap(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "failures.ndjson")
|
||||
failures := []failure{
|
||||
{Package: "example.com/pkg", Test: "TestA", Elapsed: 0.10, Output: strings.Repeat("a", 1000)},
|
||||
{Package: "example.com/pkg", Test: "TestB", Elapsed: 0.20, Output: "second"},
|
||||
}
|
||||
summaryLine, err := marshalRecord(truncationRecord{Truncated: true, RemainingFailures: 1})
|
||||
require.NoError(t, err)
|
||||
minimumLine, err := marshalRecord(failureRecord{
|
||||
Package: failures[0].Package,
|
||||
Test: failures[0].Test,
|
||||
ElapsedS: failures[0].Elapsed,
|
||||
Output: "",
|
||||
OutputTruncated: true,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
capBytes := len(summaryLine) + len(minimumLine) + 20
|
||||
|
||||
require.NoError(t, writeFailuresNDJSON(path, failures, capBytes))
|
||||
content, err := os.ReadFile(path)
|
||||
require.NoError(t, err)
|
||||
require.LessOrEqual(t, len(content), capBytes)
|
||||
|
||||
lines := strings.Split(strings.TrimSpace(string(content)), "\n")
|
||||
require.Len(t, lines, 2)
|
||||
var first map[string]any
|
||||
require.NoError(t, json.Unmarshal([]byte(lines[0]), &first))
|
||||
require.Equal(t, true, first["output_truncated"])
|
||||
require.Equal(t, "TestA", first["test"])
|
||||
require.Less(t, len(first["output"].(string)), 1000)
|
||||
var second map[string]any
|
||||
require.NoError(t, json.Unmarshal([]byte(lines[1]), &second))
|
||||
require.Equal(t, true, second["truncated"])
|
||||
require.Equal(t, float64(1), second["remaining_failures"])
|
||||
}
|
||||
|
||||
func TestRunPackageLevelFailure(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
jsonFile := writeEvents(t,
|
||||
testEvent{Action: "output", Package: "example.com/pkg", Output: "setup failed\n"},
|
||||
testEvent{Action: "fail", Package: "example.com/pkg", Elapsed: 0.30},
|
||||
)
|
||||
|
||||
markdown := runMarkdown(t, jsonFile, config{MaxOutputBytes: 8192})
|
||||
require.Contains(t, markdown, "(package)")
|
||||
require.Contains(t, markdown, "setup failed")
|
||||
}
|
||||
|
||||
func runMarkdown(t *testing.T, jsonFile string, cfg config) string {
|
||||
t.Helper()
|
||||
cfg.JSONFile = jsonFile
|
||||
cfg.MarkdownOut = "-"
|
||||
if cfg.MaxOutputBytes == 0 {
|
||||
cfg.MaxOutputBytes = 8192
|
||||
}
|
||||
var stdout bytes.Buffer
|
||||
err := run(context.Background(), cfg, &stdout, ioDiscard{}, emptyEnv)
|
||||
require.NoError(t, err)
|
||||
return stdout.String()
|
||||
}
|
||||
|
||||
func writeEvents(t *testing.T, events ...testEvent) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "go-test.json")
|
||||
var content strings.Builder
|
||||
for _, event := range events {
|
||||
line, err := json.Marshal(event)
|
||||
require.NoError(t, err)
|
||||
_, _ = content.Write(line)
|
||||
_ = content.WriteByte('\n')
|
||||
}
|
||||
require.NoError(t, os.WriteFile(path, []byte(content.String()), 0o600))
|
||||
return path
|
||||
}
|
||||
|
||||
func assertFileContent(t *testing.T, path string, expected string) {
|
||||
t.Helper()
|
||||
content, err := os.ReadFile(path)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, expected, string(content))
|
||||
}
|
||||
|
||||
func emptyEnv(string) string { return "" }
|
||||
|
||||
type ioDiscard struct{}
|
||||
|
||||
func (ioDiscard) Write(p []byte) (int, error) { return len(p), nil }
|
||||
Reference in New Issue
Block a user