mirror of
https://github.com/coder/coder.git
synced 2026-06-02 20:48:20 +00:00
85792d08bc
This PR adds an opinionated harness-engineering layer for agent-driven workflows: a small set of agent-readable docs, mechanical structure checks, structured CI failure summaries, an architecture-lint umbrella, and per-worktree dev-server isolation. The goal is to make local dev, tests, and CI mechanically inspectable by agents without changing app runtime behavior. ## What landed **Agent docs and navigation** - `.claude/docs/OBSERVABILITY.md`, `.claude/docs/DEV_ISOLATION.md`, `.claude/docs/AGENT_FAILURES.md`: task-oriented guides for logs, tracing, Prometheus, dev-server isolation, and a seeded failure catalog. - `AGENTS.md`: added an `Agent navigation` block, then trimmed the file from 375 to 229 lines by migrating duplicated detail into `WORKFLOWS.md`, `GO.md`, `TESTING.md`, and `DATABASE.md`. The user-managed custom-instructions block is preserved. - `.agents/docs`: symlink mirror of `.claude/docs` for agent runtimes that look under `.agents`. **Mechanical checks** - `scripts/check_agents_structure.sh`: validates `@...` references in tracked `AGENTS.md` files and warns when root grows past 600 lines. Wired as `make lint/agents` and into `make lint`. - `scripts/audit-agent-readiness.sh`: report-first audit of harness readiness. Currently `10 ok, 0 warn, 0 fail`. - `scripts/check_architecture.sh` / `make lint/architecture`: umbrella architecture-lint target. Consolidates the existing `check_enterprise_imports.sh` and `check_codersdk_imports.sh` so they run exactly once via the umbrella. Slot is open for new high-confidence rules. **Structured CI failure summaries** - `scripts/playwright-failure-summary.sh`: parses `site/test-results/results.json` and writes Markdown to `$GITHUB_STEP_SUMMARY` on failure. Wired into the `test-e2e` matrix job. - `scripts/go-test-failure-summary.sh`: parses `go test -json` line-delimited output the same way. Wired into `test-go-pg`, `test-go-pg-17`, and `test-go-race-pg` by injecting `gotestsum --jsonfile` in the workflow without touching `Makefile`. JSON also uploaded as a CI artifact on failure. - `site/e2e/playwright.config.ts`: enables `screenshot: only-on-failure`, `trace: retain-on-failure`, JSON reporter, and HTML reporter alongside existing reporters. - `.github/workflows/ci.yaml`: failure artifact uploads for Playwright now use `if: failure()` and predictable names (`playwright-artifacts-<variant>-<sha>`). **Per-worktree dev-server isolation** (`scripts/develop/main.go`) - Deterministic FNV-64a hash of the worktree path produces a port offset in `[0, 1000)` (50 buckets, step 20 to avoid API/proxy overlap across adjacent buckets). - Offset is applied only to defaults; both env vars (`CODER_DEV_PORT`, `CODER_DEV_WEB_PORT`, `CODER_DEV_PROXY_PORT`, `CODER_DEV_PROMETHEUS_PORT`) and CLI flags retain priority. - Hardcoded ports `9090` (embedded Prometheus UI) and `12345` (Delve) are unchanged by design. - Startup banner shows each port's source: `default`, `offset`, or `explicit`. - Unit tests in `scripts/develop/main_test.go` cover determinism, bounds, no-overlap across the four ports, and explicit-skip behavior. - State (`.coderv2/`) was already worktree-isolated via `os.Getwd()`, so no state-dir changes were needed. ## Validation `make lint/agents`, `make lint/architecture`, `make lint/emdash`, `bash scripts/audit-agent-readiness.sh` (10 ok, 0 warn, 0 fail), `shellcheck` on all 5 new scripts, `go test ./scripts/develop/...`, and `js-yaml` parse of `ci.yaml` all pass. Synthetic fixtures verify both failure-summary scripts handle empty/missing input (silent exit 0), ANSI-stripped output, and parent/subtest formatting. ## Known follow-ups (deferred) - Frontend Storybook/Vitest failure summary: lowest-leverage slice of the failure-summary work. Skipping until observed pain. - Architecture lint currently only delegates to existing import checks; new rules (`InTx` outer-store detection, swagger-annotation lint) plug in as needed. - 50 port-offset buckets means two worktree paths can occasionally collide. The DEV_ISOLATION doc tells users to set the relevant env var when this happens. > Mux opened this PR on Mike's behalf.
101 lines
2.4 KiB
Bash
Executable File
101 lines
2.4 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
# Summarize failed Go tests from go test JSON output.
|
|
|
|
set -euo pipefail
|
|
# shellcheck source=scripts/lib.sh
|
|
# shellcheck disable=SC1091
|
|
source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
|
|
cdroot
|
|
|
|
if [[ $# -ne 1 ]]; then
|
|
error "Usage: go-test-failure-summary.sh <go-test.json>"
|
|
fi
|
|
|
|
results_file=$1
|
|
if [[ ! -s "$results_file" ]]; then
|
|
exit 0
|
|
fi
|
|
|
|
if ! command -v jq >/dev/null; then
|
|
error "jq is required to summarize Go test failures."
|
|
fi
|
|
|
|
jq -sr '
|
|
def clean_block:
|
|
tostring
|
|
| gsub("\u001b\\[[0-9;?]*[ -/]*[@-~]"; "")
|
|
| gsub("```"; "``");
|
|
def clean_inline:
|
|
tostring | gsub("`"; "") | gsub("[\r\n]"; " ");
|
|
def truncate($max):
|
|
if length > $max then .[0:$max] + "..." else . end;
|
|
def terminal_action:
|
|
.Action == "pass" or .Action == "fail" or .Action == "skip";
|
|
def test_key:
|
|
(.Package // "") + "\u0000" + (.Test // "");
|
|
def output_for($events; $package; $test):
|
|
[
|
|
$events[]
|
|
| select(.Action == "output")
|
|
| select((.Package // "") == $package)
|
|
| select((.Test // "") == $test)
|
|
| .Output // ""
|
|
]
|
|
| join("")
|
|
| clean_block
|
|
| if . == "" then "No output recorded." else . end
|
|
| truncate(600);
|
|
|
|
map(select(type == "object")) as $events
|
|
| [
|
|
$events
|
|
| to_entries[]
|
|
| .value + {idx: .key}
|
|
| select((.Test // "") != "")
|
|
| select(terminal_action)
|
|
] as $terminal_tests
|
|
| [
|
|
$terminal_tests
|
|
| group_by(test_key)
|
|
| .[]
|
|
| max_by(.idx)
|
|
| select(.Action == "fail")
|
|
| {
|
|
package: ((.Package // "unknown") | clean_inline),
|
|
test: ((.Test // "unknown") | clean_inline),
|
|
elapsed: (.Elapsed // 0),
|
|
output: output_for($events; (.Package // ""); (.Test // ""))
|
|
}
|
|
] as $failures
|
|
| if ($failures | length) == 0 then
|
|
empty
|
|
else
|
|
($failures | length) as $failed
|
|
| ($failures | map(.package) | unique | length) as $packages
|
|
| ([
|
|
$events[]
|
|
| select((.Test // "") == "")
|
|
| select(.Action == "pass" or .Action == "fail")
|
|
| .Elapsed // 0
|
|
] | add // 0) as $duration
|
|
| ([
|
|
$events[]
|
|
| select((.Test // "") == "")
|
|
| select(.Action == "fail")
|
|
| .Package // empty
|
|
] | unique | length) as $package_failures
|
|
| [
|
|
"## Go test failures (\($failed) in \($packages))",
|
|
"- Duration: \($duration)s",
|
|
"- Package failures: \($package_failures)",
|
|
"",
|
|
($failures[]
|
|
| "### \(.package) :: \(.test)\n"
|
|
+ "- Elapsed: \(.elapsed)s\n\n"
|
|
+ "```\n\(.output)\n```\n")
|
|
]
|
|
| join("\n")
|
|
end
|
|
' "$results_file"
|