mirror of
https://github.com/coder/coder.git
synced 2026-06-02 20:48:20 +00:00
85792d08bc
This PR adds an opinionated harness-engineering layer for agent-driven workflows: a small set of agent-readable docs, mechanical structure checks, structured CI failure summaries, an architecture-lint umbrella, and per-worktree dev-server isolation. The goal is to make local dev, tests, and CI mechanically inspectable by agents without changing app runtime behavior. ## What landed **Agent docs and navigation** - `.claude/docs/OBSERVABILITY.md`, `.claude/docs/DEV_ISOLATION.md`, `.claude/docs/AGENT_FAILURES.md`: task-oriented guides for logs, tracing, Prometheus, dev-server isolation, and a seeded failure catalog. - `AGENTS.md`: added an `Agent navigation` block, then trimmed the file from 375 to 229 lines by migrating duplicated detail into `WORKFLOWS.md`, `GO.md`, `TESTING.md`, and `DATABASE.md`. The user-managed custom-instructions block is preserved. - `.agents/docs`: symlink mirror of `.claude/docs` for agent runtimes that look under `.agents`. **Mechanical checks** - `scripts/check_agents_structure.sh`: validates `@...` references in tracked `AGENTS.md` files and warns when root grows past 600 lines. Wired as `make lint/agents` and into `make lint`. - `scripts/audit-agent-readiness.sh`: report-first audit of harness readiness. Currently `10 ok, 0 warn, 0 fail`. - `scripts/check_architecture.sh` / `make lint/architecture`: umbrella architecture-lint target. Consolidates the existing `check_enterprise_imports.sh` and `check_codersdk_imports.sh` so they run exactly once via the umbrella. Slot is open for new high-confidence rules. **Structured CI failure summaries** - `scripts/playwright-failure-summary.sh`: parses `site/test-results/results.json` and writes Markdown to `$GITHUB_STEP_SUMMARY` on failure. Wired into the `test-e2e` matrix job. - `scripts/go-test-failure-summary.sh`: parses `go test -json` line-delimited output the same way. Wired into `test-go-pg`, `test-go-pg-17`, and `test-go-race-pg` by injecting `gotestsum --jsonfile` in the workflow without touching `Makefile`. JSON also uploaded as a CI artifact on failure. - `site/e2e/playwright.config.ts`: enables `screenshot: only-on-failure`, `trace: retain-on-failure`, JSON reporter, and HTML reporter alongside existing reporters. - `.github/workflows/ci.yaml`: failure artifact uploads for Playwright now use `if: failure()` and predictable names (`playwright-artifacts-<variant>-<sha>`). **Per-worktree dev-server isolation** (`scripts/develop/main.go`) - Deterministic FNV-64a hash of the worktree path produces a port offset in `[0, 1000)` (50 buckets, step 20 to avoid API/proxy overlap across adjacent buckets). - Offset is applied only to defaults; both env vars (`CODER_DEV_PORT`, `CODER_DEV_WEB_PORT`, `CODER_DEV_PROXY_PORT`, `CODER_DEV_PROMETHEUS_PORT`) and CLI flags retain priority. - Hardcoded ports `9090` (embedded Prometheus UI) and `12345` (Delve) are unchanged by design. - Startup banner shows each port's source: `default`, `offset`, or `explicit`. - Unit tests in `scripts/develop/main_test.go` cover determinism, bounds, no-overlap across the four ports, and explicit-skip behavior. - State (`.coderv2/`) was already worktree-isolated via `os.Getwd()`, so no state-dir changes were needed. ## Validation `make lint/agents`, `make lint/architecture`, `make lint/emdash`, `bash scripts/audit-agent-readiness.sh` (10 ok, 0 warn, 0 fail), `shellcheck` on all 5 new scripts, `go test ./scripts/develop/...`, and `js-yaml` parse of `ci.yaml` all pass. Synthetic fixtures verify both failure-summary scripts handle empty/missing input (silent exit 0), ANSI-stripped output, and parent/subtest formatting. ## Known follow-ups (deferred) - Frontend Storybook/Vitest failure summary: lowest-leverage slice of the failure-summary work. Skipping until observed pain. - Architecture lint currently only delegates to existing import checks; new rules (`InTx` outer-store detection, swagger-annotation lint) plug in as needed. - 50 port-offset buckets means two worktree paths can occasionally collide. The DEV_ISOLATION doc tells users to set the relevant env var when this happens. > Mux opened this PR on Mike's behalf.
163 lines
4.5 KiB
TypeScript
163 lines
4.5 KiB
TypeScript
import * as path from "node:path";
|
|
import { defineConfig } from "@playwright/test";
|
|
import {
|
|
coderdPProfPort,
|
|
coderPort,
|
|
e2eFakeExperiment1,
|
|
e2eFakeExperiment2,
|
|
gitAuth,
|
|
requireTerraformTests,
|
|
} from "./constants";
|
|
|
|
export const wsEndpoint = process.env.CODER_E2E_WS_ENDPOINT;
|
|
export const retries = (() => {
|
|
if (process.env.CODER_E2E_TEST_RETRIES === undefined) {
|
|
return undefined;
|
|
}
|
|
const count = Number.parseInt(process.env.CODER_E2E_TEST_RETRIES, 10);
|
|
if (Number.isNaN(count)) {
|
|
throw new Error(
|
|
`CODER_E2E_TEST_RETRIES is not a number: ${process.env.CODER_E2E_TEST_RETRIES}`,
|
|
);
|
|
}
|
|
if (count < 0) {
|
|
throw new Error(
|
|
`CODER_E2E_TEST_RETRIES is less than 0: ${process.env.CODER_E2E_TEST_RETRIES}`,
|
|
);
|
|
}
|
|
return count;
|
|
})();
|
|
|
|
const localURL = (port: number, path: string): string => {
|
|
return `http://localhost:${port}${path}`;
|
|
};
|
|
|
|
export default defineConfig({
|
|
retries,
|
|
globalSetup: require.resolve("./setup/preflight"),
|
|
outputDir: "../test-results",
|
|
projects: [
|
|
{
|
|
name: "testsSetup",
|
|
testMatch: /setup\/.*\.spec\.ts/,
|
|
},
|
|
{
|
|
name: "tests",
|
|
testMatch: /tests\/.*\.spec\.ts/,
|
|
dependencies: ["testsSetup"],
|
|
timeout: 30_000,
|
|
},
|
|
],
|
|
reporter: [
|
|
["list"],
|
|
["html", { open: "never" }],
|
|
[
|
|
"json",
|
|
{ outputFile: path.join(__dirname, "../test-results/results.json") },
|
|
],
|
|
["./reporter.ts"],
|
|
],
|
|
use: {
|
|
actionTimeout: 5000,
|
|
baseURL: `http://localhost:${coderPort}`,
|
|
screenshot: "only-on-failure",
|
|
trace: "retain-on-failure",
|
|
video: "retain-on-failure",
|
|
...(wsEndpoint
|
|
? {
|
|
connectOptions: {
|
|
wsEndpoint: wsEndpoint,
|
|
},
|
|
}
|
|
: {
|
|
launchOptions: {
|
|
args: ["--disable-webgl"],
|
|
},
|
|
}),
|
|
},
|
|
webServer: {
|
|
url: `http://localhost:${coderPort}/api/v2/deployment/config`,
|
|
// The default timeout is 60s, but `go run` compilation with the
|
|
// embed tag can take longer on CI.
|
|
timeout: 120_000,
|
|
command: [
|
|
`go run -tags embed ${path.join(__dirname, "../../enterprise/cmd/coder")}`,
|
|
"server",
|
|
"--global-config $(mktemp -d -t e2e-XXXXXXXXXX)",
|
|
`--access-url=http://localhost:${coderPort}`,
|
|
`--http-address=0.0.0.0:${coderPort}`,
|
|
"--ephemeral",
|
|
"--telemetry=false",
|
|
"--dangerous-disable-rate-limits",
|
|
"--provisioner-daemons 10",
|
|
// TODO: Enable some terraform provisioners
|
|
`--provisioner-types=echo${requireTerraformTests ? ",terraform" : ""}`,
|
|
"--provisioner-daemons=10",
|
|
"--web-terminal-renderer=dom",
|
|
"--pprof-enable",
|
|
"--log-filter=.*",
|
|
`--log-human=${path.join(__dirname, "test-results/debug.log")}`,
|
|
]
|
|
.filter(Boolean)
|
|
.join(" "),
|
|
stdout: "pipe",
|
|
env: {
|
|
...process.env,
|
|
// Otherwise, the runner fails on Mac with: could not determine kind of name for C.uuid_string_t
|
|
CGO_ENABLED: "0",
|
|
|
|
// This is the test provider for git auth with devices!
|
|
CODER_GITAUTH_0_ID: gitAuth.deviceProvider,
|
|
CODER_GITAUTH_0_TYPE: "github",
|
|
CODER_GITAUTH_0_CLIENT_ID: "client",
|
|
CODER_GITAUTH_0_CLIENT_SECRET: "secret",
|
|
CODER_GITAUTH_0_DEVICE_FLOW: "true",
|
|
CODER_GITAUTH_0_APP_INSTALL_URL:
|
|
"https://github.com/apps/coder/installations/new",
|
|
CODER_GITAUTH_0_APP_INSTALLATIONS_URL: localURL(
|
|
gitAuth.devicePort,
|
|
gitAuth.installationsPath,
|
|
),
|
|
CODER_GITAUTH_0_TOKEN_URL: localURL(
|
|
gitAuth.devicePort,
|
|
gitAuth.tokenPath,
|
|
),
|
|
CODER_GITAUTH_0_DEVICE_CODE_URL: localURL(
|
|
gitAuth.devicePort,
|
|
gitAuth.codePath,
|
|
),
|
|
CODER_GITAUTH_0_VALIDATE_URL: localURL(
|
|
gitAuth.devicePort,
|
|
gitAuth.validatePath,
|
|
),
|
|
|
|
CODER_GITAUTH_1_ID: gitAuth.webProvider,
|
|
CODER_GITAUTH_1_TYPE: "github",
|
|
CODER_GITAUTH_1_CLIENT_ID: "client",
|
|
CODER_GITAUTH_1_CLIENT_SECRET: "secret",
|
|
CODER_GITAUTH_1_AUTH_URL: localURL(gitAuth.webPort, gitAuth.authPath),
|
|
CODER_GITAUTH_1_TOKEN_URL: localURL(gitAuth.webPort, gitAuth.tokenPath),
|
|
CODER_GITAUTH_1_DEVICE_CODE_URL: localURL(
|
|
gitAuth.webPort,
|
|
gitAuth.codePath,
|
|
),
|
|
CODER_GITAUTH_1_VALIDATE_URL: localURL(
|
|
gitAuth.webPort,
|
|
gitAuth.validatePath,
|
|
),
|
|
CODER_PPROF_ADDRESS: `127.0.0.1:${coderdPProfPort}`,
|
|
CODER_EXPERIMENTS: `${e2eFakeExperiment1},${e2eFakeExperiment2}`,
|
|
|
|
// Tests for Deployment / User Authentication / OIDC
|
|
CODER_OIDC_ISSUER_URL: "https://accounts.google.com",
|
|
CODER_OIDC_EMAIL_DOMAIN: "coder.com",
|
|
CODER_OIDC_CLIENT_ID: "1234567890",
|
|
CODER_OIDC_CLIENT_SECRET: "1234567890Secret",
|
|
CODER_OIDC_ALLOW_SIGNUPS: "false",
|
|
CODER_OIDC_SIGN_IN_TEXT: "Hello",
|
|
CODER_OIDC_ICON_URL: "/icon/google.svg",
|
|
},
|
|
reuseExistingServer: false,
|
|
},
|
|
});
|