From a6a8fd94d7fd7e1232f3c0b10daaaa2e96a8ae0e Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Thu, 5 Mar 2026 13:58:10 +0200 Subject: [PATCH] build(Makefile): enable parallel `make -j gen` with correct dependency graph (#22612) `make gen` could not run with `-j` because inter-target dependency edges were missing. Multiple recipes compile `coderd/rbac` (which includes generated files like `object_gen.go`), and without explicit ordering, parallel runs produced syntax errors from mid-write reads. Three main changes: **Dependency graph fixes** declare the compile-time chain through `coderd/rbac` so that `object_gen.go` is written before anything that imports it is compiled. The DB generation targets use a GNU Make 4.3+ grouped target (`&:`) so Make knows `generate.sh` co-produces `querier.go`, `unique_constraint.go`, `dbmetrics`, and `dbauthz` in a single invocation. `SKIP_DUMP_SQL=1` avoids re-entrant `make` inside `generate.sh` when the Makefile already guarantees `dump.sql` is fresh. **`scripts/atomicwrite` package** replaces `os.WriteFile` in all gen scripts with a temp-file-in-same-dir + rename pattern, preventing interrupted runs from leaving partial files. **`.PRECIOUS` and shell atomic writes** protect git-tracked generated files from Make's default delete-on-error behavior. Since these files are committed, deletion is worse than staleness -- `git restore` is the recovery path. CI now runs `make -j --output-sync -B gen` (~32s, down from ~85s serial). | Scenario | Before | After | |-----------------------------------|--------------------|----------| | `make gen` (serial) | 95s | 95s | | `make -j gen` (parallel) | race error | **22s** | | CI `make -j --output-sync -B gen` | forced serial ~85s | **~32s** | --- .github/workflows/ci.yaml | 4 +- Makefile | 170 +++++++++++++++++++------- coderd/database/gen/dump/main.go | 3 +- coderd/database/generate.sh | 14 ++- scripts/apidocgen/generate.sh | 11 +- scripts/apidocgen/postprocess/main.go | 8 +- scripts/apidocgen/swaginit/main.go | 8 +- scripts/atomicwrite/atomicwrite.go | 32 +++++ scripts/auditdocgen/main.go | 5 +- scripts/clidocgen/gen.go | 14 +-- scripts/clidocgen/main.go | 8 +- scripts/dbgen/constraint.go | 4 +- scripts/dbgen/main.go | 4 +- scripts/gensite/generate_icon_list.go | 17 +-- scripts/metricsdocgen/main.go | 10 +- 15 files changed, 218 insertions(+), 94 deletions(-) create mode 100644 scripts/atomicwrite/atomicwrite.go diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 48ca6ade61..58203e2123 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -315,9 +315,7 @@ jobs: # Notifications require DB, we could start a DB instance here but # let's just restore for now. git checkout -- coderd/notifications/testdata/rendered-templates - # no `-j` flag as `make` fails with: - # coderd/rbac/object_gen.go:1:1: syntax error: package statement must be first - make --output-sync -B gen + make -j --output-sync -B gen - name: Check for unstaged files run: ./scripts/check_unstaged.sh diff --git a/Makefile b/Makefile index 6202ab0211..3095ad07d0 100644 --- a/Makefile +++ b/Makefile @@ -23,6 +23,33 @@ SHELL := bash # See https://stackoverflow.com/questions/25752543/make-delete-on-error-for-directory-targets .DELETE_ON_ERROR: +# Protect git-tracked generated files from deletion on interrupt. +# .DELETE_ON_ERROR is desirable for most targets but for files that +# are committed to git and serve as inputs to other rules, deletion +# is worse than a stale file — `git restore` is the recovery path. +.PRECIOUS: \ + coderd/database/dump.sql \ + coderd/database/querier.go \ + coderd/database/unique_constraint.go \ + coderd/database/dbmetrics/querymetrics.go \ + coderd/database/dbauthz/dbauthz.go \ + site/src/api/typesGenerated.ts \ + site/e2e/provisionerGenerated.ts \ + site/src/api/chatModelOptionsGenerated.json \ + site/src/api/rbacresourcesGenerated.ts \ + site/src/api/countriesGenerated.ts \ + site/src/theme/icons.json \ + examples/examples.gen.json \ + docs/manifest.json \ + docs/admin/integrations/prometheus.md \ + docs/admin/security/audit-logs.md \ + docs/reference/cli/index.md \ + coderd/apidoc/swagger.json \ + coderd/rbac/object_gen.go \ + coderd/rbac/scopes_constants_gen.go \ + codersdk/rbacresources_gen.go \ + codersdk/apikey_scopes_gen.go + # Don't print the commands in the file unless you specify VERBOSE. This is # essentially the same as putting "@" at the start of each line. ifndef VERBOSE @@ -613,7 +640,7 @@ DB_GEN_FILES := \ coderd/database/dump.sql \ coderd/database/querier.go \ coderd/database/unique_constraint.go \ - coderd/database/dbmetrics/dbmetrics.go \ + coderd/database/dbmetrics/querymetrics.go \ coderd/database/dbauthz/dbauthz.go \ coderd/database/dbmock/dbmock.go @@ -648,6 +675,7 @@ GEN_FILES := \ coderd/apidoc/swagger.json \ docs/manifest.json \ provisioner/terraform/testdata/version \ + scripts/metricsdocgen/generated_metrics \ site/e2e/provisionerGenerated.ts \ examples/examples.gen.json \ $(TAILNETTEST_MOCKS) \ @@ -691,11 +719,17 @@ gen/mark-fresh: vpn/vpn.pb.go \ enterprise/aibridged/proto/aibridged.pb.go \ coderd/database/dump.sql \ - $(DB_GEN_FILES) \ + coderd/database/querier.go \ + coderd/database/unique_constraint.go \ + coderd/database/dbmetrics/querymetrics.go \ + coderd/database/dbauthz/dbauthz.go \ + coderd/database/dbmock/dbmock.go \ + coderd/database/pubsub/psmock/psmock.go \ site/src/api/typesGenerated.ts \ coderd/rbac/object_gen.go \ codersdk/rbacresources_gen.go \ coderd/rbac/scopes_constants_gen.go \ + codersdk/apikey_scopes_gen.go \ site/src/api/rbacresourcesGenerated.ts \ site/src/api/countriesGenerated.ts \ site/src/api/chatModelOptionsGenerated.json \ @@ -707,8 +741,8 @@ gen/mark-fresh: site/e2e/provisionerGenerated.ts \ site/src/theme/icons.json \ examples/examples.gen.json \ + scripts/metricsdocgen/generated_metrics \ $(TAILNETTEST_MOCKS) \ - coderd/database/pubsub/psmock/psmock.go \ agent/agentcontainers/acmock/acmock.go \ agent/agentcontainers/dcspec/dcspec_gen.go \ coderd/httpmw/loggermw/loggermock/loggermock.go \ @@ -737,9 +771,19 @@ coderd/database/dump.sql: coderd/database/gen/dump/main.go $(wildcard coderd/dat # Generates Go code for querying the database. # coderd/database/queries.sql.go # coderd/database/models.go -coderd/database/querier.go: coderd/database/sqlc.yaml coderd/database/dump.sql $(wildcard coderd/database/queries/*.sql) - ./coderd/database/generate.sh - touch "$@" +# +# NOTE: grouped target (&:) ensures generate.sh runs only once even +# with -j and all outputs are considered produced together. These +# files are all written by generate.sh (via sqlc and scripts/dbgen). +coderd/database/querier.go \ +coderd/database/unique_constraint.go \ +coderd/database/dbmetrics/querymetrics.go \ +coderd/database/dbauthz/dbauthz.go &: \ + coderd/database/sqlc.yaml \ + coderd/database/dump.sql \ + $(wildcard coderd/database/queries/*.sql) + SKIP_DUMP_SQL=1 ./coderd/database/generate.sh + touch coderd/database/querier.go coderd/database/unique_constraint.go coderd/database/dbmetrics/querymetrics.go coderd/database/dbauthz/dbauthz.go coderd/database/dbmock/dbmock.go: coderd/database/db.go coderd/database/querier.go go generate ./coderd/database/dbmock/ @@ -838,23 +882,26 @@ enterprise/aibridged/proto/aibridged.pb.go: enterprise/aibridged/proto/aibridged ./enterprise/aibridged/proto/aibridged.proto site/src/api/typesGenerated.ts: site/node_modules/.installed $(wildcard scripts/apitypings/*) $(shell find ./codersdk $(FIND_EXCLUSIONS) -type f -name '*.go') - # -C sets the directory for the go run command - go run -C ./scripts/apitypings main.go > $@ - ./scripts/biome_format.sh src/api/typesGenerated.ts - touch "$@" + # Generate to a temp file, format it, then atomically move to + # the target so that an interrupt never leaves a partial or + # unformatted file in the working tree. + tmpfile=$$(mktemp -d)/$(notdir $@) && \ + go run -C ./scripts/apitypings main.go > "$$tmpfile" && \ + ./scripts/biome_format.sh "$$tmpfile" && \ + mv "$$tmpfile" "$@" site/e2e/provisionerGenerated.ts: site/node_modules/.installed provisionerd/proto/provisionerd.pb.go provisionersdk/proto/provisioner.pb.go (cd site/ && pnpm run gen:provisioner) touch "$@" site/src/theme/icons.json: site/node_modules/.installed $(wildcard scripts/gensite/*) $(wildcard site/static/icon/*) - go run ./scripts/gensite/ -icons "$@" - ./scripts/biome_format.sh src/theme/icons.json - touch "$@" + tmpfile=$$(mktemp -d)/$(notdir $@) && \ + go run ./scripts/gensite/ -icons "$$tmpfile" && \ + ./scripts/biome_format.sh "$$tmpfile" && \ + mv "$$tmpfile" "$@" examples/examples.gen.json: scripts/examplegen/main.go examples/examples.go $(shell find ./examples/templates) - go run ./scripts/examplegen/main.go > examples/examples.gen.json - touch "$@" + go run ./scripts/examplegen/main.go > "$@.tmp" && mv "$@.tmp" "$@" coderd/rbac/object_gen.go: scripts/typegen/rbacobject.gotmpl scripts/typegen/main.go coderd/rbac/object.go coderd/rbac/policy/policy.go tempdir=$(shell mktemp -d /tmp/typegen_rbac_object.XXXXXX) @@ -863,7 +910,10 @@ coderd/rbac/object_gen.go: scripts/typegen/rbacobject.gotmpl scripts/typegen/mai rmdir -v "$$tempdir" touch "$@" -coderd/rbac/scopes_constants_gen.go: scripts/typegen/scopenames.gotmpl scripts/typegen/main.go coderd/rbac/policy/policy.go +# NOTE: depends on object_gen.go because `go run` compiles +# coderd/rbac which includes it. +coderd/rbac/scopes_constants_gen.go: scripts/typegen/scopenames.gotmpl scripts/typegen/main.go coderd/rbac/policy/policy.go \ + coderd/rbac/object_gen.go # Generate typed low-level ScopeName constants from RBACPermissions # Write to a temp file first to avoid truncating the package during build # since the generator imports the rbac package. @@ -872,53 +922,72 @@ coderd/rbac/scopes_constants_gen.go: scripts/typegen/scopenames.gotmpl scripts/t mv -v "$$tempfile" coderd/rbac/scopes_constants_gen.go touch "$@" -codersdk/rbacresources_gen.go: scripts/typegen/codersdk.gotmpl scripts/typegen/main.go coderd/rbac/object.go coderd/rbac/policy/policy.go +# NOTE: depends on object_gen.go and scopes_constants_gen.go because +# `go run` compiles coderd/rbac which includes both. +codersdk/rbacresources_gen.go: scripts/typegen/codersdk.gotmpl scripts/typegen/main.go coderd/rbac/object.go coderd/rbac/policy/policy.go \ + coderd/rbac/object_gen.go coderd/rbac/scopes_constants_gen.go # Do no overwrite codersdk/rbacresources_gen.go directly, as it would make the file empty, breaking # the `codersdk` package and any parallel build targets. go run scripts/typegen/main.go rbac codersdk > /tmp/rbacresources_gen.go mv /tmp/rbacresources_gen.go codersdk/rbacresources_gen.go touch "$@" -codersdk/apikey_scopes_gen.go: scripts/apikeyscopesgen/main.go coderd/rbac/scopes_catalog.go coderd/rbac/scopes.go +# NOTE: depends on object_gen.go and scopes_constants_gen.go because +# `go run` compiles coderd/rbac which includes both. +codersdk/apikey_scopes_gen.go: scripts/apikeyscopesgen/main.go coderd/rbac/scopes_catalog.go coderd/rbac/scopes.go \ + coderd/rbac/object_gen.go coderd/rbac/scopes_constants_gen.go # Generate SDK constants for external API key scopes. go run ./scripts/apikeyscopesgen > /tmp/apikey_scopes_gen.go mv /tmp/apikey_scopes_gen.go codersdk/apikey_scopes_gen.go touch "$@" -site/src/api/rbacresourcesGenerated.ts: site/node_modules/.installed scripts/typegen/codersdk.gotmpl scripts/typegen/main.go coderd/rbac/object.go coderd/rbac/policy/policy.go - go run scripts/typegen/main.go rbac typescript > "$@" - ./scripts/biome_format.sh src/api/rbacresourcesGenerated.ts - touch "$@" +# NOTE: depends on object_gen.go and scopes_constants_gen.go because +# `go run` compiles coderd/rbac which includes both. +site/src/api/rbacresourcesGenerated.ts: site/node_modules/.installed scripts/typegen/codersdk.gotmpl scripts/typegen/main.go coderd/rbac/object.go coderd/rbac/policy/policy.go \ + coderd/rbac/object_gen.go coderd/rbac/scopes_constants_gen.go + tmpfile=$$(mktemp -d)/$(notdir $@) && \ + go run scripts/typegen/main.go rbac typescript > "$$tmpfile" && \ + ./scripts/biome_format.sh "$$tmpfile" && \ + mv "$$tmpfile" "$@" site/src/api/countriesGenerated.ts: site/node_modules/.installed scripts/typegen/countries.tstmpl scripts/typegen/main.go codersdk/countries.go - go run scripts/typegen/main.go countries > "$@" - ./scripts/biome_format.sh src/api/countriesGenerated.ts - touch "$@" + tmpfile=$$(mktemp -d)/$(notdir $@) && \ + go run scripts/typegen/main.go countries > "$$tmpfile" && \ + ./scripts/biome_format.sh "$$tmpfile" && \ + mv "$$tmpfile" "$@" site/src/api/chatModelOptionsGenerated.json: scripts/modeloptionsgen/main.go codersdk/chats.go - go run ./scripts/modeloptionsgen/main.go | tail -n +2 > "$@" - cd site && pnpm biome format --write src/api/chatModelOptionsGenerated.json + tmpfile=$$(mktemp -d)/$(notdir $@) && \ + go run ./scripts/modeloptionsgen/main.go | tail -n +2 > "$$tmpfile" && \ + ./scripts/biome_format.sh "$$tmpfile" && \ + mv "$$tmpfile" "$@" scripts/metricsdocgen/generated_metrics: $(GO_SRC_FILES) - go run ./scripts/metricsdocgen/scanner > $@ + go run ./scripts/metricsdocgen/scanner > $@.tmp && mv $@.tmp $@ docs/admin/integrations/prometheus.md: node_modules/.installed scripts/metricsdocgen/main.go scripts/metricsdocgen/metrics scripts/metricsdocgen/generated_metrics - go run scripts/metricsdocgen/main.go - pnpm exec markdownlint-cli2 --fix ./docs/admin/integrations/prometheus.md - pnpm exec markdown-table-formatter ./docs/admin/integrations/prometheus.md - touch "$@" + tmpfile=$$(mktemp -d)/$(notdir $@) && cp "$@" "$$tmpfile" && \ + go run scripts/metricsdocgen/main.go --prometheus-doc-file="$$tmpfile" && \ + pnpm exec markdownlint-cli2 --fix "$$tmpfile" && \ + pnpm exec markdown-table-formatter "$$tmpfile" && \ + mv "$$tmpfile" "$@" docs/reference/cli/index.md: node_modules/.installed scripts/clidocgen/main.go examples/examples.gen.json $(GO_SRC_FILES) - CI=true BASE_PATH="." go run ./scripts/clidocgen - pnpm exec markdownlint-cli2 --fix ./docs/reference/cli/*.md - pnpm exec markdown-table-formatter ./docs/reference/cli/*.md - touch "$@" + tmpdir=$$(mktemp -d) && \ + mkdir -p "$$tmpdir/docs/reference/cli" && \ + cp docs/manifest.json "$$tmpdir/docs/manifest.json" && \ + CI=true DOCS_DIR="$$tmpdir/docs" go run ./scripts/clidocgen && \ + pnpm exec markdownlint-cli2 --fix "$$tmpdir/docs/reference/cli/*.md" && \ + pnpm exec markdown-table-formatter "$$tmpdir/docs/reference/cli/*.md" && \ + cp "$$tmpdir/docs/reference/cli/"*.md docs/reference/cli/ && \ + rm -rf "$$tmpdir" docs/admin/security/audit-logs.md: node_modules/.installed coderd/database/querier.go scripts/auditdocgen/main.go enterprise/audit/table.go coderd/rbac/object_gen.go - go run scripts/auditdocgen/main.go - pnpm exec markdownlint-cli2 --fix ./docs/admin/security/audit-logs.md - pnpm exec markdown-table-formatter ./docs/admin/security/audit-logs.md - touch "$@" + tmpfile=$$(mktemp -d)/$(notdir $@) && cp "$@" "$$tmpfile" && \ + go run scripts/auditdocgen/main.go --audit-doc-file="$$tmpfile" && \ + pnpm exec markdownlint-cli2 --fix "$$tmpfile" && \ + pnpm exec markdown-table-formatter "$$tmpfile" && \ + mv "$$tmpfile" "$@" coderd/apidoc/.gen: \ node_modules/.installed \ @@ -934,17 +1003,26 @@ coderd/apidoc/.gen: \ scripts/apidocgen/swaginit/main.go \ $(wildcard scripts/apidocgen/postprocess/*) \ $(wildcard scripts/apidocgen/markdown-template/*) - ./scripts/apidocgen/generate.sh - pnpm exec markdownlint-cli2 --fix ./docs/reference/api/*.md - pnpm exec markdown-table-formatter ./docs/reference/api/*.md + tmpdir=$$(mktemp -d) && swagtmp=$$(mktemp -d) && \ + mkdir -p "$$tmpdir/reference/api" && \ + cp docs/manifest.json "$$tmpdir/manifest.json" && \ + SWAG_OUTPUT_DIR="$$swagtmp" APIDOCGEN_DOCS_DIR="$$tmpdir" ./scripts/apidocgen/generate.sh && \ + pnpm exec markdownlint-cli2 --fix "$$tmpdir/reference/api/*.md" && \ + pnpm exec markdown-table-formatter "$$tmpdir/reference/api/*.md" && \ + ./scripts/biome_format.sh "$$swagtmp/swagger.json" && \ + cp "$$tmpdir/reference/api/"*.md docs/reference/api/ && \ + cp "$$tmpdir/manifest.json" docs/manifest.json && \ + cp "$$swagtmp/docs.go" coderd/apidoc/docs.go && \ + cp "$$swagtmp/swagger.json" coderd/apidoc/swagger.json && \ + rm -rf "$$tmpdir" "$$swagtmp" touch "$@" docs/manifest.json: site/node_modules/.installed coderd/apidoc/.gen docs/reference/cli/index.md - ./scripts/biome_format.sh ../docs/manifest.json - touch "$@" + tmpfile=$$(mktemp -d)/$(notdir $@) && cp "$@" "$$tmpfile" && \ + ./scripts/biome_format.sh "$$tmpfile" && \ + mv "$$tmpfile" "$@" coderd/apidoc/swagger.json: site/node_modules/.installed coderd/apidoc/.gen - ./scripts/biome_format.sh ../coderd/apidoc/swagger.json touch "$@" update-golden-files: diff --git a/coderd/database/gen/dump/main.go b/coderd/database/gen/dump/main.go index 25bcbcd396..1f87c94f0e 100644 --- a/coderd/database/gen/dump/main.go +++ b/coderd/database/gen/dump/main.go @@ -11,6 +11,7 @@ import ( "github.com/coder/coder/v2/coderd/database/dbtestutil" "github.com/coder/coder/v2/coderd/database/migrations" + "github.com/coder/coder/v2/scripts/atomicwrite" ) var preamble = []byte("-- Code generated by 'make coderd/database/generate'. DO NOT EDIT.") @@ -82,7 +83,7 @@ func main() { if !ok { panic("couldn't get caller path") } - err = os.WriteFile(filepath.Join(mainPath, "..", "..", "..", "dump.sql"), append(preamble, dumpBytes...), 0o600) + err = atomicwrite.File(filepath.Join(mainPath, "..", "..", "..", "dump.sql"), append(preamble, dumpBytes...)) if err != nil { err = xerrors.Errorf("write dump failed: %w", err) panic(err) diff --git a/coderd/database/generate.sh b/coderd/database/generate.sh index 66f6da39ed..ddab225e1b 100755 --- a/coderd/database/generate.sh +++ b/coderd/database/generate.sh @@ -16,10 +16,15 @@ SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}") echo generate 1>&2 # Dump the updated schema (use make to utilize caching). - make -C ../.. --no-print-directory coderd/database/dump.sql + if [[ "${SKIP_DUMP_SQL:-0}" != 1 ]]; then + make -C ../.. --no-print-directory coderd/database/dump.sql + fi # The logic below depends on the exact version being correct :( sqlc generate + tmpfile=$(mktemp "${TMPDIR:-/tmp}/queries.sql.go.XXXXXX") + trap 'rm -f "$tmpfile"' EXIT + first=true files=$(find ./queries/ -type f -name "*.sql.go" | LC_ALL=C sort) for fi in $files; do @@ -33,14 +38,17 @@ SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}") # Copy the header from the first file only, ignoring the source comment. if $first; then - head -n 6 <"$fi" | grep -v "source" >queries.sql.go + head -n 6 <"$fi" | grep -v "source" >"$tmpfile" first=false fi # Append the file past the imports section into queries.sql.go. - tail -n "+$cut" <"$fi" >>queries.sql.go + tail -n "+$cut" <"$fi" >>"$tmpfile" done + # Atomically replace the target file. + mv "$tmpfile" queries.sql.go + # Move the files we want. mv queries/querier.go . mv queries/models.go . diff --git a/scripts/apidocgen/generate.sh b/scripts/apidocgen/generate.sh index 22e6edded3..38f0b5c4df 100755 --- a/scripts/apidocgen/generate.sh +++ b/scripts/apidocgen/generate.sh @@ -10,6 +10,11 @@ source "$(dirname "$(dirname "${BASH_SOURCE[0]}")")/lib.sh" APIDOCGEN_DIR=$(dirname "${BASH_SOURCE[0]}") API_MD_TMP_FILE=$(mktemp /tmp/coder-apidocgen.XXXXXX) +# SWAG_OUTPUT_DIR controls where swag writes swagger.json and docs.go. +# The caller may set it to a temp directory to avoid writing directly +# into the working tree. +SWAG_OUTPUT_DIR="${SWAG_OUTPUT_DIR:-./coderd/apidoc}" + cleanup() { rm -f "${API_MD_TMP_FILE}" } @@ -28,14 +33,14 @@ pushd "${APIDOCGEN_DIR}" # Make sure that widdershins is installed correctly. pnpm exec -- widdershins --version -# Render the Markdown file. +# Render the Markdown file from the swagger output. pnpm exec -- widdershins \ --user_templates "./markdown-template" \ --search false \ --omitHeader true \ --language_tabs "shell:curl" \ - --summary "../../coderd/apidoc/swagger.json" \ + --summary "${SWAG_OUTPUT_DIR}/swagger.json" \ --outfile "${API_MD_TMP_FILE}" # Perform the postprocessing -go run postprocess/main.go -in-md-file-single "${API_MD_TMP_FILE}" +go run postprocess/main.go -in-md-file-single "${API_MD_TMP_FILE}" -docs-directory "${APIDOCGEN_DOCS_DIR:-../../docs}" popd diff --git a/scripts/apidocgen/postprocess/main.go b/scripts/apidocgen/postprocess/main.go index c4bc3f19ea..3d7a13d434 100644 --- a/scripts/apidocgen/postprocess/main.go +++ b/scripts/apidocgen/postprocess/main.go @@ -13,6 +13,8 @@ import ( "strings" "golang.org/x/xerrors" + + "github.com/coder/coder/v2/scripts/atomicwrite" ) const ( @@ -126,7 +128,7 @@ func writeDocs(sections [][]byte) error { log.Println("Write docs to destination") apiDir := path.Join(docsDirectory, apiSubdir) - err := os.WriteFile(path.Join(apiDir, apiIndexFile), []byte(apiIndexContent), 0o644) // #nosec + err := atomicwrite.File(path.Join(apiDir, apiIndexFile), []byte(apiIndexContent)) if err != nil { return xerrors.Errorf(`can't write the index file: %w`, err) } @@ -147,7 +149,7 @@ func writeDocs(sections [][]byte) error { mdFilename := toMdFilename(sectionName) docPath := path.Join(apiDir, mdFilename) - err = os.WriteFile(docPath, section, 0o644) // #nosec + err = atomicwrite.File(docPath, section) if err != nil { return xerrors.Errorf(`can't write doc file "%s": %w`, docPath, err) } @@ -226,7 +228,7 @@ func writeDocs(sections [][]byte) error { return xerrors.Errorf("json.Marshal failed: %w", err) } - err = os.WriteFile(manifestPath, manifestFile, 0o644) // #nosec + err = atomicwrite.File(manifestPath, manifestFile) if err != nil { return xerrors.Errorf("can't write manifest file: %w", err) } diff --git a/scripts/apidocgen/swaginit/main.go b/scripts/apidocgen/swaginit/main.go index f38f9d5d0c..b6a60bb59e 100644 --- a/scripts/apidocgen/swaginit/main.go +++ b/scripts/apidocgen/swaginit/main.go @@ -16,11 +16,17 @@ import ( func main() { logger := log.New(os.Stdout, "", log.LstdFlags) + outputDir := "./coderd/apidoc" + if d := os.Getenv("SWAG_OUTPUT_DIR"); d != "" { + outputDir = d + } + err := gen.New().Build(&gen.Config{ SearchDir: "./coderd,./codersdk,./enterprise/coderd,./enterprise/wsproxy/wsproxysdk", MainAPIFile: "coderd.go", - OutputDir: "./coderd/apidoc", + OutputDir: outputDir, OutputTypes: []string{"go", "json"}, + PackageName: "apidoc", ParseDependency: 1, Strict: true, OverridesFile: gen.DefaultOverridesFile, diff --git a/scripts/atomicwrite/atomicwrite.go b/scripts/atomicwrite/atomicwrite.go new file mode 100644 index 0000000000..bea6b898ed --- /dev/null +++ b/scripts/atomicwrite/atomicwrite.go @@ -0,0 +1,32 @@ +package atomicwrite + +import ( + "os" + "path/filepath" + + "golang.org/x/xerrors" +) + +// File atomically writes data to the named file. It writes to a +// temporary file in the same directory and renames it so that an +// interrupted write never leaves a partially-written target. +func File(path string, data []byte) error { + dir := filepath.Dir(path) + tmp, err := os.CreateTemp(dir, filepath.Base(path)+".tmp.*") + if err != nil { + return xerrors.Errorf("create temp file: %w", err) + } + defer os.Remove(tmp.Name()) + + if _, err := tmp.Write(data); err != nil { + _ = tmp.Close() + return xerrors.Errorf("write temp file: %w", err) + } + if err := tmp.Close(); err != nil { + return xerrors.Errorf("close temp file: %w", err) + } + if err := os.Rename(tmp.Name(), path); err != nil { + return xerrors.Errorf("rename temp file: %w", err) + } + return nil +} diff --git a/scripts/auditdocgen/main.go b/scripts/auditdocgen/main.go index bc9eab2b0d..98748fb4c1 100644 --- a/scripts/auditdocgen/main.go +++ b/scripts/auditdocgen/main.go @@ -12,6 +12,7 @@ import ( "golang.org/x/xerrors" "github.com/coder/coder/v2/enterprise/audit" + "github.com/coder/coder/v2/scripts/atomicwrite" ) var ( @@ -150,9 +151,7 @@ func updateAuditDoc(doc []byte, auditableResourcesMap AuditableResourcesMap) ([] } func writeAuditDoc(doc []byte) error { - // G306: Expect WriteFile permissions to be 0600 or less - /* #nosec G306 */ - return os.WriteFile(auditDocFile, doc, 0o644) + return atomicwrite.File(auditDocFile, doc) } func sortKeys[T any](stringMap map[string]T) []string { diff --git a/scripts/clidocgen/gen.go b/scripts/clidocgen/gen.go index d48c5a0890..6679fb6853 100644 --- a/scripts/clidocgen/gen.go +++ b/scripts/clidocgen/gen.go @@ -12,6 +12,7 @@ import ( "github.com/acarl005/stripansi" "github.com/coder/coder/v2/buildinfo" + "github.com/coder/coder/v2/scripts/atomicwrite" "github.com/coder/flog" "github.com/coder/serpent" ) @@ -125,24 +126,21 @@ func genTree(dir string, cmd *serpent.Command, wroteLog map[string]*serpent.Comm } path := filepath.Join(dir, fmtDocFilename(cmd)) - // Write out root. - fi, err := os.OpenFile( - path, - os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644, - ) + + var buf strings.Builder + err := writeCommand(&buf, cmd) if err != nil { return err } - defer fi.Close() - err = writeCommand(fi, cmd) + err = atomicwrite.File(path, []byte(buf.String())) if err != nil { return err } flog.Successf( "wrote\t%s", - fi.Name(), + path, ) wroteLog[path] = cmd for _, sub := range cmd.Children { diff --git a/scripts/clidocgen/main.go b/scripts/clidocgen/main.go index da8452c7ce..47998fca17 100644 --- a/scripts/clidocgen/main.go +++ b/scripts/clidocgen/main.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/coder/coder/v2/enterprise/cli" + "github.com/coder/coder/v2/scripts/atomicwrite" "github.com/coder/flog" "github.com/coder/serpent" ) @@ -94,6 +95,11 @@ func main() { cliMarkdownDir = filepath.Join(docsDir, "reference/cli") ) + if d := os.Getenv("DOCS_DIR"); d != "" { + docsDir = d + cliMarkdownDir = filepath.Join(docsDir, "reference/cli") + } + cmd, err := root.Command(root.EnterpriseSubcommands()) if err != nil { flog.Fatalf("creating command: %v", err) @@ -188,7 +194,7 @@ func main() { flog.Fatalf("marshaling manifest: %v", err) } - err = os.WriteFile(manifestPath, manifestByt, 0o600) + err = atomicwrite.File(manifestPath, manifestByt) if err != nil { flog.Fatalf("writing manifest: %v", err) } diff --git a/scripts/dbgen/constraint.go b/scripts/dbgen/constraint.go index 6853f9bb26..fb752b9434 100644 --- a/scripts/dbgen/constraint.go +++ b/scripts/dbgen/constraint.go @@ -11,6 +11,8 @@ import ( "golang.org/x/tools/imports" "golang.org/x/xerrors" + + "github.com/coder/coder/v2/scripts/atomicwrite" ) type constraintType string @@ -135,7 +137,7 @@ const ( if err != nil { return err } - return os.WriteFile(outputPath, data, 0o600) + return atomicwrite.File(outputPath, data) } // generateUniqueConstraints generates the UniqueConstraint enum. diff --git a/scripts/dbgen/main.go b/scripts/dbgen/main.go index 246c4c4038..71fdcbbeef 100644 --- a/scripts/dbgen/main.go +++ b/scripts/dbgen/main.go @@ -17,6 +17,8 @@ import ( "github.com/dave/dst/decorator/resolver/guess" "golang.org/x/tools/imports" "golang.org/x/xerrors" + + "github.com/coder/coder/v2/scripts/atomicwrite" ) var ( @@ -245,7 +247,7 @@ func orderAndStubDatabaseFunctions(filePath, receiver, structName string, stub f if err != nil { return xerrors.Errorf("process imports: %w", err) } - return os.WriteFile(filePath, data, 0o600) + return atomicwrite.File(filePath, data) } // compileFuncDecl extracts the function declaration from the given code. diff --git a/scripts/gensite/generate_icon_list.go b/scripts/gensite/generate_icon_list.go index ec3f91c1ab..a6aee66c2a 100644 --- a/scripts/gensite/generate_icon_list.go +++ b/scripts/gensite/generate_icon_list.go @@ -4,6 +4,8 @@ import ( "encoding/json" "fmt" "os" + + "github.com/coder/coder/v2/scripts/atomicwrite" ) func generateIconList(path string) int { @@ -30,14 +32,6 @@ func generateIconList(path string) int { } icons = icons[:i] - outputFile, err := os.Create(path) - if err != nil { - _, _ = fmt.Println("failed to create file") - _, _ = fmt.Println("err:", err.Error()) - return 73 // CANTCREAT - } - defer outputFile.Close() - iconsJSON, err := json.Marshal(icons) if err != nil { _, _ = fmt.Println("failed to serialize JSON") @@ -45,12 +39,9 @@ func generateIconList(path string) int { return 70 // SOFTWARE } - written, err := outputFile.Write(iconsJSON) - if err != nil || written != len(iconsJSON) { + if err := atomicwrite.File(path, iconsJSON); err != nil { _, _ = fmt.Println("failed to write JSON") - if err != nil { - _, _ = fmt.Println("err:", err.Error()) - } + _, _ = fmt.Println("err:", err.Error()) return 74 // IOERR } diff --git a/scripts/metricsdocgen/main.go b/scripts/metricsdocgen/main.go index 7576225036..d320b60c6a 100644 --- a/scripts/metricsdocgen/main.go +++ b/scripts/metricsdocgen/main.go @@ -13,6 +13,8 @@ import ( dto "github.com/prometheus/client_model/go" "github.com/prometheus/common/expfmt" "golang.org/x/xerrors" + + "github.com/coder/coder/v2/scripts/atomicwrite" ) var ( @@ -186,13 +188,7 @@ func updatePrometheusDoc(doc []byte, metricFamilies []*dto.MetricFamily) ([]byte } func writePrometheusDoc(doc []byte) error { - // G306: Expect WriteFile permissions to be 0600 or less - /* #nosec G306 */ - err := os.WriteFile(prometheusDocFile, doc, 0o644) - if err != nil { - return err - } - return nil + return atomicwrite.File(prometheusDocFile, doc) } func sortedKeys(m map[string]struct{}) []string {