build(Makefile): enable parallel make -j gen with correct dependency graph (#22612)

`make gen` could not run with `-j` because inter-target dependency edges
were missing. Multiple recipes compile `coderd/rbac` (which includes
generated files like `object_gen.go`), and without explicit ordering,
parallel runs produced syntax errors from mid-write reads.

Three main changes:

**Dependency graph fixes** declare the compile-time chain through
`coderd/rbac` so that `object_gen.go` is written before anything that
imports it is compiled. The DB generation targets use a GNU Make 4.3+
grouped target (`&:`) so Make knows `generate.sh` co-produces
`querier.go`, `unique_constraint.go`, `dbmetrics`, and `dbauthz` in a
single invocation. `SKIP_DUMP_SQL=1` avoids re-entrant `make` inside
`generate.sh` when the Makefile already guarantees `dump.sql` is fresh.

**`scripts/atomicwrite` package** replaces `os.WriteFile` in all gen
scripts with a temp-file-in-same-dir + rename pattern, preventing
interrupted runs from leaving partial files.

**`.PRECIOUS` and shell atomic writes** protect git-tracked generated
files from Make's default delete-on-error behavior. Since these files
are committed, deletion is worse than staleness -- `git restore` is the
recovery path.

CI now runs `make -j --output-sync -B gen` (~32s, down from ~85s
serial).

| Scenario                          | Before             | After    |
|-----------------------------------|--------------------|----------|
| `make gen` (serial)               | 95s                | 95s      |
| `make -j gen` (parallel)          | race error         | **22s**  |
| CI `make -j --output-sync -B gen` | forced serial ~85s | **~32s** |
This commit is contained in:
Mathias Fredriksson
2026-03-05 13:58:10 +02:00
committed by GitHub
parent b0e10402c8
commit a6a8fd94d7
15 changed files with 218 additions and 94 deletions
+8 -3
View File
@@ -10,6 +10,11 @@ source "$(dirname "$(dirname "${BASH_SOURCE[0]}")")/lib.sh"
APIDOCGEN_DIR=$(dirname "${BASH_SOURCE[0]}")
API_MD_TMP_FILE=$(mktemp /tmp/coder-apidocgen.XXXXXX)
# SWAG_OUTPUT_DIR controls where swag writes swagger.json and docs.go.
# The caller may set it to a temp directory to avoid writing directly
# into the working tree.
SWAG_OUTPUT_DIR="${SWAG_OUTPUT_DIR:-./coderd/apidoc}"
cleanup() {
rm -f "${API_MD_TMP_FILE}"
}
@@ -28,14 +33,14 @@ pushd "${APIDOCGEN_DIR}"
# Make sure that widdershins is installed correctly.
pnpm exec -- widdershins --version
# Render the Markdown file.
# Render the Markdown file from the swagger output.
pnpm exec -- widdershins \
--user_templates "./markdown-template" \
--search false \
--omitHeader true \
--language_tabs "shell:curl" \
--summary "../../coderd/apidoc/swagger.json" \
--summary "${SWAG_OUTPUT_DIR}/swagger.json" \
--outfile "${API_MD_TMP_FILE}"
# Perform the postprocessing
go run postprocess/main.go -in-md-file-single "${API_MD_TMP_FILE}"
go run postprocess/main.go -in-md-file-single "${API_MD_TMP_FILE}" -docs-directory "${APIDOCGEN_DOCS_DIR:-../../docs}"
popd
+5 -3
View File
@@ -13,6 +13,8 @@ import (
"strings"
"golang.org/x/xerrors"
"github.com/coder/coder/v2/scripts/atomicwrite"
)
const (
@@ -126,7 +128,7 @@ func writeDocs(sections [][]byte) error {
log.Println("Write docs to destination")
apiDir := path.Join(docsDirectory, apiSubdir)
err := os.WriteFile(path.Join(apiDir, apiIndexFile), []byte(apiIndexContent), 0o644) // #nosec
err := atomicwrite.File(path.Join(apiDir, apiIndexFile), []byte(apiIndexContent))
if err != nil {
return xerrors.Errorf(`can't write the index file: %w`, err)
}
@@ -147,7 +149,7 @@ func writeDocs(sections [][]byte) error {
mdFilename := toMdFilename(sectionName)
docPath := path.Join(apiDir, mdFilename)
err = os.WriteFile(docPath, section, 0o644) // #nosec
err = atomicwrite.File(docPath, section)
if err != nil {
return xerrors.Errorf(`can't write doc file "%s": %w`, docPath, err)
}
@@ -226,7 +228,7 @@ func writeDocs(sections [][]byte) error {
return xerrors.Errorf("json.Marshal failed: %w", err)
}
err = os.WriteFile(manifestPath, manifestFile, 0o644) // #nosec
err = atomicwrite.File(manifestPath, manifestFile)
if err != nil {
return xerrors.Errorf("can't write manifest file: %w", err)
}
+7 -1
View File
@@ -16,11 +16,17 @@ import (
func main() {
logger := log.New(os.Stdout, "", log.LstdFlags)
outputDir := "./coderd/apidoc"
if d := os.Getenv("SWAG_OUTPUT_DIR"); d != "" {
outputDir = d
}
err := gen.New().Build(&gen.Config{
SearchDir: "./coderd,./codersdk,./enterprise/coderd,./enterprise/wsproxy/wsproxysdk",
MainAPIFile: "coderd.go",
OutputDir: "./coderd/apidoc",
OutputDir: outputDir,
OutputTypes: []string{"go", "json"},
PackageName: "apidoc",
ParseDependency: 1,
Strict: true,
OverridesFile: gen.DefaultOverridesFile,
+32
View File
@@ -0,0 +1,32 @@
package atomicwrite
import (
"os"
"path/filepath"
"golang.org/x/xerrors"
)
// File atomically writes data to the named file. It writes to a
// temporary file in the same directory and renames it so that an
// interrupted write never leaves a partially-written target.
func File(path string, data []byte) error {
dir := filepath.Dir(path)
tmp, err := os.CreateTemp(dir, filepath.Base(path)+".tmp.*")
if err != nil {
return xerrors.Errorf("create temp file: %w", err)
}
defer os.Remove(tmp.Name())
if _, err := tmp.Write(data); err != nil {
_ = tmp.Close()
return xerrors.Errorf("write temp file: %w", err)
}
if err := tmp.Close(); err != nil {
return xerrors.Errorf("close temp file: %w", err)
}
if err := os.Rename(tmp.Name(), path); err != nil {
return xerrors.Errorf("rename temp file: %w", err)
}
return nil
}
+2 -3
View File
@@ -12,6 +12,7 @@ import (
"golang.org/x/xerrors"
"github.com/coder/coder/v2/enterprise/audit"
"github.com/coder/coder/v2/scripts/atomicwrite"
)
var (
@@ -150,9 +151,7 @@ func updateAuditDoc(doc []byte, auditableResourcesMap AuditableResourcesMap) ([]
}
func writeAuditDoc(doc []byte) error {
// G306: Expect WriteFile permissions to be 0600 or less
/* #nosec G306 */
return os.WriteFile(auditDocFile, doc, 0o644)
return atomicwrite.File(auditDocFile, doc)
}
func sortKeys[T any](stringMap map[string]T) []string {
+6 -8
View File
@@ -12,6 +12,7 @@ import (
"github.com/acarl005/stripansi"
"github.com/coder/coder/v2/buildinfo"
"github.com/coder/coder/v2/scripts/atomicwrite"
"github.com/coder/flog"
"github.com/coder/serpent"
)
@@ -125,24 +126,21 @@ func genTree(dir string, cmd *serpent.Command, wroteLog map[string]*serpent.Comm
}
path := filepath.Join(dir, fmtDocFilename(cmd))
// Write out root.
fi, err := os.OpenFile(
path,
os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644,
)
var buf strings.Builder
err := writeCommand(&buf, cmd)
if err != nil {
return err
}
defer fi.Close()
err = writeCommand(fi, cmd)
err = atomicwrite.File(path, []byte(buf.String()))
if err != nil {
return err
}
flog.Successf(
"wrote\t%s",
fi.Name(),
path,
)
wroteLog[path] = cmd
for _, sub := range cmd.Children {
+7 -1
View File
@@ -8,6 +8,7 @@ import (
"strings"
"github.com/coder/coder/v2/enterprise/cli"
"github.com/coder/coder/v2/scripts/atomicwrite"
"github.com/coder/flog"
"github.com/coder/serpent"
)
@@ -94,6 +95,11 @@ func main() {
cliMarkdownDir = filepath.Join(docsDir, "reference/cli")
)
if d := os.Getenv("DOCS_DIR"); d != "" {
docsDir = d
cliMarkdownDir = filepath.Join(docsDir, "reference/cli")
}
cmd, err := root.Command(root.EnterpriseSubcommands())
if err != nil {
flog.Fatalf("creating command: %v", err)
@@ -188,7 +194,7 @@ func main() {
flog.Fatalf("marshaling manifest: %v", err)
}
err = os.WriteFile(manifestPath, manifestByt, 0o600)
err = atomicwrite.File(manifestPath, manifestByt)
if err != nil {
flog.Fatalf("writing manifest: %v", err)
}
+3 -1
View File
@@ -11,6 +11,8 @@ import (
"golang.org/x/tools/imports"
"golang.org/x/xerrors"
"github.com/coder/coder/v2/scripts/atomicwrite"
)
type constraintType string
@@ -135,7 +137,7 @@ const (
if err != nil {
return err
}
return os.WriteFile(outputPath, data, 0o600)
return atomicwrite.File(outputPath, data)
}
// generateUniqueConstraints generates the UniqueConstraint enum.
+3 -1
View File
@@ -17,6 +17,8 @@ import (
"github.com/dave/dst/decorator/resolver/guess"
"golang.org/x/tools/imports"
"golang.org/x/xerrors"
"github.com/coder/coder/v2/scripts/atomicwrite"
)
var (
@@ -245,7 +247,7 @@ func orderAndStubDatabaseFunctions(filePath, receiver, structName string, stub f
if err != nil {
return xerrors.Errorf("process imports: %w", err)
}
return os.WriteFile(filePath, data, 0o600)
return atomicwrite.File(filePath, data)
}
// compileFuncDecl extracts the function declaration from the given code.
+4 -13
View File
@@ -4,6 +4,8 @@ import (
"encoding/json"
"fmt"
"os"
"github.com/coder/coder/v2/scripts/atomicwrite"
)
func generateIconList(path string) int {
@@ -30,14 +32,6 @@ func generateIconList(path string) int {
}
icons = icons[:i]
outputFile, err := os.Create(path)
if err != nil {
_, _ = fmt.Println("failed to create file")
_, _ = fmt.Println("err:", err.Error())
return 73 // CANTCREAT
}
defer outputFile.Close()
iconsJSON, err := json.Marshal(icons)
if err != nil {
_, _ = fmt.Println("failed to serialize JSON")
@@ -45,12 +39,9 @@ func generateIconList(path string) int {
return 70 // SOFTWARE
}
written, err := outputFile.Write(iconsJSON)
if err != nil || written != len(iconsJSON) {
if err := atomicwrite.File(path, iconsJSON); err != nil {
_, _ = fmt.Println("failed to write JSON")
if err != nil {
_, _ = fmt.Println("err:", err.Error())
}
_, _ = fmt.Println("err:", err.Error())
return 74 // IOERR
}
+3 -7
View File
@@ -13,6 +13,8 @@ import (
dto "github.com/prometheus/client_model/go"
"github.com/prometheus/common/expfmt"
"golang.org/x/xerrors"
"github.com/coder/coder/v2/scripts/atomicwrite"
)
var (
@@ -186,13 +188,7 @@ func updatePrometheusDoc(doc []byte, metricFamilies []*dto.MetricFamily) ([]byte
}
func writePrometheusDoc(doc []byte) error {
// G306: Expect WriteFile permissions to be 0600 or less
/* #nosec G306 */
err := os.WriteFile(prometheusDocFile, doc, 0o644)
if err != nil {
return err
}
return nil
return atomicwrite.File(prometheusDocFile, doc)
}
func sortedKeys(m map[string]struct{}) []string {