shishantbiswas/coder
1
0
Fork 0
mirror of https://github.com/coder/coder.git synced 2026-06-02 20:48:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

ci: add retry logic for Go module operations (#21609)

Browse Source 
## Description

Add exponential backoff retries to all `go install` and `go mod
download` commands across CI workflows and actions.

## Why

Fixes
[coder/internal#1276](https://github.com/coder/internal/issues/1276) -
CI fails when `sum.golang.org` returns 500 errors during Go module
verification. This is an infrastructure-level flake that can't be
controlled.

## Changes

- Created `.github/scripts/retry.sh` - reusable retry helper with
exponential backoff (2s, 4s, 8s delays, max 3 attempts), using
`scripts/lib.sh` helpers
- Wrapped all `go install` and `go mod download` commands with retry in:
  - `.github/actions/setup-go/action.yaml`
  - `.github/actions/setup-sqlc/action.yaml`
  - `.github/actions/setup-go-tools/action.yaml`
  - `.github/workflows/ci.yaml`
  - `.github/workflows/release.yaml`
  - `.github/workflows/security.yaml`
- Added GNU tools setup (bash 4+, GNU getopt, make 4+) for macOS in
`test-go-pg` job, since `retry.sh` uses `lib.sh` which requires these
tools
This commit is contained in:
Kacper Sawicki
2026-01-29 16:05:49 +01:00
committed by GitHub
parent 8ee4f594d5
commit 9a417df940
7 changed files with 83 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/actions/setup-go-tools/action.yaml
+2 -2
View File
@@ -7,6 +7,6 @@ runs:
- name: go install tools
shell: bash
run: |
go install tool
./.github/scripts/retry.sh -- go install tool
# NOTE: protoc-gen-go cannot be installed with `go get`
go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.30
./.github/scripts/retry.sh -- go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.30
.github/actions/setup-go/action.yaml
+3 -3
View File
@@ -22,14 +22,14 @@ runs:
- name: Install gotestsum
shell: bash
run: go install gotest.tools/gotestsum@0d9599e513d70e5792bb9334869f82f6e8b53d4d # main as of 2025-05-15
run: ./.github/scripts/retry.sh -- go install gotest.tools/gotestsum@0d9599e513d70e5792bb9334869f82f6e8b53d4d # main as of 2025-05-15
- name: Install mtimehash
shell: bash
run: go install github.com/slsyy/mtimehash/cmd/mtimehash@a6b5da4ed2c4a40e7b805534b004e9fde7b53ce0 # v1.0.0
run: ./.github/scripts/retry.sh -- go install github.com/slsyy/mtimehash/cmd/mtimehash@a6b5da4ed2c4a40e7b805534b004e9fde7b53ce0 # v1.0.0
# It isn't necessary that we ever do this, but it helps
# separate the "setup" from the "run" times.
- name: go mod download
shell: bash
run: go mod download -x
run: ./.github/scripts/retry.sh -- go mod download -x
.github/actions/setup-sqlc/action.yaml
+1 -1
View File
@@ -14,4 +14,4 @@ runs:
# - https://github.com/sqlc-dev/sqlc/pull/4159
shell: bash
run: |
CGO_ENABLED=1 go install github.com/coder/sqlc/cmd/sqlc@aab4e865a51df0c43e1839f81a9d349b41d14f05
./.github/scripts/retry.sh -- env CGO_ENABLED=1 go install github.com/coder/sqlc/cmd/sqlc@aab4e865a51df0c43e1839f81a9d349b41d14f05
.github/scripts/retry.sh
Executable
+50
View File
@@ -0,0 +1,50 @@
#!/usr/bin/env bash
# Retry a command with exponential backoff.
#
# Usage: retry.sh [--max-attempts N] -- <command...>
#
# Example:
# retry.sh --max-attempts 3 -- go install gotest.tools/gotestsum@latest
#
# This will retry the command up to 3 times with exponential backoff
# (2s, 4s, 8s delays between attempts).
set -euo pipefail
# shellcheck source=scripts/lib.sh
source "$(dirname "${BASH_SOURCE[0]}")/../../scripts/lib.sh"
max_attempts=3
args="$(getopt -o "" -l max-attempts: -- "$@")"
eval set -- "$args"
while true; do
case "$1" in
--max-attempts)
max_attempts="$2"
shift 2
;;
--)
shift
break
;;
*)
error "Unrecognized option: $1"
;;
esac
done
if [[ $# -lt 1 ]]; then
error "Usage: retry.sh [--max-attempts N] -- <command...>"
fi
attempt=1
until "$@"; do
if ((attempt >= max_attempts)); then
error "Command failed after $max_attempts attempts: $*"
fi
delay=$((2 ** attempt))
log "Attempt $attempt/$max_attempts failed, retrying in ${delay}s..."
sleep "$delay"
((attempt++))
done
.github/workflows/ci.yaml
+21 -9
View File
@@ -176,7 +176,7 @@ jobs:
- name: Get golangci-lint cache dir
run: |
linter_ver=$(grep -Eo 'GOLANGCI_LINT_VERSION=\S+' dogfood/coder/Dockerfile | cut -d '=' -f 2)
go install "github.com/golangci/golangci-lint/cmd/golangci-lint@v$linter_ver"
./.github/scripts/retry.sh -- go install "github.com/golangci/golangci-lint/cmd/golangci-lint@v$linter_ver"
dir=$(golangci-lint cache status | awk '/Dir/ { print $2 }')
echo "LINT_CACHE_DIR=$dir" >> "$GITHUB_ENV"
@@ -329,7 +329,7 @@ jobs:
uses: ./.github/actions/setup-go
- name: Install shfmt
run: go install mvdan.cc/sh/v3/cmd/shfmt@v3.7.0
run: ./.github/scripts/retry.sh -- go install mvdan.cc/sh/v3/cmd/shfmt@v3.7.0
- name: make fmt
timeout-minutes: 7
@@ -395,6 +395,18 @@ jobs:
id: go-paths
uses: ./.github/actions/setup-go-paths
# macOS default bash and coreutils are too old for our scripts
# (lib.sh requires bash 4+, GNU getopt, make 4+).
- name: Setup GNU tools (macOS)
if: runner.os == 'macOS'
run: |
brew install bash gnu-getopt make
{
echo "$(brew --prefix bash)/bin"
echo "$(brew --prefix gnu-getopt)/bin"
echo "$(brew --prefix make)/libexec/gnubin"
} >> "$GITHUB_PATH"
- name: Setup Go
uses: ./.github/actions/setup-go
with:
@@ -1068,7 +1080,7 @@ jobs:
- name: Build dylibs
run: |
set -euxo pipefail
go mod download
./.github/scripts/retry.sh -- go mod download
make gen/mark-fresh
make build/coder-dylib
@@ -1117,10 +1129,10 @@ jobs:
uses: ./.github/actions/setup-go
- name: Install go-winres
run: go install github.com/tc-hib/go-winres@d743268d7ea168077ddd443c4240562d4f5e8c3e # v0.3.3
run: ./.github/scripts/retry.sh -- go install github.com/tc-hib/go-winres@d743268d7ea168077ddd443c4240562d4f5e8c3e # v0.3.3
- name: Install nfpm
run: go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.35.1
run: ./.github/scripts/retry.sh -- go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.35.1
- name: Install zstd
run: sudo apt-get install -y zstd
@@ -1128,7 +1140,7 @@ jobs:
- name: Build
run: |
set -euxo pipefail
go mod download
./.github/scripts/retry.sh -- go mod download
make gen/mark-fresh
make build
@@ -1207,10 +1219,10 @@ jobs:
java-version: "11.0"
- name: Install go-winres
run: go install github.com/tc-hib/go-winres@d743268d7ea168077ddd443c4240562d4f5e8c3e # v0.3.3
run: ./.github/scripts/retry.sh -- go install github.com/tc-hib/go-winres@d743268d7ea168077ddd443c4240562d4f5e8c3e # v0.3.3
- name: Install nfpm
run: go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.35.1
run: ./.github/scripts/retry.sh -- go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.35.1
- name: Install zstd
run: sudo apt-get install -y zstd
@@ -1258,7 +1270,7 @@ jobs:
- name: Build
run: |
set -euxo pipefail
go mod download
./.github/scripts/retry.sh -- go mod download
version="$(./scripts/version.sh)"
tag="main-${version//+/-}"
.github/workflows/release.yaml
+3 -3
View File
@@ -121,7 +121,7 @@ jobs:
- name: Build dylibs
run: |
set -euxo pipefail
go mod download
./.github/scripts/retry.sh -- go mod download
make gen/mark-fresh
make build/coder-dylib
@@ -259,7 +259,7 @@ jobs:
java-version: "11.0"
- name: Install go-winres
run: go install github.com/tc-hib/go-winres@d743268d7ea168077ddd443c4240562d4f5e8c3e # v0.3.3
run: ./.github/scripts/retry.sh -- go install github.com/tc-hib/go-winres@d743268d7ea168077ddd443c4240562d4f5e8c3e # v0.3.3
- name: Install nsis and zstd
run: sudo apt-get install -y nsis zstd
@@ -341,7 +341,7 @@ jobs:
- name: Build binaries
run: |
set -euo pipefail
go mod download
./.github/scripts/retry.sh -- go mod download
version="$(./scripts/version.sh)"
make gen/mark-fresh
.github/workflows/security.yaml
+3 -3
View File
@@ -97,11 +97,11 @@ jobs:
- name: Install yq
run: go run github.com/mikefarah/yq/v4@v4.44.3
- name: Install mockgen
run: go install go.uber.org/mock/mockgen@v0.6.0
run: ./.github/scripts/retry.sh -- go install go.uber.org/mock/mockgen@v0.6.0
- name: Install protoc-gen-go
run: go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.30
run: ./.github/scripts/retry.sh -- go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.30
- name: Install protoc-gen-go-drpc
run: go install storj.io/drpc/cmd/protoc-gen-go-drpc@v0.0.34
run: ./.github/scripts/retry.sh -- go install storj.io/drpc/cmd/protoc-gen-go-drpc@v0.0.34
- name: Install Protoc
run: |
# protoc must be in lockstep with our dogfood Dockerfile or the