From 3f0e015fe53dca6172ebf31feafa30756ae980f8 Mon Sep 17 00:00:00 2001 From: George K Date: Tue, 28 Apr 2026 09:17:08 -0700 Subject: [PATCH] fix: allow coderd to start with an empty DERP map when built-in DERP is disabled (#24544) Allow coderd to start with an empty base DERP map when built-in DERP is disabled and no static DERP map is configured, so DERP can come from workspace proxies after startup. Also add a DERP healthcheck warning when no DERP servers are currently available at runtime. Related to: https://linear.app/codercom/issue/PLAT-43/bug-coderd-unable-to-be-started-if-built-in-derp-server-disabled-and Related to: https://github.com/coder/coder/issues/22324 --- cli/server.go | 27 ++++++++---- cli/server_test.go | 17 ++++---- coderd/apidoc/docs.go | 2 + coderd/apidoc/swagger.json | 2 + coderd/coderdtest/coderdtest.go | 19 ++++++--- coderd/healthcheck/derphealth/derp.go | 16 +++++++ coderd/healthcheck/derphealth/derp_test.go | 42 ++++++++++++++++--- coderd/healthcheck/health/model.go | 1 + docs/admin/monitoring/health-check.md | 19 +++++++++ docs/reference/api/schemas.md | 6 +-- .../coderd/coderdenttest/coderdenttest.go | 2 +- enterprise/coderd/workspaceproxy_test.go | 40 +----------------- site/src/api/typesGenerated.ts | 2 + 13 files changed, 125 insertions(+), 70 deletions(-) diff --git a/cli/server.go b/cli/server.go index 5fc25d2558..3b3d8b5a4a 100644 --- a/cli/server.go +++ b/cli/server.go @@ -599,13 +599,26 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd. defaultRegion = nil } - derpMap, err := tailnet.NewDERPMap( - ctx, defaultRegion, vals.DERP.Server.STUNAddresses, - vals.DERP.Config.URL.String(), vals.DERP.Config.Path.String(), - vals.DERP.Config.BlockDirect.Value(), - ) - if err != nil { - return xerrors.Errorf("create derp map: %w", err) + derpConfigURL := vals.DERP.Config.URL.String() + derpConfigPath := vals.DERP.Config.Path.String() + var derpMap *tailcfg.DERPMap + if defaultRegion == nil && derpConfigURL == "" && derpConfigPath == "" { + logger.Warn(ctx, + "no DERP servers are currently configured; workspace networking"+ + " will not work until you either restart coderd with the"+ + " built-in DERP server enabled, restart coderd with an"+ + " external DERP map configured, or start a workspace proxy"+ + " with its DERP server enabled") + derpMap = &tailcfg.DERPMap{Regions: map[int]*tailcfg.DERPRegion{}} + } else { + derpMap, err = tailnet.NewDERPMap( + ctx, defaultRegion, vals.DERP.Server.STUNAddresses, + derpConfigURL, derpConfigPath, + vals.DERP.Config.BlockDirect.Value(), + ) + if err != nil { + return xerrors.Errorf("create derp map: %w", err) + } } appHostname := vals.WildcardAccessURL.String() diff --git a/cli/server_test.go b/cli/server_test.go index dcba43e2f2..b7a6fc3d79 100644 --- a/cli/server_test.go +++ b/cli/server_test.go @@ -2370,27 +2370,26 @@ func TestConnectToPostgres(t *testing.T) { }) } -func TestServer_InvalidDERP(t *testing.T) { +func TestServer_DisabledDERP_EmptyBaseMap(t *testing.T) { t.Parallel() + ctx, cancelFunc := context.WithTimeout(context.Background(), testutil.WaitShort) + defer cancelFunc() + // Try to start a server with the built-in DERP server disabled and no // external DERP map. - - inv, _ := clitest.New(t, + inv, cfg := clitest.New(t, "server", dbArg(t), "--http-address", ":0", "--access-url", "http://example.com", "--derp-server-enable=false", - "--derp-server-stun-addresses", "disable", - "--block-direct-connections", ) - err := inv.Run() - require.Error(t, err) - require.ErrorContains(t, err, "A valid DERP map is required for networking to work") + clitest.Start(t, inv.WithContext(ctx)) + waitAccessURL(t, cfg) } -func TestServer_DisabledDERP(t *testing.T) { +func TestServer_DisabledDERP_ExternalMap(t *testing.T) { t.Parallel() derpMap, _ := tailnettest.RunDERPAndSTUN(t) diff --git a/coderd/apidoc/docs.go b/coderd/apidoc/docs.go index 0a43b40556..6ec51ff20f 100644 --- a/coderd/apidoc/docs.go +++ b/coderd/apidoc/docs.go @@ -23736,6 +23736,7 @@ const docTemplate = `{ "EACS04", "EDERP01", "EDERP02", + "EDERP03", "EPD01", "EPD02", "EPD03" @@ -23756,6 +23757,7 @@ const docTemplate = `{ "CodeAccessURLNotOK", "CodeDERPNodeUsesWebsocket", "CodeDERPOneNodeUnhealthy", + "CodeDERPNoNodes", "CodeProvisionerDaemonsNoProvisionerDaemons", "CodeProvisionerDaemonVersionMismatch", "CodeProvisionerDaemonAPIMajorVersionDeprecated" diff --git a/coderd/apidoc/swagger.json b/coderd/apidoc/swagger.json index 6307845378..348b8bf2cd 100644 --- a/coderd/apidoc/swagger.json +++ b/coderd/apidoc/swagger.json @@ -21863,6 +21863,7 @@ "EACS04", "EDERP01", "EDERP02", + "EDERP03", "EPD01", "EPD02", "EPD03" @@ -21883,6 +21884,7 @@ "CodeAccessURLNotOK", "CodeDERPNodeUsesWebsocket", "CodeDERPOneNodeUnhealthy", + "CodeDERPNoNodes", "CodeProvisionerDaemonsNoProvisionerDaemons", "CodeProvisionerDaemonVersionMismatch", "CodeProvisionerDaemonAPIMajorVersionDeprecated" diff --git a/coderd/coderdtest/coderdtest.go b/coderd/coderdtest/coderdtest.go index 81c7884627..cbbee1b1c2 100644 --- a/coderd/coderdtest/coderdtest.go +++ b/coderd/coderdtest/coderdtest.go @@ -560,12 +560,19 @@ func NewOptions(t testing.TB, options *Options) (func(http.Handler), context.Can if !options.DeploymentValues.DERP.Server.Enable.Value() { region = nil } - derpMap, err := tailnet.NewDERPMap(ctx, region, stunAddresses, - options.DeploymentValues.DERP.Config.URL.Value(), - options.DeploymentValues.DERP.Config.Path.Value(), - options.DeploymentValues.DERP.Config.BlockDirect.Value(), - ) - require.NoError(t, err) + derpConfigURL := options.DeploymentValues.DERP.Config.URL.Value() + derpConfigPath := options.DeploymentValues.DERP.Config.Path.Value() + var derpMap *tailcfg.DERPMap + if region == nil && derpConfigURL == "" && derpConfigPath == "" { + derpMap = &tailcfg.DERPMap{Regions: map[int]*tailcfg.DERPRegion{}} + } else { + derpMap, err = tailnet.NewDERPMap( + ctx, region, stunAddresses, + derpConfigURL, derpConfigPath, + options.DeploymentValues.DERP.Config.BlockDirect.Value(), + ) + require.NoError(t, err) + } return func(h http.Handler) { mutex.Lock() diff --git a/coderd/healthcheck/derphealth/derp.go b/coderd/healthcheck/derphealth/derp.go index 0c8cab5e9f..c85bff5776 100644 --- a/coderd/healthcheck/derphealth/derp.go +++ b/coderd/healthcheck/derphealth/derp.go @@ -34,6 +34,7 @@ const ( oneNodeUnhealthy = "Region is operational, but performance might be degraded as one node is unhealthy." missingNodeReport = "Missing node health report, probably a developer error." noSTUN = "No STUN servers are available." + noDERP = "No DERP servers are available." stunMapVaryDest = "STUN returned different addresses; you may be behind a hard NAT." ) @@ -69,11 +70,20 @@ func (r *Report) Run(ctx context.Context, opts *ReportOptions) { r.Regions = map[int]*healthsdk.DERPRegionReport{} + // Track whether the map contains any DERP nodes so we can warn if + // it does not. + hasDERP := false wg := &sync.WaitGroup{} mu := sync.Mutex{} wg.Add(len(opts.DERPMap.Regions)) for _, region := range opts.DERPMap.Regions { + for _, node := range region.Nodes { + if !node.STUNOnly { + hasDERP = true + break + } + } var ( region = region regionReport = RegionReport{ @@ -103,6 +113,12 @@ func (r *Report) Run(ctx context.Context, opts *ReportOptions) { mu.Unlock() }() } + if !hasDERP { + r.Severity = health.SeverityWarning + r.Warnings = append(r.Warnings, health.Messagef( + health.CodeDERPNoNodes, noDERP, + )) + } ncLogf := func(format string, args ...interface{}) { mu.Lock() diff --git a/coderd/healthcheck/derphealth/derp_test.go b/coderd/healthcheck/derphealth/derp_test.go index 08dc7db97f..b6177d3db8 100644 --- a/coderd/healthcheck/derphealth/derp_test.go +++ b/coderd/healthcheck/derphealth/derp_test.go @@ -64,6 +64,9 @@ func TestDERP(t *testing.T) { report.Run(ctx, opts) assert.True(t, report.Healthy) + for _, warning := range report.Warnings { + assert.NotEqual(t, health.CodeDERPNoNodes, warning.Code) + } for _, region := range report.Regions { assert.True(t, region.Healthy) for _, node := range region.NodeReports { @@ -361,7 +364,7 @@ func TestDERP(t *testing.T) { } }) - t.Run("STUNOnly/OK", func(t *testing.T) { + t.Run("STUNOnly/WarnsNoDERP", func(t *testing.T) { t.Parallel() var ( @@ -389,7 +392,9 @@ func TestDERP(t *testing.T) { report.Run(ctx, opts) assert.True(t, report.Healthy) - assert.Equal(t, health.SeverityOK, report.Severity) + assert.Equal(t, health.SeverityWarning, report.Severity) + require.Len(t, report.Warnings, 1) + assert.Equal(t, health.CodeDERPNoNodes, report.Warnings[0].Code) for _, region := range report.Regions { assert.True(t, region.Healthy) assert.Equal(t, health.SeverityOK, region.Severity) @@ -405,6 +410,27 @@ func TestDERP(t *testing.T) { } }) + t.Run("NoDERP/EmptyMap", func(t *testing.T) { + t.Parallel() + + var ( + ctx = context.Background() + report = derphealth.Report{} + opts = &derphealth.ReportOptions{ + DERPMap: &tailcfg.DERPMap{ + Regions: map[int]*tailcfg.DERPRegion{}, + }, + } + ) + + report.Run(ctx, opts) + + assert.Equal(t, health.SeverityWarning, report.Severity) + require.Len(t, report.Warnings, 1) + assert.Equal(t, health.CodeDERPNoNodes, report.Warnings[0].Code) + assert.Empty(t, report.Regions) + }) + t.Run("STUNOnly/OneBadOneGood", func(t *testing.T) { t.Parallel() @@ -443,9 +469,15 @@ func TestDERP(t *testing.T) { report.Run(ctx, opts) assert.True(t, report.Healthy) assert.Equal(t, health.SeverityWarning, report.Severity) - if assert.Len(t, report.Warnings, 1) { - assert.Equal(t, health.CodeDERPOneNodeUnhealthy, report.Warnings[0].Code) - } + assert.Len(t, report.Warnings, 2) + assert.Contains(t, []health.Code{ + report.Warnings[0].Code, + report.Warnings[1].Code, + }, health.CodeDERPOneNodeUnhealthy) + assert.Contains(t, []health.Code{ + report.Warnings[0].Code, + report.Warnings[1].Code, + }, health.CodeDERPNoNodes) for _, region := range report.Regions { assert.True(t, region.Healthy) assert.Equal(t, health.SeverityWarning, region.Severity) diff --git a/coderd/healthcheck/health/model.go b/coderd/healthcheck/health/model.go index 4b09e4b344..6fe6c152af 100644 --- a/coderd/healthcheck/health/model.go +++ b/coderd/healthcheck/health/model.go @@ -36,6 +36,7 @@ const ( CodeDERPNodeUsesWebsocket Code = `EDERP01` CodeDERPOneNodeUnhealthy Code = `EDERP02` + CodeDERPNoNodes Code = `EDERP03` CodeSTUNNoNodes = `ESTUN01` CodeSTUNMapVaryDest = `ESTUN02` diff --git a/docs/admin/monitoring/health-check.md b/docs/admin/monitoring/health-check.md index 3139697fec..ead5e210ca 100644 --- a/docs/admin/monitoring/health-check.md +++ b/docs/admin/monitoring/health-check.md @@ -173,6 +173,25 @@ curl -v "https://coder.company.com/derp" # DERP requires connection upgrade ``` +### EDERP03 + +#### No DERP servers available + +**Problem:** This is shown when Coder's effective DERP map does not contain +any DERP servers. Without at least one working DERP server, workspace +networking may not work. + +This can happen if the built-in DERP server is disabled and no external DERP +map is configured, or if workspace proxies are expected to provide DERP but no +healthy DERP-enabled proxy is currently available. + +**Solution:** Ensure that at least one DERP server is available to the +deployment. For example: + +- Restart `coderd` with the built-in DERP server enabled +- Restart `coderd` with an external DERP map configured +- Make sure a workspace proxy with DERP server enabled is running and healthy + ### ESTUN01 #### No STUN servers available diff --git a/docs/reference/api/schemas.md b/docs/reference/api/schemas.md index 659237cc80..e5788ba9d6 100644 --- a/docs/reference/api/schemas.md +++ b/docs/reference/api/schemas.md @@ -13751,9 +13751,9 @@ Zero means unspecified. There might be a limit, but the client need not try to r #### Enumerated Values -| Value(s) | -|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `EACS01`, `EACS02`, `EACS03`, `EACS04`, `EDB01`, `EDB02`, `EDERP01`, `EDERP02`, `EPD01`, `EPD02`, `EPD03`, `EUNKNOWN`, `EWP01`, `EWP02`, `EWP04`, `EWS01`, `EWS02`, `EWS03` | +| Value(s) | +|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `EACS01`, `EACS02`, `EACS03`, `EACS04`, `EDB01`, `EDB02`, `EDERP01`, `EDERP02`, `EDERP03`, `EPD01`, `EPD02`, `EPD03`, `EUNKNOWN`, `EWP01`, `EWP02`, `EWP04`, `EWS01`, `EWS02`, `EWS03` | ## health.Message diff --git a/enterprise/coderd/coderdenttest/coderdenttest.go b/enterprise/coderd/coderdenttest/coderdenttest.go index e117414e3b..a7efd1b302 100644 --- a/enterprise/coderd/coderdenttest/coderdenttest.go +++ b/enterprise/coderd/coderdenttest/coderdenttest.go @@ -109,7 +109,7 @@ func NewWithAPI(t *testing.T, options *Options) ( BrowserOnly: options.BrowserOnly, SCIMAPIKey: options.SCIMAPIKey, DERPServerRelayAddress: serverURL.String(), - DERPServerRegionID: oop.BaseDERPMap.RegionIDs()[0], + DERPServerRegionID: int(oop.DeploymentValues.DERP.Server.RegionID.Value()), ReplicaSyncUpdateInterval: options.ReplicaSyncUpdateInterval, ReplicaErrorGracePeriod: options.ReplicaErrorGracePeriod, Options: oop, diff --git a/enterprise/coderd/workspaceproxy_test.go b/enterprise/coderd/workspaceproxy_test.go index 1d0b664b5f..4195648552 100644 --- a/enterprise/coderd/workspaceproxy_test.go +++ b/enterprise/coderd/workspaceproxy_test.go @@ -2,15 +2,12 @@ package coderd_test import ( "database/sql" - "encoding/json" "fmt" "net" "net/http" "net/http/httptest" "net/http/httputil" "net/url" - "os" - "path/filepath" "runtime" "testing" "time" @@ -19,7 +16,6 @@ import ( "github.com/sqlc-dev/pqtype" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "tailscale.com/tailcfg" "cdr.dev/slog/v3/sloggers/slogtest" "github.com/coder/coder/v2/agent/agenttest" @@ -38,7 +34,6 @@ import ( "github.com/coder/coder/v2/enterprise/wsproxy/wsproxysdk" "github.com/coder/coder/v2/provisioner/echo" "github.com/coder/coder/v2/testutil" - "github.com/coder/serpent" ) func TestRegions(t *testing.T) { @@ -613,27 +608,8 @@ func TestProxyRegisterDeregister(t *testing.T) { t.Run("RegisterWithDisabledBuiltInDERP/DerpEnabled", func(t *testing.T) { t.Parallel() - // Create a DERP map file. Currently, Coder refuses to start if there - // are zero DERP regions. - // TODO: ideally coder can start without any DERP servers if the - // customer is going to be using DERPs via proxies. We could make it - // a configuration value to allow an empty DERP map on startup or - // something. - tmpDir := t.TempDir() - derpPath := filepath.Join(tmpDir, "derp.json") - content, err := json.Marshal(&tailcfg.DERPMap{ - Regions: map[int]*tailcfg.DERPRegion{ - 1: { - Nodes: []*tailcfg.DERPNode{{}}, - }, - }, - }) - require.NoError(t, err) - require.NoError(t, os.WriteFile(derpPath, content, 0o600)) - dv := coderdtest.DeploymentValues(t) dv.DERP.Server.Enable = false // disable built-in DERP server - dv.DERP.Config.Path = serpent.String(derpPath) client, _ := setupWithDeploymentValues(t, dv) ctx := testutil.Context(t, testutil.WaitLong) @@ -659,25 +635,11 @@ func TestProxyRegisterDeregister(t *testing.T) { require.Equal(t, registerRes.DERPMeshKey, coderdtest.DefaultDERPMeshKey) }) - t.Run("RegisterWithDisabledBuiltInDERP/DerpEnabled", func(t *testing.T) { + t.Run("RegisterWithDisabledBuiltInDERP/DerpDisabled", func(t *testing.T) { t.Parallel() - // Same as above. - tmpDir := t.TempDir() - derpPath := filepath.Join(tmpDir, "derp.json") - content, err := json.Marshal(&tailcfg.DERPMap{ - Regions: map[int]*tailcfg.DERPRegion{ - 1: { - Nodes: []*tailcfg.DERPNode{{}}, - }, - }, - }) - require.NoError(t, err) - require.NoError(t, os.WriteFile(derpPath, content, 0o600)) - dv := coderdtest.DeploymentValues(t) dv.DERP.Server.Enable = false // disable built-in DERP server - dv.DERP.Config.Path = serpent.String(derpPath) client, _ := setupWithDeploymentValues(t, dv) ctx := testutil.Context(t, testutil.WaitLong) diff --git a/site/src/api/typesGenerated.ts b/site/src/api/typesGenerated.ts index cb0417bef8..19e9b9d58b 100644 --- a/site/src/api/typesGenerated.ts +++ b/site/src/api/typesGenerated.ts @@ -4164,6 +4164,7 @@ export type HealthCode = | "EACS02" | "EACS04" | "EACS01" + | "EDERP03" | "EDERP01" | "EDERP02" | "EDB01" @@ -4193,6 +4194,7 @@ export const HealthCodes: HealthCode[] = [ "EACS02", "EACS04", "EACS01", + "EDERP03", "EDERP01", "EDERP02", "EDB01",