fix(coderd): extract provisionerdserver.StaleInterval to 90 seconds (#15643)

Follow-up from https://github.com/coder/coder/pull/15578

Extracts `provisionerdserver.StaleInterval` and sets it to 90 seconds by
default

coderd/coderd.go

@@ -628,7 +628,8 @@ func New(options *Options) *API {
 					CurrentVersion:         buildinfo.Version(),
 					CurrentAPIMajorVersion: proto.CurrentMajor,
 					Store:                  options.Database,
-					// TimeNow and StaleInterval set to defaults, see healthcheck/provisioner.go
+					StaleInterval:          provisionerdserver.StaleInterval,
+					// TimeNow set to default, see healthcheck/provisioner.go
 				},
 			})
 		}

coderd/healthcheck/provisioner.go

@@ -50,7 +50,7 @@ func (r *ProvisionerDaemonsReport) Run(ctx context.Context, opts *ProvisionerDae
 	now := opts.TimeNow()
 
 	if opts.StaleInterval == 0 {
-		opts.StaleInterval = provisionerdserver.DefaultHeartbeatInterval * 3
+		opts.StaleInterval = provisionerdserver.StaleInterval
 	}
 
 	if opts.CurrentVersion == "" {

coderd/healthcheck/provisioner_test.go

@@ -15,15 +15,21 @@ import (
 	"github.com/coder/coder/v2/coderd/database/dbtime"
 	"github.com/coder/coder/v2/coderd/healthcheck"
 	"github.com/coder/coder/v2/coderd/healthcheck/health"
+	"github.com/coder/coder/v2/coderd/provisionerdserver"
 	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/coder/v2/codersdk/healthsdk"
 	"github.com/coder/coder/v2/provisionerd/proto"
+	"github.com/coder/coder/v2/testutil"
 )
 
 func TestProvisionerDaemonReport(t *testing.T) {
 	t.Parallel()
 
-	now := dbtime.Now()
+	var (
+		now            = dbtime.Now()
+		oneHourAgo     = now.Add(-time.Hour)
+		staleThreshold = now.Add(-provisionerdserver.StaleInterval).Add(-time.Second)
+	)
 
 	for _, tt := range []struct {
 		name                   string
@@ -65,7 +71,9 @@ func TestProvisionerDaemonReport(t *testing.T) {
 			currentVersion:         "v1.2.3",
 			currentAPIMajorVersion: proto.CurrentMajor,
 			expectedSeverity:       health.SeverityOK,
-			provisionerDaemons:     []database.ProvisionerDaemon{fakeProvisionerDaemon(t, "pd-ok", "v1.2.3", "1.0", now)},
+			provisionerDaemons: []database.ProvisionerDaemon{
+				fakeProvisionerDaemon(t, withName("pd-ok"), withVersion("v1.2.3"), withAPIVersion("1.0"), withCreatedAt(now), withLastSeenAt(now)),
+			},
 			expectedItems: []healthsdk.ProvisionerDaemonsReportItem{
 				{
 					ProvisionerDaemon: codersdk.ProvisionerDaemon{
@@ -88,7 +96,9 @@ func TestProvisionerDaemonReport(t *testing.T) {
 			currentAPIMajorVersion: proto.CurrentMajor,
 			expectedSeverity:       health.SeverityWarning,
 			expectedWarningCode:    health.CodeProvisionerDaemonVersionMismatch,
-			provisionerDaemons:     []database.ProvisionerDaemon{fakeProvisionerDaemon(t, "pd-old", "v1.1.2", "1.0", now)},
+			provisionerDaemons: []database.ProvisionerDaemon{
+				fakeProvisionerDaemon(t, withName("pd-old"), withVersion("v1.1.2"), withAPIVersion("1.0"), withCreatedAt(now), withLastSeenAt(now)),
+			},
 			expectedItems: []healthsdk.ProvisionerDaemonsReportItem{
 				{
 					ProvisionerDaemon: codersdk.ProvisionerDaemon{
@@ -116,7 +126,9 @@ func TestProvisionerDaemonReport(t *testing.T) {
 			currentAPIMajorVersion: proto.CurrentMajor,
 			expectedSeverity:       health.SeverityError,
 			expectedWarningCode:    health.CodeUnknown,
-			provisionerDaemons:     []database.ProvisionerDaemon{fakeProvisionerDaemon(t, "pd-invalid-version", "invalid", "1.0", now)},
+			provisionerDaemons: []database.ProvisionerDaemon{
+				fakeProvisionerDaemon(t, withName("pd-invalid-version"), withVersion("invalid"), withAPIVersion("1.0"), withCreatedAt(now), withLastSeenAt(now)),
+			},
 			expectedItems: []healthsdk.ProvisionerDaemonsReportItem{
 				{
 					ProvisionerDaemon: codersdk.ProvisionerDaemon{
@@ -144,7 +156,9 @@ func TestProvisionerDaemonReport(t *testing.T) {
 			currentAPIMajorVersion: proto.CurrentMajor,
 			expectedSeverity:       health.SeverityError,
 			expectedWarningCode:    health.CodeUnknown,
-			provisionerDaemons:     []database.ProvisionerDaemon{fakeProvisionerDaemon(t, "pd-invalid-api", "v1.2.3", "invalid", now)},
+			provisionerDaemons: []database.ProvisionerDaemon{
+				fakeProvisionerDaemon(t, withName("pd-invalid-api"), withVersion("v1.2.3"), withAPIVersion("invalid"), withCreatedAt(now), withLastSeenAt(now)),
+			},
 			expectedItems: []healthsdk.ProvisionerDaemonsReportItem{
 				{
 					ProvisionerDaemon: codersdk.ProvisionerDaemon{
@@ -172,7 +186,9 @@ func TestProvisionerDaemonReport(t *testing.T) {
 			currentAPIMajorVersion: 2,
 			expectedSeverity:       health.SeverityWarning,
 			expectedWarningCode:    health.CodeProvisionerDaemonAPIMajorVersionDeprecated,
-			provisionerDaemons:     []database.ProvisionerDaemon{fakeProvisionerDaemon(t, "pd-old-api", "v2.3.4", "1.0", now)},
+			provisionerDaemons: []database.ProvisionerDaemon{
+				fakeProvisionerDaemon(t, withName("pd-old-api"), withVersion("v2.3.4"), withAPIVersion("1.0"), withCreatedAt(now), withLastSeenAt(now)),
+			},
 			expectedItems: []healthsdk.ProvisionerDaemonsReportItem{
 				{
 					ProvisionerDaemon: codersdk.ProvisionerDaemon{
@@ -200,7 +216,10 @@ func TestProvisionerDaemonReport(t *testing.T) {
 			currentAPIMajorVersion: proto.CurrentMajor,
 			expectedSeverity:       health.SeverityWarning,
 			expectedWarningCode:    health.CodeProvisionerDaemonVersionMismatch,
-			provisionerDaemons:     []database.ProvisionerDaemon{fakeProvisionerDaemon(t, "pd-ok", "v1.2.3", "1.0", now), fakeProvisionerDaemon(t, "pd-old", "v1.1.2", "1.0", now)},
+			provisionerDaemons: []database.ProvisionerDaemon{
+				fakeProvisionerDaemon(t, withName("pd-ok"), withVersion("v1.2.3"), withAPIVersion("1.0"), withCreatedAt(now), withLastSeenAt(now)),
+				fakeProvisionerDaemon(t, withName("pd-old"), withVersion("v1.1.2"), withAPIVersion("1.0"), withCreatedAt(now), withLastSeenAt(now)),
+			},
 			expectedItems: []healthsdk.ProvisionerDaemonsReportItem{
 				{
 					ProvisionerDaemon: codersdk.ProvisionerDaemon{
@@ -241,7 +260,10 @@ func TestProvisionerDaemonReport(t *testing.T) {
 			currentAPIMajorVersion: proto.CurrentMajor,
 			expectedSeverity:       health.SeverityWarning,
 			expectedWarningCode:    health.CodeProvisionerDaemonVersionMismatch,
-			provisionerDaemons:     []database.ProvisionerDaemon{fakeProvisionerDaemon(t, "pd-ok", "v1.2.3", "1.0", now), fakeProvisionerDaemon(t, "pd-new", "v2.3.4", "1.0", now)},
+			provisionerDaemons: []database.ProvisionerDaemon{
+				fakeProvisionerDaemon(t, withName("pd-ok"), withVersion("v1.2.3"), withAPIVersion("1.0"), withCreatedAt(now), withLastSeenAt(now)),
+				fakeProvisionerDaemon(t, withName("pd-new"), withVersion("v2.3.4"), withAPIVersion("1.0"), withCreatedAt(now), withLastSeenAt(now)),
+			},
 			expectedItems: []healthsdk.ProvisionerDaemonsReportItem{
 				{
 					ProvisionerDaemon: codersdk.ProvisionerDaemon{
@@ -281,7 +303,10 @@ func TestProvisionerDaemonReport(t *testing.T) {
 			currentVersion:         "v2.3.4",
 			currentAPIMajorVersion: proto.CurrentMajor,
 			expectedSeverity:       health.SeverityOK,
-			provisionerDaemons:     []database.ProvisionerDaemon{fakeProvisionerDaemonStale(t, "pd-stale", "v1.2.3", "0.9", now.Add(-5*time.Minute), now), fakeProvisionerDaemon(t, "pd-ok", "v2.3.4", "1.0", now)},
+			provisionerDaemons: []database.ProvisionerDaemon{
+				fakeProvisionerDaemon(t, withName("pd-stale"), withVersion("v1.2.3"), withAPIVersion("0.9"), withCreatedAt(oneHourAgo), withLastSeenAt(staleThreshold)),
+				fakeProvisionerDaemon(t, withName("pd-ok"), withVersion("v2.3.4"), withAPIVersion("1.0"), withCreatedAt(now), withLastSeenAt(now)),
+			},
 			expectedItems: []healthsdk.ProvisionerDaemonsReportItem{
 				{
 					ProvisionerDaemon: codersdk.ProvisionerDaemon{
@@ -304,8 +329,10 @@ func TestProvisionerDaemonReport(t *testing.T) {
 			currentAPIMajorVersion: proto.CurrentMajor,
 			expectedSeverity:       health.SeverityError,
 			expectedWarningCode:    health.CodeProvisionerDaemonsNoProvisionerDaemons,
-			provisionerDaemons:     []database.ProvisionerDaemon{fakeProvisionerDaemonStale(t, "pd-ok", "v1.2.3", "0.9", now.Add(-5*time.Minute), now)},
-			expectedItems:          []healthsdk.ProvisionerDaemonsReportItem{},
+			provisionerDaemons: []database.ProvisionerDaemon{
+				fakeProvisionerDaemon(t, withName("pd-stale"), withVersion("v1.2.3"), withAPIVersion("0.9"), withCreatedAt(oneHourAgo), withLastSeenAt(staleThreshold)),
+			},
+			expectedItems: []healthsdk.ProvisionerDaemonsReportItem{},
 		},
 	} {
 		tt := tt
@@ -353,25 +380,52 @@ func TestProvisionerDaemonReport(t *testing.T) {
 	}
 }
 
-func fakeProvisionerDaemon(t *testing.T, name, version, apiVersion string, now time.Time) database.ProvisionerDaemon {
-	t.Helper()
-	return database.ProvisionerDaemon{
-		ID:           uuid.Nil,
-		Name:         name,
-		CreatedAt:    now,
-		LastSeenAt:   sql.NullTime{Time: now, Valid: true},
-		Provisioners: []database.ProvisionerType{database.ProvisionerTypeEcho, database.ProvisionerTypeTerraform},
-		ReplicaID:    uuid.NullUUID{},
-		Tags:         map[string]string{},
-		Version:      version,
-		APIVersion:   apiVersion,
+func withName(s string) func(*database.ProvisionerDaemon) {
+	return func(pd *database.ProvisionerDaemon) {
+		pd.Name = s
 	}
 }
 
-func fakeProvisionerDaemonStale(t *testing.T, name, version, apiVersion string, lastSeenAt, now time.Time) database.ProvisionerDaemon {
-	t.Helper()
-	d := fakeProvisionerDaemon(t, name, version, apiVersion, now)
-	d.LastSeenAt.Valid = true
-	d.LastSeenAt.Time = lastSeenAt
-	return d
+func withCreatedAt(at time.Time) func(*database.ProvisionerDaemon) {
+	return func(pd *database.ProvisionerDaemon) {
+		pd.CreatedAt = at
+	}
+}
+
+func withLastSeenAt(at time.Time) func(*database.ProvisionerDaemon) {
+	return func(pd *database.ProvisionerDaemon) {
+		pd.LastSeenAt.Valid = true
+		pd.LastSeenAt.Time = at
+	}
+}
+
+func withVersion(v string) func(*database.ProvisionerDaemon) {
+	return func(pd *database.ProvisionerDaemon) {
+		pd.Version = v
+	}
+}
+
+func withAPIVersion(v string) func(*database.ProvisionerDaemon) {
+	return func(pd *database.ProvisionerDaemon) {
+		pd.APIVersion = v
+	}
+}
+
+func fakeProvisionerDaemon(t *testing.T, opts ...func(*database.ProvisionerDaemon)) database.ProvisionerDaemon {
+	t.Helper()
+	pd := database.ProvisionerDaemon{
+		ID:           uuid.Nil,
+		Name:         testutil.GetRandomName(t),
+		CreatedAt:    time.Time{},
+		LastSeenAt:   sql.NullTime{},
+		Provisioners: []database.ProvisionerType{database.ProvisionerTypeEcho, database.ProvisionerTypeTerraform},
+		ReplicaID:    uuid.NullUUID{},
+		Tags:         map[string]string{},
+		Version:      "",
+		APIVersion:   "",
+	}
+	for _, o := range opts {
+		o(&pd)
+	}
+	return pd
 }

coderd/provisionerdserver/provisionerdserver.go

@@ -57,6 +57,10 @@ const (
 	// DefaultHeartbeatInterval is the interval at which the provisioner daemon
 	// will update its last seen at timestamp in the database.
 	DefaultHeartbeatInterval = time.Minute
+
+	// StaleInterval is the amount of time after the last heartbeat for which
+	// the provisioner will be reported as 'stale'.
+	StaleInterval = 90 * time.Second
 )
 
 type Options struct {

coderd/templateversions.go

@@ -1515,7 +1515,7 @@ func (api *API) postTemplateVersionsByOrganization(rw http.ResponseWriter, r *ht
 
 		// Check for eligible provisioners. This allows us to log a message warning deployment administrators
 		// of users submitting jobs for which no provisioners are available.
-		matchedProvisioners, err = checkProvisioners(ctx, tx, organization.ID, tags, api.DeploymentValues.Provisioner.DaemonPollInterval.Value())
+		matchedProvisioners, err = checkProvisioners(ctx, tx, organization.ID, tags)
 		if err != nil {
 			api.Logger.Error(ctx, "failed to check eligible provisioner daemons for job", slog.Error(err))
 		} else if matchedProvisioners.Count == 0 {
@@ -1823,7 +1823,7 @@ func (api *API) publishTemplateUpdate(ctx context.Context, templateID uuid.UUID)
 	}
 }
 
-func checkProvisioners(ctx context.Context, store database.Store, orgID uuid.UUID, wantTags map[string]string, pollInterval time.Duration) (codersdk.MatchedProvisioners, error) {
+func checkProvisioners(ctx context.Context, store database.Store, orgID uuid.UUID, wantTags map[string]string) (codersdk.MatchedProvisioners, error) {
 	// Check for eligible provisioners. This allows us to return a warning to the user if they
 	// submit a job for which no provisioner is available.
 	eligibleProvisioners, err := store.GetProvisionerDaemonsByOrganization(ctx, database.GetProvisionerDaemonsByOrganizationParams{
@@ -1835,7 +1835,7 @@ func checkProvisioners(ctx context.Context, store database.Store, orgID uuid.UUI
 		return codersdk.MatchedProvisioners{}, xerrors.Errorf("provisioner daemons by organization: %w", err)
 	}
 
-	threePollsAgo := time.Now().Add(-3 * pollInterval)
+	staleInterval := time.Now().Add(-provisionerdserver.StaleInterval)
 	mostRecentlySeen := codersdk.NullTime{}
 	var matched codersdk.MatchedProvisioners
 	for _, provisioner := range eligibleProvisioners {
@@ -1843,7 +1843,7 @@ func checkProvisioners(ctx context.Context, store database.Store, orgID uuid.UUI
 			continue
 		}
 		matched.Count++
-		if provisioner.LastSeenAt.Time.After(threePollsAgo) {
+		if provisioner.LastSeenAt.Time.After(staleInterval) {
 			matched.Available++
 		}
 		if provisioner.LastSeenAt.Time.After(mostRecentlySeen.Time) {