chore: add tracing to prebuilds (#21443)

The implementation for prebuilt workspaces is complex and conversations
regarding edge cases and bugs frequently get bogged down by minutiae,
because it's hard to reason about the behaviour of the system.

To alleviate this, I've introduced otel tracing to the StoreReconciler
(see attached). We can now directly observe the behaviour of the
prebuilds system under load in order to inform our decisions.

Traces are terminated at the boundary between prebuilds and workspace
builder, because of prebuilt workspaces' "fire and forget" philosophy
and to prevent span explosion.

<img width="3024" height="1718" alt="image"
src="https://github.com/user-attachments/assets/f9b207be-8f2c-475e-98a8-46ef70bda446"
/>

enterprise/cli/create_test.go

@@ -17,6 +17,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel/trace/noop"
 
 	"github.com/coder/coder/v2/coderd/database"
 	"github.com/coder/coder/v2/coderd/database/dbtestutil"
@@ -370,6 +371,7 @@ func TestEnterpriseCreateWithPreset(t *testing.T) {
 			prometheus.NewRegistry(),
 			notifications.NewNoopEnqueuer(),
 			newNoopUsageCheckerPtr(),
+			noop.NewTracerProvider(),
 		)
 		var claimer agplprebuilds.Claimer = prebuilds.NewEnterpriseClaimer(db)
 		api.AGPL.PrebuildsClaimer.Store(&claimer)
@@ -481,6 +483,7 @@ func TestEnterpriseCreateWithPreset(t *testing.T) {
 			prometheus.NewRegistry(),
 			notifications.NewNoopEnqueuer(),
 			newNoopUsageCheckerPtr(),
+			noop.NewTracerProvider(),
 		)
 		var claimer agplprebuilds.Claimer = prebuilds.NewEnterpriseClaimer(db)
 		api.AGPL.PrebuildsClaimer.Store(&claimer)

enterprise/coderd/coderd.go

@@ -1307,6 +1307,6 @@ func (api *API) setupPrebuilds(featureEnabled bool) (agplprebuilds.Reconciliatio
 	}
 
 	reconciler := prebuilds.NewStoreReconciler(api.Database, api.Pubsub, api.AGPL.FileCache, api.DeploymentValues.Prebuilds,
-		api.Logger.Named("prebuilds"), quartz.NewReal(), api.PrometheusRegistry, api.NotificationsEnqueuer, api.AGPL.BuildUsageChecker)
+		api.Logger.Named("prebuilds"), quartz.NewReal(), api.PrometheusRegistry, api.NotificationsEnqueuer, api.AGPL.BuildUsageChecker, api.TracerProvider)
 	return reconciler, prebuilds.NewEnterpriseClaimer(api.Database)
 }

enterprise/coderd/prebuilds/claim_test.go

@@ -13,6 +13,7 @@ import (
 	"github.com/google/uuid"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel/trace/noop"
 	"golang.org/x/xerrors"
 
 	"github.com/coder/coder/v2/coderd/database/dbtime"
@@ -167,7 +168,7 @@ func TestClaimPrebuild(t *testing.T) {
 				defer provisionerCloser.Close()
 
 				cache := files.New(prometheus.NewRegistry(), &coderdtest.FakeAuthorizer{})
-				reconciler := prebuilds.NewStoreReconciler(spy, pubsub, cache, codersdk.PrebuildsConfig{}, logger, quartz.NewMock(t), prometheus.NewRegistry(), newNoopEnqueuer(), newNoopUsageCheckerPtr())
+				reconciler := prebuilds.NewStoreReconciler(spy, pubsub, cache, codersdk.PrebuildsConfig{}, logger, quartz.NewMock(t), prometheus.NewRegistry(), newNoopEnqueuer(), newNoopUsageCheckerPtr(), noop.NewTracerProvider())
 				var claimer agplprebuilds.Claimer = prebuilds.NewEnterpriseClaimer(spy)
 				api.AGPL.PrebuildsClaimer.Store(&claimer)
 

enterprise/coderd/prebuilds/metricscollector_test.go

@@ -6,11 +6,11 @@ import (
 	"testing"
 
 	"github.com/google/uuid"
-	"github.com/stretchr/testify/require"
-	"tailscale.com/types/ptr"
-
 	"github.com/prometheus/client_golang/prometheus"
 	prometheus_client "github.com/prometheus/client_model/go"
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel/trace/noop"
+	"tailscale.com/types/ptr"
 
 	"cdr.dev/slog/sloggers/slogtest"
 	"github.com/coder/coder/v2/coderd/coderdtest"
@@ -197,7 +197,7 @@ func TestMetricsCollector(t *testing.T) {
 									clock := quartz.NewMock(t)
 									db, pubsub := dbtestutil.NewDB(t)
 									cache := files.New(prometheus.NewRegistry(), &coderdtest.FakeAuthorizer{})
-									reconciler := prebuilds.NewStoreReconciler(db, pubsub, cache, codersdk.PrebuildsConfig{}, logger, quartz.NewMock(t), prometheus.NewRegistry(), newNoopEnqueuer(), newNoopUsageCheckerPtr())
+									reconciler := prebuilds.NewStoreReconciler(db, pubsub, cache, codersdk.PrebuildsConfig{}, logger, quartz.NewMock(t), prometheus.NewRegistry(), newNoopEnqueuer(), newNoopUsageCheckerPtr(), noop.NewTracerProvider())
 									ctx := testutil.Context(t, testutil.WaitLong)
 
 									createdUsers := []uuid.UUID{database.PrebuildsSystemUserID}
@@ -329,7 +329,7 @@ func TestMetricsCollector_DuplicateTemplateNames(t *testing.T) {
 	clock := quartz.NewMock(t)
 	db, pubsub := dbtestutil.NewDB(t)
 	cache := files.New(prometheus.NewRegistry(), &coderdtest.FakeAuthorizer{})
-	reconciler := prebuilds.NewStoreReconciler(db, pubsub, cache, codersdk.PrebuildsConfig{}, logger, quartz.NewMock(t), prometheus.NewRegistry(), newNoopEnqueuer(), newNoopUsageCheckerPtr())
+	reconciler := prebuilds.NewStoreReconciler(db, pubsub, cache, codersdk.PrebuildsConfig{}, logger, quartz.NewMock(t), prometheus.NewRegistry(), newNoopEnqueuer(), newNoopUsageCheckerPtr(), noop.NewTracerProvider())
 	ctx := testutil.Context(t, testutil.WaitLong)
 
 	collector := prebuilds.NewMetricsCollector(db, logger, reconciler)
@@ -477,7 +477,7 @@ func TestMetricsCollector_ReconciliationPausedMetric(t *testing.T) {
 		db, pubsub := dbtestutil.NewDB(t)
 		cache := files.New(prometheus.NewRegistry(), &coderdtest.FakeAuthorizer{})
 		registry := prometheus.NewPedanticRegistry()
-		reconciler := prebuilds.NewStoreReconciler(db, pubsub, cache, codersdk.PrebuildsConfig{}, logger, quartz.NewMock(t), registry, newNoopEnqueuer(), newNoopUsageCheckerPtr())
+		reconciler := prebuilds.NewStoreReconciler(db, pubsub, cache, codersdk.PrebuildsConfig{}, logger, quartz.NewMock(t), registry, newNoopEnqueuer(), newNoopUsageCheckerPtr(), noop.NewTracerProvider())
 		ctx := testutil.Context(t, testutil.WaitLong)
 
 		// Ensure no pause setting is set (default state)
@@ -506,7 +506,7 @@ func TestMetricsCollector_ReconciliationPausedMetric(t *testing.T) {
 		db, pubsub := dbtestutil.NewDB(t)
 		cache := files.New(prometheus.NewRegistry(), &coderdtest.FakeAuthorizer{})
 		registry := prometheus.NewPedanticRegistry()
-		reconciler := prebuilds.NewStoreReconciler(db, pubsub, cache, codersdk.PrebuildsConfig{}, logger, quartz.NewMock(t), registry, newNoopEnqueuer(), newNoopUsageCheckerPtr())
+		reconciler := prebuilds.NewStoreReconciler(db, pubsub, cache, codersdk.PrebuildsConfig{}, logger, quartz.NewMock(t), registry, newNoopEnqueuer(), newNoopUsageCheckerPtr(), noop.NewTracerProvider())
 		ctx := testutil.Context(t, testutil.WaitLong)
 
 		// Set reconciliation to paused
@@ -535,7 +535,7 @@ func TestMetricsCollector_ReconciliationPausedMetric(t *testing.T) {
 		db, pubsub := dbtestutil.NewDB(t)
 		cache := files.New(prometheus.NewRegistry(), &coderdtest.FakeAuthorizer{})
 		registry := prometheus.NewPedanticRegistry()
-		reconciler := prebuilds.NewStoreReconciler(db, pubsub, cache, codersdk.PrebuildsConfig{}, logger, quartz.NewMock(t), registry, newNoopEnqueuer(), newNoopUsageCheckerPtr())
+		reconciler := prebuilds.NewStoreReconciler(db, pubsub, cache, codersdk.PrebuildsConfig{}, logger, quartz.NewMock(t), registry, newNoopEnqueuer(), newNoopUsageCheckerPtr(), noop.NewTracerProvider())
 		ctx := testutil.Context(t, testutil.WaitLong)
 
 		// Set reconciliation back to not paused

enterprise/coderd/prebuilds/reconcile.go

@@ -16,6 +16,8 @@ import (
 	"github.com/hashicorp/go-multierror"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
 	"golang.org/x/sync/errgroup"
 	"golang.org/x/xerrors"
 
@@ -31,6 +33,7 @@ import (
 	"github.com/coder/coder/v2/coderd/prebuilds"
 	"github.com/coder/coder/v2/coderd/rbac"
 	"github.com/coder/coder/v2/coderd/rbac/policy"
+	"github.com/coder/coder/v2/coderd/tracing"
 	"github.com/coder/coder/v2/coderd/wsbuilder"
 	"github.com/coder/coder/v2/codersdk"
 	sdkproto "github.com/coder/coder/v2/provisionersdk/proto"
@@ -47,6 +50,7 @@ type StoreReconciler struct {
 	registerer        prometheus.Registerer
 	notifEnq          notifications.Enqueuer
 	buildUsageChecker *atomic.Pointer[wsbuilder.UsageChecker]
+	tracer            trace.Tracer
 
 	cancelFn          context.CancelCauseFunc
 	running           atomic.Bool
@@ -89,6 +93,7 @@ func NewStoreReconciler(store database.Store,
 	registerer prometheus.Registerer,
 	notifEnq notifications.Enqueuer,
 	buildUsageChecker *atomic.Pointer[wsbuilder.UsageChecker],
+	tracerProvider trace.TracerProvider,
 ) *StoreReconciler {
 	reconciler := &StoreReconciler{
 		store:             store,
@@ -100,6 +105,7 @@ func NewStoreReconciler(store database.Store,
 		registerer:        registerer,
 		notifEnq:          notifEnq,
 		buildUsageChecker: buildUsageChecker,
+		tracer:            tracerProvider.Tracer(tracing.TracerName),
 		done:              make(chan struct{}, 1),
 		provisionNotifyCh: make(chan database.ProvisionerJob, 10),
 	}
@@ -283,6 +289,9 @@ func (c *StoreReconciler) Stop(ctx context.Context, cause error) {
 // simultaneously for the same preset, but once both jobs have completed the reconciliation loop will notice the
 // extraneous instance and delete it.
 func (c *StoreReconciler) ReconcileAll(ctx context.Context) (stats prebuilds.ReconcileStats, err error) {
+	ctx, span := c.tracer.Start(ctx, "prebuilds.ReconcileAll")
+	defer span.End()
+
 	start := c.clock.Now()
 	defer func() {
 		stats.Elapsed = c.clock.Since(start)
@@ -420,6 +429,9 @@ func (c *StoreReconciler) reportHardLimitedPresets(snapshot *prebuilds.GlobalSna
 
 // SnapshotState captures the current state of all prebuilds across templates.
 func (c *StoreReconciler) SnapshotState(ctx context.Context, store database.Store) (*prebuilds.GlobalSnapshot, error) {
+	ctx, span := c.tracer.Start(ctx, "prebuilds.SnapshotState")
+	defer span.End()
+
 	if err := ctx.Err(); err != nil {
 		return nil, err
 	}
@@ -482,13 +494,21 @@ func (c *StoreReconciler) SnapshotState(ctx context.Context, store database.Stor
 	}, &database.TxOptions{
 		Isolation:    sql.LevelRepeatableRead, // This mirrors the MVCC snapshotting Postgres does when using CTEs
 		ReadOnly:     true,
-		TxIdentifier: "prebuilds_state_determination",
+		TxIdentifier: "prebuilds.SnapshotState",
 	})
 
 	return &state, err
 }
 
 func (c *StoreReconciler) ReconcilePreset(ctx context.Context, ps prebuilds.PresetSnapshot) error {
+	ctx, span := c.tracer.Start(ctx, "prebuilds.ReconcilePreset", trace.WithAttributes(
+		attribute.String("preset_id", ps.Preset.ID.String()),
+		attribute.String("preset_name", ps.Preset.Name),
+		attribute.String("template_id", ps.Preset.TemplateID.String()),
+		attribute.String("template_name", ps.Preset.TemplateName),
+	))
+	defer span.End()
+
 	logger := c.logger.With(
 		slog.F("template_id", ps.Preset.TemplateID.String()),
 		slog.F("template_name", ps.Preset.TemplateName),
@@ -554,6 +574,9 @@ func (c *StoreReconciler) WithReconciliationLock(
 	logger slog.Logger,
 	fn func(ctx context.Context, db database.Store) error,
 ) error {
+	ctx, span := c.tracer.Start(ctx, "prebuilds.WithReconciliationLock")
+	defer span.End()
+
 	// This tx holds a global lock, which prevents any other coderd replica from starting a reconciliation and
 	// possibly getting an inconsistent view of the state.
 	//
@@ -574,8 +597,10 @@ func (c *StoreReconciler) WithReconciliationLock(
 		}
 		if !acquired {
 			// Normal case: another replica has the lock
+			span.SetAttributes(attribute.Bool("lock_acquired", false))
 			return nil
 		}
+		span.SetAttributes(attribute.Bool("lock_acquired", true))
 
 		logger.Debug(ctx,
 			"acquired top-level reconciliation lock",
@@ -586,7 +611,7 @@ func (c *StoreReconciler) WithReconciliationLock(
 	}, &database.TxOptions{
 		Isolation:    sql.LevelRepeatableRead,
 		ReadOnly:     true,
-		TxIdentifier: "prebuilds",
+		TxIdentifier: "prebuilds.WithReconciliationLock",
 	})
 }
 
@@ -599,6 +624,13 @@ func (c *StoreReconciler) WithReconciliationLock(
 // This method handles logging at appropriate levels and performs the necessary operations
 // according to the action type. It returns an error if any part of the action fails.
 func (c *StoreReconciler) executeReconciliationAction(ctx context.Context, logger slog.Logger, ps prebuilds.PresetSnapshot, action *prebuilds.ReconciliationActions) error {
+	ctx, span := c.tracer.Start(ctx, "prebuilds.executeReconciliationAction", trace.WithAttributes(
+		attribute.Int("action_type", int(action.ActionType)),
+		attribute.Int("create_count", int(action.Create)),
+		attribute.Int("delete_count", len(action.DeleteIDs)),
+	))
+	defer span.End()
+
 	levelFn := logger.Debug
 
 	// Nothing has to be done.
@@ -692,6 +724,13 @@ func (c *StoreReconciler) executeReconciliationAction(ctx context.Context, logge
 }
 
 func (c *StoreReconciler) createPrebuiltWorkspace(ctx context.Context, prebuiltWorkspaceID uuid.UUID, templateID uuid.UUID, presetID uuid.UUID) error {
+	ctx, span := c.tracer.Start(ctx, "prebuilds.createPrebuiltWorkspace", trace.WithAttributes(
+		attribute.String("prebuild_id", prebuiltWorkspaceID.String()),
+		attribute.String("template_id", templateID.String()),
+		attribute.String("preset_id", presetID.String()),
+	))
+	defer span.End()
+
 	name, err := prebuilds.GenerateName()
 	if err != nil {
 		return xerrors.Errorf("failed to generate unique prebuild ID: %w", err)
@@ -736,8 +775,9 @@ func (c *StoreReconciler) createPrebuiltWorkspace(ctx context.Context, prebuiltW
 		provisionerJob, err = c.provision(ctx, db, prebuiltWorkspaceID, template, presetID, database.WorkspaceTransitionStart, workspace, DeprovisionModeNormal)
 		return err
 	}, &database.TxOptions{
-		Isolation: sql.LevelRepeatableRead,
-		ReadOnly:  false,
+		Isolation:    sql.LevelRepeatableRead,
+		ReadOnly:     false,
+		TxIdentifier: "prebuilds.createPrebuiltWorkspace",
 	})
 	if err != nil {
 		return err
@@ -788,6 +828,13 @@ func (c *StoreReconciler) provisionDelete(ctx context.Context, db database.Store
 // Since these jobs were never processed by a provisioner, no Terraform resources were created,
 // making it safe to orphan-delete the workspaces (skipping Terraform destroy).
 func (c *StoreReconciler) cancelAndOrphanDeletePendingPrebuilds(ctx context.Context, templateID uuid.UUID, templateVersionID uuid.UUID, presetID uuid.UUID) error {
+	ctx, span := c.tracer.Start(ctx, "prebuilds.cancelAndOrphanDeletePendingPrebuilds", trace.WithAttributes(
+		attribute.String("template_id", templateID.String()),
+		attribute.String("template_version_id", templateVersionID.String()),
+		attribute.String("preset_id", presetID.String()),
+	))
+	defer span.End()
+
 	var canceledProvisionerJob *database.ProvisionerJob
 	var canceledWorkspaceID uuid.UUID
 	err := c.store.InTx(func(db database.Store) error {
@@ -832,8 +879,9 @@ func (c *StoreReconciler) cancelAndOrphanDeletePendingPrebuilds(ctx context.Cont
 
 		return multiErr.ErrorOrNil()
 	}, &database.TxOptions{
-		Isolation: sql.LevelRepeatableRead,
-		ReadOnly:  false,
+		Isolation:    sql.LevelRepeatableRead,
+		ReadOnly:     false,
+		TxIdentifier: "prebuilds.cancelAndOrphanDeletePendingPrebuilds",
 	})
 	if err != nil {
 		return err
@@ -851,13 +899,21 @@ func (c *StoreReconciler) cancelAndOrphanDeletePendingPrebuilds(ctx context.Cont
 }
 
 func (c *StoreReconciler) deletePrebuiltWorkspace(ctx context.Context, prebuiltWorkspaceID uuid.UUID, templateID uuid.UUID, presetID uuid.UUID) error {
+	ctx, span := c.tracer.Start(ctx, "prebuilds.deletePrebuiltWorkspace", trace.WithAttributes(
+		attribute.String("prebuild_id", prebuiltWorkspaceID.String()),
+		attribute.String("template_id", templateID.String()),
+		attribute.String("preset_id", presetID.String()),
+	))
+	defer span.End()
+
 	var provisionerJob *database.ProvisionerJob
 	err := c.store.InTx(func(db database.Store) (err error) {
 		provisionerJob, err = c.provisionDelete(ctx, db, prebuiltWorkspaceID, templateID, presetID, DeprovisionModeNormal)
 		return err
 	}, &database.TxOptions{
-		Isolation: sql.LevelRepeatableRead,
-		ReadOnly:  false,
+		Isolation:    sql.LevelRepeatableRead,
+		ReadOnly:     false,
+		TxIdentifier: "prebuilds.deletePrebuiltWorkspace",
 	})
 	if err != nil {
 		return err
@@ -879,6 +935,16 @@ func (c *StoreReconciler) provision(
 	workspace database.Workspace,
 	mode DeprovisionMode,
 ) (*database.ProvisionerJob, error) {
+	ctx, span := c.tracer.Start(ctx, "prebuilds.provision", trace.WithAttributes(
+		attribute.String("prebuild_id", prebuildID.String()),
+		attribute.String("template_id", template.ID.String()),
+		attribute.String("preset_id", presetID.String()),
+		attribute.String("transition", string(transition)),
+		attribute.String("workspace_id", workspace.ID.String()),
+		attribute.String("mode", mode.String()),
+	))
+	defer span.End()
+
 	tvp, err := db.GetPresetParametersByTemplateVersionID(ctx, template.ActiveVersionID)
 	if err != nil {
 		return nil, xerrors.Errorf("fetch preset details: %w", err)
@@ -920,8 +986,12 @@ func (c *StoreReconciler) provision(
 		builder = builder.Orphan()
 	}
 
+	// Strip trace context - provisionerd is a separate service and should
+	// start its own trace rather than continuing the prebuilds trace.
+	buildCtx := trace.ContextWithSpan(ctx, tracing.NoopSpan)
+
 	_, provisionerJob, _, err := builder.Build(
-		ctx,
+		buildCtx,
 		db,
 		c.fileCache,
 		func(_ policy.Action, _ rbac.Objecter) bool {

enterprise/coderd/prebuilds/reconcile_test.go

@@ -13,6 +13,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel/trace/noop"
 	"golang.org/x/xerrors"
 	"tailscale.com/types/ptr"
 
@@ -52,7 +53,7 @@ func TestNoReconciliationActionsIfNoPresets(t *testing.T) {
 	}
 	logger := testutil.Logger(t)
 	cache := files.New(prometheus.NewRegistry(), &coderdtest.FakeAuthorizer{})
-	controller := prebuilds.NewStoreReconciler(db, ps, cache, cfg, logger, quartz.NewMock(t), prometheus.NewRegistry(), newNoopEnqueuer(), newNoopUsageCheckerPtr())
+	controller := prebuilds.NewStoreReconciler(db, ps, cache, cfg, logger, quartz.NewMock(t), prometheus.NewRegistry(), newNoopEnqueuer(), newNoopUsageCheckerPtr(), noop.NewTracerProvider())
 
 	// given a template version with no presets
 	org := dbgen.Organization(t, db, database.Organization{})
@@ -95,7 +96,7 @@ func TestNoReconciliationActionsIfNoPrebuilds(t *testing.T) {
 	}
 	logger := testutil.Logger(t)
 	cache := files.New(prometheus.NewRegistry(), &coderdtest.FakeAuthorizer{})
-	controller := prebuilds.NewStoreReconciler(db, ps, cache, cfg, logger, quartz.NewMock(t), prometheus.NewRegistry(), newNoopEnqueuer(), newNoopUsageCheckerPtr())
+	controller := prebuilds.NewStoreReconciler(db, ps, cache, cfg, logger, quartz.NewMock(t), prometheus.NewRegistry(), newNoopEnqueuer(), newNoopUsageCheckerPtr(), noop.NewTracerProvider())
 
 	// given there are presets, but no prebuilds
 	org := dbgen.Organization(t, db, database.Organization{})
@@ -425,7 +426,7 @@ func (tc testCase) run(t *testing.T) {
 			pubSub = &brokenPublisher{Pubsub: pubSub}
 		}
 		cache := files.New(prometheus.NewRegistry(), &coderdtest.FakeAuthorizer{})
-		controller := prebuilds.NewStoreReconciler(db, pubSub, cache, cfg, logger, quartz.NewMock(t), prometheus.NewRegistry(), newNoopEnqueuer(), newNoopUsageCheckerPtr())
+		controller := prebuilds.NewStoreReconciler(db, pubSub, cache, cfg, logger, quartz.NewMock(t), prometheus.NewRegistry(), newNoopEnqueuer(), newNoopUsageCheckerPtr(), noop.NewTracerProvider())
 
 		// Run the reconciliation multiple times to ensure idempotency
 		// 8 was arbitrary, but large enough to reasonably trust the result
@@ -494,7 +495,7 @@ func TestMultiplePresetsPerTemplateVersion(t *testing.T) {
 	).Leveled(slog.LevelDebug)
 	db, pubSub := dbtestutil.NewDB(t)
 	cache := files.New(prometheus.NewRegistry(), &coderdtest.FakeAuthorizer{})
-	controller := prebuilds.NewStoreReconciler(db, pubSub, cache, cfg, logger, quartz.NewMock(t), prometheus.NewRegistry(), newNoopEnqueuer(), newNoopUsageCheckerPtr())
+	controller := prebuilds.NewStoreReconciler(db, pubSub, cache, cfg, logger, quartz.NewMock(t), prometheus.NewRegistry(), newNoopEnqueuer(), newNoopUsageCheckerPtr(), noop.NewTracerProvider())
 
 	ownerID := uuid.New()
 	dbgen.User(t, db, database.User{
@@ -617,7 +618,7 @@ func TestPrebuildScheduling(t *testing.T) {
 			).Leveled(slog.LevelDebug)
 			db, pubSub := dbtestutil.NewDB(t)
 			cache := files.New(prometheus.NewRegistry(), &coderdtest.FakeAuthorizer{})
-			controller := prebuilds.NewStoreReconciler(db, pubSub, cache, cfg, logger, clock, prometheus.NewRegistry(), newNoopEnqueuer(), newNoopUsageCheckerPtr())
+			controller := prebuilds.NewStoreReconciler(db, pubSub, cache, cfg, logger, clock, prometheus.NewRegistry(), newNoopEnqueuer(), newNoopUsageCheckerPtr(), noop.NewTracerProvider())
 
 			ownerID := uuid.New()
 			dbgen.User(t, db, database.User{
@@ -718,7 +719,7 @@ func TestInvalidPreset(t *testing.T) {
 	).Leveled(slog.LevelDebug)
 	db, pubSub := dbtestutil.NewDB(t)
 	cache := files.New(prometheus.NewRegistry(), &coderdtest.FakeAuthorizer{})
-	controller := prebuilds.NewStoreReconciler(db, pubSub, cache, cfg, logger, quartz.NewMock(t), prometheus.NewRegistry(), newNoopEnqueuer(), newNoopUsageCheckerPtr())
+	controller := prebuilds.NewStoreReconciler(db, pubSub, cache, cfg, logger, quartz.NewMock(t), prometheus.NewRegistry(), newNoopEnqueuer(), newNoopUsageCheckerPtr(), noop.NewTracerProvider())
 
 	ownerID := uuid.New()
 	dbgen.User(t, db, database.User{
@@ -780,7 +781,7 @@ func TestDeletionOfPrebuiltWorkspaceWithInvalidPreset(t *testing.T) {
 	).Leveled(slog.LevelDebug)
 	db, pubSub := dbtestutil.NewDB(t)
 	cache := files.New(prometheus.NewRegistry(), &coderdtest.FakeAuthorizer{})
-	controller := prebuilds.NewStoreReconciler(db, pubSub, cache, cfg, logger, quartz.NewMock(t), prometheus.NewRegistry(), newNoopEnqueuer(), newNoopUsageCheckerPtr())
+	controller := prebuilds.NewStoreReconciler(db, pubSub, cache, cfg, logger, quartz.NewMock(t), prometheus.NewRegistry(), newNoopEnqueuer(), newNoopUsageCheckerPtr(), noop.NewTracerProvider())
 
 	ownerID := uuid.New()
 	dbgen.User(t, db, database.User{
@@ -874,7 +875,7 @@ func TestSkippingHardLimitedPresets(t *testing.T) {
 			fakeEnqueuer := newFakeEnqueuer()
 			registry := prometheus.NewRegistry()
 			cache := files.New(prometheus.NewRegistry(), &coderdtest.FakeAuthorizer{})
-			controller := prebuilds.NewStoreReconciler(db, pubSub, cache, cfg, logger, clock, registry, fakeEnqueuer, newNoopUsageCheckerPtr())
+			controller := prebuilds.NewStoreReconciler(db, pubSub, cache, cfg, logger, clock, registry, fakeEnqueuer, newNoopUsageCheckerPtr(), noop.NewTracerProvider())
 
 			// Set up test environment with a template, version, and preset.
 			ownerID := uuid.New()
@@ -1017,7 +1018,7 @@ func TestHardLimitedPresetShouldNotBlockDeletion(t *testing.T) {
 			fakeEnqueuer := newFakeEnqueuer()
 			registry := prometheus.NewRegistry()
 			cache := files.New(prometheus.NewRegistry(), &coderdtest.FakeAuthorizer{})
-			controller := prebuilds.NewStoreReconciler(db, pubSub, cache, cfg, logger, clock, registry, fakeEnqueuer, newNoopUsageCheckerPtr())
+			controller := prebuilds.NewStoreReconciler(db, pubSub, cache, cfg, logger, clock, registry, fakeEnqueuer, newNoopUsageCheckerPtr(), noop.NewTracerProvider())
 
 			// Set up test environment with a template, version, and preset.
 			ownerID := uuid.New()
@@ -1211,7 +1212,7 @@ func TestRunLoop(t *testing.T) {
 	).Leveled(slog.LevelDebug)
 	db, pubSub := dbtestutil.NewDB(t)
 	cache := files.New(prometheus.NewRegistry(), &coderdtest.FakeAuthorizer{})
-	reconciler := prebuilds.NewStoreReconciler(db, pubSub, cache, cfg, logger, clock, prometheus.NewRegistry(), newNoopEnqueuer(), newNoopUsageCheckerPtr())
+	reconciler := prebuilds.NewStoreReconciler(db, pubSub, cache, cfg, logger, clock, prometheus.NewRegistry(), newNoopEnqueuer(), newNoopUsageCheckerPtr(), noop.NewTracerProvider())
 
 	ownerID := uuid.New()
 	dbgen.User(t, db, database.User{
@@ -1339,7 +1340,7 @@ func TestFailedBuildBackoff(t *testing.T) {
 	).Leveled(slog.LevelDebug)
 	db, ps := dbtestutil.NewDB(t)
 	cache := files.New(prometheus.NewRegistry(), &coderdtest.FakeAuthorizer{})
-	reconciler := prebuilds.NewStoreReconciler(db, ps, cache, cfg, logger, clock, prometheus.NewRegistry(), newNoopEnqueuer(), newNoopUsageCheckerPtr())
+	reconciler := prebuilds.NewStoreReconciler(db, ps, cache, cfg, logger, clock, prometheus.NewRegistry(), newNoopEnqueuer(), newNoopUsageCheckerPtr(), noop.NewTracerProvider())
 
 	// Given: an active template version with presets and prebuilds configured.
 	const desiredInstances = 2
@@ -1461,7 +1462,7 @@ func TestReconciliationLock(t *testing.T) {
 				quartz.NewMock(t),
 				prometheus.NewRegistry(),
 				newNoopEnqueuer(),
-				newNoopUsageCheckerPtr())
+				newNoopUsageCheckerPtr(), noop.NewTracerProvider())
 			reconciler.WithReconciliationLock(ctx, logger, func(_ context.Context, _ database.Store) error {
 				lockObtained := mutex.TryLock()
 				// As long as the postgres lock is held, this mutex should always be unlocked when we get here.
@@ -1491,7 +1492,7 @@ func TestTrackResourceReplacement(t *testing.T) {
 	fakeEnqueuer := newFakeEnqueuer()
 	registry := prometheus.NewRegistry()
 	cache := files.New(registry, &coderdtest.FakeAuthorizer{})
-	reconciler := prebuilds.NewStoreReconciler(db, ps, cache, codersdk.PrebuildsConfig{}, logger, clock, registry, fakeEnqueuer, newNoopUsageCheckerPtr())
+	reconciler := prebuilds.NewStoreReconciler(db, ps, cache, codersdk.PrebuildsConfig{}, logger, clock, registry, fakeEnqueuer, newNoopUsageCheckerPtr(), noop.NewTracerProvider())
 
 	// Given: a template admin to receive a notification.
 	templateAdmin := dbgen.User(t, db, database.User{
@@ -1643,7 +1644,7 @@ func TestExpiredPrebuildsMultipleActions(t *testing.T) {
 			fakeEnqueuer := newFakeEnqueuer()
 			registry := prometheus.NewRegistry()
 			cache := files.New(registry, &coderdtest.FakeAuthorizer{})
-			controller := prebuilds.NewStoreReconciler(db, pubSub, cache, cfg, logger, clock, registry, fakeEnqueuer, newNoopUsageCheckerPtr())
+			controller := prebuilds.NewStoreReconciler(db, pubSub, cache, cfg, logger, clock, registry, fakeEnqueuer, newNoopUsageCheckerPtr(), noop.NewTracerProvider())
 
 			// Set up test environment with a template, version, and preset
 			ownerID := uuid.New()
@@ -2098,7 +2099,7 @@ func TestCancelPendingPrebuilds(t *testing.T) {
 				registry := prometheus.NewRegistry()
 				cache := files.New(registry, &coderdtest.FakeAuthorizer{})
 				logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: false}).Leveled(slog.LevelDebug)
-				reconciler := prebuilds.NewStoreReconciler(db, ps, cache, codersdk.PrebuildsConfig{}, logger, clock, registry, fakeEnqueuer, newNoopUsageCheckerPtr())
+				reconciler := prebuilds.NewStoreReconciler(db, ps, cache, codersdk.PrebuildsConfig{}, logger, clock, registry, fakeEnqueuer, newNoopUsageCheckerPtr(), noop.NewTracerProvider())
 				owner := coderdtest.CreateFirstUser(t, client)
 
 				// Given: a template with a version containing a preset with 1 prebuild instance
@@ -2335,7 +2336,7 @@ func TestCancelPendingPrebuilds(t *testing.T) {
 		registry := prometheus.NewRegistry()
 		cache := files.New(registry, &coderdtest.FakeAuthorizer{})
 		logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: false}).Leveled(slog.LevelDebug)
-		reconciler := prebuilds.NewStoreReconciler(db, ps, cache, codersdk.PrebuildsConfig{}, logger, clock, registry, fakeEnqueuer, newNoopUsageCheckerPtr())
+		reconciler := prebuilds.NewStoreReconciler(db, ps, cache, codersdk.PrebuildsConfig{}, logger, clock, registry, fakeEnqueuer, newNoopUsageCheckerPtr(), noop.NewTracerProvider())
 		owner := coderdtest.CreateFirstUser(t, client)
 
 		// Given: template A with 2 versions
@@ -2400,7 +2401,7 @@ func TestReconciliationStats(t *testing.T) {
 	registry := prometheus.NewRegistry()
 	cache := files.New(registry, &coderdtest.FakeAuthorizer{})
 	logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: false}).Leveled(slog.LevelDebug)
-	reconciler := prebuilds.NewStoreReconciler(db, ps, cache, codersdk.PrebuildsConfig{}, logger, clock, registry, fakeEnqueuer, newNoopUsageCheckerPtr())
+	reconciler := prebuilds.NewStoreReconciler(db, ps, cache, codersdk.PrebuildsConfig{}, logger, clock, registry, fakeEnqueuer, newNoopUsageCheckerPtr(), noop.NewTracerProvider())
 	owner := coderdtest.CreateFirstUser(t, client)
 
 	ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort)
@@ -2911,7 +2912,7 @@ func TestReconciliationRespectsPauseSetting(t *testing.T) {
 	}
 	logger := testutil.Logger(t)
 	cache := files.New(prometheus.NewRegistry(), &coderdtest.FakeAuthorizer{})
-	reconciler := prebuilds.NewStoreReconciler(db, ps, cache, cfg, logger, clock, prometheus.NewRegistry(), newNoopEnqueuer(), newNoopUsageCheckerPtr())
+	reconciler := prebuilds.NewStoreReconciler(db, ps, cache, cfg, logger, clock, prometheus.NewRegistry(), newNoopEnqueuer(), newNoopUsageCheckerPtr(), noop.NewTracerProvider())
 
 	// Setup a template with a preset that should create prebuilds
 	org := dbgen.Organization(t, db, database.Organization{})

enterprise/coderd/workspaces_test.go

@@ -18,6 +18,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel/trace/noop"
 
 	"cdr.dev/slog"
 	"cdr.dev/slog/sloggers/slogtest"
@@ -1986,6 +1987,7 @@ func TestPrebuildsAutobuild(t *testing.T) {
 			prometheus.NewRegistry(),
 			notificationsNoop,
 			api.AGPL.BuildUsageChecker,
+			noop.NewTracerProvider(),
 		)
 		var claimer agplprebuilds.Claimer = prebuilds.NewEnterpriseClaimer(db)
 		api.AGPL.PrebuildsClaimer.Store(&claimer)
@@ -2108,6 +2110,7 @@ func TestPrebuildsAutobuild(t *testing.T) {
 			prometheus.NewRegistry(),
 			notificationsNoop,
 			api.AGPL.BuildUsageChecker,
+			noop.NewTracerProvider(),
 		)
 		var claimer agplprebuilds.Claimer = prebuilds.NewEnterpriseClaimer(db)
 		api.AGPL.PrebuildsClaimer.Store(&claimer)
@@ -2230,6 +2233,7 @@ func TestPrebuildsAutobuild(t *testing.T) {
 			prometheus.NewRegistry(),
 			notificationsNoop,
 			api.AGPL.BuildUsageChecker,
+			noop.NewTracerProvider(),
 		)
 		var claimer agplprebuilds.Claimer = prebuilds.NewEnterpriseClaimer(db)
 		api.AGPL.PrebuildsClaimer.Store(&claimer)
@@ -2374,6 +2378,7 @@ func TestPrebuildsAutobuild(t *testing.T) {
 			prometheus.NewRegistry(),
 			notificationsNoop,
 			api.AGPL.BuildUsageChecker,
+			noop.NewTracerProvider(),
 		)
 		var claimer agplprebuilds.Claimer = prebuilds.NewEnterpriseClaimer(db)
 		api.AGPL.PrebuildsClaimer.Store(&claimer)
@@ -2519,6 +2524,7 @@ func TestPrebuildsAutobuild(t *testing.T) {
 			prometheus.NewRegistry(),
 			notificationsNoop,
 			api.AGPL.BuildUsageChecker,
+			noop.NewTracerProvider(),
 		)
 		var claimer agplprebuilds.Claimer = prebuilds.NewEnterpriseClaimer(db)
 		api.AGPL.PrebuildsClaimer.Store(&claimer)
@@ -2964,6 +2970,7 @@ func TestWorkspaceProvisionerdServerMetrics(t *testing.T) {
 		prometheus.NewRegistry(),
 		notifications.NewNoopEnqueuer(),
 		api.AGPL.BuildUsageChecker,
+		noop.NewTracerProvider(),
 	)
 	var claimer agplprebuilds.Claimer = prebuilds.NewEnterpriseClaimer(db)
 	api.AGPL.PrebuildsClaimer.Store(&claimer)