From 25a0c807cb6f0052ec2693abb85f47e0bb3b000b Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 6 Feb 2026 09:44:40 +0000 Subject: [PATCH] chore(coderd/database/dbfake): add support for provisioner job timestamp control (#21944) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Relates to https://github.com/coder/coder/pull/21922 / https://github.com/coder/internal/issues/1259 * Adds `dbfake.BuilderOption func(*WorkspaceBuildBuilder)` * Adds `BuilderOption` methods for setting various provisioner job related fields on `WorkspaceBuildBuilder`. * Migrates a number of existing tests that previously dependeded on provisioner job timing to use these updated methods in the following packages: * `coderd/jobreaper` * `coderd/notifications/reports` * `enterprise/coderd/schedule` * `enterprise/coderd/prebuilds` * `scripts/workspace-runtime-audit` 🤖 Created using Mux (Opus 4.5) --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- coderd/database/dbfake/dbfake.go | 151 +++++- coderd/jobreaper/detector_test.go | 443 +++++------------- .../reports/generator_internal_test.go | 113 +++-- enterprise/coderd/prebuilds/reconcile_test.go | 65 +-- enterprise/coderd/schedule/template_test.go | 75 +-- .../runtimeaudit_test.go | 83 ++-- 6 files changed, 412 insertions(+), 518 deletions(-) diff --git a/coderd/database/dbfake/dbfake.go b/coderd/database/dbfake/dbfake.go index f23be9ea3b..d0a019b32c 100644 --- a/coderd/database/dbfake/dbfake.go +++ b/coderd/database/dbfake/dbfake.go @@ -58,6 +58,61 @@ type WorkspaceBuildBuilder struct { jobStatus database.ProvisionerJobStatus taskAppID uuid.UUID taskSeed database.TaskTable + + // Individual timestamp fields for job customization. + jobCreatedAt time.Time + jobStartedAt time.Time + jobUpdatedAt time.Time + jobCompletedAt time.Time + + jobError string // Error message for failed jobs + jobErrorCode string // Error code for failed jobs +} + +// BuilderOption is a functional option for customizing job timestamps +// on status methods. +type BuilderOption func(*WorkspaceBuildBuilder) + +// WithJobCreatedAt sets the CreatedAt timestamp for the provisioner job. +func WithJobCreatedAt(t time.Time) BuilderOption { + return func(b *WorkspaceBuildBuilder) { + b.jobCreatedAt = t + } +} + +// WithJobStartedAt sets the StartedAt timestamp for the provisioner job. +func WithJobStartedAt(t time.Time) BuilderOption { + return func(b *WorkspaceBuildBuilder) { + b.jobStartedAt = t + } +} + +// WithJobUpdatedAt sets the UpdatedAt timestamp for the provisioner job. +func WithJobUpdatedAt(t time.Time) BuilderOption { + return func(b *WorkspaceBuildBuilder) { + b.jobUpdatedAt = t + } +} + +// WithJobCompletedAt sets the CompletedAt timestamp for the provisioner job. +func WithJobCompletedAt(t time.Time) BuilderOption { + return func(b *WorkspaceBuildBuilder) { + b.jobCompletedAt = t + } +} + +// WithJobError sets the error message for the provisioner job. +func WithJobError(msg string) BuilderOption { + return func(b *WorkspaceBuildBuilder) { + b.jobError = msg + } +} + +// WithJobErrorCode sets the error code for the provisioner job. +func WithJobErrorCode(code string) BuilderOption { + return func(b *WorkspaceBuildBuilder) { + b.jobErrorCode = code + } } // WorkspaceBuild generates a workspace build for the provided workspace. @@ -141,18 +196,59 @@ func (b WorkspaceBuildBuilder) WithTask(taskSeed database.TaskTable, appSeed *sd }) } -func (b WorkspaceBuildBuilder) Starting() WorkspaceBuildBuilder { +// Starting sets the job to running status. +func (b WorkspaceBuildBuilder) Starting(opts ...BuilderOption) WorkspaceBuildBuilder { + //nolint: revive // returns modified struct b.jobStatus = database.ProvisionerJobStatusRunning + for _, opt := range opts { + opt(&b) + } return b } -func (b WorkspaceBuildBuilder) Pending() WorkspaceBuildBuilder { +// Pending sets the job to pending status. +func (b WorkspaceBuildBuilder) Pending(opts ...BuilderOption) WorkspaceBuildBuilder { + //nolint: revive // returns modified struct b.jobStatus = database.ProvisionerJobStatusPending + for _, opt := range opts { + opt(&b) + } return b } -func (b WorkspaceBuildBuilder) Canceled() WorkspaceBuildBuilder { +// Canceled sets the job to canceled status. +func (b WorkspaceBuildBuilder) Canceled(opts ...BuilderOption) WorkspaceBuildBuilder { + //nolint: revive // returns modified struct b.jobStatus = database.ProvisionerJobStatusCanceled + for _, opt := range opts { + opt(&b) + } + return b +} + +// Succeeded sets the job to succeeded status. +// This is the default status. +func (b WorkspaceBuildBuilder) Succeeded(opts ...BuilderOption) WorkspaceBuildBuilder { + //nolint: revive // returns modified struct + b.jobStatus = database.ProvisionerJobStatusSucceeded + for _, opt := range opts { + opt(&b) + } + return b +} + +// Failed sets the provisioner job to a failed state. Use WithJobError and +// WithJobErrorCode options to set the error message and code. If no error +// message is provided, "failed" is used as the default. +func (b WorkspaceBuildBuilder) Failed(opts ...BuilderOption) WorkspaceBuildBuilder { + //nolint: revive // returns modified struct + b.jobStatus = database.ProvisionerJobStatusFailed + for _, opt := range opts { + opt(&b) + } + if b.jobError == "" { + b.jobError = "failed" + } return b } @@ -267,8 +363,8 @@ func (b WorkspaceBuildBuilder) doInTX() WorkspaceResponse { job, err := b.db.InsertProvisionerJob(ownerCtx, database.InsertProvisionerJobParams{ ID: jobID, - CreatedAt: dbtime.Now(), - UpdatedAt: dbtime.Now(), + CreatedAt: takeFirstTime(b.jobCreatedAt, b.ws.CreatedAt, dbtime.Now()), + UpdatedAt: takeFirstTime(b.jobCreatedAt, b.ws.CreatedAt, dbtime.Now()), OrganizationID: b.ws.OrganizationID, InitiatorID: b.ws.OwnerID, Provisioner: database.ProvisionerTypeEcho, @@ -291,11 +387,12 @@ func (b WorkspaceBuildBuilder) doInTX() WorkspaceResponse { // might need to do this multiple times if we got a template version // import job as well b.logger.Debug(context.Background(), "looping to acquire provisioner job") + startedAt := takeFirstTime(b.jobStartedAt, dbtime.Now()) for { j, err := b.db.AcquireProvisionerJob(ownerCtx, database.AcquireProvisionerJobParams{ OrganizationID: job.OrganizationID, StartedAt: sql.NullTime{ - Time: dbtime.Now(), + Time: startedAt, Valid: true, }, WorkerID: uuid.NullUUID{ @@ -311,32 +408,54 @@ func (b WorkspaceBuildBuilder) doInTX() WorkspaceResponse { break } } + if !b.jobUpdatedAt.IsZero() { + err = b.db.UpdateProvisionerJobByID(ownerCtx, database.UpdateProvisionerJobByIDParams{ + ID: job.ID, + UpdatedAt: b.jobUpdatedAt, + }) + require.NoError(b.t, err, "update job updated_at") + } case database.ProvisionerJobStatusCanceled: // Set provisioner job status to 'canceled' b.logger.Debug(context.Background(), "canceling the provisioner job") - now := dbtime.Now() + completedAt := takeFirstTime(b.jobCompletedAt, dbtime.Now()) err = b.db.UpdateProvisionerJobWithCancelByID(ownerCtx, database.UpdateProvisionerJobWithCancelByIDParams{ ID: jobID, CanceledAt: sql.NullTime{ - Time: now, + Time: completedAt, Valid: true, }, CompletedAt: sql.NullTime{ - Time: now, + Time: completedAt, Valid: true, }, }) require.NoError(b.t, err, "cancel job") + case database.ProvisionerJobStatusFailed: + b.logger.Debug(context.Background(), "failing the provisioner job") + completedAt := takeFirstTime(b.jobCompletedAt, dbtime.Now()) + err = b.db.UpdateProvisionerJobWithCompleteByID(ownerCtx, database.UpdateProvisionerJobWithCompleteByIDParams{ + ID: job.ID, + UpdatedAt: completedAt, + Error: sql.NullString{String: b.jobError, Valid: b.jobError != ""}, + ErrorCode: sql.NullString{String: b.jobErrorCode, Valid: b.jobErrorCode != ""}, + CompletedAt: sql.NullTime{ + Time: completedAt, + Valid: true, + }, + }) + require.NoError(b.t, err, "fail job") default: // By default, consider jobs in 'succeeded' status b.logger.Debug(context.Background(), "completing the provisioner job") + completedAt := takeFirstTime(b.jobCompletedAt, dbtime.Now()) err = b.db.UpdateProvisionerJobWithCompleteByID(ownerCtx, database.UpdateProvisionerJobWithCompleteByIDParams{ ID: job.ID, - UpdatedAt: dbtime.Now(), + UpdatedAt: completedAt, Error: sql.NullString{}, ErrorCode: sql.NullString{}, CompletedAt: sql.NullTime{ - Time: dbtime.Now(), + Time: completedAt, Valid: true, }, }) @@ -751,6 +870,16 @@ func takeFirst[Value comparable](values ...Value) Value { }) } +// takeFirstTime returns the first non-zero time.Time. +func takeFirstTime(values ...time.Time) time.Time { + for _, v := range values { + if !v.IsZero() { + return v + } + } + return time.Time{} +} + // mustWorkspaceAppByWorkspaceAndBuildAndAppID finds a workspace app by // workspace ID, build number, and app ID. It returns the workspace app // if found, otherwise fails the test. diff --git a/coderd/jobreaper/detector_test.go b/coderd/jobreaper/detector_test.go index 5d12ac34fc..d92070a0e2 100644 --- a/coderd/jobreaper/detector_test.go +++ b/coderd/jobreaper/detector_test.go @@ -8,7 +8,6 @@ import ( "testing" "time" - "github.com/google/uuid" "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -18,6 +17,7 @@ import ( "github.com/coder/coder/v2/coderd/coderdtest" "github.com/coder/coder/v2/coderd/database" "github.com/coder/coder/v2/coderd/database/dbauthz" + "github.com/coder/coder/v2/coderd/database/dbfake" "github.com/coder/coder/v2/coderd/database/dbgen" "github.com/coder/coder/v2/coderd/database/dbtestutil" "github.com/coder/coder/v2/coderd/jobreaper" @@ -113,87 +113,33 @@ func TestDetectorHungWorkspaceBuild(t *testing.T) { ) var ( - now = time.Now() - twentyMinAgo = now.Add(-time.Minute * 20) - tenMinAgo = now.Add(-time.Minute * 10) - sixMinAgo = now.Add(-time.Minute * 6) - org = dbgen.Organization(t, db, database.Organization{}) - user = dbgen.User(t, db, database.User{}) - file = dbgen.File(t, db, database.File{}) - template = dbgen.Template(t, db, database.Template{ - OrganizationID: org.ID, - CreatedBy: user.ID, - }) - templateVersion = dbgen.TemplateVersion(t, db, database.TemplateVersion{ - OrganizationID: org.ID, - TemplateID: uuid.NullUUID{ - UUID: template.ID, - Valid: true, - }, - CreatedBy: user.ID, - }) - workspace = dbgen.Workspace(t, db, database.WorkspaceTable{ - OwnerID: user.ID, - OrganizationID: org.ID, - TemplateID: template.ID, - }) - - // Previous build. + now = time.Now() + twentyMinAgo = now.Add(-time.Minute * 20) + tenMinAgo = now.Add(-time.Minute * 10) + sixMinAgo = now.Add(-time.Minute * 6) + org = dbgen.Organization(t, db, database.Organization{}) + user = dbgen.User(t, db, database.User{}) expectedWorkspaceBuildState = []byte(`{"dean":"cool","colin":"also cool"}`) - previousWorkspaceBuildJob = dbgen.ProvisionerJob(t, db, pubsub, database.ProvisionerJob{ - CreatedAt: twentyMinAgo, - UpdatedAt: twentyMinAgo, - StartedAt: sql.NullTime{ - Time: twentyMinAgo, - Valid: true, - }, - CompletedAt: sql.NullTime{ - Time: twentyMinAgo, - Valid: true, - }, - OrganizationID: org.ID, - InitiatorID: user.ID, - Provisioner: database.ProvisionerTypeEcho, - StorageMethod: database.ProvisionerStorageMethodFile, - FileID: file.ID, - Type: database.ProvisionerJobTypeWorkspaceBuild, - Input: []byte("{}"), - }) - _ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{ - WorkspaceID: workspace.ID, - TemplateVersionID: templateVersion.ID, - BuildNumber: 1, - ProvisionerState: expectedWorkspaceBuildState, - JobID: previousWorkspaceBuildJob.ID, - }) - - // Current build. - currentWorkspaceBuildJob = dbgen.ProvisionerJob(t, db, pubsub, database.ProvisionerJob{ - CreatedAt: tenMinAgo, - UpdatedAt: sixMinAgo, - StartedAt: sql.NullTime{ - Time: tenMinAgo, - Valid: true, - }, - OrganizationID: org.ID, - InitiatorID: user.ID, - Provisioner: database.ProvisionerTypeEcho, - StorageMethod: database.ProvisionerStorageMethodFile, - FileID: file.ID, - Type: database.ProvisionerJobTypeWorkspaceBuild, - Input: []byte("{}"), - }) - currentWorkspaceBuild = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{ - WorkspaceID: workspace.ID, - TemplateVersionID: templateVersion.ID, - BuildNumber: 2, - JobID: currentWorkspaceBuildJob.ID, - // No provisioner state. - }) ) - t.Log("previous job ID: ", previousWorkspaceBuildJob.ID) - t.Log("current job ID: ", currentWorkspaceBuildJob.ID) + // Previous build (completed successfully). + previousBuild := dbfake.WorkspaceBuild(t, db, database.WorkspaceTable{ + OrganizationID: org.ID, + OwnerID: user.ID, + }).Pubsub(pubsub).Seed(database.WorkspaceBuild{ + ProvisionerState: expectedWorkspaceBuildState, + }).Succeeded(dbfake.WithJobCompletedAt(twentyMinAgo)). + Do() + + // Current build (hung - running job with UpdatedAt > 5 min ago). + currentBuild := dbfake.WorkspaceBuild(t, db, previousBuild.Workspace). + Pubsub(pubsub). + Seed(database.WorkspaceBuild{BuildNumber: 2}). + Starting(dbfake.WithJobStartedAt(tenMinAgo), dbfake.WithJobUpdatedAt(sixMinAgo)). + Do() + + t.Log("previous job ID: ", previousBuild.Build.JobID) + t.Log("current job ID: ", currentBuild.Build.JobID) detector := jobreaper.New(ctx, wrapDBAuthz(db, log), pubsub, log, tickCh).WithStatsChannel(statsCh) detector.Start() @@ -202,10 +148,10 @@ func TestDetectorHungWorkspaceBuild(t *testing.T) { stats := <-statsCh require.NoError(t, stats.Error) require.Len(t, stats.TerminatedJobIDs, 1) - require.Equal(t, currentWorkspaceBuildJob.ID, stats.TerminatedJobIDs[0]) + require.Equal(t, currentBuild.Build.JobID, stats.TerminatedJobIDs[0]) // Check that the current provisioner job was updated. - job, err := db.GetProvisionerJobByID(ctx, currentWorkspaceBuildJob.ID) + job, err := db.GetProvisionerJobByID(ctx, currentBuild.Build.JobID) require.NoError(t, err) require.WithinDuration(t, now, job.UpdatedAt, 30*time.Second) require.True(t, job.CompletedAt.Valid) @@ -215,7 +161,7 @@ func TestDetectorHungWorkspaceBuild(t *testing.T) { require.False(t, job.ErrorCode.Valid) // Check that the provisioner state was copied. - build, err := db.GetWorkspaceBuildByID(ctx, currentWorkspaceBuild.ID) + build, err := db.GetWorkspaceBuildByID(ctx, currentBuild.Build.ID) require.NoError(t, err) require.Equal(t, expectedWorkspaceBuildState, build.ProvisionerState) @@ -235,88 +181,37 @@ func TestDetectorHungWorkspaceBuildNoOverrideState(t *testing.T) { ) var ( - now = time.Now() - twentyMinAgo = now.Add(-time.Minute * 20) - tenMinAgo = now.Add(-time.Minute * 10) - sixMinAgo = now.Add(-time.Minute * 6) - org = dbgen.Organization(t, db, database.Organization{}) - user = dbgen.User(t, db, database.User{}) - file = dbgen.File(t, db, database.File{}) - template = dbgen.Template(t, db, database.Template{ - OrganizationID: org.ID, - CreatedBy: user.ID, - }) - templateVersion = dbgen.TemplateVersion(t, db, database.TemplateVersion{ - OrganizationID: org.ID, - TemplateID: uuid.NullUUID{ - UUID: template.ID, - Valid: true, - }, - CreatedBy: user.ID, - }) - workspace = dbgen.Workspace(t, db, database.WorkspaceTable{ - OwnerID: user.ID, - OrganizationID: org.ID, - TemplateID: template.ID, - }) - - // Previous build. - previousWorkspaceBuildJob = dbgen.ProvisionerJob(t, db, pubsub, database.ProvisionerJob{ - CreatedAt: twentyMinAgo, - UpdatedAt: twentyMinAgo, - StartedAt: sql.NullTime{ - Time: twentyMinAgo, - Valid: true, - }, - CompletedAt: sql.NullTime{ - Time: twentyMinAgo, - Valid: true, - }, - OrganizationID: org.ID, - InitiatorID: user.ID, - Provisioner: database.ProvisionerTypeEcho, - StorageMethod: database.ProvisionerStorageMethodFile, - FileID: file.ID, - Type: database.ProvisionerJobTypeWorkspaceBuild, - Input: []byte("{}"), - }) - _ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{ - WorkspaceID: workspace.ID, - TemplateVersionID: templateVersion.ID, - BuildNumber: 1, - ProvisionerState: []byte(`{"dean":"NOT cool","colin":"also NOT cool"}`), - JobID: previousWorkspaceBuildJob.ID, - }) - - // Current build. + now = time.Now() + twentyMinAgo = now.Add(-time.Minute * 20) + tenMinAgo = now.Add(-time.Minute * 10) + sixMinAgo = now.Add(-time.Minute * 6) + org = dbgen.Organization(t, db, database.Organization{}) + user = dbgen.User(t, db, database.User{}) expectedWorkspaceBuildState = []byte(`{"dean":"cool","colin":"also cool"}`) - currentWorkspaceBuildJob = dbgen.ProvisionerJob(t, db, pubsub, database.ProvisionerJob{ - CreatedAt: tenMinAgo, - UpdatedAt: sixMinAgo, - StartedAt: sql.NullTime{ - Time: tenMinAgo, - Valid: true, - }, - OrganizationID: org.ID, - InitiatorID: user.ID, - Provisioner: database.ProvisionerTypeEcho, - StorageMethod: database.ProvisionerStorageMethodFile, - FileID: file.ID, - Type: database.ProvisionerJobTypeWorkspaceBuild, - Input: []byte("{}"), - }) - currentWorkspaceBuild = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{ - WorkspaceID: workspace.ID, - TemplateVersionID: templateVersion.ID, - BuildNumber: 2, - JobID: currentWorkspaceBuildJob.ID, - // Should not be overridden. - ProvisionerState: expectedWorkspaceBuildState, - }) ) - t.Log("previous job ID: ", previousWorkspaceBuildJob.ID) - t.Log("current job ID: ", currentWorkspaceBuildJob.ID) + // Previous build (completed successfully). + previousBuild := dbfake.WorkspaceBuild(t, db, database.WorkspaceTable{ + OrganizationID: org.ID, + OwnerID: user.ID, + }).Pubsub(pubsub).Seed(database.WorkspaceBuild{ + ProvisionerState: []byte(`{"dean":"NOT cool","colin":"also NOT cool"}`), + }).Succeeded(dbfake.WithJobCompletedAt(twentyMinAgo)). + Do() + + // Current build (hung - running job with UpdatedAt > 5 min ago). + // This build already has provisioner state, which should NOT be overridden. + currentBuild := dbfake.WorkspaceBuild(t, db, previousBuild.Workspace). + Pubsub(pubsub). + Seed(database.WorkspaceBuild{ + BuildNumber: 2, + ProvisionerState: expectedWorkspaceBuildState, + }). + Starting(dbfake.WithJobStartedAt(tenMinAgo), dbfake.WithJobUpdatedAt(sixMinAgo)). + Do() + + t.Log("previous job ID: ", previousBuild.Build.JobID) + t.Log("current job ID: ", currentBuild.Build.JobID) detector := jobreaper.New(ctx, wrapDBAuthz(db, log), pubsub, log, tickCh).WithStatsChannel(statsCh) detector.Start() @@ -325,10 +220,10 @@ func TestDetectorHungWorkspaceBuildNoOverrideState(t *testing.T) { stats := <-statsCh require.NoError(t, stats.Error) require.Len(t, stats.TerminatedJobIDs, 1) - require.Equal(t, currentWorkspaceBuildJob.ID, stats.TerminatedJobIDs[0]) + require.Equal(t, currentBuild.Build.JobID, stats.TerminatedJobIDs[0]) // Check that the current provisioner job was updated. - job, err := db.GetProvisionerJobByID(ctx, currentWorkspaceBuildJob.ID) + job, err := db.GetProvisionerJobByID(ctx, currentBuild.Build.JobID) require.NoError(t, err) require.WithinDuration(t, now, job.UpdatedAt, 30*time.Second) require.True(t, job.CompletedAt.Valid) @@ -338,7 +233,7 @@ func TestDetectorHungWorkspaceBuildNoOverrideState(t *testing.T) { require.False(t, job.ErrorCode.Valid) // Check that the provisioner state was NOT copied. - build, err := db.GetWorkspaceBuildByID(ctx, currentWorkspaceBuild.ID) + build, err := db.GetWorkspaceBuildByID(ctx, currentBuild.Build.ID) require.NoError(t, err) require.Equal(t, expectedWorkspaceBuildState, build.ProvisionerState) @@ -358,58 +253,25 @@ func TestDetectorHungWorkspaceBuildNoOverrideStateIfNoExistingBuild(t *testing.T ) var ( - now = time.Now() - tenMinAgo = now.Add(-time.Minute * 10) - sixMinAgo = now.Add(-time.Minute * 6) - org = dbgen.Organization(t, db, database.Organization{}) - user = dbgen.User(t, db, database.User{}) - file = dbgen.File(t, db, database.File{}) - template = dbgen.Template(t, db, database.Template{ - OrganizationID: org.ID, - CreatedBy: user.ID, - }) - templateVersion = dbgen.TemplateVersion(t, db, database.TemplateVersion{ - OrganizationID: org.ID, - TemplateID: uuid.NullUUID{ - UUID: template.ID, - Valid: true, - }, - CreatedBy: user.ID, - }) - workspace = dbgen.Workspace(t, db, database.WorkspaceTable{ - OwnerID: user.ID, - OrganizationID: org.ID, - TemplateID: template.ID, - }) - - // First build. + now = time.Now() + tenMinAgo = now.Add(-time.Minute * 10) + sixMinAgo = now.Add(-time.Minute * 6) + org = dbgen.Organization(t, db, database.Organization{}) + user = dbgen.User(t, db, database.User{}) expectedWorkspaceBuildState = []byte(`{"dean":"cool","colin":"also cool"}`) - currentWorkspaceBuildJob = dbgen.ProvisionerJob(t, db, pubsub, database.ProvisionerJob{ - CreatedAt: tenMinAgo, - UpdatedAt: sixMinAgo, - StartedAt: sql.NullTime{ - Time: tenMinAgo, - Valid: true, - }, - OrganizationID: org.ID, - InitiatorID: user.ID, - Provisioner: database.ProvisionerTypeEcho, - StorageMethod: database.ProvisionerStorageMethodFile, - FileID: file.ID, - Type: database.ProvisionerJobTypeWorkspaceBuild, - Input: []byte("{}"), - }) - currentWorkspaceBuild = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{ - WorkspaceID: workspace.ID, - TemplateVersionID: templateVersion.ID, - BuildNumber: 1, - JobID: currentWorkspaceBuildJob.ID, - // Should not be overridden. - ProvisionerState: expectedWorkspaceBuildState, - }) ) - t.Log("current job ID: ", currentWorkspaceBuildJob.ID) + // First build (hung - no previous build exists). + // This build has provisioner state, which should NOT be overridden. + currentBuild := dbfake.WorkspaceBuild(t, db, database.WorkspaceTable{ + OrganizationID: org.ID, + OwnerID: user.ID, + }).Pubsub(pubsub).Seed(database.WorkspaceBuild{ + ProvisionerState: expectedWorkspaceBuildState, + }).Starting(dbfake.WithJobStartedAt(tenMinAgo), dbfake.WithJobUpdatedAt(sixMinAgo)). + Do() + + t.Log("current job ID: ", currentBuild.Build.JobID) detector := jobreaper.New(ctx, wrapDBAuthz(db, log), pubsub, log, tickCh).WithStatsChannel(statsCh) detector.Start() @@ -418,10 +280,10 @@ func TestDetectorHungWorkspaceBuildNoOverrideStateIfNoExistingBuild(t *testing.T stats := <-statsCh require.NoError(t, stats.Error) require.Len(t, stats.TerminatedJobIDs, 1) - require.Equal(t, currentWorkspaceBuildJob.ID, stats.TerminatedJobIDs[0]) + require.Equal(t, currentBuild.Build.JobID, stats.TerminatedJobIDs[0]) // Check that the current provisioner job was updated. - job, err := db.GetProvisionerJobByID(ctx, currentWorkspaceBuildJob.ID) + job, err := db.GetProvisionerJobByID(ctx, currentBuild.Build.JobID) require.NoError(t, err) require.WithinDuration(t, now, job.UpdatedAt, 30*time.Second) require.True(t, job.CompletedAt.Valid) @@ -431,7 +293,7 @@ func TestDetectorHungWorkspaceBuildNoOverrideStateIfNoExistingBuild(t *testing.T require.False(t, job.ErrorCode.Valid) // Check that the provisioner state was NOT updated. - build, err := db.GetWorkspaceBuildByID(ctx, currentWorkspaceBuild.ID) + build, err := db.GetWorkspaceBuildByID(ctx, currentBuild.Build.ID) require.NoError(t, err) require.Equal(t, expectedWorkspaceBuildState, build.ProvisionerState) @@ -451,57 +313,24 @@ func TestDetectorPendingWorkspaceBuildNoOverrideStateIfNoExistingBuild(t *testin ) var ( - now = time.Now() - thirtyFiveMinAgo = now.Add(-time.Minute * 35) - org = dbgen.Organization(t, db, database.Organization{}) - user = dbgen.User(t, db, database.User{}) - file = dbgen.File(t, db, database.File{}) - template = dbgen.Template(t, db, database.Template{ - OrganizationID: org.ID, - CreatedBy: user.ID, - }) - templateVersion = dbgen.TemplateVersion(t, db, database.TemplateVersion{ - OrganizationID: org.ID, - TemplateID: uuid.NullUUID{ - UUID: template.ID, - Valid: true, - }, - CreatedBy: user.ID, - }) - workspace = dbgen.Workspace(t, db, database.WorkspaceTable{ - OwnerID: user.ID, - OrganizationID: org.ID, - TemplateID: template.ID, - }) - - // First build. + now = time.Now() + thirtyFiveMinAgo = now.Add(-time.Minute * 35) + org = dbgen.Organization(t, db, database.Organization{}) + user = dbgen.User(t, db, database.User{}) expectedWorkspaceBuildState = []byte(`{"dean":"cool","colin":"also cool"}`) - currentWorkspaceBuildJob = dbgen.ProvisionerJob(t, db, pubsub, database.ProvisionerJob{ - CreatedAt: thirtyFiveMinAgo, - UpdatedAt: thirtyFiveMinAgo, - StartedAt: sql.NullTime{ - Time: time.Time{}, - Valid: false, - }, - OrganizationID: org.ID, - InitiatorID: user.ID, - Provisioner: database.ProvisionerTypeEcho, - StorageMethod: database.ProvisionerStorageMethodFile, - FileID: file.ID, - Type: database.ProvisionerJobTypeWorkspaceBuild, - Input: []byte("{}"), - }) - currentWorkspaceBuild = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{ - WorkspaceID: workspace.ID, - TemplateVersionID: templateVersion.ID, - BuildNumber: 1, - JobID: currentWorkspaceBuildJob.ID, - // Should not be overridden. - ProvisionerState: expectedWorkspaceBuildState, - }) ) - t.Log("current job ID: ", currentWorkspaceBuildJob.ID) + // First build (hung pending - no previous build exists). + // This build has provisioner state, which should NOT be overridden. + currentBuild := dbfake.WorkspaceBuild(t, db, database.WorkspaceTable{ + OrganizationID: org.ID, + OwnerID: user.ID, + }).Pubsub(pubsub).Seed(database.WorkspaceBuild{ + ProvisionerState: expectedWorkspaceBuildState, + }).Pending(dbfake.WithJobCreatedAt(thirtyFiveMinAgo), dbfake.WithJobUpdatedAt(thirtyFiveMinAgo)). + Do() + + t.Log("current job ID: ", currentBuild.Build.JobID) detector := jobreaper.New(ctx, wrapDBAuthz(db, log), pubsub, log, tickCh).WithStatsChannel(statsCh) detector.Start() @@ -510,10 +339,10 @@ func TestDetectorPendingWorkspaceBuildNoOverrideStateIfNoExistingBuild(t *testin stats := <-statsCh require.NoError(t, stats.Error) require.Len(t, stats.TerminatedJobIDs, 1) - require.Equal(t, currentWorkspaceBuildJob.ID, stats.TerminatedJobIDs[0]) + require.Equal(t, currentBuild.Build.JobID, stats.TerminatedJobIDs[0]) // Check that the current provisioner job was updated. - job, err := db.GetProvisionerJobByID(ctx, currentWorkspaceBuildJob.ID) + job, err := db.GetProvisionerJobByID(ctx, currentBuild.Build.JobID) require.NoError(t, err) require.WithinDuration(t, now, job.UpdatedAt, 30*time.Second) require.True(t, job.CompletedAt.Valid) @@ -525,7 +354,7 @@ func TestDetectorPendingWorkspaceBuildNoOverrideStateIfNoExistingBuild(t *testin require.False(t, job.ErrorCode.Valid) // Check that the provisioner state was NOT updated. - build, err := db.GetWorkspaceBuildByID(ctx, currentWorkspaceBuild.ID) + build, err := db.GetWorkspaceBuildByID(ctx, currentBuild.Build.ID) require.NoError(t, err) require.Equal(t, expectedWorkspaceBuildState, build.ProvisionerState) @@ -551,66 +380,34 @@ func TestDetectorWorkspaceBuildForDormantWorkspace(t *testing.T) { ) var ( - now = time.Now() - tenMinAgo = now.Add(-time.Minute * 10) - sixMinAgo = now.Add(-time.Minute * 6) - org = dbgen.Organization(t, db, database.Organization{}) - user = dbgen.User(t, db, database.User{}) - file = dbgen.File(t, db, database.File{}) - template = dbgen.Template(t, db, database.Template{ - OrganizationID: org.ID, - CreatedBy: user.ID, - }) - templateVersion = dbgen.TemplateVersion(t, db, database.TemplateVersion{ - OrganizationID: org.ID, - TemplateID: uuid.NullUUID{ - UUID: template.ID, - Valid: true, - }, - CreatedBy: user.ID, - }) - workspace = dbgen.Workspace(t, db, database.WorkspaceTable{ - OwnerID: user.ID, - OrganizationID: org.ID, - TemplateID: template.ID, - DormantAt: sql.NullTime{ - Time: now.Add(-time.Hour), - Valid: true, - }, - }) - - // First build. + now = time.Now() + tenMinAgo = now.Add(-time.Minute * 10) + sixMinAgo = now.Add(-time.Minute * 6) + org = dbgen.Organization(t, db, database.Organization{}) + user = dbgen.User(t, db, database.User{}) expectedWorkspaceBuildState = []byte(`{"dean":"cool","colin":"also cool"}`) - currentWorkspaceBuildJob = dbgen.ProvisionerJob(t, db, pubsub, database.ProvisionerJob{ - CreatedAt: tenMinAgo, - UpdatedAt: sixMinAgo, - StartedAt: sql.NullTime{ - Time: tenMinAgo, - Valid: true, - }, - OrganizationID: org.ID, - InitiatorID: user.ID, - Provisioner: database.ProvisionerTypeEcho, - StorageMethod: database.ProvisionerStorageMethodFile, - FileID: file.ID, - Type: database.ProvisionerJobTypeWorkspaceBuild, - Input: []byte("{}"), - }) - _ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{ - WorkspaceID: workspace.ID, - TemplateVersionID: templateVersion.ID, - BuildNumber: 1, - JobID: currentWorkspaceBuildJob.ID, - // Should not be overridden. - ProvisionerState: expectedWorkspaceBuildState, - }) ) - t.Log("current job ID: ", currentWorkspaceBuildJob.ID) + // First build (hung - running job with UpdatedAt > 5 min ago). + // This build has provisioner state, which should NOT be overridden. + // The workspace is dormant from the start. + currentBuild := dbfake.WorkspaceBuild(t, db, database.WorkspaceTable{ + OrganizationID: org.ID, + OwnerID: user.ID, + DormantAt: sql.NullTime{ + Time: now.Add(-time.Hour), + Valid: true, + }, + }).Pubsub(pubsub).Seed(database.WorkspaceBuild{ + ProvisionerState: expectedWorkspaceBuildState, + }).Starting(dbfake.WithJobStartedAt(tenMinAgo), dbfake.WithJobUpdatedAt(sixMinAgo)). + Do() + + t.Log("current job ID: ", currentBuild.Build.JobID) // Ensure the RBAC is the dormant type to ensure we're testing the right // thing. - require.Equal(t, rbac.ResourceWorkspaceDormant.Type, workspace.RBACObject().Type) + require.Equal(t, rbac.ResourceWorkspaceDormant.Type, currentBuild.Workspace.RBACObject().Type) detector := jobreaper.New(ctx, wrapDBAuthz(db, log), pubsub, log, tickCh).WithStatsChannel(statsCh) detector.Start() @@ -619,10 +416,10 @@ func TestDetectorWorkspaceBuildForDormantWorkspace(t *testing.T) { stats := <-statsCh require.NoError(t, stats.Error) require.Len(t, stats.TerminatedJobIDs, 1) - require.Equal(t, currentWorkspaceBuildJob.ID, stats.TerminatedJobIDs[0]) + require.Equal(t, currentBuild.Build.JobID, stats.TerminatedJobIDs[0]) // Check that the current provisioner job was updated. - job, err := db.GetProvisionerJobByID(ctx, currentWorkspaceBuildJob.ID) + job, err := db.GetProvisionerJobByID(ctx, currentBuild.Build.JobID) require.NoError(t, err) require.WithinDuration(t, now, job.UpdatedAt, 30*time.Second) require.True(t, job.CompletedAt.Valid) diff --git a/coderd/notifications/reports/generator_internal_test.go b/coderd/notifications/reports/generator_internal_test.go index 5cc7b3e9df..30749c62c7 100644 --- a/coderd/notifications/reports/generator_internal_test.go +++ b/coderd/notifications/reports/generator_internal_test.go @@ -16,6 +16,7 @@ import ( "github.com/coder/coder/v2/coderd/coderdtest" "github.com/coder/coder/v2/coderd/database" "github.com/coder/coder/v2/coderd/database/dbauthz" + "github.com/coder/coder/v2/coderd/database/dbfake" "github.com/coder/coder/v2/coderd/database/dbgen" "github.com/coder/coder/v2/coderd/database/dbtestutil" "github.com/coder/coder/v2/coderd/database/pubsub" @@ -92,8 +93,11 @@ func TestReportFailedWorkspaceBuilds(t *testing.T) { // Workspaces w1 := dbgen.Workspace(t, db, database.WorkspaceTable{TemplateID: t1.ID, OwnerID: user1.ID, OrganizationID: org.ID}) - w1wb1pj := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{OrganizationID: org.ID, Error: jobError, ErrorCode: jobErrorCode, CompletedAt: sql.NullTime{Time: now.Add(-6 * dayDuration), Valid: true}}) - _ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{WorkspaceID: w1.ID, BuildNumber: 1, TemplateVersionID: t1v1.ID, JobID: w1wb1pj.ID, CreatedAt: now.Add(-2 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator}) + _ = dbfake.WorkspaceBuild(t, db, w1). + Pubsub(ps). + Seed(database.WorkspaceBuild{BuildNumber: 1, TemplateVersionID: t1v1.ID, CreatedAt: now.Add(-2 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator}). + Failed(dbfake.WithJobError(jobError.String), dbfake.WithJobErrorCode(jobErrorCode.String), dbfake.WithJobCompletedAt(now.Add(-6*dayDuration))). + Do() // When: first run notifEnq.Clear() @@ -178,27 +182,54 @@ func TestReportFailedWorkspaceBuilds(t *testing.T) { now := clk.Now() // Workspace builds - w1wb1pj := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{OrganizationID: org.ID, Error: jobError, ErrorCode: jobErrorCode, CompletedAt: sql.NullTime{Time: now.Add(-6 * dayDuration), Valid: true}}) - _ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{WorkspaceID: w1.ID, BuildNumber: 1, TemplateVersionID: t1v1.ID, JobID: w1wb1pj.ID, CreatedAt: now.Add(-6 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator}) - w1wb2pj := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{OrganizationID: org.ID, CompletedAt: sql.NullTime{Time: now.Add(-5 * dayDuration), Valid: true}}) - _ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{WorkspaceID: w1.ID, BuildNumber: 2, TemplateVersionID: t1v2.ID, JobID: w1wb2pj.ID, CreatedAt: now.Add(-5 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator}) - w1wb3pj := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{OrganizationID: org.ID, Error: jobError, ErrorCode: jobErrorCode, CompletedAt: sql.NullTime{Time: now.Add(-4 * dayDuration), Valid: true}}) - _ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{WorkspaceID: w1.ID, BuildNumber: 3, TemplateVersionID: t1v2.ID, JobID: w1wb3pj.ID, CreatedAt: now.Add(-4 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator}) + _ = dbfake.WorkspaceBuild(t, db, w1). + Pubsub(ps). + Seed(database.WorkspaceBuild{BuildNumber: 1, TemplateVersionID: t1v1.ID, CreatedAt: now.Add(-6 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator}). + Failed(dbfake.WithJobError(jobError.String), dbfake.WithJobErrorCode(jobErrorCode.String), dbfake.WithJobCompletedAt(now.Add(-6*dayDuration))). + Do() + _ = dbfake.WorkspaceBuild(t, db, w1). + Pubsub(ps). + Seed(database.WorkspaceBuild{BuildNumber: 2, TemplateVersionID: t1v2.ID, CreatedAt: now.Add(-5 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator}). + Succeeded(dbfake.WithJobCompletedAt(now.Add(-5 * dayDuration))). + Do() + _ = dbfake.WorkspaceBuild(t, db, w1). + Pubsub(ps). + Seed(database.WorkspaceBuild{BuildNumber: 3, TemplateVersionID: t1v2.ID, CreatedAt: now.Add(-4 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator}). + Failed(dbfake.WithJobError(jobError.String), dbfake.WithJobErrorCode(jobErrorCode.String), dbfake.WithJobCompletedAt(now.Add(-4*dayDuration))). + Do() - w2wb1pj := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{OrganizationID: org.ID, CompletedAt: sql.NullTime{Time: now.Add(-5 * dayDuration), Valid: true}}) - _ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{WorkspaceID: w2.ID, BuildNumber: 4, TemplateVersionID: t2v1.ID, JobID: w2wb1pj.ID, CreatedAt: now.Add(-5 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator}) - w2wb2pj := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{OrganizationID: org.ID, Error: jobError, ErrorCode: jobErrorCode, CompletedAt: sql.NullTime{Time: now.Add(-4 * dayDuration), Valid: true}}) - _ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{WorkspaceID: w2.ID, BuildNumber: 5, TemplateVersionID: t2v2.ID, JobID: w2wb2pj.ID, CreatedAt: now.Add(-4 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator}) - w2wb3pj := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{OrganizationID: org.ID, Error: jobError, ErrorCode: jobErrorCode, CompletedAt: sql.NullTime{Time: now.Add(-3 * dayDuration), Valid: true}}) - _ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{WorkspaceID: w2.ID, BuildNumber: 6, TemplateVersionID: t2v2.ID, JobID: w2wb3pj.ID, CreatedAt: now.Add(-3 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator}) + _ = dbfake.WorkspaceBuild(t, db, w2). + Pubsub(ps). + Seed(database.WorkspaceBuild{BuildNumber: 4, TemplateVersionID: t2v1.ID, CreatedAt: now.Add(-5 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator}). + Succeeded(dbfake.WithJobCompletedAt(now.Add(-5 * dayDuration))). + Do() + _ = dbfake.WorkspaceBuild(t, db, w2). + Pubsub(ps). + Seed(database.WorkspaceBuild{BuildNumber: 5, TemplateVersionID: t2v2.ID, CreatedAt: now.Add(-4 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator}). + Failed(dbfake.WithJobError(jobError.String), dbfake.WithJobErrorCode(jobErrorCode.String), dbfake.WithJobCompletedAt(now.Add(-4*dayDuration))). + Do() + _ = dbfake.WorkspaceBuild(t, db, w2). + Pubsub(ps). + Seed(database.WorkspaceBuild{BuildNumber: 6, TemplateVersionID: t2v2.ID, CreatedAt: now.Add(-3 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator}). + Failed(dbfake.WithJobError(jobError.String), dbfake.WithJobErrorCode(jobErrorCode.String), dbfake.WithJobCompletedAt(now.Add(-3*dayDuration))). + Do() - w3wb1pj := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{OrganizationID: org.ID, Error: jobError, ErrorCode: jobErrorCode, CompletedAt: sql.NullTime{Time: now.Add(-3 * dayDuration), Valid: true}}) - _ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{WorkspaceID: w3.ID, BuildNumber: 7, TemplateVersionID: t1v1.ID, JobID: w3wb1pj.ID, CreatedAt: now.Add(-3 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator}) + _ = dbfake.WorkspaceBuild(t, db, w3). + Pubsub(ps). + Seed(database.WorkspaceBuild{BuildNumber: 7, TemplateVersionID: t1v1.ID, CreatedAt: now.Add(-3 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator}). + Failed(dbfake.WithJobError(jobError.String), dbfake.WithJobErrorCode(jobErrorCode.String), dbfake.WithJobCompletedAt(now.Add(-3*dayDuration))). + Do() - w4wb1pj := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{OrganizationID: org.ID, Error: jobError, ErrorCode: jobErrorCode, CompletedAt: sql.NullTime{Time: now.Add(-6 * dayDuration), Valid: true}}) - _ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{WorkspaceID: w4.ID, BuildNumber: 8, TemplateVersionID: t2v1.ID, JobID: w4wb1pj.ID, CreatedAt: now.Add(-6 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator}) - w4wb2pj := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{OrganizationID: org.ID, CompletedAt: sql.NullTime{Time: now.Add(-dayDuration), Valid: true}}) - _ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{WorkspaceID: w4.ID, BuildNumber: 9, TemplateVersionID: t2v2.ID, JobID: w4wb2pj.ID, CreatedAt: now.Add(-dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator}) + _ = dbfake.WorkspaceBuild(t, db, w4). + Pubsub(ps). + Seed(database.WorkspaceBuild{BuildNumber: 8, TemplateVersionID: t2v1.ID, CreatedAt: now.Add(-6 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator}). + Failed(dbfake.WithJobError(jobError.String), dbfake.WithJobErrorCode(jobErrorCode.String), dbfake.WithJobCompletedAt(now.Add(-6*dayDuration))). + Do() + _ = dbfake.WorkspaceBuild(t, db, w4). + Pubsub(ps). + Seed(database.WorkspaceBuild{BuildNumber: 9, TemplateVersionID: t2v2.ID, CreatedAt: now.Add(-dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator}). + Succeeded(dbfake.WithJobCompletedAt(now.Add(-dayDuration))). + Do() // When notifEnq.Clear() @@ -275,8 +306,11 @@ func TestReportFailedWorkspaceBuilds(t *testing.T) { clk.Advance(6 * dayDuration).MustWait(context.Background()) now = clk.Now() - w1wb4pj := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{OrganizationID: org.ID, Error: jobError, ErrorCode: jobErrorCode, CompletedAt: sql.NullTime{Time: now.Add(-dayDuration), Valid: true}}) - _ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{WorkspaceID: w1.ID, BuildNumber: 77, TemplateVersionID: t1v2.ID, JobID: w1wb4pj.ID, CreatedAt: now.Add(-dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator}) + _ = dbfake.WorkspaceBuild(t, db, w1). + Pubsub(ps). + Seed(database.WorkspaceBuild{BuildNumber: 77, TemplateVersionID: t1v2.ID, CreatedAt: now.Add(-dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator}). + Failed(dbfake.WithJobError(jobError.String), dbfake.WithJobErrorCode(jobErrorCode.String), dbfake.WithJobCompletedAt(now.Add(-dayDuration))). + Do() // When notifEnq.Clear() @@ -380,17 +414,26 @@ func TestReportFailedWorkspaceBuilds(t *testing.T) { now := clk.Now() // Workspace builds - pj0 := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{OrganizationID: org.ID, CompletedAt: sql.NullTime{Time: now.Add(-24 * time.Hour), Valid: true}}) - _ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{WorkspaceID: w1.ID, BuildNumber: 777, TemplateVersionID: t1v1.ID, JobID: pj0.ID, CreatedAt: now.Add(-24 * time.Hour), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator}) + _ = dbfake.WorkspaceBuild(t, db, w1). + Pubsub(ps). + Seed(database.WorkspaceBuild{BuildNumber: 777, TemplateVersionID: t1v1.ID, CreatedAt: now.Add(-24 * time.Hour), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator}). + Succeeded(dbfake.WithJobCompletedAt(now.Add(-24 * time.Hour))). + Do() for i := 1; i <= 23; i++ { at := now.Add(-time.Duration(i) * time.Hour) - pj1 := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{OrganizationID: org.ID, Error: jobError, ErrorCode: jobErrorCode, CompletedAt: sql.NullTime{Time: at, Valid: true}}) - _ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{WorkspaceID: w1.ID, BuildNumber: int32(i), TemplateVersionID: t1v1.ID, JobID: pj1.ID, CreatedAt: at, Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator}) // nolint:gosec + _ = dbfake.WorkspaceBuild(t, db, w1). + Pubsub(ps). + Seed(database.WorkspaceBuild{BuildNumber: int32(i), TemplateVersionID: t1v1.ID, CreatedAt: at, Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator}). // nolint:gosec + Failed(dbfake.WithJobError(jobError.String), dbfake.WithJobErrorCode(jobErrorCode.String), dbfake.WithJobCompletedAt(at)). + Do() - pj2 := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{OrganizationID: org.ID, Error: jobError, ErrorCode: jobErrorCode, CompletedAt: sql.NullTime{Time: at, Valid: true}}) - _ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{WorkspaceID: w1.ID, BuildNumber: int32(i) + 100, TemplateVersionID: t1v2.ID, JobID: pj2.ID, CreatedAt: at, Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator}) // nolint:gosec + _ = dbfake.WorkspaceBuild(t, db, w1). + Pubsub(ps). + Seed(database.WorkspaceBuild{BuildNumber: int32(i) + 100, TemplateVersionID: t1v2.ID, CreatedAt: at, Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator}). // nolint:gosec + Failed(dbfake.WithJobError(jobError.String), dbfake.WithJobErrorCode(jobErrorCode.String), dbfake.WithJobCompletedAt(at)). + Do() } // When @@ -486,10 +529,16 @@ func TestReportFailedWorkspaceBuilds(t *testing.T) { now := clk.Now() // Workspace builds - w1wb1pj := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{OrganizationID: org.ID, CompletedAt: sql.NullTime{Time: now.Add(-6 * dayDuration), Valid: true}}) - _ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{WorkspaceID: w1.ID, BuildNumber: 1, TemplateVersionID: t1v1.ID, JobID: w1wb1pj.ID, CreatedAt: now.Add(-2 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator}) - w1wb2pj := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{OrganizationID: org.ID, CompletedAt: sql.NullTime{Time: now.Add(-5 * dayDuration), Valid: true}}) - _ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{WorkspaceID: w1.ID, BuildNumber: 2, TemplateVersionID: t1v1.ID, JobID: w1wb2pj.ID, CreatedAt: now.Add(-1 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator}) + _ = dbfake.WorkspaceBuild(t, db, w1). + Pubsub(ps). + Seed(database.WorkspaceBuild{BuildNumber: 1, TemplateVersionID: t1v1.ID, CreatedAt: now.Add(-2 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator}). + Succeeded(dbfake.WithJobCompletedAt(now.Add(-6 * dayDuration))). + Do() + _ = dbfake.WorkspaceBuild(t, db, w1). + Pubsub(ps). + Seed(database.WorkspaceBuild{BuildNumber: 2, TemplateVersionID: t1v1.ID, CreatedAt: now.Add(-1 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator}). + Succeeded(dbfake.WithJobCompletedAt(now.Add(-5 * dayDuration))). + Do() // When notifEnq.Clear() diff --git a/enterprise/coderd/prebuilds/reconcile_test.go b/enterprise/coderd/prebuilds/reconcile_test.go index f896cf6b8f..393322cb1b 100644 --- a/enterprise/coderd/prebuilds/reconcile_test.go +++ b/enterprise/coderd/prebuilds/reconcile_test.go @@ -1830,25 +1830,27 @@ func TestExpiredPrebuildsMultipleActions(t *testing.T) { expiredCount++ } - workspace, _ := setupTestDBPrebuild( - t, - clock, - db, - pubSub, - database.WorkspaceTransitionStart, - database.ProvisionerJobStatusSucceeded, - org.ID, - preset, - template.ID, - templateVersionID, - withCreatedAt(clock.Now().Add(createdAt)), - ) + jobCreatedAt := clock.Now().Add(createdAt) + resp := dbfake.WorkspaceBuild(t, db, database.WorkspaceTable{ + OwnerID: database.PrebuildsSystemUserID, + OrganizationID: org.ID, + TemplateID: template.ID, + CreatedAt: jobCreatedAt, + }).Pubsub(pubSub).Seed(database.WorkspaceBuild{ + InitiatorID: database.PrebuildsSystemUserID, + TemplateVersionID: templateVersionID, + TemplateVersionPresetID: uuid.NullUUID{UUID: preset.ID, Valid: true}, + Transition: database.WorkspaceTransitionStart, + }).Params(database.WorkspaceBuildParameter{ + Name: "test", + Value: "test", + }).Do() if isExpired { - expiredWorkspaces = append(expiredWorkspaces, workspace) + expiredWorkspaces = append(expiredWorkspaces, resp.Workspace) } else { - nonExpiredWorkspaces = append(nonExpiredWorkspaces, workspace) + nonExpiredWorkspaces = append(nonExpiredWorkspaces, resp.Workspace) } - runningWorkspaces[workspace.ID.String()] = workspace + runningWorkspaces[resp.Workspace.ID.String()] = resp.Workspace } getJobStatusMap := func(workspaces []database.WorkspaceTable) map[database.ProvisionerJobStatus]int { @@ -2791,21 +2793,6 @@ func setupTestDBPresetWithScheduling( return preset } -// prebuildOptions holds optional parameters for creating a prebuild workspace. -type prebuildOptions struct { - createdAt *time.Time -} - -// prebuildOption defines a function type to apply optional settings to prebuildOptions. -type prebuildOption func(*prebuildOptions) - -// withCreatedAt returns a prebuildOption that sets the CreatedAt timestamp. -func withCreatedAt(createdAt time.Time) prebuildOption { - return func(opts *prebuildOptions) { - opts.createdAt = &createdAt - } -} - func setupTestDBPrebuild( t *testing.T, clock quartz.Clock, @@ -2817,10 +2804,9 @@ func setupTestDBPrebuild( preset database.TemplateVersionPreset, templateID uuid.UUID, templateVersionID uuid.UUID, - opts ...prebuildOption, ) (database.WorkspaceTable, database.WorkspaceBuild) { t.Helper() - return setupTestDBWorkspace(t, clock, db, ps, transition, prebuildStatus, orgID, preset, templateID, templateVersionID, database.PrebuildsSystemUserID, database.PrebuildsSystemUserID, opts...) + return setupTestDBWorkspace(t, clock, db, ps, transition, prebuildStatus, orgID, preset, templateID, templateVersionID, database.PrebuildsSystemUserID, database.PrebuildsSystemUserID) } func setupTestDBWorkspace( @@ -2836,7 +2822,6 @@ func setupTestDBWorkspace( templateVersionID uuid.UUID, initiatorID uuid.UUID, ownerID uuid.UUID, - opts ...prebuildOption, ) (database.WorkspaceTable, database.WorkspaceBuild) { t.Helper() cancelledAt := sql.NullTime{} @@ -2864,19 +2849,7 @@ func setupTestDBWorkspace( default: } - // Apply all provided prebuild options. - prebuiltOptions := &prebuildOptions{} - for _, opt := range opts { - opt(prebuiltOptions) - } - - // Set createdAt to default value if not overridden by options. createdAt := clock.Now().Add(muchEarlier) - if prebuiltOptions.createdAt != nil { - createdAt = *prebuiltOptions.createdAt - // Ensure startedAt matches createdAt for consistency. - startedAt = sql.NullTime{Time: createdAt, Valid: true} - } workspace := dbgen.Workspace(t, db, database.WorkspaceTable{ TemplateID: templateID, diff --git a/enterprise/coderd/schedule/template_test.go b/enterprise/coderd/schedule/template_test.go index c03a1fcd22..6c6a6c6a18 100644 --- a/enterprise/coderd/schedule/template_test.go +++ b/enterprise/coderd/schedule/template_test.go @@ -242,73 +242,32 @@ func TestTemplateUpdateBuildDeadlines(t *testing.T) { t.Log("newMaxDeadline", c.newMaxDeadline) t.Log("ttl", c.ttl) - var ( - template = dbgen.Template(t, db, database.Template{ - OrganizationID: organizationID, - ActiveVersionID: templateVersion.ID, - CreatedBy: user.ID, - }) - ws = dbgen.Workspace(t, db, database.WorkspaceTable{ - OrganizationID: organizationID, - OwnerID: user.ID, - TemplateID: template.ID, - }) - job = dbgen.ProvisionerJob(t, db, nil, database.ProvisionerJob{ - OrganizationID: organizationID, - FileID: file.ID, - InitiatorID: user.ID, - Provisioner: database.ProvisionerTypeEcho, - Tags: database.StringMap{ - c.name: "yeah", - }, - }) - wsBuild = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{ - WorkspaceID: ws.ID, - BuildNumber: 1, - JobID: job.ID, - InitiatorID: user.ID, - TemplateVersionID: templateVersion.ID, - ProvisionerState: []byte(must(cryptorand.String(64))), - }) - ) + template := dbgen.Template(t, db, database.Template{ + OrganizationID: organizationID, + ActiveVersionID: templateVersion.ID, + CreatedBy: user.ID, + }) + buildResp := dbfake.WorkspaceBuild(t, db, database.WorkspaceTable{ + OrganizationID: organizationID, + OwnerID: user.ID, + TemplateID: template.ID, + }).Seed(database.WorkspaceBuild{ + TemplateVersionID: templateVersion.ID, + ProvisionerState: []byte(must(cryptorand.String(64))), + }).Succeeded(dbfake.WithJobCompletedAt(buildTime)).Do() // Assert test invariant: workspace build state must not be empty - require.NotEmpty(t, wsBuild.ProvisionerState, "provisioner state must not be empty") + require.NotEmpty(t, buildResp.Build.ProvisionerState, "provisioner state must not be empty") - acquiredJob, err := db.AcquireProvisionerJob(ctx, database.AcquireProvisionerJobParams{ - OrganizationID: job.OrganizationID, - StartedAt: sql.NullTime{ - Time: buildTime, - Valid: true, - }, - WorkerID: uuid.NullUUID{ - UUID: uuid.New(), - Valid: true, - }, - Types: []database.ProvisionerType{database.ProvisionerTypeEcho}, - ProvisionerTags: json.RawMessage(fmt.Sprintf(`{%q: "yeah"}`, c.name)), - }) - require.NoError(t, err) - require.Equal(t, job.ID, acquiredJob.ID) - err = db.UpdateProvisionerJobWithCompleteByID(ctx, database.UpdateProvisionerJobWithCompleteByIDParams{ - ID: job.ID, - CompletedAt: sql.NullTime{ - Time: buildTime, - Valid: true, - }, - UpdatedAt: buildTime, - }) - require.NoError(t, err) - - err = db.UpdateWorkspaceBuildDeadlineByID(ctx, database.UpdateWorkspaceBuildDeadlineByIDParams{ - ID: wsBuild.ID, + err := db.UpdateWorkspaceBuildDeadlineByID(ctx, database.UpdateWorkspaceBuildDeadlineByIDParams{ + ID: buildResp.Build.ID, UpdatedAt: buildTime, Deadline: c.deadline, MaxDeadline: c.maxDeadline, }) require.NoError(t, err) - wsBuild, err = db.GetWorkspaceBuildByID(ctx, wsBuild.ID) + wsBuild, err := db.GetWorkspaceBuildByID(ctx, buildResp.Build.ID) require.NoError(t, err) logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}).Leveled(slog.LevelDebug) diff --git a/scripts/workspace-runtime-audit/runtimeaudit_test.go b/scripts/workspace-runtime-audit/runtimeaudit_test.go index 926fa79a63..ac524850c4 100644 --- a/scripts/workspace-runtime-audit/runtimeaudit_test.go +++ b/scripts/workspace-runtime-audit/runtimeaudit_test.go @@ -5,7 +5,6 @@ package runtimeaudit_test import ( - "database/sql" _ "embed" "math" "strings" @@ -258,8 +257,8 @@ func TestRuntimeAudit(t *testing.T) { name: "canceled_start_does_not_count_usage", // Only start+succeeded counts; canceled start is ignored. builds: []workspaceBuildArgs{ - {at: decUTC(8, 9, 0), canceled: true, transition: database.WorkspaceTransitionStart, jobStatus: database.ProvisionerJobStatusCanceled}, - {at: decUTC(8, 10, 0), canceled: false, transition: database.WorkspaceTransitionStop, jobStatus: database.ProvisionerJobStatusSucceeded}, + {at: decUTC(8, 9, 0), transition: database.WorkspaceTransitionStart, jobStatus: database.ProvisionerJobStatusCanceled}, + {at: decUTC(8, 10, 0), transition: database.WorkspaceTransitionStop, jobStatus: database.ProvisionerJobStatusSucceeded}, }, expect: func(_ time.Time, _ []workspaceBuildArgs) int { return 0 }, }, @@ -267,8 +266,8 @@ func TestRuntimeAudit(t *testing.T) { name: "failed_start_does_not_count_even_if_later_stop_occurs", // Start failed => never turns on => later stop does nothing. builds: []workspaceBuildArgs{ - {at: decUTC(9, 9, 0), canceled: false, transition: database.WorkspaceTransitionStart, jobStatus: database.ProvisionerJobStatusFailed}, - {at: decUTC(9, 12, 0), canceled: false, transition: database.WorkspaceTransitionStop, jobStatus: database.ProvisionerJobStatusSucceeded}, + {at: decUTC(9, 9, 0), transition: database.WorkspaceTransitionStart, jobStatus: database.ProvisionerJobStatusFailed}, + {at: decUTC(9, 12, 0), transition: database.WorkspaceTransitionStop, jobStatus: database.ProvisionerJobStatusSucceeded}, }, expect: func(_ time.Time, _ []workspaceBuildArgs) int { return 0 }, }, @@ -276,8 +275,8 @@ func TestRuntimeAudit(t *testing.T) { name: "canceled_stop_still_stops_timer_and_counts_time", // Any non-(start+succeeded) is treated as stop while running, regardless of status/canceled. builds: []workspaceBuildArgs{ - {at: decUTC(10, 9, 0), canceled: false, transition: database.WorkspaceTransitionStart, jobStatus: database.ProvisionerJobStatusSucceeded}, - {at: decUTC(10, 9, 40), canceled: true, transition: database.WorkspaceTransitionStop, jobStatus: database.ProvisionerJobStatusCanceled}, + {at: decUTC(10, 9, 0), transition: database.WorkspaceTransitionStart, jobStatus: database.ProvisionerJobStatusSucceeded}, + {at: decUTC(10, 9, 40), transition: database.WorkspaceTransitionStop, jobStatus: database.ProvisionerJobStatusCanceled}, }, expect: func(_ time.Time, in []workspaceBuildArgs) int { return roundUpHours(in[1].at, in[0].at) @@ -287,8 +286,8 @@ func TestRuntimeAudit(t *testing.T) { name: "failed_stop_still_stops_timer_and_counts_time", // Same as above: stop is stop even if job failed (ELSE path). builds: []workspaceBuildArgs{ - {at: decUTC(11, 10, 0), canceled: false, transition: database.WorkspaceTransitionStart, jobStatus: database.ProvisionerJobStatusSucceeded}, - {at: decUTC(11, 10, 10), canceled: false, transition: database.WorkspaceTransitionStop, jobStatus: database.ProvisionerJobStatusFailed}, + {at: decUTC(11, 10, 0), transition: database.WorkspaceTransitionStart, jobStatus: database.ProvisionerJobStatusSucceeded}, + {at: decUTC(11, 10, 10), transition: database.WorkspaceTransitionStop, jobStatus: database.ProvisionerJobStatusFailed}, }, expect: func(_ time.Time, in []workspaceBuildArgs) int { return roundUpHours(in[1].at, in[0].at) @@ -298,8 +297,8 @@ func TestRuntimeAudit(t *testing.T) { name: "failed_transition_stops_timer_and_counts_time", // A failed *non-stop* transition (e.g. delete) still stops if currently on. builds: []workspaceBuildArgs{ - {at: decUTC(12, 8, 0), canceled: false, transition: database.WorkspaceTransitionStart, jobStatus: database.ProvisionerJobStatusSucceeded}, - {at: decUTC(12, 8, 5), canceled: false, transition: database.WorkspaceTransitionDelete, jobStatus: database.ProvisionerJobStatusFailed}, + {at: decUTC(12, 8, 0), transition: database.WorkspaceTransitionStart, jobStatus: database.ProvisionerJobStatusSucceeded}, + {at: decUTC(12, 8, 5), transition: database.WorkspaceTransitionDelete, jobStatus: database.ProvisionerJobStatusFailed}, }, expect: func(_ time.Time, in []workspaceBuildArgs) int { return roundUpHours(in[1].at, in[0].at) @@ -310,11 +309,11 @@ func TestRuntimeAudit(t *testing.T) { // When already on, a subsequent non-(start+succeeded) build triggers stop logic. // This verifies you *do not* treat start+failed as a "start"; it will stop the running timer. builds: []workspaceBuildArgs{ - {at: decUTC(13, 9, 0), canceled: false, transition: database.WorkspaceTransitionStart, jobStatus: database.ProvisionerJobStatusSucceeded}, + {at: decUTC(13, 9, 0), transition: database.WorkspaceTransitionStart, jobStatus: database.ProvisionerJobStatusSucceeded}, // This goes to ELSE branch (because job_status != succeeded) and will stop the timer. - {at: decUTC(13, 9, 30), canceled: false, transition: database.WorkspaceTransitionStart, jobStatus: database.ProvisionerJobStatusFailed}, + {at: decUTC(13, 9, 30), transition: database.WorkspaceTransitionStart, jobStatus: database.ProvisionerJobStatusFailed}, // Subsequent stop should not add more time because timer was reset. - {at: decUTC(13, 10, 0), canceled: false, transition: database.WorkspaceTransitionStop, jobStatus: database.ProvisionerJobStatusSucceeded}, + {at: decUTC(13, 10, 0), transition: database.WorkspaceTransitionStop, jobStatus: database.ProvisionerJobStatusSucceeded}, }, expect: func(_ time.Time, in []workspaceBuildArgs) int { // Only counts from first start to failed-start event. @@ -368,13 +367,12 @@ func initSetup(t *testing.T, db database.Store) *setup { type workspaceBuildArgs struct { at time.Time - canceled bool transition database.WorkspaceTransition jobStatus database.ProvisionerJobStatus } func (s *setup) createWorkspace(t *testing.T, db database.Store, builds []workspaceBuildArgs) database.WorkspaceTable { - // Insert the first build + // Create template version first tv := dbfake.TemplateVersion(t, db). Seed(database.TemplateVersion{ OrganizationID: s.org.ID, @@ -390,39 +388,28 @@ func (s *setup) createWorkspace(t *testing.T, db database.Store, builds []worksp }) for i, b := range builds { - job := dbgen.ProvisionerJob(t, db, nil, database.ProvisionerJob{ - CreatedAt: b.at, - UpdatedAt: b.at, - StartedAt: sql.NullTime{ - Time: b.at, - Valid: true, - }, - CanceledAt: sql.NullTime{ - Time: b.at, - Valid: b.canceled, - }, - CompletedAt: sql.NullTime{ - Time: b.at, - Valid: true, - }, - Error: sql.NullString{}, - OrganizationID: s.org.ID, - InitiatorID: s.usr.ID, - Type: database.ProvisionerJobTypeWorkspaceBuild, - JobStatus: b.jobStatus, - }) + builder := dbfake.WorkspaceBuild(t, db, wrk). + Seed(database.WorkspaceBuild{ + CreatedAt: b.at, + UpdatedAt: b.at, + TemplateVersionID: tv.TemplateVersion.ID, + //nolint:gosec // this will not overflow + BuildNumber: int32(i) + 1, + Transition: b.transition, + InitiatorID: s.usr.ID, + }). + Succeeded(dbfake.WithJobCompletedAt(b.at)) - dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{ - CreatedAt: b.at, - UpdatedAt: b.at, - WorkspaceID: wrk.ID, - TemplateVersionID: tv.TemplateVersion.ID, - ///nolint:gosec // this will not overflow - BuildNumber: int32(i) + 1, - Transition: b.transition, - InitiatorID: s.usr.ID, - JobID: job.ID, - }) + // Set job status based on the build args + switch b.jobStatus { + case database.ProvisionerJobStatusCanceled: + builder = builder.Canceled(dbfake.WithJobCompletedAt(b.at)) + case database.ProvisionerJobStatusFailed: + builder = builder.Failed(dbfake.WithJobError("fake error"), dbfake.WithJobCompletedAt(b.at)) + // default: Succeeded (the builder's default) + } + + builder.Do() } return wrk