fix: add constraint and runtime check for provisioner logs size limit (#18893)

This PR sets a constraint of 1MB on the provisioner job logs written to
the database. This is consistent with the constraint we place on
workspace agent logs:
https://github.com/coder/coder/blob/4ac6be6d835dc36c242e35a26b584b784040bf28/coderd/database/dump.sql#L2030

It also adds a message printed to the front end about the provisioner
log overflow, and updates the message printed to the front end when
workspace startup logs exceed the max, as it was causing some customers
to think their startup script had failed to run.

cli/testdata/coder_list_--output_json.golden

@@ -55,7 +55,8 @@
           "template_name": "",
           "template_display_name": "",
           "template_icon": ""
-        }
+        },
+        "logs_overflowed": false
       },
       "reason": "initiator",
       "resources": [],

cli/testdata/coder_provisioner_jobs_list_--help.golden

@@ -11,7 +11,7 @@ OPTIONS:
   -O, --org string, $CODER_ORGANIZATION
           Select which organization (uuid or name) to use.
 
-  -c, --column [id|created at|started at|completed at|canceled at|error|error code|status|worker id|worker name|file id|tags|queue position|queue size|organization id|template version id|workspace build id|type|available workers|template version name|template id|template name|template display name|template icon|workspace id|workspace name|organization|queue] (default: created at,id,type,template display name,status,queue,tags)
+  -c, --column [id|created at|started at|completed at|canceled at|error|error code|status|worker id|worker name|file id|tags|queue position|queue size|organization id|template version id|workspace build id|type|available workers|template version name|template id|template name|template display name|template icon|workspace id|workspace name|logs overflowed|organization|queue] (default: created at,id,type,template display name,status,queue,tags)
           Columns to display in table output.
 
   -l, --limit int, $CODER_PROVISIONER_JOB_LIST_LIMIT (default: 50)

cli/testdata/coder_provisioner_jobs_list_--output_json.golden

@@ -26,6 +26,7 @@
       "template_display_name": "",
       "template_icon": ""
     },
+    "logs_overflowed": false,
     "organization_name": "Coder"
   },
   {
@@ -57,6 +58,7 @@
       "workspace_id": "===========[workspace ID]===========",
       "workspace_name": "test-workspace"
     },
+    "logs_overflowed": false,
     "organization_name": "Coder"
   }
 ]

coderd/apidoc/docs.go

@@ -15261,6 +15261,9 @@ const docTemplate = `{
                 "input": {
                     "$ref": "#/definitions/codersdk.ProvisionerJobInput"
                 },
+                "logs_overflowed": {
+                    "type": "boolean"
+                },
                 "metadata": {
                     "$ref": "#/definitions/codersdk.ProvisionerJobMetadata"
                 },

coderd/apidoc/swagger.json

@@ -13852,6 +13852,9 @@
 				"input": {
 					"$ref": "#/definitions/codersdk.ProvisionerJobInput"
 				},
+				"logs_overflowed": {
+					"type": "boolean"
+				},
 				"metadata": {
 					"$ref": "#/definitions/codersdk.ProvisionerJobMetadata"
 				},

coderd/database/dbauthz/dbauthz.go

@@ -4489,6 +4489,22 @@ func (q *querier) UpdateProvisionerJobByID(ctx context.Context, arg database.Upd
 	return q.db.UpdateProvisionerJobByID(ctx, arg)
 }
 
+func (q *querier) UpdateProvisionerJobLogsLength(ctx context.Context, arg database.UpdateProvisionerJobLogsLengthParams) error {
+	// Not sure what the rbac should be here, going with this for now
+	if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceProvisionerJobs); err != nil {
+		return err
+	}
+	return q.db.UpdateProvisionerJobLogsLength(ctx, arg)
+}
+
+func (q *querier) UpdateProvisionerJobLogsOverflowed(ctx context.Context, arg database.UpdateProvisionerJobLogsOverflowedParams) error {
+	// Not sure what the rbac should be here, going with this for now
+	if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceProvisionerJobs); err != nil {
+		return err
+	}
+	return q.db.UpdateProvisionerJobLogsOverflowed(ctx, arg)
+}
+
 func (q *querier) UpdateProvisionerJobWithCancelByID(ctx context.Context, arg database.UpdateProvisionerJobWithCancelByIDParams) error {
 	// TODO: Remove this once we have a proper rbac check for provisioner jobs.
 	// Details in https://github.com/coder/coder/issues/16160

coderd/database/dbauthz/dbauthz_test.go

@@ -4341,6 +4341,20 @@ func (s *MethodTestSuite) TestSystemFunctions() {
 			UpdatedAt: time.Now(),
 		}).Asserts(rbac.ResourceProvisionerJobs, policy.ActionUpdate)
 	}))
+	s.Run("UpdateProvisionerJobLogsLength", s.Subtest(func(db database.Store, check *expects) {
+		j := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{})
+		check.Args(database.UpdateProvisionerJobLogsLengthParams{
+			ID:         j.ID,
+			LogsLength: 100,
+		}).Asserts(rbac.ResourceProvisionerJobs, policy.ActionUpdate)
+	}))
+	s.Run("UpdateProvisionerJobLogsOverflowed", s.Subtest(func(db database.Store, check *expects) {
+		j := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{})
+		check.Args(database.UpdateProvisionerJobLogsOverflowedParams{
+			ID:             j.ID,
+			LogsOverflowed: true,
+		}).Asserts(rbac.ResourceProvisionerJobs, policy.ActionUpdate)
+	}))
 	s.Run("InsertProvisionerJob", s.Subtest(func(db database.Store, check *expects) {
 		dbtestutil.DisableForeignKeysAndTriggers(s.T(), db)
 		check.Args(database.InsertProvisionerJobParams{

coderd/database/dbfake/dbfake.go

@@ -179,6 +179,7 @@ func (b WorkspaceBuildBuilder) Do() WorkspaceResponse {
 		Input:          payload,
 		Tags:           map[string]string{},
 		TraceMetadata:  pqtype.NullRawMessage{},
+		LogsOverflowed: false,
 	})
 	require.NoError(b.t, err, "insert job")
 

coderd/database/dbgen/dbgen.go

@@ -775,6 +775,7 @@ func ProvisionerJob(t testing.TB, db database.Store, ps pubsub.Pubsub, orig data
 		Input:          takeFirstSlice(orig.Input, []byte("{}")),
 		Tags:           tags,
 		TraceMetadata:  pqtype.NullRawMessage{},
+		LogsOverflowed: false,
 	})
 	require.NoError(t, err, "insert job")
 	if ps != nil {

coderd/database/dbmetrics/querymetrics.go

@@ -2784,6 +2784,20 @@ func (m queryMetricsStore) UpdateProvisionerJobByID(ctx context.Context, arg dat
 	return err
 }
 
+func (m queryMetricsStore) UpdateProvisionerJobLogsLength(ctx context.Context, arg database.UpdateProvisionerJobLogsLengthParams) error {
+	start := time.Now()
+	r0 := m.s.UpdateProvisionerJobLogsLength(ctx, arg)
+	m.queryLatencies.WithLabelValues("UpdateProvisionerJobLogsLength").Observe(time.Since(start).Seconds())
+	return r0
+}
+
+func (m queryMetricsStore) UpdateProvisionerJobLogsOverflowed(ctx context.Context, arg database.UpdateProvisionerJobLogsOverflowedParams) error {
+	start := time.Now()
+	r0 := m.s.UpdateProvisionerJobLogsOverflowed(ctx, arg)
+	m.queryLatencies.WithLabelValues("UpdateProvisionerJobLogsOverflowed").Observe(time.Since(start).Seconds())
+	return r0
+}
+
 func (m queryMetricsStore) UpdateProvisionerJobWithCancelByID(ctx context.Context, arg database.UpdateProvisionerJobWithCancelByIDParams) error {
 	start := time.Now()
 	err := m.s.UpdateProvisionerJobWithCancelByID(ctx, arg)

coderd/database/dbmock/dbmock.go

@@ -5958,6 +5958,34 @@ func (mr *MockStoreMockRecorder) UpdateProvisionerJobByID(ctx, arg any) *gomock.
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateProvisionerJobByID", reflect.TypeOf((*MockStore)(nil).UpdateProvisionerJobByID), ctx, arg)
 }
 
+// UpdateProvisionerJobLogsLength mocks base method.
+func (m *MockStore) UpdateProvisionerJobLogsLength(ctx context.Context, arg database.UpdateProvisionerJobLogsLengthParams) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "UpdateProvisionerJobLogsLength", ctx, arg)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// UpdateProvisionerJobLogsLength indicates an expected call of UpdateProvisionerJobLogsLength.
+func (mr *MockStoreMockRecorder) UpdateProvisionerJobLogsLength(ctx, arg any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateProvisionerJobLogsLength", reflect.TypeOf((*MockStore)(nil).UpdateProvisionerJobLogsLength), ctx, arg)
+}
+
+// UpdateProvisionerJobLogsOverflowed mocks base method.
+func (m *MockStore) UpdateProvisionerJobLogsOverflowed(ctx context.Context, arg database.UpdateProvisionerJobLogsOverflowedParams) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "UpdateProvisionerJobLogsOverflowed", ctx, arg)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// UpdateProvisionerJobLogsOverflowed indicates an expected call of UpdateProvisionerJobLogsOverflowed.
+func (mr *MockStoreMockRecorder) UpdateProvisionerJobLogsOverflowed(ctx, arg any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateProvisionerJobLogsOverflowed", reflect.TypeOf((*MockStore)(nil).UpdateProvisionerJobLogsOverflowed), ctx, arg)
+}
+
 // UpdateProvisionerJobWithCancelByID mocks base method.
 func (m *MockStore) UpdateProvisionerJobWithCancelByID(ctx context.Context, arg database.UpdateProvisionerJobWithCancelByIDParams) error {
 	m.ctrl.T.Helper()

coderd/database/dump.sql

@@ -1419,11 +1419,18 @@ CASE
         WHEN (started_at IS NULL) THEN 'pending'::provisioner_job_status
         ELSE 'running'::provisioner_job_status
     END
-END) STORED NOT NULL
+END) STORED NOT NULL,
+    logs_length integer DEFAULT 0 NOT NULL,
+    logs_overflowed boolean DEFAULT false NOT NULL,
+    CONSTRAINT max_provisioner_logs_length CHECK ((logs_length <= 1048576))
 );
 
 COMMENT ON COLUMN provisioner_jobs.job_status IS 'Computed column to track the status of the job.';
 
+COMMENT ON COLUMN provisioner_jobs.logs_length IS 'Total length of provisioner logs';
+
+COMMENT ON COLUMN provisioner_jobs.logs_overflowed IS 'Whether the provisioner logs overflowed in length';
+
 CREATE TABLE provisioner_keys (
     id uuid NOT NULL,
     created_at timestamp with time zone NOT NULL,

coderd/database/errors.go

@@ -79,3 +79,11 @@ func IsWorkspaceAgentLogsLimitError(err error) bool {
 
 	return false
 }
+
+func IsProvisionerJobLogsLimitError(err error) bool {
+	var pqErr *pq.Error
+	if errors.As(err, &pqErr) {
+		return pqErr.Constraint == "max_provisioner_logs_length" && pqErr.Table == "provisioner_jobs"
+	}
+	return false
+}

coderd/database/migrations/000355_add_provisioner_logs_overflowed.down.sql

@@ -0,0 +1,2 @@
+ALTER TABLE provisioner_jobs DROP COLUMN logs_length;
+ALTER TABLE provisioner_jobs DROP COLUMN logs_overflowed;

coderd/database/migrations/000355_add_provisioner_logs_overflowed.up.sql

@@ -0,0 +1,6 @@
+ -- Add logs length tracking and overflow flag, similar to workspace agents
+  ALTER TABLE provisioner_jobs ADD COLUMN logs_length integer NOT NULL DEFAULT 0 CONSTRAINT max_provisioner_logs_length CHECK (logs_length <= 1048576);
+  ALTER TABLE provisioner_jobs ADD COLUMN logs_overflowed boolean NOT NULL DEFAULT false;
+
+  COMMENT ON COLUMN provisioner_jobs.logs_length IS 'Total length of provisioner logs';
+  COMMENT ON COLUMN provisioner_jobs.logs_overflowed IS 'Whether the provisioner logs overflowed in length';

coderd/database/models.go

@@ -3384,6 +3384,10 @@ type ProvisionerJob struct {
 	TraceMetadata  pqtype.NullRawMessage    `db:"trace_metadata" json:"trace_metadata"`
 	// Computed column to track the status of the job.
 	JobStatus ProvisionerJobStatus `db:"job_status" json:"job_status"`
+	// Total length of provisioner logs
+	LogsLength int32 `db:"logs_length" json:"logs_length"`
+	// Whether the provisioner logs overflowed in length
+	LogsOverflowed bool `db:"logs_overflowed" json:"logs_overflowed"`
 }
 
 type ProvisionerJobLog struct {

coderd/database/querier.go

@@ -593,6 +593,8 @@ type sqlcQuerier interface {
 	UpdatePresetPrebuildStatus(ctx context.Context, arg UpdatePresetPrebuildStatusParams) error
 	UpdateProvisionerDaemonLastSeenAt(ctx context.Context, arg UpdateProvisionerDaemonLastSeenAtParams) error
 	UpdateProvisionerJobByID(ctx context.Context, arg UpdateProvisionerJobByIDParams) error
+	UpdateProvisionerJobLogsLength(ctx context.Context, arg UpdateProvisionerJobLogsLengthParams) error
+	UpdateProvisionerJobLogsOverflowed(ctx context.Context, arg UpdateProvisionerJobLogsOverflowedParams) error
 	UpdateProvisionerJobWithCancelByID(ctx context.Context, arg UpdateProvisionerJobWithCancelByIDParams) error
 	UpdateProvisionerJobWithCompleteByID(ctx context.Context, arg UpdateProvisionerJobWithCompleteByIDParams) error
 	UpdateProvisionerJobWithCompleteWithStartedAtByID(ctx context.Context, arg UpdateProvisionerJobWithCompleteWithStartedAtByIDParams) error

coderd/database/queries.sql.go

@@ -8514,6 +8514,44 @@ func (q *sqlQuerier) InsertProvisionerJobLogs(ctx context.Context, arg InsertPro
 	return items, nil
 }
 
+const updateProvisionerJobLogsLength = `-- name: UpdateProvisionerJobLogsLength :exec
+UPDATE 
+	provisioner_jobs
+SET 
+	logs_length = logs_length + $2
+WHERE 
+	id = $1
+`
+
+type UpdateProvisionerJobLogsLengthParams struct {
+	ID         uuid.UUID `db:"id" json:"id"`
+	LogsLength int32     `db:"logs_length" json:"logs_length"`
+}
+
+func (q *sqlQuerier) UpdateProvisionerJobLogsLength(ctx context.Context, arg UpdateProvisionerJobLogsLengthParams) error {
+	_, err := q.db.ExecContext(ctx, updateProvisionerJobLogsLength, arg.ID, arg.LogsLength)
+	return err
+}
+
+const updateProvisionerJobLogsOverflowed = `-- name: UpdateProvisionerJobLogsOverflowed :exec
+UPDATE 
+	provisioner_jobs
+SET 
+	logs_overflowed = $2
+WHERE 
+	id = $1
+`
+
+type UpdateProvisionerJobLogsOverflowedParams struct {
+	ID             uuid.UUID `db:"id" json:"id"`
+	LogsOverflowed bool      `db:"logs_overflowed" json:"logs_overflowed"`
+}
+
+func (q *sqlQuerier) UpdateProvisionerJobLogsOverflowed(ctx context.Context, arg UpdateProvisionerJobLogsOverflowedParams) error {
+	_, err := q.db.ExecContext(ctx, updateProvisionerJobLogsOverflowed, arg.ID, arg.LogsOverflowed)
+	return err
+}
+
 const acquireProvisionerJob = `-- name: AcquireProvisionerJob :one
 UPDATE
 	provisioner_jobs
@@ -8543,7 +8581,7 @@ WHERE
 		SKIP LOCKED
 		LIMIT
 			1
-	) RETURNING id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status
+	) RETURNING id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status, logs_length, logs_overflowed
 `
 
 type AcquireProvisionerJobParams struct {
@@ -8589,13 +8627,15 @@ func (q *sqlQuerier) AcquireProvisionerJob(ctx context.Context, arg AcquireProvi
 		&i.ErrorCode,
 		&i.TraceMetadata,
 		&i.JobStatus,
+		&i.LogsLength,
+		&i.LogsOverflowed,
 	)
 	return i, err
 }
 
 const getProvisionerJobByID = `-- name: GetProvisionerJobByID :one
 SELECT
-	id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status
+	id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status, logs_length, logs_overflowed
 FROM
 	provisioner_jobs
 WHERE
@@ -8625,13 +8665,15 @@ func (q *sqlQuerier) GetProvisionerJobByID(ctx context.Context, id uuid.UUID) (P
 		&i.ErrorCode,
 		&i.TraceMetadata,
 		&i.JobStatus,
+		&i.LogsLength,
+		&i.LogsOverflowed,
 	)
 	return i, err
 }
 
 const getProvisionerJobByIDForUpdate = `-- name: GetProvisionerJobByIDForUpdate :one
 SELECT
-	id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status
+	id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status, logs_length, logs_overflowed
 FROM
 	provisioner_jobs
 WHERE
@@ -8665,6 +8707,8 @@ func (q *sqlQuerier) GetProvisionerJobByIDForUpdate(ctx context.Context, id uuid
 		&i.ErrorCode,
 		&i.TraceMetadata,
 		&i.JobStatus,
+		&i.LogsLength,
+		&i.LogsOverflowed,
 	)
 	return i, err
 }
@@ -8708,7 +8752,7 @@ func (q *sqlQuerier) GetProvisionerJobTimingsByJobID(ctx context.Context, jobID
 
 const getProvisionerJobsByIDs = `-- name: GetProvisionerJobsByIDs :many
 SELECT
-	id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status
+	id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status, logs_length, logs_overflowed
 FROM
 	provisioner_jobs
 WHERE
@@ -8744,6 +8788,8 @@ func (q *sqlQuerier) GetProvisionerJobsByIDs(ctx context.Context, ids []uuid.UUI
 			&i.ErrorCode,
 			&i.TraceMetadata,
 			&i.JobStatus,
+			&i.LogsLength,
+			&i.LogsOverflowed,
 		); err != nil {
 			return nil, err
 		}
@@ -8811,7 +8857,7 @@ SELECT
 	-- Step 5: Final SELECT with INNER JOIN provisioner_jobs
 	fj.id,
 	fj.created_at,
-	pj.id, pj.created_at, pj.updated_at, pj.started_at, pj.canceled_at, pj.completed_at, pj.error, pj.organization_id, pj.initiator_id, pj.provisioner, pj.storage_method, pj.type, pj.input, pj.worker_id, pj.file_id, pj.tags, pj.error_code, pj.trace_metadata, pj.job_status,
+	pj.id, pj.created_at, pj.updated_at, pj.started_at, pj.canceled_at, pj.completed_at, pj.error, pj.organization_id, pj.initiator_id, pj.provisioner, pj.storage_method, pj.type, pj.input, pj.worker_id, pj.file_id, pj.tags, pj.error_code, pj.trace_metadata, pj.job_status, pj.logs_length, pj.logs_overflowed,
 	fj.queue_position,
 	fj.queue_size
 FROM
@@ -8867,6 +8913,8 @@ func (q *sqlQuerier) GetProvisionerJobsByIDsWithQueuePosition(ctx context.Contex
 			&i.ProvisionerJob.ErrorCode,
 			&i.ProvisionerJob.TraceMetadata,
 			&i.ProvisionerJob.JobStatus,
+			&i.ProvisionerJob.LogsLength,
+			&i.ProvisionerJob.LogsOverflowed,
 			&i.QueuePosition,
 			&i.QueueSize,
 		); err != nil {
@@ -8909,7 +8957,7 @@ queue_size AS (
 	SELECT COUNT(*) AS count FROM pending_jobs
 )
 SELECT
-	pj.id, pj.created_at, pj.updated_at, pj.started_at, pj.canceled_at, pj.completed_at, pj.error, pj.organization_id, pj.initiator_id, pj.provisioner, pj.storage_method, pj.type, pj.input, pj.worker_id, pj.file_id, pj.tags, pj.error_code, pj.trace_metadata, pj.job_status,
+	pj.id, pj.created_at, pj.updated_at, pj.started_at, pj.canceled_at, pj.completed_at, pj.error, pj.organization_id, pj.initiator_id, pj.provisioner, pj.storage_method, pj.type, pj.input, pj.worker_id, pj.file_id, pj.tags, pj.error_code, pj.trace_metadata, pj.job_status, pj.logs_length, pj.logs_overflowed,
     COALESCE(qp.queue_position, 0) AS queue_position,
     COALESCE(qs.count, 0) AS queue_size,
 	-- Use subquery to utilize ORDER BY in array_agg since it cannot be
@@ -9045,6 +9093,8 @@ func (q *sqlQuerier) GetProvisionerJobsByOrganizationAndStatusWithQueuePositionA
 			&i.ProvisionerJob.ErrorCode,
 			&i.ProvisionerJob.TraceMetadata,
 			&i.ProvisionerJob.JobStatus,
+			&i.ProvisionerJob.LogsLength,
+			&i.ProvisionerJob.LogsOverflowed,
 			&i.QueuePosition,
 			&i.QueueSize,
 			pq.Array(&i.AvailableWorkers),
@@ -9071,7 +9121,7 @@ func (q *sqlQuerier) GetProvisionerJobsByOrganizationAndStatusWithQueuePositionA
 }
 
 const getProvisionerJobsCreatedAfter = `-- name: GetProvisionerJobsCreatedAfter :many
-SELECT id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status FROM provisioner_jobs WHERE created_at > $1
+SELECT id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status, logs_length, logs_overflowed FROM provisioner_jobs WHERE created_at > $1
 `
 
 func (q *sqlQuerier) GetProvisionerJobsCreatedAfter(ctx context.Context, createdAt time.Time) ([]ProvisionerJob, error) {
@@ -9103,6 +9153,8 @@ func (q *sqlQuerier) GetProvisionerJobsCreatedAfter(ctx context.Context, created
 			&i.ErrorCode,
 			&i.TraceMetadata,
 			&i.JobStatus,
+			&i.LogsLength,
+			&i.LogsOverflowed,
 		); err != nil {
 			return nil, err
 		}
@@ -9119,7 +9171,7 @@ func (q *sqlQuerier) GetProvisionerJobsCreatedAfter(ctx context.Context, created
 
 const getProvisionerJobsToBeReaped = `-- name: GetProvisionerJobsToBeReaped :many
 SELECT
-	id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status
+	id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status, logs_length, logs_overflowed
 FROM
 	provisioner_jobs
 WHERE
@@ -9176,6 +9228,8 @@ func (q *sqlQuerier) GetProvisionerJobsToBeReaped(ctx context.Context, arg GetPr
 			&i.ErrorCode,
 			&i.TraceMetadata,
 			&i.JobStatus,
+			&i.LogsLength,
+			&i.LogsOverflowed,
 		); err != nil {
 			return nil, err
 		}
@@ -9204,10 +9258,11 @@ INSERT INTO
 		"type",
 		"input",
 		tags,
-		trace_metadata
+		trace_metadata,
+		logs_overflowed
 	)
 VALUES
-	($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) RETURNING id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status
+	($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) RETURNING id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status, logs_length, logs_overflowed
 `
 
 type InsertProvisionerJobParams struct {
@@ -9223,6 +9278,7 @@ type InsertProvisionerJobParams struct {
 	Input          json.RawMessage          `db:"input" json:"input"`
 	Tags           StringMap                `db:"tags" json:"tags"`
 	TraceMetadata  pqtype.NullRawMessage    `db:"trace_metadata" json:"trace_metadata"`
+	LogsOverflowed bool                     `db:"logs_overflowed" json:"logs_overflowed"`
 }
 
 func (q *sqlQuerier) InsertProvisionerJob(ctx context.Context, arg InsertProvisionerJobParams) (ProvisionerJob, error) {
@@ -9239,6 +9295,7 @@ func (q *sqlQuerier) InsertProvisionerJob(ctx context.Context, arg InsertProvisi
 		arg.Input,
 		arg.Tags,
 		arg.TraceMetadata,
+		arg.LogsOverflowed,
 	)
 	var i ProvisionerJob
 	err := row.Scan(
@@ -9261,6 +9318,8 @@ func (q *sqlQuerier) InsertProvisionerJob(ctx context.Context, arg InsertProvisi
 		&i.ErrorCode,
 		&i.TraceMetadata,
 		&i.JobStatus,
+		&i.LogsLength,
+		&i.LogsOverflowed,
 	)
 	return i, err
 }

coderd/database/queries/provisionerjoblogs.sql

@@ -19,3 +19,19 @@ SELECT
 	unnest(@level :: log_level [ ]) AS LEVEL,
 	unnest(@stage :: VARCHAR(128) [ ]) AS stage,
 	unnest(@output :: VARCHAR(1024) [ ]) AS output RETURNING *;
+	
+-- name: UpdateProvisionerJobLogsOverflowed :exec
+UPDATE 
+	provisioner_jobs
+SET 
+	logs_overflowed = $2
+WHERE 
+	id = $1;
+	
+-- name: UpdateProvisionerJobLogsLength :exec
+UPDATE 
+	provisioner_jobs
+SET 
+	logs_length = logs_length + $2
+WHERE 
+	id = $1;

coderd/database/queries/provisionerjobs.sql

@@ -247,10 +247,11 @@ INSERT INTO
 		"type",
 		"input",
 		tags,
-		trace_metadata
+		trace_metadata,
+		logs_overflowed
 	)
 VALUES
-	($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) RETURNING *;
+	($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) RETURNING *;
 
 -- name: UpdateProvisionerJobByID :exec
 UPDATE

coderd/provisionerdserver/provisionerdserver.go

@@ -902,29 +902,93 @@ func (s *server) UpdateJob(ctx context.Context, request *proto.UpdateJobRequest)
 		return nil, xerrors.Errorf("update job: %w", err)
 	}
 
-	if len(request.Logs) > 0 {
+	if len(request.Logs) > 0 && !job.LogsOverflowed {
 		//nolint:exhaustruct // We append to the additional fields below.
 		insertParams := database.InsertProvisionerJobLogsParams{
 			JobID: parsedID,
 		}
+
+		newLogSize := 0
+		overflowedErrorMsg := "Provisioner logs exceeded the max size of 1MB. Will not continue to write provisioner logs for workspace build."
+		lenErrMsg := len(overflowedErrorMsg)
+
+		var (
+			createdAt time.Time
+			level     database.LogLevel
+			stage     string
+			source    database.LogSource
+			output    string
+		)
+
 		for _, log := range request.Logs {
-			logLevel, err := convertLogLevel(log.Level)
+			// Build our log params
+			level, err = convertLogLevel(log.Level)
 			if err != nil {
 				return nil, xerrors.Errorf("convert log level: %w", err)
 			}
-			logSource, err := convertLogSource(log.Source)
+			source, err = convertLogSource(log.Source)
 			if err != nil {
 				return nil, xerrors.Errorf("convert log source: %w", err)
 			}
-			insertParams.CreatedAt = append(insertParams.CreatedAt, time.UnixMilli(log.CreatedAt))
-			insertParams.Level = append(insertParams.Level, logLevel)
-			insertParams.Stage = append(insertParams.Stage, log.Stage)
-			insertParams.Source = append(insertParams.Source, logSource)
-			insertParams.Output = append(insertParams.Output, log.Output)
+			createdAt = time.UnixMilli(log.CreatedAt)
+			stage = log.Stage
+			output = log.Output
+
+			// Check if we would overflow the job logs (not leaving enough room for the error message)
+			willOverflow := int64(job.LogsLength)+int64(newLogSize)+int64(lenErrMsg)+int64(len(output)) > 1048576
+			if willOverflow {
+				s.Logger.Debug(ctx, "provisioner job logs overflowed 1MB size limit in database", slog.F("job_id", parsedID))
+				err = s.Database.UpdateProvisionerJobLogsOverflowed(ctx, database.UpdateProvisionerJobLogsOverflowedParams{
+					ID:             parsedID,
+					LogsOverflowed: true,
+				})
+				if err != nil {
+					s.Logger.Error(ctx, "failed to set logs overflowed flag", slog.F("job_id", parsedID), slog.Error(err))
+				}
+
+				level = database.LogLevelWarn
+				output = overflowedErrorMsg
+			}
+
+			newLogSize += len(output)
+
+			insertParams.CreatedAt = append(insertParams.CreatedAt, createdAt)
+			insertParams.Level = append(insertParams.Level, level)
+			insertParams.Stage = append(insertParams.Stage, stage)
+			insertParams.Source = append(insertParams.Source, source)
+			insertParams.Output = append(insertParams.Output, output)
 			s.Logger.Debug(ctx, "job log",
 				slog.F("job_id", parsedID),
-				slog.F("stage", log.Stage),
-				slog.F("output", log.Output))
+				slog.F("stage", stage),
+				slog.F("output", output))
+
+			// Don't write any more logs because there's no room.
+			if willOverflow {
+				break
+			}
+		}
+
+		err = s.Database.UpdateProvisionerJobLogsLength(ctx, database.UpdateProvisionerJobLogsLengthParams{
+			ID:         parsedID,
+			LogsLength: int32(newLogSize), // #nosec G115 - Log output length is limited to 1MB (2^20) which fits in an int32.
+		})
+		if err != nil {
+			// Even though we do the runtime check for the overflow, we still check for the database error
+			// as well.
+			if database.IsProvisionerJobLogsLimitError(err) {
+				err = s.Database.UpdateProvisionerJobLogsOverflowed(ctx, database.UpdateProvisionerJobLogsOverflowedParams{
+					ID:             parsedID,
+					LogsOverflowed: true,
+				})
+				if err != nil {
+					s.Logger.Error(ctx, "failed to set logs overflowed flag", slog.F("job_id", parsedID), slog.Error(err))
+				}
+				return &proto.UpdateJobResponse{
+					Canceled: job.CanceledAt.Valid,
+				}, nil
+			}
+			s.Logger.Error(ctx, "failed to update logs length", slog.F("job_id", parsedID), slog.Error(err))
+			return nil, xerrors.Errorf("update logs length: %w", err)
 		}
 
 		logs, err := s.Database.InsertProvisionerJobLogs(ctx, insertParams)
@@ -932,6 +996,7 @@ func (s *server) UpdateJob(ctx context.Context, request *proto.UpdateJobRequest)
 			s.Logger.Error(ctx, "failed to insert job logs", slog.F("job_id", parsedID), slog.Error(err))
 			return nil, xerrors.Errorf("insert job logs: %w", err)
 		}
+
 		// Publish by the lowest log ID inserted so the log stream will fetch
 		// everything from that point.
 		lowestID := logs[0].ID

coderd/provisionerdserver/provisionerdserver_test.go

@@ -928,6 +928,141 @@ func TestUpdateJob(t *testing.T) {
 		require.Equal(t, workspaceTags[1].Key, "cat")
 		require.Equal(t, workspaceTags[1].Value, "jinx")
 	})
+
+	t.Run("LogSizeLimit", func(t *testing.T) {
+		t.Parallel()
+		srv, db, _, pd := setup(t, false, &overrides{})
+		job := setupJob(t, db, pd.ID, pd.Tags)
+
+		// Create a log message that exceeds the 1MB limit
+		largeOutput := strings.Repeat("a", 1048577) // 1MB + 1 byte
+
+		_, err := srv.UpdateJob(ctx, &proto.UpdateJobRequest{
+			JobId: job.String(),
+			Logs: []*proto.Log{{
+				Source: proto.LogSource_PROVISIONER,
+				Level:  sdkproto.LogLevel_INFO,
+				Output: largeOutput,
+			}},
+		})
+		require.NoError(t, err) // Should succeed but trigger overflow
+
+		// Verify the overflow flag is set
+		jobResult, err := db.GetProvisionerJobByID(ctx, job)
+		require.NoError(t, err)
+		require.True(t, jobResult.LogsOverflowed)
+	})
+
+	t.Run("IncrementalLogSizeOverflow", func(t *testing.T) {
+		t.Parallel()
+		srv, db, _, pd := setup(t, false, &overrides{})
+		job := setupJob(t, db, pd.ID, pd.Tags)
+
+		// Send logs that together exceed the limit
+		mediumOutput := strings.Repeat("b", 524289) // Half a MB + 1 byte
+
+		// First log - should succeed
+		_, err := srv.UpdateJob(ctx, &proto.UpdateJobRequest{
+			JobId: job.String(),
+			Logs: []*proto.Log{{
+				Source: proto.LogSource_PROVISIONER,
+				Level:  sdkproto.LogLevel_INFO,
+				Output: mediumOutput,
+			}},
+		})
+		require.NoError(t, err)
+
+		// Verify overflow flag not yet set
+		jobResult, err := db.GetProvisionerJobByID(ctx, job)
+		require.NoError(t, err)
+		require.False(t, jobResult.LogsOverflowed)
+
+		// Second log - should trigger overflow
+		_, err = srv.UpdateJob(ctx, &proto.UpdateJobRequest{
+			JobId: job.String(),
+			Logs: []*proto.Log{{
+				Source: proto.LogSource_PROVISIONER,
+				Level:  sdkproto.LogLevel_INFO,
+				Output: mediumOutput,
+			}},
+		})
+		require.NoError(t, err)
+
+		// Verify overflow flag is set
+		jobResult, err = db.GetProvisionerJobByID(ctx, job)
+		require.NoError(t, err)
+		require.True(t, jobResult.LogsOverflowed)
+	})
+
+	t.Run("LogSizeTracking", func(t *testing.T) {
+		t.Parallel()
+		srv, db, _, pd := setup(t, false, &overrides{})
+		job := setupJob(t, db, pd.ID, pd.Tags)
+
+		logOutput := "test log message"
+		expectedSize := int32(len(logOutput)) // #nosec G115 - Log length is 16.
+
+		_, err := srv.UpdateJob(ctx, &proto.UpdateJobRequest{
+			JobId: job.String(),
+			Logs: []*proto.Log{{
+				Source: proto.LogSource_PROVISIONER,
+				Level:  sdkproto.LogLevel_INFO,
+				Output: logOutput,
+			}},
+		})
+		require.NoError(t, err)
+
+		// Verify the logs_length is correctly tracked
+		jobResult, err := db.GetProvisionerJobByID(ctx, job)
+		require.NoError(t, err)
+		require.Equal(t, expectedSize, jobResult.LogsLength)
+		require.False(t, jobResult.LogsOverflowed)
+	})
+
+	t.Run("LogOverflowStopsProcessing", func(t *testing.T) {
+		t.Parallel()
+		srv, db, _, pd := setup(t, false, &overrides{})
+		job := setupJob(t, db, pd.ID, pd.Tags)
+
+		// First: trigger overflow
+		largeOutput := strings.Repeat("a", 1048577) // 1MB + 1 byte
+		_, err := srv.UpdateJob(ctx, &proto.UpdateJobRequest{
+			JobId: job.String(),
+			Logs: []*proto.Log{{
+				Source: proto.LogSource_PROVISIONER,
+				Level:  sdkproto.LogLevel_INFO,
+				Output: largeOutput,
+			}},
+		})
+		require.NoError(t, err)
+
+		// Get the initial log count
+		initialLogs, err := db.GetProvisionerLogsAfterID(ctx, database.GetProvisionerLogsAfterIDParams{
+			JobID:        job,
+			CreatedAfter: -1,
+		})
+		require.NoError(t, err)
+		initialCount := len(initialLogs)
+
+		// Second: try to send more logs - should be ignored
+		_, err = srv.UpdateJob(ctx, &proto.UpdateJobRequest{
+			JobId: job.String(),
+			Logs: []*proto.Log{{
+				Source: proto.LogSource_PROVISIONER,
+				Level:  sdkproto.LogLevel_INFO,
+				Output: "this should be ignored",
+			}},
+		})
+		require.NoError(t, err)
+
+		// Verify no new logs were added
+		finalLogs, err := db.GetProvisionerLogsAfterID(ctx, database.GetProvisionerLogsAfterIDParams{
+			JobID:        job,
+			CreatedAfter: -1,
+		})
+		require.NoError(t, err)
+		require.Equal(t, initialCount, len(finalLogs))
+	})
 }
 
 func TestFailJob(t *testing.T) {

coderd/provisionerjobs.go

@@ -363,6 +363,7 @@ func convertProvisionerJob(pj database.GetProvisionerJobsByIDsWithQueuePositionR
 		Tags:           provisionerJob.Tags,
 		QueuePosition:  int(pj.QueuePosition),
 		QueueSize:      int(pj.QueueSize),
+		LogsOverflowed: provisionerJob.LogsOverflowed,
 	}
 	// Applying values optional to the struct.
 	if provisionerJob.StartedAt.Valid {

coderd/templateversions.go

@@ -552,6 +552,7 @@ func (api *API) postTemplateVersionDryRun(rw http.ResponseWriter, r *http.Reques
 			Valid:      true,
 			RawMessage: metadataRaw,
 		},
+		LogsOverflowed: false,
 	})
 	if err != nil {
 		httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{
@@ -1646,6 +1647,7 @@ func (api *API) postTemplateVersionsByOrganization(rw http.ResponseWriter, r *ht
 				Valid:      true,
 				RawMessage: traceMetadataRaw,
 			},
+			LogsOverflowed: false,
 		})
 		if err != nil {
 			httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{

coderd/wsbuilder/wsbuilder.go

@@ -409,6 +409,7 @@ func (b *Builder) buildTx(authFunc func(action policy.Action, object rbac.Object
 			Valid:      true,
 			RawMessage: traceMetadataRaw,
 		},
+		LogsOverflowed: false,
 	})
 	if err != nil {
 		return nil, nil, nil, BuildError{http.StatusInternalServerError, "insert provisioner job", err}

codersdk/provisionerdaemons.go

@@ -188,6 +188,7 @@ type ProvisionerJob struct {
 	Type             ProvisionerJobType     `json:"type" table:"type"`
 	AvailableWorkers []uuid.UUID            `json:"available_workers,omitempty" format:"uuid" table:"available workers"`
 	Metadata         ProvisionerJobMetadata `json:"metadata" table:"metadata,recursive_inline"`
+	LogsOverflowed   bool                   `json:"logs_overflowed" table:"logs overflowed"`
 }
 
 // ProvisionerJobLog represents the provisioner log entry annotated with source and level.

docs/reference/api/builds.md

@@ -52,6 +52,7 @@ curl -X GET http://coder-server:8080/api/v2/users/{user}/workspace/{workspacenam
       "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1",
       "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478"
     },
+    "logs_overflowed": true,
     "metadata": {
       "template_display_name": "string",
       "template_icon": "string",
@@ -289,6 +290,7 @@ curl -X GET http://coder-server:8080/api/v2/workspacebuilds/{workspacebuild} \
       "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1",
       "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478"
     },
+    "logs_overflowed": true,
     "metadata": {
       "template_display_name": "string",
       "template_icon": "string",
@@ -1015,6 +1017,7 @@ curl -X GET http://coder-server:8080/api/v2/workspacebuilds/{workspacebuild}/sta
       "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1",
       "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478"
     },
+    "logs_overflowed": true,
     "metadata": {
       "template_display_name": "string",
       "template_icon": "string",
@@ -1325,6 +1328,7 @@ curl -X GET http://coder-server:8080/api/v2/workspaces/{workspace}/builds \
         "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1",
         "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478"
       },
+      "logs_overflowed": true,
       "metadata": {
         "template_display_name": "string",
         "template_icon": "string",
@@ -1540,6 +1544,7 @@ Status Code **200**
 | `»»» error`                      | string                                                                                                 | false    |              |                                                                                                                                                                                                                                                |
 | `»»» template_version_id`        | string(uuid)                                                                                           | false    |              |                                                                                                                                                                                                                                                |
 | `»»» workspace_build_id`         | string(uuid)                                                                                           | false    |              |                                                                                                                                                                                                                                                |
+| `»» logs_overflowed`             | boolean                                                                                                | false    |              |                                                                                                                                                                                                                                                |
 | `»» metadata`                    | [codersdk.ProvisionerJobMetadata](schemas.md#codersdkprovisionerjobmetadata)                           | false    |              |                                                                                                                                                                                                                                                |
 | `»»» template_display_name`      | string                                                                                                 | false    |              |                                                                                                                                                                                                                                                |
 | `»»» template_icon`              | string                                                                                                 | false    |              |                                                                                                                                                                                                                                                |
@@ -1816,6 +1821,7 @@ curl -X POST http://coder-server:8080/api/v2/workspaces/{workspace}/builds \
       "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1",
       "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478"
     },
+    "logs_overflowed": true,
     "metadata": {
       "template_display_name": "string",
       "template_icon": "string",

docs/reference/api/organizations.md

@@ -407,6 +407,7 @@ curl -X GET http://coder-server:8080/api/v2/organizations/{organization}/provisi
       "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1",
       "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478"
     },
+    "logs_overflowed": true,
     "metadata": {
       "template_display_name": "string",
       "template_icon": "string",
@@ -457,6 +458,7 @@ Status Code **200**
 | `»» error`                 | string                                                                       | false    |              |             |
 | `»» template_version_id`   | string(uuid)                                                                 | false    |              |             |
 | `»» workspace_build_id`    | string(uuid)                                                                 | false    |              |             |
+| `» logs_overflowed`        | boolean                                                                      | false    |              |             |
 | `» metadata`               | [codersdk.ProvisionerJobMetadata](schemas.md#codersdkprovisionerjobmetadata) | false    |              |             |
 | `»» template_display_name` | string                                                                       | false    |              |             |
 | `»» template_icon`         | string                                                                       | false    |              |             |
@@ -534,6 +536,7 @@ curl -X GET http://coder-server:8080/api/v2/organizations/{organization}/provisi
     "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1",
     "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478"
   },
+  "logs_overflowed": true,
   "metadata": {
     "template_display_name": "string",
     "template_icon": "string",

docs/reference/api/schemas.md

@@ -5906,6 +5906,7 @@ Only certain features set these fields: - FeatureManagedAgentLimit|
     "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1",
     "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478"
   },
+  "logs_overflowed": true,
   "metadata": {
     "template_display_name": "string",
     "template_icon": "string",
@@ -5943,6 +5944,7 @@ Only certain features set these fields: - FeatureManagedAgentLimit|
 | `file_id`           | string                                                             | false    |              |             |
 | `id`                | string                                                             | false    |              |             |
 | `input`             | [codersdk.ProvisionerJobInput](#codersdkprovisionerjobinput)       | false    |              |             |
+| `logs_overflowed`   | boolean                                                            | false    |              |             |
 | `metadata`          | [codersdk.ProvisionerJobMetadata](#codersdkprovisionerjobmetadata) | false    |              |             |
 | `organization_id`   | string                                                             | false    |              |             |
 | `queue_position`    | integer                                                            | false    |              |             |
@@ -7626,6 +7628,7 @@ Restarts will only happen on weekdays in this list on weeks which line up with W
       "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1",
       "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478"
     },
+    "logs_overflowed": true,
     "metadata": {
       "template_display_name": "string",
       "template_icon": "string",
@@ -8802,6 +8805,7 @@ If the schedule is empty, the user will be updated to use the default schedule.|
         "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1",
         "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478"
       },
+      "logs_overflowed": true,
       "metadata": {
         "template_display_name": "string",
         "template_icon": "string",
@@ -9911,6 +9915,7 @@ If the schedule is empty, the user will be updated to use the default schedule.|
       "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1",
       "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478"
     },
+    "logs_overflowed": true,
     "metadata": {
       "template_display_name": "string",
       "template_icon": "string",
@@ -10642,6 +10647,7 @@ If the schedule is empty, the user will be updated to use the default schedule.|
             "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1",
             "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478"
           },
+          "logs_overflowed": true,
           "metadata": {
             "template_display_name": "string",
             "template_icon": "string",

docs/reference/api/templates.md

@@ -479,6 +479,7 @@ curl -X GET http://coder-server:8080/api/v2/organizations/{organization}/templat
       "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1",
       "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478"
     },
+    "logs_overflowed": true,
     "metadata": {
       "template_display_name": "string",
       "template_icon": "string",
@@ -577,6 +578,7 @@ curl -X GET http://coder-server:8080/api/v2/organizations/{organization}/templat
       "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1",
       "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478"
     },
+    "logs_overflowed": true,
     "metadata": {
       "template_display_name": "string",
       "template_icon": "string",
@@ -699,6 +701,7 @@ curl -X POST http://coder-server:8080/api/v2/organizations/{organization}/templa
       "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1",
       "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478"
     },
+    "logs_overflowed": true,
     "metadata": {
       "template_display_name": "string",
       "template_icon": "string",
@@ -1264,6 +1267,7 @@ curl -X GET http://coder-server:8080/api/v2/templates/{template}/versions \
         "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1",
         "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478"
       },
+      "logs_overflowed": true,
       "metadata": {
         "template_display_name": "string",
         "template_icon": "string",
@@ -1337,6 +1341,7 @@ Status Code **200**
 | `»»» error`                 | string                                                                       | false    |              |                                                                                                                                                                     |
 | `»»» template_version_id`   | string(uuid)                                                                 | false    |              |                                                                                                                                                                     |
 | `»»» workspace_build_id`    | string(uuid)                                                                 | false    |              |                                                                                                                                                                     |
+| `»» logs_overflowed`        | boolean                                                                      | false    |              |                                                                                                                                                                     |
 | `»» metadata`               | [codersdk.ProvisionerJobMetadata](schemas.md#codersdkprovisionerjobmetadata) | false    |              |                                                                                                                                                                     |
 | `»»» template_display_name` | string                                                                       | false    |              |                                                                                                                                                                     |
 | `»»» template_icon`         | string                                                                       | false    |              |                                                                                                                                                                     |
@@ -1543,6 +1548,7 @@ curl -X GET http://coder-server:8080/api/v2/templates/{template}/versions/{templ
         "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1",
         "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478"
       },
+      "logs_overflowed": true,
       "metadata": {
         "template_display_name": "string",
         "template_icon": "string",
@@ -1616,6 +1622,7 @@ Status Code **200**
 | `»»» error`                 | string                                                                       | false    |              |                                                                                                                                                                     |
 | `»»» template_version_id`   | string(uuid)                                                                 | false    |              |                                                                                                                                                                     |
 | `»»» workspace_build_id`    | string(uuid)                                                                 | false    |              |                                                                                                                                                                     |
+| `»» logs_overflowed`        | boolean                                                                      | false    |              |                                                                                                                                                                     |
 | `»» metadata`               | [codersdk.ProvisionerJobMetadata](schemas.md#codersdkprovisionerjobmetadata) | false    |              |                                                                                                                                                                     |
 | `»»» template_display_name` | string                                                                       | false    |              |                                                                                                                                                                     |
 | `»»» template_icon`         | string                                                                       | false    |              |                                                                                                                                                                     |
@@ -1712,6 +1719,7 @@ curl -X GET http://coder-server:8080/api/v2/templateversions/{templateversion} \
       "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1",
       "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478"
     },
+    "logs_overflowed": true,
     "metadata": {
       "template_display_name": "string",
       "template_icon": "string",
@@ -1819,6 +1827,7 @@ curl -X PATCH http://coder-server:8080/api/v2/templateversions/{templateversion}
       "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1",
       "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478"
     },
+    "logs_overflowed": true,
     "metadata": {
       "template_display_name": "string",
       "template_icon": "string",
@@ -2016,6 +2025,7 @@ curl -X POST http://coder-server:8080/api/v2/templateversions/{templateversion}/
     "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1",
     "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478"
   },
+  "logs_overflowed": true,
   "metadata": {
     "template_display_name": "string",
     "template_icon": "string",
@@ -2089,6 +2099,7 @@ curl -X GET http://coder-server:8080/api/v2/templateversions/{templateversion}/d
     "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1",
     "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478"
   },
+  "logs_overflowed": true,
   "metadata": {
     "template_display_name": "string",
     "template_icon": "string",

docs/reference/api/workspaces.md

@@ -107,6 +107,7 @@ of the template will be used.
         "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1",
         "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478"
       },
+      "logs_overflowed": true,
       "metadata": {
         "template_display_name": "string",
         "template_icon": "string",
@@ -394,6 +395,7 @@ curl -X GET http://coder-server:8080/api/v2/users/{user}/workspace/{workspacenam
         "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1",
         "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478"
       },
+      "logs_overflowed": true,
       "metadata": {
         "template_display_name": "string",
         "template_icon": "string",
@@ -706,6 +708,7 @@ of the template will be used.
         "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1",
         "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478"
       },
+      "logs_overflowed": true,
       "metadata": {
         "template_display_name": "string",
         "template_icon": "string",
@@ -996,6 +999,7 @@ curl -X GET http://coder-server:8080/api/v2/workspaces \
             "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1",
             "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478"
           },
+          "logs_overflowed": true,
           "metadata": {
             "template_display_name": "string",
             "template_icon": "string",
@@ -1267,6 +1271,7 @@ curl -X GET http://coder-server:8080/api/v2/workspaces/{workspace} \
         "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1",
         "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478"
       },
+      "logs_overflowed": true,
       "metadata": {
         "template_display_name": "string",
         "template_icon": "string",
@@ -1670,6 +1675,7 @@ curl -X PUT http://coder-server:8080/api/v2/workspaces/{workspace}/dormant \
         "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1",
         "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478"
       },
+      "logs_overflowed": true,
       "metadata": {
         "template_display_name": "string",
         "template_icon": "string",

docs/reference/cli/provisioner_jobs_list.md

@@ -45,10 +45,10 @@ Select which organization (uuid or name) to use.
 
 ### -c, --column
 
-|         |                                                                                                                                                                                                                                                                                                                                                                                                   |
-|---------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| Type    | <code>[id\|created at\|started at\|completed at\|canceled at\|error\|error code\|status\|worker id\|worker name\|file id\|tags\|queue position\|queue size\|organization id\|template version id\|workspace build id\|type\|available workers\|template version name\|template id\|template name\|template display name\|template icon\|workspace id\|workspace name\|organization\|queue]</code> |
-| Default | <code>created at,id,type,template display name,status,queue,tags</code>                                                                                                                                                                                                                                                                                                                           |
+|         |                                                                                                                                                                                                                                                                                                                                                                                                                    |
+|---------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| Type    | <code>[id\|created at\|started at\|completed at\|canceled at\|error\|error code\|status\|worker id\|worker name\|file id\|tags\|queue position\|queue size\|organization id\|template version id\|workspace build id\|type\|available workers\|template version name\|template id\|template name\|template display name\|template icon\|workspace id\|workspace name\|logs overflowed\|organization\|queue]</code> |
+| Default | <code>created at,id,type,template display name,status,queue,tags</code>                                                                                                                                                                                                                                                                                                                                            |
 
 Columns to display in table output.
 

enterprise/cli/testdata/coder_provisioner_jobs_list_--help.golden

@@ -11,7 +11,7 @@ OPTIONS:
   -O, --org string, $CODER_ORGANIZATION
           Select which organization (uuid or name) to use.
 
-  -c, --column [id|created at|started at|completed at|canceled at|error|error code|status|worker id|worker name|file id|tags|queue position|queue size|organization id|template version id|workspace build id|type|available workers|template version name|template id|template name|template display name|template icon|workspace id|workspace name|organization|queue] (default: created at,id,type,template display name,status,queue,tags)
+  -c, --column [id|created at|started at|completed at|canceled at|error|error code|status|worker id|worker name|file id|tags|queue position|queue size|organization id|template version id|workspace build id|type|available workers|template version name|template id|template name|template display name|template icon|workspace id|workspace name|logs overflowed|organization|queue] (default: created at,id,type,template display name,status,queue,tags)
           Columns to display in table output.
 
   -l, --limit int, $CODER_PROVISIONER_JOB_LIST_LIMIT (default: 50)

site/src/api/typesGenerated.ts

@@ -2156,6 +2156,7 @@ export interface ProvisionerJob {
 	readonly type: ProvisionerJobType;
 	readonly available_workers?: readonly string[];
 	readonly metadata: ProvisionerJobMetadata;
+	readonly logs_overflowed: boolean;
 }
 
 // From codersdk/provisionerdaemons.go

site/src/modules/resources/AgentRow.tsx

@@ -87,7 +87,8 @@ export const AgentRow: FC<AgentRowProps> = ({
 			logs.push({
 				id: -1,
 				level: "error",
-				output: "Startup logs exceeded the max size of 1MB!",
+				output:
+					"Startup logs exceeded the max size of 1MB, and will not continue to be written to the database! Logs will continue to be written to the /tmp/coder-startup-script.log file in the workspace.",
 				created_at: new Date().toISOString(),
 				source_id: "",
 			});

site/src/modules/workspaces/WorkspaceBuildLogs/WorkspaceBuildLogs.tsx

@@ -1,9 +1,9 @@
 import { type Interpolation, type Theme, useTheme } from "@emotion/react";
-import type { ProvisionerJobLog } from "api/typesGenerated";
+import type { ProvisionerJobLog, WorkspaceBuild } from "api/typesGenerated";
 import type { Line } from "components/Logs/LogLine";
 import { DEFAULT_LOG_LINE_SIDE_PADDING, Logs } from "components/Logs/Logs";
 import dayjs from "dayjs";
-import { type FC, Fragment, type HTMLAttributes } from "react";
+import { type FC, Fragment, type HTMLAttributes, useMemo } from "react";
 import { BODY_FONT_FAMILY, MONOSPACE_FONT_FAMILY } from "theme/constants";
 
 const Language = {
@@ -42,15 +42,37 @@ interface WorkspaceBuildLogsProps extends HTMLAttributes<HTMLDivElement> {
 	hideTimestamps?: boolean;
 	sticky?: boolean;
 	logs: ProvisionerJobLog[];
+	build?: WorkspaceBuild;
 }
 
 export const WorkspaceBuildLogs: FC<WorkspaceBuildLogsProps> = ({
 	hideTimestamps,
 	sticky,
 	logs,
+	build,
 	...attrs
 }) => {
 	const theme = useTheme();
+
+	const processedLogs = useMemo(() => {
+		const allLogs = logs || [];
+
+		// Add synthetic overflow message if needed
+		if (build?.job?.logs_overflowed) {
+			allLogs.push({
+				id: -1,
+				created_at: new Date().toISOString(),
+				log_level: "error",
+				log_source: "provisioner",
+				output:
+					"Provisioner logs exceeded the max size of 1MB. Will not continue to write provisioner logs for workspace build.",
+				stage: "overflow",
+			});
+		}
+
+		return allLogs;
+	}, [logs, build?.job?.logs_overflowed]);
+
 	const groupedLogsByStage = groupLogsByStage(logs);
 
 	return (

site/src/pages/WorkspaceBuildPage/WorkspaceBuildPageView.tsx

@@ -212,7 +212,24 @@ export const WorkspaceBuildPageView: FC<WorkspaceBuildPageViewProps> = ({
 						</Alert>
 					)}
 
-					{tabState.value === "build" && <BuildLogsContent logs={logs} />}
+					{build?.job?.logs_overflowed && (
+						<Alert
+							severity="warning"
+							css={{
+								borderRadius: 0,
+								border: 0,
+								background: theme.roles.warning.background,
+								borderBottom: `1px solid ${theme.palette.divider}`,
+							}}
+						>
+							Provisioner logs exceeded the max size of 1MB. Will not continue
+							to write provisioner logs for workspace build.
+						</Alert>
+					)}
+
+					{tabState.value === "build" && (
+						<BuildLogsContent logs={logs} build={build} />
+					)}
 					{tabState.value !== "build" && selectedAgent && (
 						<AgentLogsContent agent={selectedAgent} />
 					)}
@@ -261,7 +278,10 @@ const ScrollArea: FC<HTMLProps<HTMLDivElement>> = (props) => {
 	);
 };
 
-const BuildLogsContent: FC<{ logs?: ProvisionerJobLog[] }> = ({ logs }) => {
+const BuildLogsContent: FC<{
+	logs?: ProvisionerJobLog[];
+	build?: WorkspaceBuild;
+}> = ({ logs, build }) => {
 	if (!logs) {
 		return <Loader />;
 	}
@@ -278,6 +298,7 @@ const BuildLogsContent: FC<{ logs?: ProvisionerJobLog[] }> = ({ logs }) => {
 				},
 			}}
 			logs={sortLogsByCreatedAt(logs)}
+			build={build}
 		/>
 	);
 };

site/src/testHelpers/entities.ts

@@ -689,6 +689,7 @@ export const MockProvisionerJob: TypesGen.ProvisionerJob = {
 		template_version_name: "test-version",
 		workspace_name: "test-workspace",
 	},
+	logs_overflowed: false,
 };
 
 export const MockFailedProvisionerJob: TypesGen.ProvisionerJob = {