From 233343c010cabfa8d353945667c8a4db4b420216 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Wed, 8 Apr 2026 11:08:09 +0100 Subject: [PATCH] feat: add chat and chat_files cleanup to dbpurge (#23833) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes https://github.com/coder/coder/issues/23910 Adds periodic cleanup of chats and chat files to the dbpurge background goroutine, with a configurable retention period exposed in the Agent settings UI. > 🤖 Written by a Coder Agent. Reviewed by a human. --- coderd/apidoc/docs.go | 78 +++ coderd/apidoc/swagger.json | 70 +++ coderd/coderd.go | 2 + coderd/database/dbauthz/dbauthz.go | 30 ++ coderd/database/dbauthz/dbauthz_test.go | 16 + coderd/database/dbmetrics/querymetrics.go | 32 ++ coderd/database/dbmock/dbmock.go | 59 +++ coderd/database/dbpurge/dbpurge.go | 49 ++ coderd/database/dbpurge/dbpurge_test.go | 498 ++++++++++++++++++ coderd/database/querier.go | 26 + coderd/database/queries.sql.go | 121 ++++- coderd/database/queries/chatfiles.sql | 34 ++ coderd/database/queries/chats.sql | 28 +- coderd/database/queries/siteconfig.sql | 17 + coderd/exp_chats.go | 64 +++ coderd/exp_chats_test.go | 56 ++ codersdk/chats.go | 38 ++ docs/ai-coder/agents/chat-retention.md | 44 ++ docs/manifest.json | 6 + docs/reference/api/schemas.md | 28 + site/src/api/api.ts | 14 + site/src/api/queries/chats.ts | 16 + site/src/api/typesGenerated.ts | 17 + .../AgentsPage/AgentSettingsBehaviorPage.tsx | 13 + .../AgentSettingsBehaviorPageView.tsx | 156 ++++++ .../AgentsPage/AgentsPageView.stories.tsx | 6 + 26 files changed, 1514 insertions(+), 4 deletions(-) create mode 100644 docs/ai-coder/agents/chat-retention.md diff --git a/coderd/apidoc/docs.go b/coderd/apidoc/docs.go index fec76b6a2d..662ce7464b 100644 --- a/coderd/apidoc/docs.go +++ b/coderd/apidoc/docs.go @@ -1266,6 +1266,68 @@ const docTemplate = `{ ] } }, + "/experimental/chats/config/retention-days": { + "get": { + "produces": [ + "application/json" + ], + "tags": [ + "Chats" + ], + "summary": "Get chat retention days", + "operationId": "get-chat-retention-days", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/codersdk.ChatRetentionDaysResponse" + } + } + }, + "security": [ + { + "CoderSessionToken": [] + } + ], + "x-apidocgen": { + "skip": true + } + }, + "put": { + "consumes": [ + "application/json" + ], + "tags": [ + "Chats" + ], + "summary": "Update chat retention days", + "operationId": "update-chat-retention-days", + "parameters": [ + { + "description": "Request body", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/codersdk.UpdateChatRetentionDaysRequest" + } + } + ], + "responses": { + "204": { + "description": "No Content" + } + }, + "security": [ + { + "CoderSessionToken": [] + } + ], + "x-apidocgen": { + "skip": true + } + } + }, "/experimental/watch-all-workspacebuilds": { "get": { "produces": [ @@ -14420,6 +14482,14 @@ const docTemplate = `{ } } }, + "codersdk.ChatRetentionDaysResponse": { + "type": "object", + "properties": { + "retention_days": { + "type": "integer" + } + } + }, "codersdk.ConnectionLatency": { "type": "object", "properties": { @@ -20952,6 +21022,14 @@ const docTemplate = `{ } } }, + "codersdk.UpdateChatRetentionDaysRequest": { + "type": "object", + "properties": { + "retention_days": { + "type": "integer" + } + } + }, "codersdk.UpdateCheckResponse": { "type": "object", "properties": { diff --git a/coderd/apidoc/swagger.json b/coderd/apidoc/swagger.json index fd22aa91b9..91fe3e819b 100644 --- a/coderd/apidoc/swagger.json +++ b/coderd/apidoc/swagger.json @@ -1103,6 +1103,60 @@ ] } }, + "/experimental/chats/config/retention-days": { + "get": { + "produces": ["application/json"], + "tags": ["Chats"], + "summary": "Get chat retention days", + "operationId": "get-chat-retention-days", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/codersdk.ChatRetentionDaysResponse" + } + } + }, + "security": [ + { + "CoderSessionToken": [] + } + ], + "x-apidocgen": { + "skip": true + } + }, + "put": { + "consumes": ["application/json"], + "tags": ["Chats"], + "summary": "Update chat retention days", + "operationId": "update-chat-retention-days", + "parameters": [ + { + "description": "Request body", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/codersdk.UpdateChatRetentionDaysRequest" + } + } + ], + "responses": { + "204": { + "description": "No Content" + } + }, + "security": [ + { + "CoderSessionToken": [] + } + ], + "x-apidocgen": { + "skip": true + } + } + }, "/experimental/watch-all-workspacebuilds": { "get": { "produces": ["application/json"], @@ -12963,6 +13017,14 @@ } } }, + "codersdk.ChatRetentionDaysResponse": { + "type": "object", + "properties": { + "retention_days": { + "type": "integer" + } + } + }, "codersdk.ConnectionLatency": { "type": "object", "properties": { @@ -19243,6 +19305,14 @@ } } }, + "codersdk.UpdateChatRetentionDaysRequest": { + "type": "object", + "properties": { + "retention_days": { + "type": "integer" + } + } + }, "codersdk.UpdateCheckResponse": { "type": "object", "properties": { diff --git a/coderd/coderd.go b/coderd/coderd.go index aaabf5873b..746959cb85 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -1189,6 +1189,8 @@ func New(options *Options) *API { r.Delete("/user-compaction-thresholds/{modelConfig}", api.deleteUserChatCompactionThreshold) r.Get("/workspace-ttl", api.getChatWorkspaceTTL) r.Put("/workspace-ttl", api.putChatWorkspaceTTL) + r.Get("/retention-days", api.getChatRetentionDays) + r.Put("/retention-days", api.putChatRetentionDays) r.Get("/template-allowlist", api.getChatTemplateAllowlist) r.Put("/template-allowlist", api.putChatTemplateAllowlist) }) diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go index f2020172f2..e03a99dead 100644 --- a/coderd/database/dbauthz/dbauthz.go +++ b/coderd/database/dbauthz/dbauthz.go @@ -2031,6 +2031,20 @@ func (q *querier) DeleteOldAuditLogs(ctx context.Context, arg database.DeleteOld return q.db.DeleteOldAuditLogs(ctx, arg) } +func (q *querier) DeleteOldChatFiles(ctx context.Context, arg database.DeleteOldChatFilesParams) (int64, error) { + if err := q.authorizeContext(ctx, policy.ActionDelete, rbac.ResourceSystem); err != nil { + return 0, err + } + return q.db.DeleteOldChatFiles(ctx, arg) +} + +func (q *querier) DeleteOldChats(ctx context.Context, arg database.DeleteOldChatsParams) (int64, error) { + if err := q.authorizeContext(ctx, policy.ActionDelete, rbac.ResourceSystem); err != nil { + return 0, err + } + return q.db.DeleteOldChats(ctx, arg) +} + func (q *querier) DeleteOldConnectionLogs(ctx context.Context, arg database.DeleteOldConnectionLogsParams) (int64, error) { if err := q.authorizeContext(ctx, policy.ActionDelete, rbac.ResourceSystem); err != nil { return 0, err @@ -2699,6 +2713,15 @@ func (q *querier) GetChatQueuedMessages(ctx context.Context, chatID uuid.UUID) ( return q.db.GetChatQueuedMessages(ctx, chatID) } +func (q *querier) GetChatRetentionDays(ctx context.Context) (int32, error) { + // Chat retention is a deployment-wide config read by dbpurge. + // Only requires a valid actor in context. + if _, ok := ActorFromContext(ctx); !ok { + return 0, ErrNoActor + } + return q.db.GetChatRetentionDays(ctx) +} + func (q *querier) GetChatSystemPrompt(ctx context.Context) (string, error) { // The system prompt is a deployment-wide setting read during chat // creation by every authenticated user, so no RBAC policy check @@ -7031,6 +7054,13 @@ func (q *querier) UpsertChatIncludeDefaultSystemPrompt(ctx context.Context, incl return q.db.UpsertChatIncludeDefaultSystemPrompt(ctx, includeDefaultSystemPrompt) } +func (q *querier) UpsertChatRetentionDays(ctx context.Context, retentionDays int32) error { + if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceDeploymentConfig); err != nil { + return err + } + return q.db.UpsertChatRetentionDays(ctx, retentionDays) +} + func (q *querier) UpsertChatSystemPrompt(ctx context.Context, value string) error { if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceDeploymentConfig); err != nil { return err diff --git a/coderd/database/dbauthz/dbauthz_test.go b/coderd/database/dbauthz/dbauthz_test.go index 5e0cf7dbf7..6a0a965443 100644 --- a/coderd/database/dbauthz/dbauthz_test.go +++ b/coderd/database/dbauthz/dbauthz_test.go @@ -600,6 +600,22 @@ func (s *MethodTestSuite) TestChats() { dbm.EXPECT().GetChatFileMetadataByChatID(gomock.Any(), file.ID).Return(rows, nil).AnyTimes() check.Args(file.ID).Asserts(rbac.ResourceChat.WithOwner(file.OwnerID.String()).InOrg(file.OrganizationID).WithID(file.ID), policy.ActionRead).Returns(rows) })) + s.Run("DeleteOldChatFiles", s.Mocked(func(dbm *dbmock.MockStore, _ *gofakeit.Faker, check *expects) { + dbm.EXPECT().DeleteOldChatFiles(gomock.Any(), database.DeleteOldChatFilesParams{}).Return(int64(0), nil).AnyTimes() + check.Args(database.DeleteOldChatFilesParams{}).Asserts(rbac.ResourceSystem, policy.ActionDelete) + })) + s.Run("DeleteOldChats", s.Mocked(func(dbm *dbmock.MockStore, _ *gofakeit.Faker, check *expects) { + dbm.EXPECT().DeleteOldChats(gomock.Any(), database.DeleteOldChatsParams{}).Return(int64(0), nil).AnyTimes() + check.Args(database.DeleteOldChatsParams{}).Asserts(rbac.ResourceSystem, policy.ActionDelete) + })) + s.Run("GetChatRetentionDays", s.Mocked(func(dbm *dbmock.MockStore, _ *gofakeit.Faker, check *expects) { + dbm.EXPECT().GetChatRetentionDays(gomock.Any()).Return(int32(30), nil).AnyTimes() + check.Args().Asserts() + })) + s.Run("UpsertChatRetentionDays", s.Mocked(func(dbm *dbmock.MockStore, _ *gofakeit.Faker, check *expects) { + dbm.EXPECT().UpsertChatRetentionDays(gomock.Any(), int32(30)).Return(nil).AnyTimes() + check.Args(int32(30)).Asserts(rbac.ResourceDeploymentConfig, policy.ActionUpdate) + })) s.Run("GetChatMessageByID", s.Mocked(func(dbm *dbmock.MockStore, faker *gofakeit.Faker, check *expects) { chat := testutil.Fake(s.T(), faker, database.Chat{}) msg := testutil.Fake(s.T(), faker, database.ChatMessage{ChatID: chat.ID}) diff --git a/coderd/database/dbmetrics/querymetrics.go b/coderd/database/dbmetrics/querymetrics.go index 1bae104b09..629fcf25d5 100644 --- a/coderd/database/dbmetrics/querymetrics.go +++ b/coderd/database/dbmetrics/querymetrics.go @@ -592,6 +592,22 @@ func (m queryMetricsStore) DeleteOldAuditLogs(ctx context.Context, arg database. return r0, r1 } +func (m queryMetricsStore) DeleteOldChatFiles(ctx context.Context, arg database.DeleteOldChatFilesParams) (int64, error) { + start := time.Now() + r0, r1 := m.s.DeleteOldChatFiles(ctx, arg) + m.queryLatencies.WithLabelValues("DeleteOldChatFiles").Observe(time.Since(start).Seconds()) + m.queryCounts.WithLabelValues(httpmw.ExtractHTTPRoute(ctx), httpmw.ExtractHTTPMethod(ctx), "DeleteOldChatFiles").Inc() + return r0, r1 +} + +func (m queryMetricsStore) DeleteOldChats(ctx context.Context, arg database.DeleteOldChatsParams) (int64, error) { + start := time.Now() + r0, r1 := m.s.DeleteOldChats(ctx, arg) + m.queryLatencies.WithLabelValues("DeleteOldChats").Observe(time.Since(start).Seconds()) + m.queryCounts.WithLabelValues(httpmw.ExtractHTTPRoute(ctx), httpmw.ExtractHTTPMethod(ctx), "DeleteOldChats").Inc() + return r0, r1 +} + func (m queryMetricsStore) DeleteOldConnectionLogs(ctx context.Context, arg database.DeleteOldConnectionLogsParams) (int64, error) { start := time.Now() r0, r1 := m.s.DeleteOldConnectionLogs(ctx, arg) @@ -1240,6 +1256,14 @@ func (m queryMetricsStore) GetChatQueuedMessages(ctx context.Context, chatID uui return r0, r1 } +func (m queryMetricsStore) GetChatRetentionDays(ctx context.Context) (int32, error) { + start := time.Now() + r0, r1 := m.s.GetChatRetentionDays(ctx) + m.queryLatencies.WithLabelValues("GetChatRetentionDays").Observe(time.Since(start).Seconds()) + m.queryCounts.WithLabelValues(httpmw.ExtractHTTPRoute(ctx), httpmw.ExtractHTTPMethod(ctx), "GetChatRetentionDays").Inc() + return r0, r1 +} + func (m queryMetricsStore) GetChatSystemPrompt(ctx context.Context) (string, error) { start := time.Now() r0, r1 := m.s.GetChatSystemPrompt(ctx) @@ -5000,6 +5024,14 @@ func (m queryMetricsStore) UpsertChatIncludeDefaultSystemPrompt(ctx context.Cont return r0 } +func (m queryMetricsStore) UpsertChatRetentionDays(ctx context.Context, retentionDays int32) error { + start := time.Now() + r0 := m.s.UpsertChatRetentionDays(ctx, retentionDays) + m.queryLatencies.WithLabelValues("UpsertChatRetentionDays").Observe(time.Since(start).Seconds()) + m.queryCounts.WithLabelValues(httpmw.ExtractHTTPRoute(ctx), httpmw.ExtractHTTPMethod(ctx), "UpsertChatRetentionDays").Inc() + return r0 +} + func (m queryMetricsStore) UpsertChatSystemPrompt(ctx context.Context, value string) error { start := time.Now() r0 := m.s.UpsertChatSystemPrompt(ctx, value) diff --git a/coderd/database/dbmock/dbmock.go b/coderd/database/dbmock/dbmock.go index 494961d08c..4644124bde 100644 --- a/coderd/database/dbmock/dbmock.go +++ b/coderd/database/dbmock/dbmock.go @@ -984,6 +984,36 @@ func (mr *MockStoreMockRecorder) DeleteOldAuditLogs(ctx, arg any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteOldAuditLogs", reflect.TypeOf((*MockStore)(nil).DeleteOldAuditLogs), ctx, arg) } +// DeleteOldChatFiles mocks base method. +func (m *MockStore) DeleteOldChatFiles(ctx context.Context, arg database.DeleteOldChatFilesParams) (int64, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteOldChatFiles", ctx, arg) + ret0, _ := ret[0].(int64) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteOldChatFiles indicates an expected call of DeleteOldChatFiles. +func (mr *MockStoreMockRecorder) DeleteOldChatFiles(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteOldChatFiles", reflect.TypeOf((*MockStore)(nil).DeleteOldChatFiles), ctx, arg) +} + +// DeleteOldChats mocks base method. +func (m *MockStore) DeleteOldChats(ctx context.Context, arg database.DeleteOldChatsParams) (int64, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteOldChats", ctx, arg) + ret0, _ := ret[0].(int64) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteOldChats indicates an expected call of DeleteOldChats. +func (mr *MockStoreMockRecorder) DeleteOldChats(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteOldChats", reflect.TypeOf((*MockStore)(nil).DeleteOldChats), ctx, arg) +} + // DeleteOldConnectionLogs mocks base method. func (m *MockStore) DeleteOldConnectionLogs(ctx context.Context, arg database.DeleteOldConnectionLogsParams) (int64, error) { m.ctrl.T.Helper() @@ -2282,6 +2312,21 @@ func (mr *MockStoreMockRecorder) GetChatQueuedMessages(ctx, chatID any) *gomock. return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetChatQueuedMessages", reflect.TypeOf((*MockStore)(nil).GetChatQueuedMessages), ctx, chatID) } +// GetChatRetentionDays mocks base method. +func (m *MockStore) GetChatRetentionDays(ctx context.Context) (int32, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetChatRetentionDays", ctx) + ret0, _ := ret[0].(int32) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetChatRetentionDays indicates an expected call of GetChatRetentionDays. +func (mr *MockStoreMockRecorder) GetChatRetentionDays(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetChatRetentionDays", reflect.TypeOf((*MockStore)(nil).GetChatRetentionDays), ctx) +} + // GetChatSystemPrompt mocks base method. func (m *MockStore) GetChatSystemPrompt(ctx context.Context) (string, error) { m.ctrl.T.Helper() @@ -9399,6 +9444,20 @@ func (mr *MockStoreMockRecorder) UpsertChatIncludeDefaultSystemPrompt(ctx, inclu return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertChatIncludeDefaultSystemPrompt", reflect.TypeOf((*MockStore)(nil).UpsertChatIncludeDefaultSystemPrompt), ctx, includeDefaultSystemPrompt) } +// UpsertChatRetentionDays mocks base method. +func (m *MockStore) UpsertChatRetentionDays(ctx context.Context, retentionDays int32) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpsertChatRetentionDays", ctx, retentionDays) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpsertChatRetentionDays indicates an expected call of UpsertChatRetentionDays. +func (mr *MockStoreMockRecorder) UpsertChatRetentionDays(ctx, retentionDays any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertChatRetentionDays", reflect.TypeOf((*MockStore)(nil).UpsertChatRetentionDays), ctx, retentionDays) +} + // UpsertChatSystemPrompt mocks base method. func (m *MockStore) UpsertChatSystemPrompt(ctx context.Context, value string) error { m.ctrl.T.Helper() diff --git a/coderd/database/dbpurge/dbpurge.go b/coderd/database/dbpurge/dbpurge.go index ba3df7236c..6c649b2145 100644 --- a/coderd/database/dbpurge/dbpurge.go +++ b/coderd/database/dbpurge/dbpurge.go @@ -34,6 +34,11 @@ const ( // long enough to cover the maximum interval of a heartbeat event (currently // 1 hour) plus some buffer. maxTelemetryHeartbeatAge = 24 * time.Hour + // Batch sizes for chat purging. Both use 1000, which is smaller + // than audit/connection log batches (10000), because chat_files + // rows contain bytea blob data that make large batches heavier. + chatsBatchSize = 1000 + chatFilesBatchSize = 1000 ) // New creates a new periodically purging database instance. @@ -109,6 +114,17 @@ func New(ctx context.Context, logger slog.Logger, db database.Store, vals *coder // purgeTick performs a single purge iteration. It returns an error if the // purge fails. func (i *instance) purgeTick(ctx context.Context, db database.Store, start time.Time) error { + // Read chat retention config outside the transaction to + // avoid poisoning the tx if the stored value is corrupt. + // A SQL-level cast error (e.g. non-numeric text) puts PG + // into error state, failing all subsequent queries in the + // same transaction. + chatRetentionDays, err := db.GetChatRetentionDays(ctx) + if err != nil { + i.logger.Warn(ctx, "failed to read chat retention config, skipping chat purge", slog.Error(err)) + chatRetentionDays = 0 + } + // Start a transaction to grab advisory lock, we don't want to run // multiple purges at the same time (multiple replicas). return db.InTx(func(tx database.Store) error { @@ -213,12 +229,43 @@ func (i *instance) purgeTick(ctx context.Context, db database.Store, start time. } } + // Chat retention is configured via site_configs. When + // enabled, old archived chats are deleted first, then + // orphaned chat files. Deleting a chat cascades to + // chat_file_links (removing references) but not to + // chat_files directly, so files from deleted chats + // become orphaned and are caught by DeleteOldChatFiles + // in the same tick. + var purgedChats int64 + var purgedChatFiles int64 + if chatRetentionDays > 0 { + chatRetention := time.Duration(chatRetentionDays) * 24 * time.Hour + deleteChatsBefore := start.Add(-chatRetention) + + purgedChats, err = tx.DeleteOldChats(ctx, database.DeleteOldChatsParams{ + BeforeTime: deleteChatsBefore, + LimitCount: chatsBatchSize, + }) + if err != nil { + return xerrors.Errorf("failed to delete old chats: %w", err) + } + + purgedChatFiles, err = tx.DeleteOldChatFiles(ctx, database.DeleteOldChatFilesParams{ + BeforeTime: deleteChatsBefore, + LimitCount: chatFilesBatchSize, + }) + if err != nil { + return xerrors.Errorf("failed to delete old chat files: %w", err) + } + } i.logger.Debug(ctx, "purged old database entries", slog.F("workspace_agent_logs", purgedWorkspaceAgentLogs), slog.F("expired_api_keys", expiredAPIKeys), slog.F("aibridge_records", purgedAIBridgeRecords), slog.F("connection_logs", purgedConnectionLogs), slog.F("audit_logs", purgedAuditLogs), + slog.F("chats", purgedChats), + slog.F("chat_files", purgedChatFiles), slog.F("duration", i.clk.Since(start)), ) @@ -232,6 +279,8 @@ func (i *instance) purgeTick(ctx context.Context, db database.Store, start time. i.recordsPurged.WithLabelValues("aibridge_records").Add(float64(purgedAIBridgeRecords)) i.recordsPurged.WithLabelValues("connection_logs").Add(float64(purgedConnectionLogs)) i.recordsPurged.WithLabelValues("audit_logs").Add(float64(purgedAuditLogs)) + i.recordsPurged.WithLabelValues("chats").Add(float64(purgedChats)) + i.recordsPurged.WithLabelValues("chat_files").Add(float64(purgedChatFiles)) } return nil diff --git a/coderd/database/dbpurge/dbpurge_test.go b/coderd/database/dbpurge/dbpurge_test.go index 5aba49edf7..c63e1daeb2 100644 --- a/coderd/database/dbpurge/dbpurge_test.go +++ b/coderd/database/dbpurge/dbpurge_test.go @@ -12,6 +12,7 @@ import ( "time" "github.com/google/uuid" + "github.com/lib/pq" "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -53,6 +54,7 @@ func TestPurge(t *testing.T) { clk := quartz.NewMock(t) done := awaitDoTick(ctx, t, clk) mDB := dbmock.NewMockStore(gomock.NewController(t)) + mDB.EXPECT().GetChatRetentionDays(gomock.Any()).Return(int32(0), nil).AnyTimes() mDB.EXPECT().InTx(gomock.Any(), database.DefaultTXOptions().WithID("db_purge")).Return(nil).Times(2) purger := dbpurge.New(context.Background(), testutil.Logger(t), mDB, &codersdk.DeploymentValues{}, clk, prometheus.NewRegistry()) <-done // wait for doTick() to run. @@ -125,6 +127,16 @@ func TestMetrics(t *testing.T) { "record_type": "audit_logs", }) require.GreaterOrEqual(t, auditLogs, 0) + + chats := promhelp.CounterValue(t, reg, "coderd_dbpurge_records_purged_total", prometheus.Labels{ + "record_type": "chats", + }) + require.GreaterOrEqual(t, chats, 0) + + chatFiles := promhelp.CounterValue(t, reg, "coderd_dbpurge_records_purged_total", prometheus.Labels{ + "record_type": "chat_files", + }) + require.GreaterOrEqual(t, chatFiles, 0) }) t.Run("FailedIteration", func(t *testing.T) { @@ -138,6 +150,7 @@ func TestMetrics(t *testing.T) { ctrl := gomock.NewController(t) mDB := dbmock.NewMockStore(ctrl) + mDB.EXPECT().GetChatRetentionDays(gomock.Any()).Return(int32(0), nil).AnyTimes() mDB.EXPECT().InTx(gomock.Any(), database.DefaultTXOptions().WithID("db_purge")). Return(xerrors.New("simulated database error")). MinTimes(1) @@ -1634,3 +1647,488 @@ func TestDeleteExpiredAPIKeys(t *testing.T) { func ptr[T any](v T) *T { return &v } + +//nolint:paralleltest // It uses LockIDDBPurge. +func TestDeleteOldChatFiles(t *testing.T) { + now := time.Date(2025, 6, 15, 12, 0, 0, 0, time.UTC) + + // createChatFile inserts a chat file and backdates created_at. + createChatFile := func(ctx context.Context, t *testing.T, db database.Store, rawDB *sql.DB, ownerID, orgID uuid.UUID, createdAt time.Time) uuid.UUID { + t.Helper() + row, err := db.InsertChatFile(ctx, database.InsertChatFileParams{ + OwnerID: ownerID, + OrganizationID: orgID, + Name: "test.png", + Mimetype: "image/png", + Data: []byte("fake-image-data"), + }) + require.NoError(t, err) + _, err = rawDB.ExecContext(ctx, "UPDATE chat_files SET created_at = $1 WHERE id = $2", createdAt, row.ID) + require.NoError(t, err) + return row.ID + } + + // createChat inserts a chat and optionally archives it, then + // backdates updated_at to control the "archived since" window. + createChat := func(ctx context.Context, t *testing.T, db database.Store, rawDB *sql.DB, ownerID, modelConfigID uuid.UUID, archived bool, updatedAt time.Time) database.Chat { + t.Helper() + chat, err := db.InsertChat(ctx, database.InsertChatParams{ + OwnerID: ownerID, + LastModelConfigID: modelConfigID, + Title: "test-chat", + Status: database.ChatStatusWaiting, + }) + require.NoError(t, err) + if archived { + _, err = db.ArchiveChatByID(ctx, chat.ID) + require.NoError(t, err) + } + _, err = rawDB.ExecContext(ctx, "UPDATE chats SET updated_at = $1 WHERE id = $2", updatedAt, chat.ID) + require.NoError(t, err) + return chat + } + // setupChatDeps creates the common dependencies needed for + // chat-related tests: user, org, org member, provider, model config. + type chatDeps struct { + user database.User + org database.Organization + modelConfig database.ChatModelConfig + } + setupChatDeps := func(ctx context.Context, t *testing.T, db database.Store) chatDeps { + t.Helper() + user := dbgen.User(t, db, database.User{}) + org := dbgen.Organization(t, db, database.Organization{}) + _ = dbgen.OrganizationMember(t, db, database.OrganizationMember{UserID: user.ID, OrganizationID: org.ID}) + _, err := db.InsertChatProvider(ctx, database.InsertChatProviderParams{ + Provider: "openai", + DisplayName: "OpenAI", + Enabled: true, + CentralApiKeyEnabled: true, + }) + require.NoError(t, err) + mc, err := db.InsertChatModelConfig(ctx, database.InsertChatModelConfigParams{ + Provider: "openai", + Model: "test-model", + ContextLimit: 8192, + Options: json.RawMessage("{}"), + }) + require.NoError(t, err) + return chatDeps{user: user, org: org, modelConfig: mc} + } + + tests := []struct { + name string + run func(t *testing.T) + }{ + { + name: "ChatRetentionDisabled", + run: func(t *testing.T) { + ctx := testutil.Context(t, testutil.WaitLong) + clk := quartz.NewMock(t) + clk.Set(now).MustWait(ctx) + + db, _, rawDB := dbtestutil.NewDBWithSQLDB(t, dbtestutil.WithDumpOnFailure()) + logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}) + deps := setupChatDeps(ctx, t, db) + + // Disable retention. + err := db.UpsertChatRetentionDays(ctx, int32(0)) + require.NoError(t, err) + + // Create an old archived chat and an orphaned old file. + oldChat := createChat(ctx, t, db, rawDB, deps.user.ID, deps.modelConfig.ID, true, now.Add(-31*24*time.Hour)) + oldFileID := createChatFile(ctx, t, db, rawDB, deps.user.ID, deps.org.ID, now.Add(-31*24*time.Hour)) + + done := awaitDoTick(ctx, t, clk) + closer := dbpurge.New(ctx, logger, db, &codersdk.DeploymentValues{}, clk, prometheus.NewRegistry()) + defer closer.Close() + testutil.TryReceive(ctx, t, done) + + // Both should still exist. + _, err = db.GetChatByID(ctx, oldChat.ID) + require.NoError(t, err, "chat should not be deleted when retention is disabled") + _, err = db.GetChatFileByID(ctx, oldFileID) + require.NoError(t, err, "chat file should not be deleted when retention is disabled") + }, + }, + { + name: "OldArchivedChatsDeleted", + run: func(t *testing.T) { + ctx := testutil.Context(t, testutil.WaitLong) + clk := quartz.NewMock(t) + clk.Set(now).MustWait(ctx) + + db, _, rawDB := dbtestutil.NewDBWithSQLDB(t, dbtestutil.WithDumpOnFailure()) + logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}) + deps := setupChatDeps(ctx, t, db) + + err := db.UpsertChatRetentionDays(ctx, int32(30)) + require.NoError(t, err) + + // Old archived chat (31 days) — should be deleted. + oldChat := createChat(ctx, t, db, rawDB, deps.user.ID, deps.modelConfig.ID, true, now.Add(-31*24*time.Hour)) + // Insert a message so we can verify CASCADE. + _, err = db.InsertChatMessages(ctx, database.InsertChatMessagesParams{ + ChatID: oldChat.ID, + CreatedBy: []uuid.UUID{deps.user.ID}, + ModelConfigID: []uuid.UUID{deps.modelConfig.ID}, + Role: []database.ChatMessageRole{database.ChatMessageRoleUser}, + Content: []string{`[{"type":"text","text":"hello"}]`}, + ContentVersion: []int16{0}, + Visibility: []database.ChatMessageVisibility{database.ChatMessageVisibilityBoth}, + InputTokens: []int64{0}, + OutputTokens: []int64{0}, + TotalTokens: []int64{0}, + ReasoningTokens: []int64{0}, + CacheCreationTokens: []int64{0}, + CacheReadTokens: []int64{0}, + ContextLimit: []int64{0}, + Compressed: []bool{false}, + TotalCostMicros: []int64{0}, + RuntimeMs: []int64{0}, + ProviderResponseID: []string{""}, + }) + require.NoError(t, err) + + // Recently archived chat (10 days) — should be retained. + recentChat := createChat(ctx, t, db, rawDB, deps.user.ID, deps.modelConfig.ID, true, now.Add(-10*24*time.Hour)) + + // Active chat — should be retained. + activeChat := createChat(ctx, t, db, rawDB, deps.user.ID, deps.modelConfig.ID, false, now) + + done := awaitDoTick(ctx, t, clk) + closer := dbpurge.New(ctx, logger, db, &codersdk.DeploymentValues{}, clk, prometheus.NewRegistry()) + defer closer.Close() + testutil.TryReceive(ctx, t, done) + + // Old archived chat should be gone. + _, err = db.GetChatByID(ctx, oldChat.ID) + require.Error(t, err, "old archived chat should be deleted") + + // Its messages should be gone too (CASCADE). + msgs, err := db.GetChatMessagesByChatID(ctx, database.GetChatMessagesByChatIDParams{ + ChatID: oldChat.ID, + AfterID: 0, + }) + require.NoError(t, err) + require.Empty(t, msgs, "messages should be cascade-deleted") + + // Recent archived and active chats should remain. + _, err = db.GetChatByID(ctx, recentChat.ID) + require.NoError(t, err, "recently archived chat should be retained") + _, err = db.GetChatByID(ctx, activeChat.ID) + require.NoError(t, err, "active chat should be retained") + }, + }, + { + name: "OrphanedOldFilesDeleted", + run: func(t *testing.T) { + ctx := testutil.Context(t, testutil.WaitLong) + clk := quartz.NewMock(t) + clk.Set(now).MustWait(ctx) + + db, _, rawDB := dbtestutil.NewDBWithSQLDB(t, dbtestutil.WithDumpOnFailure()) + logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}) + deps := setupChatDeps(ctx, t, db) + + err := db.UpsertChatRetentionDays(ctx, int32(30)) + require.NoError(t, err) + + // File A: 31 days old, NOT in any chat -> should be deleted. + fileA := createChatFile(ctx, t, db, rawDB, deps.user.ID, deps.org.ID, now.Add(-31*24*time.Hour)) + + // File B: 31 days old, in an active chat -> should be retained. + fileB := createChatFile(ctx, t, db, rawDB, deps.user.ID, deps.org.ID, now.Add(-31*24*time.Hour)) + activeChat := createChat(ctx, t, db, rawDB, deps.user.ID, deps.modelConfig.ID, false, now) + _, err = db.LinkChatFiles(ctx, database.LinkChatFilesParams{ + ChatID: activeChat.ID, + MaxFileLinks: 100, + FileIds: []uuid.UUID{fileB}, + }) + require.NoError(t, err) + + // File C: 10 days old, NOT in any chat -> should be retained (too young). + fileC := createChatFile(ctx, t, db, rawDB, deps.user.ID, deps.org.ID, now.Add(-10*24*time.Hour)) + + // File near boundary: 29d23h old — close to threshold. + fileBoundary := createChatFile(ctx, t, db, rawDB, deps.user.ID, deps.org.ID, now.Add(-30*24*time.Hour).Add(time.Hour)) + + done := awaitDoTick(ctx, t, clk) + closer := dbpurge.New(ctx, logger, db, &codersdk.DeploymentValues{}, clk, prometheus.NewRegistry()) + defer closer.Close() + testutil.TryReceive(ctx, t, done) + + _, err = db.GetChatFileByID(ctx, fileA) + require.Error(t, err, "orphaned old file A should be deleted") + + _, err = db.GetChatFileByID(ctx, fileB) + require.NoError(t, err, "file B in active chat should be retained") + + _, err = db.GetChatFileByID(ctx, fileC) + require.NoError(t, err, "young file C should be retained") + + _, err = db.GetChatFileByID(ctx, fileBoundary) + require.NoError(t, err, "file near 30d boundary should be retained") + }, + }, + { + name: "ArchivedChatFilesDeleted", + run: func(t *testing.T) { + ctx := testutil.Context(t, testutil.WaitLong) + clk := quartz.NewMock(t) + clk.Set(now).MustWait(ctx) + + db, _, rawDB := dbtestutil.NewDBWithSQLDB(t, dbtestutil.WithDumpOnFailure()) + logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}) + deps := setupChatDeps(ctx, t, db) + + err := db.UpsertChatRetentionDays(ctx, int32(30)) + require.NoError(t, err) + + // File D: 31 days old, in a chat archived 31 days ago -> should be deleted. + fileD := createChatFile(ctx, t, db, rawDB, deps.user.ID, deps.org.ID, now.Add(-31*24*time.Hour)) + oldArchivedChat := createChat(ctx, t, db, rawDB, deps.user.ID, deps.modelConfig.ID, true, now.Add(-31*24*time.Hour)) + _, err = db.LinkChatFiles(ctx, database.LinkChatFilesParams{ + ChatID: oldArchivedChat.ID, + MaxFileLinks: 100, + FileIds: []uuid.UUID{fileD}, + }) + require.NoError(t, err) + // LinkChatFiles does not update chats.updated_at, so backdate. + _, err = rawDB.ExecContext(ctx, "UPDATE chats SET updated_at = $1 WHERE id = $2", + now.Add(-31*24*time.Hour), oldArchivedChat.ID) + require.NoError(t, err) + + // File E: 31 days old, in a chat archived 10 days ago -> should be retained. + fileE := createChatFile(ctx, t, db, rawDB, deps.user.ID, deps.org.ID, now.Add(-31*24*time.Hour)) + recentArchivedChat := createChat(ctx, t, db, rawDB, deps.user.ID, deps.modelConfig.ID, true, now.Add(-10*24*time.Hour)) + _, err = db.LinkChatFiles(ctx, database.LinkChatFilesParams{ + ChatID: recentArchivedChat.ID, + MaxFileLinks: 100, + FileIds: []uuid.UUID{fileE}, + }) + require.NoError(t, err) + _, err = rawDB.ExecContext(ctx, "UPDATE chats SET updated_at = $1 WHERE id = $2", + now.Add(-10*24*time.Hour), recentArchivedChat.ID) + require.NoError(t, err) + + // File F: 31 days old, in BOTH an active chat AND an old archived chat -> should be retained. + fileF := createChatFile(ctx, t, db, rawDB, deps.user.ID, deps.org.ID, now.Add(-31*24*time.Hour)) + anotherOldArchivedChat := createChat(ctx, t, db, rawDB, deps.user.ID, deps.modelConfig.ID, true, now.Add(-31*24*time.Hour)) + _, err = db.LinkChatFiles(ctx, database.LinkChatFilesParams{ + ChatID: anotherOldArchivedChat.ID, + MaxFileLinks: 100, + FileIds: []uuid.UUID{fileF}, + }) + require.NoError(t, err) + _, err = rawDB.ExecContext(ctx, "UPDATE chats SET updated_at = $1 WHERE id = $2", + now.Add(-31*24*time.Hour), anotherOldArchivedChat.ID) + require.NoError(t, err) + + activeChatForF := createChat(ctx, t, db, rawDB, deps.user.ID, deps.modelConfig.ID, false, now) + _, err = db.LinkChatFiles(ctx, database.LinkChatFilesParams{ + ChatID: activeChatForF.ID, + MaxFileLinks: 100, + FileIds: []uuid.UUID{fileF}, + }) + require.NoError(t, err) + + done := awaitDoTick(ctx, t, clk) + closer := dbpurge.New(ctx, logger, db, &codersdk.DeploymentValues{}, clk, prometheus.NewRegistry()) + defer closer.Close() + testutil.TryReceive(ctx, t, done) + + _, err = db.GetChatFileByID(ctx, fileD) + require.Error(t, err, "file D in old archived chat should be deleted") + + _, err = db.GetChatFileByID(ctx, fileE) + require.NoError(t, err, "file E in recently archived chat should be retained") + + _, err = db.GetChatFileByID(ctx, fileF) + require.NoError(t, err, "file F in active + old archived chat should be retained") + }, + }, + { + name: "UnarchiveAfterFilePurge", + run: func(t *testing.T) { + // Validates that when dbpurge deletes chat_files rows, + // the FK cascade on chat_file_links automatically + // removes the stale links. Unarchiving a chat after + // file purge should show only surviving files. + ctx := testutil.Context(t, testutil.WaitLong) + db, _, rawDB := dbtestutil.NewDBWithSQLDB(t, dbtestutil.WithDumpOnFailure()) + deps := setupChatDeps(ctx, t, db) + + // Create a chat with three attached files. + fileA := createChatFile(ctx, t, db, rawDB, deps.user.ID, deps.org.ID, now) + fileB := createChatFile(ctx, t, db, rawDB, deps.user.ID, deps.org.ID, now) + fileC := createChatFile(ctx, t, db, rawDB, deps.user.ID, deps.org.ID, now) + + chat := createChat(ctx, t, db, rawDB, deps.user.ID, deps.modelConfig.ID, false, now) + _, err := db.LinkChatFiles(ctx, database.LinkChatFilesParams{ + ChatID: chat.ID, + MaxFileLinks: 100, + FileIds: []uuid.UUID{fileA, fileB, fileC}, + }) + require.NoError(t, err) + + // Archive the chat. + _, err = db.ArchiveChatByID(ctx, chat.ID) + require.NoError(t, err) + + // Simulate dbpurge deleting files A and B. The FK + // cascade on chat_file_links_file_id_fkey should + // automatically remove the corresponding link rows. + _, err = rawDB.ExecContext(ctx, "DELETE FROM chat_files WHERE id = ANY($1)", pq.Array([]uuid.UUID{fileA, fileB})) + require.NoError(t, err) + + // Unarchive the chat. + _, err = db.UnarchiveChatByID(ctx, chat.ID) + require.NoError(t, err) + + // Only file C should remain linked (FK cascade + // removed the links for deleted files A and B). + files, err := db.GetChatFileMetadataByChatID(ctx, chat.ID) + require.NoError(t, err) + require.Len(t, files, 1, "only surviving file should be linked") + require.Equal(t, fileC, files[0].ID) + + // Edge case: delete the last file too. The chat + // should have zero linked files, not an error. + _, err = db.ArchiveChatByID(ctx, chat.ID) + require.NoError(t, err) + _, err = rawDB.ExecContext(ctx, "DELETE FROM chat_files WHERE id = $1", fileC) + require.NoError(t, err) + _, err = db.UnarchiveChatByID(ctx, chat.ID) + require.NoError(t, err) + + files, err = db.GetChatFileMetadataByChatID(ctx, chat.ID) + require.NoError(t, err) + require.Empty(t, files, "all-files-deleted should yield empty result") + + // Test parent+child cascade: deleting files should + // clean up links for both parent and child chats + // independently via FK cascade. + parentChat := createChat(ctx, t, db, rawDB, deps.user.ID, deps.modelConfig.ID, false, now) + childChat, err := db.InsertChat(ctx, database.InsertChatParams{ + OwnerID: deps.user.ID, + LastModelConfigID: deps.modelConfig.ID, + Title: "child-chat", + Status: database.ChatStatusWaiting, + }) + require.NoError(t, err) + // Set root_chat_id to link child to parent. + _, err = rawDB.ExecContext(ctx, "UPDATE chats SET root_chat_id = $1 WHERE id = $2", parentChat.ID, childChat.ID) + require.NoError(t, err) + + // Attach different files to parent and child. + parentFileKeep := createChatFile(ctx, t, db, rawDB, deps.user.ID, deps.org.ID, now) + parentFileStale := createChatFile(ctx, t, db, rawDB, deps.user.ID, deps.org.ID, now) + childFileKeep := createChatFile(ctx, t, db, rawDB, deps.user.ID, deps.org.ID, now) + childFileStale := createChatFile(ctx, t, db, rawDB, deps.user.ID, deps.org.ID, now) + + _, err = db.LinkChatFiles(ctx, database.LinkChatFilesParams{ + ChatID: parentChat.ID, + MaxFileLinks: 100, + FileIds: []uuid.UUID{parentFileKeep, parentFileStale}, + }) + require.NoError(t, err) + _, err = db.LinkChatFiles(ctx, database.LinkChatFilesParams{ + ChatID: childChat.ID, + MaxFileLinks: 100, + FileIds: []uuid.UUID{childFileKeep, childFileStale}, + }) + require.NoError(t, err) + + // Archive via parent (cascades to child). + _, err = db.ArchiveChatByID(ctx, parentChat.ID) + require.NoError(t, err) + + // Delete one file from each chat. + _, err = rawDB.ExecContext(ctx, "DELETE FROM chat_files WHERE id = ANY($1)", + pq.Array([]uuid.UUID{parentFileStale, childFileStale})) + require.NoError(t, err) + + // Unarchive via parent. + _, err = db.UnarchiveChatByID(ctx, parentChat.ID) + require.NoError(t, err) + + parentFiles, err := db.GetChatFileMetadataByChatID(ctx, parentChat.ID) + require.NoError(t, err) + require.Len(t, parentFiles, 1) + require.Equal(t, parentFileKeep, parentFiles[0].ID, + "parent should retain only non-stale file") + + childFiles, err := db.GetChatFileMetadataByChatID(ctx, childChat.ID) + require.NoError(t, err) + require.Len(t, childFiles, 1) + require.Equal(t, childFileKeep, childFiles[0].ID, + "child should retain only non-stale file") + }, + }, + { + name: "BatchLimitFiles", + run: func(t *testing.T) { + ctx := testutil.Context(t, testutil.WaitLong) + db, _, rawDB := dbtestutil.NewDBWithSQLDB(t, dbtestutil.WithDumpOnFailure()) + deps := setupChatDeps(ctx, t, db) + + // Create 3 deletable orphaned files (all 31 days old). + for range 3 { + createChatFile(ctx, t, db, rawDB, deps.user.ID, deps.org.ID, now.Add(-31*24*time.Hour)) + } + + // Delete with limit 2 — should delete 2, leave 1. + deleted, err := db.DeleteOldChatFiles(ctx, database.DeleteOldChatFilesParams{ + BeforeTime: now.Add(-30 * 24 * time.Hour), + LimitCount: 2, + }) + require.NoError(t, err) + require.Equal(t, int64(2), deleted, "should delete exactly 2 files") + + // Delete again — should delete the remaining 1. + deleted, err = db.DeleteOldChatFiles(ctx, database.DeleteOldChatFilesParams{ + BeforeTime: now.Add(-30 * 24 * time.Hour), + LimitCount: 2, + }) + require.NoError(t, err) + require.Equal(t, int64(1), deleted, "should delete remaining 1 file") + }, + }, + { + name: "BatchLimitChats", + run: func(t *testing.T) { + ctx := testutil.Context(t, testutil.WaitLong) + db, _, rawDB := dbtestutil.NewDBWithSQLDB(t, dbtestutil.WithDumpOnFailure()) + deps := setupChatDeps(ctx, t, db) + + // Create 3 deletable old archived chats. + for range 3 { + createChat(ctx, t, db, rawDB, deps.user.ID, deps.modelConfig.ID, true, now.Add(-31*24*time.Hour)) + } + + // Delete with limit 2 — should delete 2, leave 1. + deleted, err := db.DeleteOldChats(ctx, database.DeleteOldChatsParams{ + BeforeTime: now.Add(-30 * 24 * time.Hour), + LimitCount: 2, + }) + require.NoError(t, err) + require.Equal(t, int64(2), deleted, "should delete exactly 2 chats") + + // Delete again — should delete the remaining 1. + deleted, err = db.DeleteOldChats(ctx, database.DeleteOldChatsParams{ + BeforeTime: now.Add(-30 * 24 * time.Hour), + LimitCount: 2, + }) + require.NoError(t, err) + require.Equal(t, int64(1), deleted, "should delete remaining 1 chat") + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + tc.run(t) + }) + } +} diff --git a/coderd/database/querier.go b/coderd/database/querier.go index 049cde3c0f..26e2983642 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -128,6 +128,22 @@ type sqlcQuerier interface { // connection events (connect, disconnect, open, close) which are handled // separately by DeleteOldAuditLogConnectionEvents. DeleteOldAuditLogs(ctx context.Context, arg DeleteOldAuditLogsParams) (int64, error) + // TODO(cian): Add indexes on chats(archived, updated_at) and + // chat_files(created_at) for purge query performance. + // See: https://github.com/coder/internal/issues/1438 + // Deletes chat files that are older than the given threshold and are + // not referenced by any chat that is still active or was archived + // within the same threshold window. This covers two cases: + // 1. Orphaned files not linked to any chat. + // 2. Files whose every referencing chat has been archived for longer + // than the retention period. + DeleteOldChatFiles(ctx context.Context, arg DeleteOldChatFilesParams) (int64, error) + // Deletes chats that have been archived for longer than the given + // threshold. Active (non-archived) chats are never deleted. + // Related chat_messages, chat_diff_statuses, and + // chat_queued_messages are removed via ON DELETE CASCADE. + // Parent/root references on child chats are SET NULL. + DeleteOldChats(ctx context.Context, arg DeleteOldChatsParams) (int64, error) DeleteOldConnectionLogs(ctx context.Context, arg DeleteOldConnectionLogsParams) (int64, error) // Delete all notification messages which have not been updated for over a week. DeleteOldNotificationMessages(ctx context.Context) error @@ -265,6 +281,11 @@ type sqlcQuerier interface { GetChatProviderByProvider(ctx context.Context, provider string) (ChatProvider, error) GetChatProviders(ctx context.Context) ([]ChatProvider, error) GetChatQueuedMessages(ctx context.Context, chatID uuid.UUID) ([]ChatQueuedMessage, error) + // Returns the chat retention period in days. Chats archived longer + // than this and orphaned chat files older than this are purged by + // dbpurge. Returns 30 (days) when no value has been configured. + // A value of 0 disables chat purging entirely. + GetChatRetentionDays(ctx context.Context) (int32, error) GetChatSystemPrompt(ctx context.Context) (string, error) // GetChatSystemPromptConfig returns both chat system prompt settings in a // single read to avoid torn reads between separate site-config lookups. @@ -865,6 +886,10 @@ type sqlcQuerier interface { // This must be called from within a transaction. The lock will be automatically // released when the transaction ends. TryAcquireLock(ctx context.Context, pgTryAdvisoryXactLock int64) (bool, error) + // Unarchives a chat (and its children). Stale file references are + // handled automatically by FK cascades on chat_file_links: when + // dbpurge deletes a chat_files row, the corresponding + // chat_file_links rows are cascade-deleted by PostgreSQL. UnarchiveChatByID(ctx context.Context, id uuid.UUID) ([]Chat, error) // This will always work regardless of the current state of the template version. UnarchiveTemplateVersion(ctx context.Context, arg UnarchiveTemplateVersionParams) error @@ -1006,6 +1031,7 @@ type sqlcQuerier interface { UpsertChatDiffStatus(ctx context.Context, arg UpsertChatDiffStatusParams) (ChatDiffStatus, error) UpsertChatDiffStatusReference(ctx context.Context, arg UpsertChatDiffStatusReferenceParams) (ChatDiffStatus, error) UpsertChatIncludeDefaultSystemPrompt(ctx context.Context, includeDefaultSystemPrompt bool) error + UpsertChatRetentionDays(ctx context.Context, retentionDays int32) error UpsertChatSystemPrompt(ctx context.Context, value string) error UpsertChatTemplateAllowlist(ctx context.Context, templateAllowlist string) error UpsertChatUsageLimitConfig(ctx context.Context, arg UpsertChatUsageLimitConfigParams) (ChatUsageLimitConfig, error) diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 9aa787c6f4..2af895b725 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -2884,6 +2884,54 @@ func (q *sqlQuerier) UpsertBoundaryUsageStats(ctx context.Context, arg UpsertBou return new_period, err } +const deleteOldChatFiles = `-- name: DeleteOldChatFiles :execrows +WITH kept_file_ids AS ( + -- NOTE: This uses updated_at as a proxy for archive time + -- because there is no archived_at column. Correctness + -- requires that updated_at is never backdated on archived + -- chats. See ArchiveChatByID. + SELECT DISTINCT cfl.file_id + FROM chat_file_links cfl + JOIN chats c ON c.id = cfl.chat_id + WHERE c.archived = false + OR c.updated_at >= $1::timestamptz +), +deletable AS ( + SELECT cf.id + FROM chat_files cf + LEFT JOIN kept_file_ids k ON cf.id = k.file_id + WHERE cf.created_at < $1::timestamptz + AND k.file_id IS NULL + ORDER BY cf.created_at ASC + LIMIT $2 +) +DELETE FROM chat_files +USING deletable +WHERE chat_files.id = deletable.id +` + +type DeleteOldChatFilesParams struct { + BeforeTime time.Time `db:"before_time" json:"before_time"` + LimitCount int32 `db:"limit_count" json:"limit_count"` +} + +// TODO(cian): Add indexes on chats(archived, updated_at) and +// chat_files(created_at) for purge query performance. +// See: https://github.com/coder/internal/issues/1438 +// Deletes chat files that are older than the given threshold and are +// not referenced by any chat that is still active or was archived +// within the same threshold window. This covers two cases: +// 1. Orphaned files not linked to any chat. +// 2. Files whose every referencing chat has been archived for longer +// than the retention period. +func (q *sqlQuerier) DeleteOldChatFiles(ctx context.Context, arg DeleteOldChatFilesParams) (int64, error) { + result, err := q.db.ExecContext(ctx, deleteOldChatFiles, arg.BeforeTime, arg.LimitCount) + if err != nil { + return 0, err + } + return result.RowsAffected() +} + const getChatFileByID = `-- name: GetChatFileByID :one SELECT id, owner_id, organization_id, created_at, name, mimetype, data FROM chat_files WHERE id = $1::uuid ` @@ -4504,6 +4552,39 @@ func (q *sqlQuerier) DeleteChatUsageLimitUserOverride(ctx context.Context, userI return err } +const deleteOldChats = `-- name: DeleteOldChats :execrows +WITH deletable AS ( + SELECT id + FROM chats + WHERE archived = true + AND updated_at < $1::timestamptz + ORDER BY updated_at ASC + LIMIT $2 +) +DELETE FROM chats +USING deletable +WHERE chats.id = deletable.id + AND chats.archived = true +` + +type DeleteOldChatsParams struct { + BeforeTime time.Time `db:"before_time" json:"before_time"` + LimitCount int32 `db:"limit_count" json:"limit_count"` +} + +// Deletes chats that have been archived for longer than the given +// threshold. Active (non-archived) chats are never deleted. +// Related chat_messages, chat_diff_statuses, and +// chat_queued_messages are removed via ON DELETE CASCADE. +// Parent/root references on child chats are SET NULL. +func (q *sqlQuerier) DeleteOldChats(ctx context.Context, arg DeleteOldChatsParams) (int64, error) { + result, err := q.db.ExecContext(ctx, deleteOldChats, arg.BeforeTime, arg.LimitCount) + if err != nil { + return 0, err + } + return result.RowsAffected() +} + const getChatByID = `-- name: GetChatByID :one SELECT id, owner_id, workspace_id, title, status, worker_id, started_at, heartbeat_at, created_at, updated_at, parent_chat_id, root_chat_id, last_model_config_id, archived, last_error, mode, mcp_server_ids, labels, build_id, agent_id, pin_order, last_read_message_id, last_injected_context @@ -6404,8 +6485,9 @@ func (q *sqlQuerier) SoftDeleteChatMessagesAfterID(ctx context.Context, arg Soft const unarchiveChatByID = `-- name: UnarchiveChatByID :many WITH chats AS ( - UPDATE chats - SET archived = false, updated_at = NOW() + UPDATE chats SET + archived = false, + updated_at = NOW() WHERE id = $1::uuid OR root_chat_id = $1::uuid RETURNING id, owner_id, workspace_id, title, status, worker_id, started_at, heartbeat_at, created_at, updated_at, parent_chat_id, root_chat_id, last_model_config_id, archived, last_error, mode, mcp_server_ids, labels, build_id, agent_id, pin_order, last_read_message_id, last_injected_context ) @@ -6414,6 +6496,10 @@ FROM chats ORDER BY (id = $1::uuid) DESC, created_at ASC, id ASC ` +// Unarchives a chat (and its children). Stale file references are +// handled automatically by FK cascades on chat_file_links: when +// dbpurge deletes a chat_files row, the corresponding +// chat_file_links rows are cascade-deleted by PostgreSQL. func (q *sqlQuerier) UnarchiveChatByID(ctx context.Context, id uuid.UUID) ([]Chat, error) { rows, err := q.db.QueryContext(ctx, unarchiveChatByID, id) if err != nil { @@ -18774,6 +18860,25 @@ func (q *sqlQuerier) GetChatIncludeDefaultSystemPrompt(ctx context.Context) (boo return include_default_system_prompt, err } +const getChatRetentionDays = `-- name: GetChatRetentionDays :one +SELECT COALESCE( + (SELECT value::integer FROM site_configs + WHERE key = 'agents_chat_retention_days'), + 30 +) :: integer AS retention_days +` + +// Returns the chat retention period in days. Chats archived longer +// than this and orphaned chat files older than this are purged by +// dbpurge. Returns 30 (days) when no value has been configured. +// A value of 0 disables chat purging entirely. +func (q *sqlQuerier) GetChatRetentionDays(ctx context.Context) (int32, error) { + row := q.db.QueryRowContext(ctx, getChatRetentionDays) + var retention_days int32 + err := row.Scan(&retention_days) + return retention_days, err +} + const getChatSystemPrompt = `-- name: GetChatSystemPrompt :one SELECT COALESCE((SELECT value FROM site_configs WHERE key = 'agents_chat_system_prompt'), '') :: text AS chat_system_prompt @@ -19074,6 +19179,18 @@ func (q *sqlQuerier) UpsertChatIncludeDefaultSystemPrompt(ctx context.Context, i return err } +const upsertChatRetentionDays = `-- name: UpsertChatRetentionDays :exec +INSERT INTO site_configs (key, value) +VALUES ('agents_chat_retention_days', CAST($1 AS integer)::text) +ON CONFLICT (key) DO UPDATE SET value = CAST($1 AS integer)::text +WHERE site_configs.key = 'agents_chat_retention_days' +` + +func (q *sqlQuerier) UpsertChatRetentionDays(ctx context.Context, retentionDays int32) error { + _, err := q.db.ExecContext(ctx, upsertChatRetentionDays, retentionDays) + return err +} + const upsertChatSystemPrompt = `-- name: UpsertChatSystemPrompt :exec INSERT INTO site_configs (key, value) VALUES ('agents_chat_system_prompt', $1) ON CONFLICT (key) DO UPDATE SET value = $1 WHERE site_configs.key = 'agents_chat_system_prompt' diff --git a/coderd/database/queries/chatfiles.sql b/coderd/database/queries/chatfiles.sql index ac0ec0782e..7ebf8713fc 100644 --- a/coderd/database/queries/chatfiles.sql +++ b/coderd/database/queries/chatfiles.sql @@ -18,3 +18,37 @@ FROM chat_files cf JOIN chat_file_links cfl ON cfl.file_id = cf.id WHERE cfl.chat_id = @chat_id::uuid ORDER BY cf.created_at ASC; + +-- TODO(cian): Add indexes on chats(archived, updated_at) and +-- chat_files(created_at) for purge query performance. +-- See: https://github.com/coder/internal/issues/1438 +-- name: DeleteOldChatFiles :execrows +-- Deletes chat files that are older than the given threshold and are +-- not referenced by any chat that is still active or was archived +-- within the same threshold window. This covers two cases: +-- 1. Orphaned files not linked to any chat. +-- 2. Files whose every referencing chat has been archived for longer +-- than the retention period. +WITH kept_file_ids AS ( + -- NOTE: This uses updated_at as a proxy for archive time + -- because there is no archived_at column. Correctness + -- requires that updated_at is never backdated on archived + -- chats. See ArchiveChatByID. + SELECT DISTINCT cfl.file_id + FROM chat_file_links cfl + JOIN chats c ON c.id = cfl.chat_id + WHERE c.archived = false + OR c.updated_at >= @before_time::timestamptz +), +deletable AS ( + SELECT cf.id + FROM chat_files cf + LEFT JOIN kept_file_ids k ON cf.id = k.file_id + WHERE cf.created_at < @before_time::timestamptz + AND k.file_id IS NULL + ORDER BY cf.created_at ASC + LIMIT @limit_count +) +DELETE FROM chat_files +USING deletable +WHERE chat_files.id = deletable.id; diff --git a/coderd/database/queries/chats.sql b/coderd/database/queries/chats.sql index 31d606d70c..45b54475b6 100644 --- a/coderd/database/queries/chats.sql +++ b/coderd/database/queries/chats.sql @@ -10,9 +10,14 @@ FROM chats ORDER BY (id = @id::uuid) DESC, created_at ASC, id ASC; -- name: UnarchiveChatByID :many +-- Unarchives a chat (and its children). Stale file references are +-- handled automatically by FK cascades on chat_file_links: when +-- dbpurge deletes a chat_files row, the corresponding +-- chat_file_links rows are cascade-deleted by PostgreSQL. WITH chats AS ( - UPDATE chats - SET archived = false, updated_at = NOW() + UPDATE chats SET + archived = false, + updated_at = NOW() WHERE id = @id::uuid OR root_chat_id = @id::uuid RETURNING * ) @@ -1220,3 +1225,22 @@ LIMIT 1; UPDATE chats SET last_read_message_id = @last_read_message_id::bigint WHERE id = @id::uuid; + +-- name: DeleteOldChats :execrows +-- Deletes chats that have been archived for longer than the given +-- threshold. Active (non-archived) chats are never deleted. +-- Related chat_messages, chat_diff_statuses, and +-- chat_queued_messages are removed via ON DELETE CASCADE. +-- Parent/root references on child chats are SET NULL. +WITH deletable AS ( + SELECT id + FROM chats + WHERE archived = true + AND updated_at < @before_time::timestamptz + ORDER BY updated_at ASC + LIMIT @limit_count +) +DELETE FROM chats +USING deletable +WHERE chats.id = deletable.id + AND chats.archived = true; diff --git a/coderd/database/queries/siteconfig.sql b/coderd/database/queries/siteconfig.sql index 3d1fc91686..eccd636abd 100644 --- a/coderd/database/queries/siteconfig.sql +++ b/coderd/database/queries/siteconfig.sql @@ -236,3 +236,20 @@ VALUES ('agents_workspace_ttl', @workspace_ttl::text) ON CONFLICT (key) DO UPDATE SET value = @workspace_ttl::text WHERE site_configs.key = 'agents_workspace_ttl'; + +-- name: GetChatRetentionDays :one +-- Returns the chat retention period in days. Chats archived longer +-- than this and orphaned chat files older than this are purged by +-- dbpurge. Returns 30 (days) when no value has been configured. +-- A value of 0 disables chat purging entirely. +SELECT COALESCE( + (SELECT value::integer FROM site_configs + WHERE key = 'agents_chat_retention_days'), + 30 +) :: integer AS retention_days; + +-- name: UpsertChatRetentionDays :exec +INSERT INTO site_configs (key, value) +VALUES ('agents_chat_retention_days', CAST(@retention_days AS integer)::text) +ON CONFLICT (key) DO UPDATE SET value = CAST(@retention_days AS integer)::text +WHERE site_configs.key = 'agents_chat_retention_days'; diff --git a/coderd/exp_chats.go b/coderd/exp_chats.go index c870e2e56a..3804d694e6 100644 --- a/coderd/exp_chats.go +++ b/coderd/exp_chats.go @@ -3183,6 +3183,70 @@ func (api *API) putChatWorkspaceTTL(rw http.ResponseWriter, r *http.Request) { rw.WriteHeader(http.StatusNoContent) } +// @Summary Get chat retention days +// @ID get-chat-retention-days +// @Security CoderSessionToken +// @Tags Chats +// @Produce json +// @Success 200 {object} codersdk.ChatRetentionDaysResponse +// @Router /experimental/chats/config/retention-days [get] +// @x-apidocgen {"skip": true} +// +//nolint:revive // get-return: revive assumes get* must be a getter, but this is an HTTP handler. +func (api *API) getChatRetentionDays(rw http.ResponseWriter, r *http.Request) { + ctx := r.Context() + retentionDays, err := api.Database.GetChatRetentionDays(ctx) + if err != nil { + httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ + Message: "Failed to get chat retention days.", + Detail: err.Error(), + }) + return + } + httpapi.Write(ctx, rw, http.StatusOK, codersdk.ChatRetentionDaysResponse{ + RetentionDays: retentionDays, + }) +} + +// Keep in sync with retentionDaysMaximum in +// site/src/pages/AgentsPage/AgentSettingsBehaviorPageView.tsx. +const retentionDaysMaximum = 3650 // ~10 years + +// @Summary Update chat retention days +// @ID update-chat-retention-days +// @Security CoderSessionToken +// @Tags Chats +// @Accept json +// @Param request body codersdk.UpdateChatRetentionDaysRequest true "Request body" +// @Success 204 +// @Router /experimental/chats/config/retention-days [put] +// @x-apidocgen {"skip": true} +func (api *API) putChatRetentionDays(rw http.ResponseWriter, r *http.Request) { + ctx := r.Context() + if !api.Authorize(r, policy.ActionUpdate, rbac.ResourceDeploymentConfig) { + httpapi.Forbidden(rw) + return + } + var req codersdk.UpdateChatRetentionDaysRequest + if !httpapi.Read(ctx, rw, r, &req) { + return + } + if req.RetentionDays < 0 || req.RetentionDays > retentionDaysMaximum { + httpapi.Write(ctx, rw, http.StatusBadRequest, codersdk.Response{ + Message: fmt.Sprintf("Retention days must be between 0 and %d.", retentionDaysMaximum), + }) + return + } + if err := api.Database.UpsertChatRetentionDays(ctx, req.RetentionDays); err != nil { + httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ + Message: "Failed to update chat retention days.", + Detail: err.Error(), + }) + return + } + rw.WriteHeader(http.StatusNoContent) +} + // EXPERIMENTAL: this endpoint is experimental and is subject to change. // //nolint:revive // get-return: revive assumes get* must be a getter, but this is an HTTP handler. diff --git a/coderd/exp_chats_test.go b/coderd/exp_chats_test.go index a32d117ccf..cebe1fde9c 100644 --- a/coderd/exp_chats_test.go +++ b/coderd/exp_chats_test.go @@ -7747,6 +7747,62 @@ func TestChatWorkspaceTTL(t *testing.T) { requireSDKError(t, err, http.StatusBadRequest) } +func TestChatRetentionDays(t *testing.T) { + t.Parallel() + ctx := testutil.Context(t, testutil.WaitLong) + + adminClient := newChatClient(t) + firstUser := coderdtest.CreateFirstUser(t, adminClient.Client) + memberClientRaw, _ := coderdtest.CreateAnotherUser(t, adminClient.Client, firstUser.OrganizationID) + memberClient := codersdk.NewExperimentalClient(memberClientRaw) + + // Default value is 30 (days) when nothing has been configured. + resp, err := adminClient.GetChatRetentionDays(ctx) + require.NoError(t, err, "get default") + require.Equal(t, int32(30), resp.RetentionDays, "default should be 30") + + // Admin can set retention days to 90. + err = adminClient.UpdateChatRetentionDays(ctx, codersdk.UpdateChatRetentionDaysRequest{ + RetentionDays: 90, + }) + require.NoError(t, err, "admin set 90") + + resp, err = adminClient.GetChatRetentionDays(ctx) + require.NoError(t, err, "get after set") + require.Equal(t, int32(90), resp.RetentionDays, "should return 90") + + // Non-admin member can read the value. + resp, err = memberClient.GetChatRetentionDays(ctx) + require.NoError(t, err, "member get") + require.Equal(t, int32(90), resp.RetentionDays, "member should see same value") + + // Non-admin member cannot write. + err = memberClient.UpdateChatRetentionDays(ctx, codersdk.UpdateChatRetentionDaysRequest{RetentionDays: 7}) + requireSDKError(t, err, http.StatusForbidden) + + // Admin can disable purge by setting 0. + err = adminClient.UpdateChatRetentionDays(ctx, codersdk.UpdateChatRetentionDaysRequest{ + RetentionDays: 0, + }) + require.NoError(t, err, "admin set 0") + + resp, err = adminClient.GetChatRetentionDays(ctx) + require.NoError(t, err, "get after zero") + require.Equal(t, int32(0), resp.RetentionDays, "should be 0 after disable") + + // Validation: negative value is rejected. + err = adminClient.UpdateChatRetentionDays(ctx, codersdk.UpdateChatRetentionDaysRequest{ + RetentionDays: -1, + }) + requireSDKError(t, err, http.StatusBadRequest) + + // Validation: exceeding the 3650-day maximum is rejected. + err = adminClient.UpdateChatRetentionDays(ctx, codersdk.UpdateChatRetentionDaysRequest{ + RetentionDays: 3651, // retentionDaysMaximum + 1; keep in sync with coderd/exp_chats.go. + }) + requireSDKError(t, err, http.StatusBadRequest) +} + //nolint:tparallel,paralleltest // Subtests share a single coderdtest instance. func TestUserChatCompactionThresholds(t *testing.T) { t.Parallel() diff --git a/codersdk/chats.go b/codersdk/chats.go index ec1d575152..dc3dc53046 100644 --- a/codersdk/chats.go +++ b/codersdk/chats.go @@ -545,6 +545,17 @@ type UpdateChatWorkspaceTTLRequest struct { WorkspaceTTLMillis int64 `json:"workspace_ttl_ms"` } +// ChatRetentionDaysResponse contains the current chat retention setting. +type ChatRetentionDaysResponse struct { + RetentionDays int32 `json:"retention_days"` +} + +// UpdateChatRetentionDaysRequest is a request to update the chat +// retention period. +type UpdateChatRetentionDaysRequest struct { + RetentionDays int32 `json:"retention_days"` +} + // ParseChatWorkspaceTTL parses a stored TTL string, returning the // default when the value is empty. func ParseChatWorkspaceTTL(s string) (time.Duration, error) { @@ -1667,6 +1678,33 @@ func (c *ExperimentalClient) UpdateChatWorkspaceTTL(ctx context.Context, req Upd return nil } +// GetChatRetentionDays returns the configured chat retention period. +func (c *ExperimentalClient) GetChatRetentionDays(ctx context.Context) (ChatRetentionDaysResponse, error) { + res, err := c.Request(ctx, http.MethodGet, "/api/experimental/chats/config/retention-days", nil) + if err != nil { + return ChatRetentionDaysResponse{}, err + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + return ChatRetentionDaysResponse{}, ReadBodyAsError(res) + } + var resp ChatRetentionDaysResponse + return resp, json.NewDecoder(res.Body).Decode(&resp) +} + +// UpdateChatRetentionDays updates the chat retention period. +func (c *ExperimentalClient) UpdateChatRetentionDays(ctx context.Context, req UpdateChatRetentionDaysRequest) error { + res, err := c.Request(ctx, http.MethodPut, "/api/experimental/chats/config/retention-days", req) + if err != nil { + return err + } + defer res.Body.Close() + if res.StatusCode != http.StatusNoContent { + return ReadBodyAsError(res) + } + return nil +} + // GetChatTemplateAllowlist returns the deployment-wide chat template allowlist. func (c *ExperimentalClient) GetChatTemplateAllowlist(ctx context.Context) (ChatTemplateAllowlist, error) { res, err := c.Request(ctx, http.MethodGet, "/api/experimental/chats/config/template-allowlist", nil) diff --git a/docs/ai-coder/agents/chat-retention.md b/docs/ai-coder/agents/chat-retention.md new file mode 100644 index 0000000000..a14c777d89 --- /dev/null +++ b/docs/ai-coder/agents/chat-retention.md @@ -0,0 +1,44 @@ +# Conversation Data Retention + +Coder Agents automatically cleans up old conversation data to manage database +growth. Archived conversations and their associated files are periodically +purged based on a configurable retention period. + +## How it works + +A background process runs approximately every 10 minutes to remove expired +conversation data. Only archived conversations are eligible for deletion — +active (non-archived) conversations are never purged. + +When an archived conversation exceeds the retention period, it is deleted along +with its messages, diff statuses, and queued messages via cascade. Orphaned +files (not referenced by any active or recently-archived conversation) are also +deleted. Both operations run in batches of 1,000 rows per cycle. + +## Configuration + +Navigate to **Deployment Settings** > **Agents** > **Behavior** to configure +the conversation retention period. The default is 30 days. Use the toggle to +disable retention entirely. + +The retention period is stored as the `agents_chat_retention_days` key in the +`site_configs` table and can also be managed via the API at +`/api/experimental/chats/config/retention-days`. + +## What gets deleted + +| Data | Condition | Cascade | +|------------------------|------------------------------------------------------------------------------------------------|---------------------------------------------------------------| +| Archived conversations | Archived longer than retention period | Messages, diff statuses, queued messages deleted via CASCADE. | +| Conversation files | Older than retention period AND not referenced by any active or recently-archived conversation | — | + +## Unarchive safety + +If a user unarchives a conversation whose files were purged, stale file +references are automatically cleaned up by FK cascades. The conversation +remains usable but previously attached files are no longer available. + +## Related links + +- [Coder Agents](./index.md) +- [Data Retention](../../admin/setup/data-retention.md) diff --git a/docs/manifest.json b/docs/manifest.json index 57d5ae6e8c..b5334195fe 100644 --- a/docs/manifest.json +++ b/docs/manifest.json @@ -1264,6 +1264,12 @@ "description": "Programmatic access to Coder Agents via the experimental Chats API", "path": "./ai-coder/agents/chats-api.md", "state": ["early access"] + }, + { + "title": "Chat Data Retention", + "description": "Automatic cleanup of old chat data", + "path": "./ai-coder/agents/chat-retention.md", + "state": ["early access"] } ] } diff --git a/docs/reference/api/schemas.md b/docs/reference/api/schemas.md index 7b37a35624..33dc2a0b25 100644 --- a/docs/reference/api/schemas.md +++ b/docs/reference/api/schemas.md @@ -2025,6 +2025,20 @@ AuthorizationObject can represent a "set" of objects, such as: all workspaces in |----------------------|---------|----------|--------------|-------------| | `acquire_batch_size` | integer | false | | | +## codersdk.ChatRetentionDaysResponse + +```json +{ + "retention_days": 0 +} +``` + +### Properties + +| Name | Type | Required | Restrictions | Description | +|------------------|---------|----------|--------------|-------------| +| `retention_days` | integer | false | | | + ## codersdk.ConnectionLatency ```json @@ -10299,6 +10313,20 @@ Restarts will only happen on weekdays in this list on weeks which line up with W | `logo_url` | string | false | | | | `service_banner` | [codersdk.BannerConfig](#codersdkbannerconfig) | false | | Deprecated: ServiceBanner has been replaced by AnnouncementBanners. | +## codersdk.UpdateChatRetentionDaysRequest + +```json +{ + "retention_days": 0 +} +``` + +### Properties + +| Name | Type | Required | Restrictions | Description | +|------------------|---------|----------|--------------|-------------| +| `retention_days` | integer | false | | | + ## codersdk.UpdateCheckResponse ```json diff --git a/site/src/api/api.ts b/site/src/api/api.ts index 260d7ab57a..31eae1e2ba 100644 --- a/site/src/api/api.ts +++ b/site/src/api/api.ts @@ -3279,6 +3279,20 @@ class ExperimentalApiMethods { await this.axios.put("/api/experimental/chats/config/workspace-ttl", req); }; + getChatRetentionDays = + async (): Promise => { + const response = await this.axios.get( + "/api/experimental/chats/config/retention-days", + ); + return response.data; + }; + + updateChatRetentionDays = async ( + req: TypesGen.UpdateChatRetentionDaysRequest, + ): Promise => { + await this.axios.put("/api/experimental/chats/config/retention-days", req); + }; + updateChatTemplateAllowlist = async ( req: TypesGen.ChatTemplateAllowlist, ): Promise => { diff --git a/site/src/api/queries/chats.ts b/site/src/api/queries/chats.ts index e5613aec8b..4ad2b03b11 100644 --- a/site/src/api/queries/chats.ts +++ b/site/src/api/queries/chats.ts @@ -770,6 +770,22 @@ export const updateChatWorkspaceTTL = (queryClient: QueryClient) => ({ }, }); +const chatRetentionDaysKey = ["chat-retention-days"] as const; + +export const chatRetentionDays = () => ({ + queryKey: chatRetentionDaysKey, + queryFn: () => API.experimental.getChatRetentionDays(), +}); + +export const updateChatRetentionDays = (queryClient: QueryClient) => ({ + mutationFn: API.experimental.updateChatRetentionDays, + onSuccess: async () => { + await queryClient.invalidateQueries({ + queryKey: chatRetentionDaysKey, + }); + }, +}); + const chatTemplateAllowlistKey = ["chat-template-allowlist"] as const; export const chatTemplateAllowlist = () => ({ diff --git a/site/src/api/typesGenerated.ts b/site/src/api/typesGenerated.ts index 36d75c217e..6fe7278af1 100644 --- a/site/src/api/typesGenerated.ts +++ b/site/src/api/typesGenerated.ts @@ -1903,6 +1903,14 @@ export interface ChatReasoningPart { readonly text: string; } +// From codersdk/chats.go +/** + * ChatRetentionDaysResponse contains the current chat retention setting. + */ +export interface ChatRetentionDaysResponse { + readonly retention_days: number; +} + // From codersdk/chats.go export interface ChatSkillPart { readonly type: "skill"; @@ -7261,6 +7269,15 @@ export interface UpdateChatRequest { readonly labels?: Record; } +// From codersdk/chats.go +/** + * UpdateChatRetentionDaysRequest is a request to update the chat + * retention period. + */ +export interface UpdateChatRetentionDaysRequest { + readonly retention_days: number; +} + // From codersdk/chats.go /** * UpdateChatSystemPromptRequest is the request body for updating the chat diff --git a/site/src/pages/AgentsPage/AgentSettingsBehaviorPage.tsx b/site/src/pages/AgentsPage/AgentSettingsBehaviorPage.tsx index 123051a7da..cb6d3634b5 100644 --- a/site/src/pages/AgentsPage/AgentSettingsBehaviorPage.tsx +++ b/site/src/pages/AgentsPage/AgentSettingsBehaviorPage.tsx @@ -3,11 +3,13 @@ import { useMutation, useQuery, useQueryClient } from "react-query"; import { chatDesktopEnabled, chatModelConfigs, + chatRetentionDays, chatSystemPrompt, chatUserCustomPrompt, chatWorkspaceTTL, deleteUserCompactionThreshold, updateChatDesktopEnabled, + updateChatRetentionDays, updateChatSystemPrompt, updateChatWorkspaceTTL, updateUserChatCustomPrompt, @@ -44,6 +46,11 @@ const AgentSettingsBehaviorPage: FC = () => { updateChatWorkspaceTTL(queryClient), ); + const retentionDaysQuery = useQuery(chatRetentionDays()); + const saveRetentionDaysMutation = useMutation( + updateChatRetentionDays(queryClient), + ); + const modelConfigsQuery = useQuery(chatModelConfigs()); const thresholdsQuery = useQuery(userCompactionThresholds()); @@ -95,6 +102,12 @@ const AgentSettingsBehaviorPage: FC = () => { onSaveWorkspaceTTL={saveWorkspaceTTLMutation.mutate} isSavingWorkspaceTTL={saveWorkspaceTTLMutation.isPending} isSaveWorkspaceTTLError={saveWorkspaceTTLMutation.isError} + retentionDaysData={retentionDaysQuery.data} + isRetentionDaysLoading={retentionDaysQuery.isLoading} + isRetentionDaysLoadError={retentionDaysQuery.isError} + onSaveRetentionDays={saveRetentionDaysMutation.mutate} + isSavingRetentionDays={saveRetentionDaysMutation.isPending} + isSaveRetentionDaysError={saveRetentionDaysMutation.isError} /> ); }; diff --git a/site/src/pages/AgentsPage/AgentSettingsBehaviorPageView.tsx b/site/src/pages/AgentsPage/AgentSettingsBehaviorPageView.tsx index c34dea248b..2999f56113 100644 --- a/site/src/pages/AgentsPage/AgentSettingsBehaviorPageView.tsx +++ b/site/src/pages/AgentsPage/AgentSettingsBehaviorPageView.tsx @@ -39,6 +39,9 @@ interface AgentSettingsBehaviorPageViewProps { workspaceTTLData: TypesGen.ChatWorkspaceTTLResponse | undefined; isWorkspaceTTLLoading: boolean; isWorkspaceTTLLoadError: boolean; + retentionDaysData: TypesGen.ChatRetentionDaysResponse | undefined; + isRetentionDaysLoading: boolean; + isRetentionDaysLoadError: boolean; modelConfigsData: TypesGen.ChatModelConfig[] | undefined; modelConfigsError: unknown; isLoadingModelConfigs: boolean; @@ -81,6 +84,13 @@ interface AgentSettingsBehaviorPageViewProps { ) => void; isSavingWorkspaceTTL: boolean; isSaveWorkspaceTTLError: boolean; + + onSaveRetentionDays: ( + req: TypesGen.UpdateChatRetentionDaysRequest, + options?: MutationCallbacks, + ) => void; + isSavingRetentionDays: boolean; + isSaveRetentionDaysError: boolean; } export const AgentSettingsBehaviorPageView: FC< @@ -93,6 +103,9 @@ export const AgentSettingsBehaviorPageView: FC< workspaceTTLData, isWorkspaceTTLLoading, isWorkspaceTTLLoadError, + retentionDaysData, + isRetentionDaysLoading, + isRetentionDaysLoadError, modelConfigsData, modelConfigsError, isLoadingModelConfigs, @@ -113,6 +126,9 @@ export const AgentSettingsBehaviorPageView: FC< onSaveWorkspaceTTL, isSavingWorkspaceTTL, isSaveWorkspaceTTLError, + onSaveRetentionDays, + isSavingRetentionDays, + isSaveRetentionDaysError, }) => { // ── Local form state ── const [localEdit, setLocalEdit] = useState(null); @@ -124,6 +140,12 @@ export const AgentSettingsBehaviorPageView: FC< const [localUserEdit, setLocalUserEdit] = useState(null); const [localTTLMs, setLocalTTLMs] = useState(null); const [autostopToggled, setAutostopToggled] = useState(null); + const [localRetentionDays, setLocalRetentionDays] = useState( + null, + ); + const [retentionToggled, setRetentionToggled] = useState( + null, + ); // Overflow states are pure UI — managed locally in the view. const [isUserPromptOverflowing, setIsUserPromptOverflowing] = useState(false); @@ -171,6 +193,18 @@ export const AgentSettingsBehaviorPageView: FC< const isTTLOverMax = ttlMs > maxTTLMs; const isTTLZero = isAutostopEnabled && ttlMs === 0; + // ── Retention days derived state ── + const serverRetentionDays = retentionDaysData?.retention_days ?? 30; + const retentionDays = localRetentionDays ?? serverRetentionDays; + const isRetentionEnabled = retentionToggled ?? serverRetentionDays > 0; + const isRetentionDaysDirty = + localRetentionDays !== null && localRetentionDays !== serverRetentionDays; + const isRetentionDaysNegative = isRetentionEnabled && retentionDays < 0; + // Keep in sync with retentionDaysMaximum in coderd/exp_chats.go. + const retentionDaysMaximum = 3650; + const isRetentionDaysOverMax = retentionDays > retentionDaysMaximum; + const isRetentionDaysZero = isRetentionEnabled && retentionDays === 0; + // ── Event handlers ── const handleSaveSystemPrompt = (event: FormEvent) => { event.preventDefault(); @@ -245,6 +279,45 @@ export const AgentSettingsBehaviorPageView: FC< } }; + const resetRetentionState = () => { + setLocalRetentionDays(null); + setRetentionToggled(null); + }; + + const handleToggleRetention = (checked: boolean) => { + if (checked) { + setRetentionToggled(true); + setLocalRetentionDays(serverRetentionDays > 0 ? serverRetentionDays : 30); + onSaveRetentionDays( + { retention_days: serverRetentionDays > 0 ? serverRetentionDays : 30 }, + { onSuccess: resetRetentionState, onError: resetRetentionState }, + ); + } else { + setRetentionToggled(false); + setLocalRetentionDays(0); + onSaveRetentionDays( + { retention_days: 0 }, + { onSuccess: resetRetentionState, onError: resetRetentionState }, + ); + } + }; + + const handleRetentionDaysChange = (value: number) => { + setLocalRetentionDays(value); + if (retentionToggled === null) { + setRetentionToggled(true); + } + }; + + const handleSaveRetentionDays = (event: FormEvent) => { + event.preventDefault(); + if (!isRetentionDaysDirty || isSavingRetentionDays) return; + onSaveRetentionDays( + { retention_days: localRetentionDays ?? 30 }, + { onSuccess: resetRetentionState }, + ); + }; + return ( <> )} +
+
void handleSaveRetentionDays(event)} + > +
+

+ Conversation Retention Period +

+ +
+
+

+ Archived conversations and orphaned files older than this are + automatically deleted. +

+ +
+ {isRetentionEnabled && ( + <> + + handleRetentionDaysChange( + Number.parseInt(event.target.value, 10) || 0, + ) + } + disabled={isSavingRetentionDays || isRetentionDaysLoading} + className="w-full rounded-lg border border-border bg-surface-primary px-4 py-2 text-[13px] text-content-primary placeholder:text-content-secondary focus:outline-none focus:ring-2 focus:ring-content-link/30" + /> + {isRetentionDaysZero && ( +

+ Retention period must be at least 1 day. +

+ )} + {isRetentionDaysNegative && ( +

+ Retention days cannot be negative. +

+ )} + {isRetentionDaysOverMax && ( +

+ Must not exceed {retentionDaysMaximum} days (~10 years). +

+ )} +
+ +
+ + )} + {isSaveRetentionDaysError && ( +

+ Failed to save retention setting. +

+ )} + {isRetentionDaysLoadError && ( +

+ Failed to load retention setting. +

+ )} +
)}
diff --git a/site/src/pages/AgentsPage/AgentsPageView.stories.tsx b/site/src/pages/AgentsPage/AgentsPageView.stories.tsx index e7c37e801a..e4c9a8b485 100644 --- a/site/src/pages/AgentsPage/AgentsPageView.stories.tsx +++ b/site/src/pages/AgentsPage/AgentsPageView.stories.tsx @@ -182,6 +182,12 @@ const BehaviorRouteElement = () => { onSaveWorkspaceTTL={fn()} isSavingWorkspaceTTL={false} isSaveWorkspaceTTLError={false} + retentionDaysData={{ retention_days: 30 }} + isRetentionDaysLoading={false} + isRetentionDaysLoadError={false} + onSaveRetentionDays={fn()} + isSavingRetentionDays={false} + isSaveRetentionDaysError={false} onSaveThreshold={fn(async () => undefined)} onResetThreshold={fn(async () => undefined)} />