From c5d720f73d6acb2b6d843d30cdc68fbeceac8a3e Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Wed, 8 Apr 2026 09:47:44 -0400 Subject: [PATCH] feat(coderd): add telemetry for agents chats and messages (#24068) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds telemetry collection for the agents chat system (`/agents`) to the existing telemetry snapshot pipeline. Three new snapshot fields: - **`Chats`** — per-chat metadata (id, owner, status, mode, workspace_id, root_chat_id, has_parent, archived, model config) collected time-windowed via `createdAfter` - **`ChatMessageSummaries`** — per-chat aggregated message metrics (counts by role, token sums by type, cost, runtime, model count, compression count) collected time-windowed - **`ChatModelConfigs`** — model configuration metadata (provider, model, context limit, enabled, default) collected as full dump No PII is included — titles, message content, and URLs are excluded at the SQL level. Only structural metadata flows through telemetry.
Implementation plan ### SQL Queries (`coderd/database/queries/chats.sql`) - `GetChatsCreatedAfter` — time-windowed chat metadata - `GetChatMessageSummariesPerChat` — per-chat message aggregates via `GROUP BY` - `GetChatModelConfigsForTelemetry` — full dump of model configs ### Telemetry (`coderd/telemetry/telemetry.go`) - `Chat`, `ChatMessageSummary`, `ChatModelConfig` structs - `ConvertChat`, `ConvertChatMessageSummary`, `ConvertChatModelConfig` conversion functions - Three `eg.Go()` blocks in `createSnapshot()` following the existing collection pattern ### Authorization (`coderd/database/dbauthz/dbauthz.go`) - System-only access for all three queries via `rbac.ResourceSystem` ### Tests - `TestChatsTelemetry` in `coderd/telemetry/telemetry_test.go` — creates chats (root + child), messages with token/cost data, model configs; verifies all snapshot fields - dbauthz test entries for all three queries in `coderd/database/dbauthz/dbauthz_test.go`
> 🤖 Generated by Coder Agents --- coderd/database/dbauthz/dbauthz.go | 24 ++ coderd/database/dbauthz/dbauthz_test.go | 14 + coderd/database/dbmetrics/querymetrics.go | 24 ++ coderd/database/dbmock/dbmock.go | 45 ++++ coderd/database/querier.go | 10 + coderd/database/queries.sql.go | 191 ++++++++++++++ coderd/database/queries/chats.sql | 43 ++++ coderd/telemetry/telemetry.go | 144 +++++++++++ coderd/telemetry/telemetry_test.go | 300 ++++++++++++++++++++++ 9 files changed, 795 insertions(+) diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go index e03a99dead..6e2834305d 100644 --- a/coderd/database/dbauthz/dbauthz.go +++ b/coderd/database/dbauthz/dbauthz.go @@ -2636,6 +2636,14 @@ func (q *querier) GetChatMessageByID(ctx context.Context, id int64) (database.Ch return msg, nil } +func (q *querier) GetChatMessageSummariesPerChat(ctx context.Context, createdAfter time.Time) ([]database.GetChatMessageSummariesPerChatRow, error) { + // Telemetry queries are called from system contexts only. + if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil { + return nil, err + } + return q.db.GetChatMessageSummariesPerChat(ctx, createdAfter) +} + func (q *querier) GetChatMessagesByChatID(ctx context.Context, arg database.GetChatMessagesByChatIDParams) ([]database.ChatMessage, error) { // Authorize read on the parent chat. _, err := q.GetChatByID(ctx, arg.ChatID) @@ -2684,6 +2692,14 @@ func (q *querier) GetChatModelConfigs(ctx context.Context) ([]database.ChatModel return q.db.GetChatModelConfigs(ctx) } +func (q *querier) GetChatModelConfigsForTelemetry(ctx context.Context) ([]database.GetChatModelConfigsForTelemetryRow, error) { + // Telemetry queries are called from system contexts only. + if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil { + return nil, err + } + return q.db.GetChatModelConfigsForTelemetry(ctx) +} + func (q *querier) GetChatProviderByID(ctx context.Context, id uuid.UUID) (database.ChatProvider, error) { if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceDeploymentConfig); err != nil { return database.ChatProvider{}, err @@ -2800,6 +2816,14 @@ func (q *querier) GetChatsByWorkspaceIDs(ctx context.Context, ids []uuid.UUID) ( return fetchWithPostFilter(q.auth, policy.ActionRead, q.db.GetChatsByWorkspaceIDs)(ctx, ids) } +func (q *querier) GetChatsUpdatedAfter(ctx context.Context, updatedAfter time.Time) ([]database.GetChatsUpdatedAfterRow, error) { + // Telemetry queries are called from system contexts only. + if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil { + return nil, err + } + return q.db.GetChatsUpdatedAfter(ctx, updatedAfter) +} + func (q *querier) GetConnectionLogsOffset(ctx context.Context, arg database.GetConnectionLogsOffsetParams) ([]database.GetConnectionLogsOffsetRow, error) { // Just like with the audit logs query, shortcut if the user is an owner. err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceConnectionLog) diff --git a/coderd/database/dbauthz/dbauthz_test.go b/coderd/database/dbauthz/dbauthz_test.go index 6a0a965443..e9add7a2a7 100644 --- a/coderd/database/dbauthz/dbauthz_test.go +++ b/coderd/database/dbauthz/dbauthz_test.go @@ -4012,6 +4012,20 @@ func (s *MethodTestSuite) TestSystemFunctions() { dbm.EXPECT().GetWorkspaceAgentsCreatedAfter(gomock.Any(), ts).Return([]database.WorkspaceAgent{}, nil).AnyTimes() check.Args(ts).Asserts(rbac.ResourceSystem, policy.ActionRead) })) + s.Run("GetChatsUpdatedAfter", s.Mocked(func(dbm *dbmock.MockStore, _ *gofakeit.Faker, check *expects) { + ts := dbtime.Now() + dbm.EXPECT().GetChatsUpdatedAfter(gomock.Any(), ts).Return([]database.GetChatsUpdatedAfterRow{}, nil).AnyTimes() + check.Args(ts).Asserts(rbac.ResourceSystem, policy.ActionRead) + })) + s.Run("GetChatMessageSummariesPerChat", s.Mocked(func(dbm *dbmock.MockStore, _ *gofakeit.Faker, check *expects) { + ts := dbtime.Now() + dbm.EXPECT().GetChatMessageSummariesPerChat(gomock.Any(), ts).Return([]database.GetChatMessageSummariesPerChatRow{}, nil).AnyTimes() + check.Args(ts).Asserts(rbac.ResourceSystem, policy.ActionRead) + })) + s.Run("GetChatModelConfigsForTelemetry", s.Mocked(func(dbm *dbmock.MockStore, _ *gofakeit.Faker, check *expects) { + dbm.EXPECT().GetChatModelConfigsForTelemetry(gomock.Any()).Return([]database.GetChatModelConfigsForTelemetryRow{}, nil).AnyTimes() + check.Args().Asserts(rbac.ResourceSystem, policy.ActionRead) + })) s.Run("GetWorkspaceAppsCreatedAfter", s.Mocked(func(dbm *dbmock.MockStore, _ *gofakeit.Faker, check *expects) { ts := dbtime.Now() dbm.EXPECT().GetWorkspaceAppsCreatedAfter(gomock.Any(), ts).Return([]database.WorkspaceApp{}, nil).AnyTimes() diff --git a/coderd/database/dbmetrics/querymetrics.go b/coderd/database/dbmetrics/querymetrics.go index 629fcf25d5..664285853c 100644 --- a/coderd/database/dbmetrics/querymetrics.go +++ b/coderd/database/dbmetrics/querymetrics.go @@ -1176,6 +1176,14 @@ func (m queryMetricsStore) GetChatMessageByID(ctx context.Context, id int64) (da return r0, r1 } +func (m queryMetricsStore) GetChatMessageSummariesPerChat(ctx context.Context, createdAfter time.Time) ([]database.GetChatMessageSummariesPerChatRow, error) { + start := time.Now() + r0, r1 := m.s.GetChatMessageSummariesPerChat(ctx, createdAfter) + m.queryLatencies.WithLabelValues("GetChatMessageSummariesPerChat").Observe(time.Since(start).Seconds()) + m.queryCounts.WithLabelValues(httpmw.ExtractHTTPRoute(ctx), httpmw.ExtractHTTPMethod(ctx), "GetChatMessageSummariesPerChat").Inc() + return r0, r1 +} + func (m queryMetricsStore) GetChatMessagesByChatID(ctx context.Context, chatID database.GetChatMessagesByChatIDParams) ([]database.ChatMessage, error) { start := time.Now() r0, r1 := m.s.GetChatMessagesByChatID(ctx, chatID) @@ -1224,6 +1232,14 @@ func (m queryMetricsStore) GetChatModelConfigs(ctx context.Context) ([]database. return r0, r1 } +func (m queryMetricsStore) GetChatModelConfigsForTelemetry(ctx context.Context) ([]database.GetChatModelConfigsForTelemetryRow, error) { + start := time.Now() + r0, r1 := m.s.GetChatModelConfigsForTelemetry(ctx) + m.queryLatencies.WithLabelValues("GetChatModelConfigsForTelemetry").Observe(time.Since(start).Seconds()) + m.queryCounts.WithLabelValues(httpmw.ExtractHTTPRoute(ctx), httpmw.ExtractHTTPMethod(ctx), "GetChatModelConfigsForTelemetry").Inc() + return r0, r1 +} + func (m queryMetricsStore) GetChatProviderByID(ctx context.Context, id uuid.UUID) (database.ChatProvider, error) { start := time.Now() r0, r1 := m.s.GetChatProviderByID(ctx, id) @@ -1336,6 +1352,14 @@ func (m queryMetricsStore) GetChatsByWorkspaceIDs(ctx context.Context, ids []uui return r0, r1 } +func (m queryMetricsStore) GetChatsUpdatedAfter(ctx context.Context, updatedAfter time.Time) ([]database.GetChatsUpdatedAfterRow, error) { + start := time.Now() + r0, r1 := m.s.GetChatsUpdatedAfter(ctx, updatedAfter) + m.queryLatencies.WithLabelValues("GetChatsUpdatedAfter").Observe(time.Since(start).Seconds()) + m.queryCounts.WithLabelValues(httpmw.ExtractHTTPRoute(ctx), httpmw.ExtractHTTPMethod(ctx), "GetChatsUpdatedAfter").Inc() + return r0, r1 +} + func (m queryMetricsStore) GetConnectionLogsOffset(ctx context.Context, arg database.GetConnectionLogsOffsetParams) ([]database.GetConnectionLogsOffsetRow, error) { start := time.Now() r0, r1 := m.s.GetConnectionLogsOffset(ctx, arg) diff --git a/coderd/database/dbmock/dbmock.go b/coderd/database/dbmock/dbmock.go index 4644124bde..30472ca133 100644 --- a/coderd/database/dbmock/dbmock.go +++ b/coderd/database/dbmock/dbmock.go @@ -2162,6 +2162,21 @@ func (mr *MockStoreMockRecorder) GetChatMessageByID(ctx, id any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetChatMessageByID", reflect.TypeOf((*MockStore)(nil).GetChatMessageByID), ctx, id) } +// GetChatMessageSummariesPerChat mocks base method. +func (m *MockStore) GetChatMessageSummariesPerChat(ctx context.Context, createdAfter time.Time) ([]database.GetChatMessageSummariesPerChatRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetChatMessageSummariesPerChat", ctx, createdAfter) + ret0, _ := ret[0].([]database.GetChatMessageSummariesPerChatRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetChatMessageSummariesPerChat indicates an expected call of GetChatMessageSummariesPerChat. +func (mr *MockStoreMockRecorder) GetChatMessageSummariesPerChat(ctx, createdAfter any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetChatMessageSummariesPerChat", reflect.TypeOf((*MockStore)(nil).GetChatMessageSummariesPerChat), ctx, createdAfter) +} + // GetChatMessagesByChatID mocks base method. func (m *MockStore) GetChatMessagesByChatID(ctx context.Context, arg database.GetChatMessagesByChatIDParams) ([]database.ChatMessage, error) { m.ctrl.T.Helper() @@ -2252,6 +2267,21 @@ func (mr *MockStoreMockRecorder) GetChatModelConfigs(ctx any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetChatModelConfigs", reflect.TypeOf((*MockStore)(nil).GetChatModelConfigs), ctx) } +// GetChatModelConfigsForTelemetry mocks base method. +func (m *MockStore) GetChatModelConfigsForTelemetry(ctx context.Context) ([]database.GetChatModelConfigsForTelemetryRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetChatModelConfigsForTelemetry", ctx) + ret0, _ := ret[0].([]database.GetChatModelConfigsForTelemetryRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetChatModelConfigsForTelemetry indicates an expected call of GetChatModelConfigsForTelemetry. +func (mr *MockStoreMockRecorder) GetChatModelConfigsForTelemetry(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetChatModelConfigsForTelemetry", reflect.TypeOf((*MockStore)(nil).GetChatModelConfigsForTelemetry), ctx) +} + // GetChatProviderByID mocks base method. func (m *MockStore) GetChatProviderByID(ctx context.Context, id uuid.UUID) (database.ChatProvider, error) { m.ctrl.T.Helper() @@ -2462,6 +2492,21 @@ func (mr *MockStoreMockRecorder) GetChatsByWorkspaceIDs(ctx, ids any) *gomock.Ca return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetChatsByWorkspaceIDs", reflect.TypeOf((*MockStore)(nil).GetChatsByWorkspaceIDs), ctx, ids) } +// GetChatsUpdatedAfter mocks base method. +func (m *MockStore) GetChatsUpdatedAfter(ctx context.Context, updatedAfter time.Time) ([]database.GetChatsUpdatedAfterRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetChatsUpdatedAfter", ctx, updatedAfter) + ret0, _ := ret[0].([]database.GetChatsUpdatedAfterRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetChatsUpdatedAfter indicates an expected call of GetChatsUpdatedAfter. +func (mr *MockStoreMockRecorder) GetChatsUpdatedAfter(ctx, updatedAfter any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetChatsUpdatedAfter", reflect.TypeOf((*MockStore)(nil).GetChatsUpdatedAfter), ctx, updatedAfter) +} + // GetConnectionLogsOffset mocks base method. func (m *MockStore) GetConnectionLogsOffset(ctx context.Context, arg database.GetConnectionLogsOffsetParams) ([]database.GetConnectionLogsOffsetRow, error) { m.ctrl.T.Helper() diff --git a/coderd/database/querier.go b/coderd/database/querier.go index 26e2983642..80dfc4070e 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -271,12 +271,18 @@ type sqlcQuerier interface { // otherwise the setting defaults to true. GetChatIncludeDefaultSystemPrompt(ctx context.Context) (bool, error) GetChatMessageByID(ctx context.Context, id int64) (ChatMessage, error) + // Aggregates message-level metrics per chat for messages created + // after the given timestamp. Uses message created_at so that + // ongoing activity in long-running chats is captured each window. + GetChatMessageSummariesPerChat(ctx context.Context, createdAfter time.Time) ([]GetChatMessageSummariesPerChatRow, error) GetChatMessagesByChatID(ctx context.Context, arg GetChatMessagesByChatIDParams) ([]ChatMessage, error) GetChatMessagesByChatIDAscPaginated(ctx context.Context, arg GetChatMessagesByChatIDAscPaginatedParams) ([]ChatMessage, error) GetChatMessagesByChatIDDescPaginated(ctx context.Context, arg GetChatMessagesByChatIDDescPaginatedParams) ([]ChatMessage, error) GetChatMessagesForPromptByChatID(ctx context.Context, chatID uuid.UUID) ([]ChatMessage, error) GetChatModelConfigByID(ctx context.Context, id uuid.UUID) (ChatModelConfig, error) GetChatModelConfigs(ctx context.Context) ([]ChatModelConfig, error) + // Returns all model configurations for telemetry snapshot collection. + GetChatModelConfigsForTelemetry(ctx context.Context) ([]GetChatModelConfigsForTelemetryRow, error) GetChatProviderByID(ctx context.Context, id uuid.UUID) (ChatProvider, error) GetChatProviderByProvider(ctx context.Context, provider string) (ChatProvider, error) GetChatProviders(ctx context.Context) ([]ChatProvider, error) @@ -304,6 +310,10 @@ type sqlcQuerier interface { GetChatWorkspaceTTL(ctx context.Context) (string, error) GetChats(ctx context.Context, arg GetChatsParams) ([]GetChatsRow, error) GetChatsByWorkspaceIDs(ctx context.Context, ids []uuid.UUID) ([]Chat, error) + // Retrieves chats updated after the given timestamp for telemetry + // snapshot collection. Uses updated_at so that long-running chats + // still appear in each snapshot window while they are active. + GetChatsUpdatedAfter(ctx context.Context, updatedAfter time.Time) ([]GetChatsUpdatedAfterRow, error) GetConnectionLogsOffset(ctx context.Context, arg GetConnectionLogsOffsetParams) ([]GetConnectionLogsOffsetRow, error) GetCryptoKeyByFeatureAndSequence(ctx context.Context, arg GetCryptoKeyByFeatureAndSequenceParams) (CryptoKey, error) GetCryptoKeys(ctx context.Context) ([]CryptoKey, error) diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 2af895b725..c734ccdfc2 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -5185,6 +5185,89 @@ func (q *sqlQuerier) GetChatMessageByID(ctx context.Context, id int64) (ChatMess return i, err } +const getChatMessageSummariesPerChat = `-- name: GetChatMessageSummariesPerChat :many +SELECT + cm.chat_id, + COUNT(*)::bigint AS message_count, + COUNT(*) FILTER (WHERE cm.role = 'user')::bigint AS user_message_count, + COUNT(*) FILTER (WHERE cm.role = 'assistant')::bigint AS assistant_message_count, + COUNT(*) FILTER (WHERE cm.role = 'tool')::bigint AS tool_message_count, + COUNT(*) FILTER (WHERE cm.role = 'system')::bigint AS system_message_count, + COALESCE(SUM(cm.input_tokens), 0)::bigint AS total_input_tokens, + COALESCE(SUM(cm.output_tokens), 0)::bigint AS total_output_tokens, + COALESCE(SUM(cm.reasoning_tokens), 0)::bigint AS total_reasoning_tokens, + COALESCE(SUM(cm.cache_creation_tokens), 0)::bigint AS total_cache_creation_tokens, + COALESCE(SUM(cm.cache_read_tokens), 0)::bigint AS total_cache_read_tokens, + COALESCE(SUM(cm.total_cost_micros), 0)::bigint AS total_cost_micros, + COALESCE(SUM(cm.runtime_ms), 0)::bigint AS total_runtime_ms, + COUNT(DISTINCT cm.model_config_id)::bigint AS distinct_model_count, + COUNT(*) FILTER (WHERE cm.compressed)::bigint AS compressed_message_count +FROM chat_messages cm +WHERE cm.created_at > $1 + AND cm.deleted = false +GROUP BY cm.chat_id +` + +type GetChatMessageSummariesPerChatRow struct { + ChatID uuid.UUID `db:"chat_id" json:"chat_id"` + MessageCount int64 `db:"message_count" json:"message_count"` + UserMessageCount int64 `db:"user_message_count" json:"user_message_count"` + AssistantMessageCount int64 `db:"assistant_message_count" json:"assistant_message_count"` + ToolMessageCount int64 `db:"tool_message_count" json:"tool_message_count"` + SystemMessageCount int64 `db:"system_message_count" json:"system_message_count"` + TotalInputTokens int64 `db:"total_input_tokens" json:"total_input_tokens"` + TotalOutputTokens int64 `db:"total_output_tokens" json:"total_output_tokens"` + TotalReasoningTokens int64 `db:"total_reasoning_tokens" json:"total_reasoning_tokens"` + TotalCacheCreationTokens int64 `db:"total_cache_creation_tokens" json:"total_cache_creation_tokens"` + TotalCacheReadTokens int64 `db:"total_cache_read_tokens" json:"total_cache_read_tokens"` + TotalCostMicros int64 `db:"total_cost_micros" json:"total_cost_micros"` + TotalRuntimeMs int64 `db:"total_runtime_ms" json:"total_runtime_ms"` + DistinctModelCount int64 `db:"distinct_model_count" json:"distinct_model_count"` + CompressedMessageCount int64 `db:"compressed_message_count" json:"compressed_message_count"` +} + +// Aggregates message-level metrics per chat for messages created +// after the given timestamp. Uses message created_at so that +// ongoing activity in long-running chats is captured each window. +func (q *sqlQuerier) GetChatMessageSummariesPerChat(ctx context.Context, createdAfter time.Time) ([]GetChatMessageSummariesPerChatRow, error) { + rows, err := q.db.QueryContext(ctx, getChatMessageSummariesPerChat, createdAfter) + if err != nil { + return nil, err + } + defer rows.Close() + var items []GetChatMessageSummariesPerChatRow + for rows.Next() { + var i GetChatMessageSummariesPerChatRow + if err := rows.Scan( + &i.ChatID, + &i.MessageCount, + &i.UserMessageCount, + &i.AssistantMessageCount, + &i.ToolMessageCount, + &i.SystemMessageCount, + &i.TotalInputTokens, + &i.TotalOutputTokens, + &i.TotalReasoningTokens, + &i.TotalCacheCreationTokens, + &i.TotalCacheReadTokens, + &i.TotalCostMicros, + &i.TotalRuntimeMs, + &i.DistinctModelCount, + &i.CompressedMessageCount, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const getChatMessagesByChatID = `-- name: GetChatMessagesByChatID :many SELECT id, chat_id, model_config_id, created_at, role, content, visibility, input_tokens, output_tokens, total_tokens, reasoning_tokens, cache_creation_tokens, cache_read_tokens, context_limit, compressed, created_by, content_version, total_cost_micros, runtime_ms, deleted, provider_response_id @@ -5490,6 +5573,52 @@ func (q *sqlQuerier) GetChatMessagesForPromptByChatID(ctx context.Context, chatI return items, nil } +const getChatModelConfigsForTelemetry = `-- name: GetChatModelConfigsForTelemetry :many +SELECT id, provider, model, context_limit, enabled, is_default +FROM chat_model_configs +WHERE deleted = false +` + +type GetChatModelConfigsForTelemetryRow struct { + ID uuid.UUID `db:"id" json:"id"` + Provider string `db:"provider" json:"provider"` + Model string `db:"model" json:"model"` + ContextLimit int64 `db:"context_limit" json:"context_limit"` + Enabled bool `db:"enabled" json:"enabled"` + IsDefault bool `db:"is_default" json:"is_default"` +} + +// Returns all model configurations for telemetry snapshot collection. +func (q *sqlQuerier) GetChatModelConfigsForTelemetry(ctx context.Context) ([]GetChatModelConfigsForTelemetryRow, error) { + rows, err := q.db.QueryContext(ctx, getChatModelConfigsForTelemetry) + if err != nil { + return nil, err + } + defer rows.Close() + var items []GetChatModelConfigsForTelemetryRow + for rows.Next() { + var i GetChatModelConfigsForTelemetryRow + if err := rows.Scan( + &i.ID, + &i.Provider, + &i.Model, + &i.ContextLimit, + &i.Enabled, + &i.IsDefault, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const getChatQueuedMessages = `-- name: GetChatQueuedMessages :many SELECT id, chat_id, content, created_at FROM chat_queued_messages WHERE chat_id = $1 @@ -5759,6 +5888,68 @@ func (q *sqlQuerier) GetChatsByWorkspaceIDs(ctx context.Context, ids []uuid.UUID return items, nil } +const getChatsUpdatedAfter = `-- name: GetChatsUpdatedAfter :many +SELECT + id, owner_id, created_at, updated_at, status, + (parent_chat_id IS NOT NULL)::bool AS has_parent, + root_chat_id, workspace_id, + mode, archived, last_model_config_id +FROM chats +WHERE updated_at > $1 +` + +type GetChatsUpdatedAfterRow struct { + ID uuid.UUID `db:"id" json:"id"` + OwnerID uuid.UUID `db:"owner_id" json:"owner_id"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` + Status ChatStatus `db:"status" json:"status"` + HasParent bool `db:"has_parent" json:"has_parent"` + RootChatID uuid.NullUUID `db:"root_chat_id" json:"root_chat_id"` + WorkspaceID uuid.NullUUID `db:"workspace_id" json:"workspace_id"` + Mode NullChatMode `db:"mode" json:"mode"` + Archived bool `db:"archived" json:"archived"` + LastModelConfigID uuid.UUID `db:"last_model_config_id" json:"last_model_config_id"` +} + +// Retrieves chats updated after the given timestamp for telemetry +// snapshot collection. Uses updated_at so that long-running chats +// still appear in each snapshot window while they are active. +func (q *sqlQuerier) GetChatsUpdatedAfter(ctx context.Context, updatedAfter time.Time) ([]GetChatsUpdatedAfterRow, error) { + rows, err := q.db.QueryContext(ctx, getChatsUpdatedAfter, updatedAfter) + if err != nil { + return nil, err + } + defer rows.Close() + var items []GetChatsUpdatedAfterRow + for rows.Next() { + var i GetChatsUpdatedAfterRow + if err := rows.Scan( + &i.ID, + &i.OwnerID, + &i.CreatedAt, + &i.UpdatedAt, + &i.Status, + &i.HasParent, + &i.RootChatID, + &i.WorkspaceID, + &i.Mode, + &i.Archived, + &i.LastModelConfigID, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const getLastChatMessageByRole = `-- name: GetLastChatMessageByRole :one SELECT id, chat_id, model_config_id, created_at, role, content, visibility, input_tokens, output_tokens, total_tokens, reasoning_tokens, cache_creation_tokens, cache_read_tokens, context_limit, compressed, created_by, content_version, total_cost_micros, runtime_ms, deleted, provider_response_id diff --git a/coderd/database/queries/chats.sql b/coderd/database/queries/chats.sql index 45b54475b6..49094e7841 100644 --- a/coderd/database/queries/chats.sql +++ b/coderd/database/queries/chats.sql @@ -1244,3 +1244,46 @@ DELETE FROM chats USING deletable WHERE chats.id = deletable.id AND chats.archived = true; + +-- name: GetChatsUpdatedAfter :many +-- Retrieves chats updated after the given timestamp for telemetry +-- snapshot collection. Uses updated_at so that long-running chats +-- still appear in each snapshot window while they are active. +SELECT + id, owner_id, created_at, updated_at, status, + (parent_chat_id IS NOT NULL)::bool AS has_parent, + root_chat_id, workspace_id, + mode, archived, last_model_config_id +FROM chats +WHERE updated_at > @updated_after; + +-- name: GetChatMessageSummariesPerChat :many +-- Aggregates message-level metrics per chat for messages created +-- after the given timestamp. Uses message created_at so that +-- ongoing activity in long-running chats is captured each window. +SELECT + cm.chat_id, + COUNT(*)::bigint AS message_count, + COUNT(*) FILTER (WHERE cm.role = 'user')::bigint AS user_message_count, + COUNT(*) FILTER (WHERE cm.role = 'assistant')::bigint AS assistant_message_count, + COUNT(*) FILTER (WHERE cm.role = 'tool')::bigint AS tool_message_count, + COUNT(*) FILTER (WHERE cm.role = 'system')::bigint AS system_message_count, + COALESCE(SUM(cm.input_tokens), 0)::bigint AS total_input_tokens, + COALESCE(SUM(cm.output_tokens), 0)::bigint AS total_output_tokens, + COALESCE(SUM(cm.reasoning_tokens), 0)::bigint AS total_reasoning_tokens, + COALESCE(SUM(cm.cache_creation_tokens), 0)::bigint AS total_cache_creation_tokens, + COALESCE(SUM(cm.cache_read_tokens), 0)::bigint AS total_cache_read_tokens, + COALESCE(SUM(cm.total_cost_micros), 0)::bigint AS total_cost_micros, + COALESCE(SUM(cm.runtime_ms), 0)::bigint AS total_runtime_ms, + COUNT(DISTINCT cm.model_config_id)::bigint AS distinct_model_count, + COUNT(*) FILTER (WHERE cm.compressed)::bigint AS compressed_message_count +FROM chat_messages cm +WHERE cm.created_at > @created_after + AND cm.deleted = false +GROUP BY cm.chat_id; + +-- name: GetChatModelConfigsForTelemetry :many +-- Returns all model configurations for telemetry snapshot collection. +SELECT id, provider, model, context_limit, enabled, is_default +FROM chat_model_configs +WHERE deleted = false; diff --git a/coderd/telemetry/telemetry.go b/coderd/telemetry/telemetry.go index 7478be3626..f59343b6e5 100644 --- a/coderd/telemetry/telemetry.go +++ b/coderd/telemetry/telemetry.go @@ -776,6 +776,40 @@ func (r *remoteReporter) createSnapshot() (*Snapshot, error) { return nil }) + eg.Go(func() error { + chats, err := r.options.Database.GetChatsUpdatedAfter(ctx, createdAfter) + if err != nil { + return xerrors.Errorf("get chats updated after: %w", err) + } + snapshot.Chats = make([]Chat, 0, len(chats)) + for _, chat := range chats { + snapshot.Chats = append(snapshot.Chats, ConvertChat(chat)) + } + return nil + }) + eg.Go(func() error { + summaries, err := r.options.Database.GetChatMessageSummariesPerChat(ctx, createdAfter) + if err != nil { + return xerrors.Errorf("get chat message summaries: %w", err) + } + snapshot.ChatMessageSummaries = make([]ChatMessageSummary, 0, len(summaries)) + for _, s := range summaries { + snapshot.ChatMessageSummaries = append(snapshot.ChatMessageSummaries, ConvertChatMessageSummary(s)) + } + return nil + }) + eg.Go(func() error { + configs, err := r.options.Database.GetChatModelConfigsForTelemetry(ctx) + if err != nil { + return xerrors.Errorf("get chat model configs: %w", err) + } + snapshot.ChatModelConfigs = make([]ChatModelConfig, 0, len(configs)) + for _, c := range configs { + snapshot.ChatModelConfigs = append(snapshot.ChatModelConfigs, ConvertChatModelConfig(c)) + } + return nil + }) + err := eg.Wait() if err != nil { return nil, err @@ -1503,6 +1537,9 @@ type Snapshot struct { AIBridgeInterceptionsSummaries []AIBridgeInterceptionsSummary `json:"aibridge_interceptions_summaries"` BoundaryUsageSummary *BoundaryUsageSummary `json:"boundary_usage_summary"` FirstUserOnboarding *FirstUserOnboarding `json:"first_user_onboarding"` + Chats []Chat `json:"chats"` + ChatMessageSummaries []ChatMessageSummary `json:"chat_message_summaries"` + ChatModelConfigs []ChatModelConfig `json:"chat_model_configs"` } // Deployment contains information about the host running Coder. @@ -2113,6 +2150,66 @@ func ConvertTask(task database.Task) Task { return t } +// ConvertChat converts a database chat row to a telemetry Chat. +func ConvertChat(dbChat database.GetChatsUpdatedAfterRow) Chat { + c := Chat{ + ID: dbChat.ID, + OwnerID: dbChat.OwnerID, + CreatedAt: dbChat.CreatedAt, + UpdatedAt: dbChat.UpdatedAt, + Status: string(dbChat.Status), + HasParent: dbChat.HasParent, + Archived: dbChat.Archived, + LastModelConfigID: dbChat.LastModelConfigID, + } + if dbChat.RootChatID.Valid { + c.RootChatID = &dbChat.RootChatID.UUID + } + if dbChat.WorkspaceID.Valid { + c.WorkspaceID = &dbChat.WorkspaceID.UUID + } + if dbChat.Mode.Valid { + mode := string(dbChat.Mode.ChatMode) + c.Mode = &mode + } + return c +} + +// ConvertChatMessageSummary converts a database chat message +// summary row to a telemetry ChatMessageSummary. +func ConvertChatMessageSummary(dbRow database.GetChatMessageSummariesPerChatRow) ChatMessageSummary { + return ChatMessageSummary{ + ChatID: dbRow.ChatID, + MessageCount: dbRow.MessageCount, + UserMessageCount: dbRow.UserMessageCount, + AssistantMessageCount: dbRow.AssistantMessageCount, + ToolMessageCount: dbRow.ToolMessageCount, + SystemMessageCount: dbRow.SystemMessageCount, + TotalInputTokens: dbRow.TotalInputTokens, + TotalOutputTokens: dbRow.TotalOutputTokens, + TotalReasoningTokens: dbRow.TotalReasoningTokens, + TotalCacheCreationTokens: dbRow.TotalCacheCreationTokens, + TotalCacheReadTokens: dbRow.TotalCacheReadTokens, + TotalCostMicros: dbRow.TotalCostMicros, + TotalRuntimeMs: dbRow.TotalRuntimeMs, + DistinctModelCount: dbRow.DistinctModelCount, + CompressedMessageCount: dbRow.CompressedMessageCount, + } +} + +// ConvertChatModelConfig converts a database model config row to a +// telemetry ChatModelConfig. +func ConvertChatModelConfig(dbRow database.GetChatModelConfigsForTelemetryRow) ChatModelConfig { + return ChatModelConfig{ + ID: dbRow.ID, + Provider: dbRow.Provider, + Model: dbRow.Model, + ContextLimit: dbRow.ContextLimit, + Enabled: dbRow.Enabled, + IsDefault: dbRow.IsDefault, + } +} + type telemetryItemKey string // The comment below gets rid of the warning that the name "TelemetryItemKey" has @@ -2234,6 +2331,53 @@ type BoundaryUsageSummary struct { PeriodDurationMilliseconds int64 `json:"period_duration_ms"` } +// Chat contains anonymized metadata about a chat for telemetry. +// Titles and message content are excluded to avoid PII leakage. +type Chat struct { + ID uuid.UUID `json:"id"` + OwnerID uuid.UUID `json:"owner_id"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + Status string `json:"status"` + HasParent bool `json:"has_parent"` + RootChatID *uuid.UUID `json:"root_chat_id"` + WorkspaceID *uuid.UUID `json:"workspace_id"` + Mode *string `json:"mode"` + Archived bool `json:"archived"` + LastModelConfigID uuid.UUID `json:"last_model_config_id"` +} + +// ChatMessageSummary contains per-chat aggregated message metrics +// for telemetry. Individual message content is never included. +type ChatMessageSummary struct { + ChatID uuid.UUID `json:"chat_id"` + MessageCount int64 `json:"message_count"` + UserMessageCount int64 `json:"user_message_count"` + AssistantMessageCount int64 `json:"assistant_message_count"` + ToolMessageCount int64 `json:"tool_message_count"` + SystemMessageCount int64 `json:"system_message_count"` + TotalInputTokens int64 `json:"total_input_tokens"` + TotalOutputTokens int64 `json:"total_output_tokens"` + TotalReasoningTokens int64 `json:"total_reasoning_tokens"` + TotalCacheCreationTokens int64 `json:"total_cache_creation_tokens"` + TotalCacheReadTokens int64 `json:"total_cache_read_tokens"` + TotalCostMicros int64 `json:"total_cost_micros"` + TotalRuntimeMs int64 `json:"total_runtime_ms"` + DistinctModelCount int64 `json:"distinct_model_count"` + CompressedMessageCount int64 `json:"compressed_message_count"` +} + +// ChatModelConfig contains model configuration metadata for +// telemetry. Sensitive fields like API keys are excluded. +type ChatModelConfig struct { + ID uuid.UUID `json:"id"` + Provider string `json:"provider"` + Model string `json:"model"` + ContextLimit int64 `json:"context_limit"` + Enabled bool `json:"enabled"` + IsDefault bool `json:"is_default"` +} + func ConvertAIBridgeInterceptionsSummary(endTime time.Time, provider, model, client string, summary database.CalculateAIBridgeInterceptionsTelemetrySummaryRow) AIBridgeInterceptionsSummary { return AIBridgeInterceptionsSummary{ ID: uuid.New(), diff --git a/coderd/telemetry/telemetry_test.go b/coderd/telemetry/telemetry_test.go index 005839cc26..56d22f5172 100644 --- a/coderd/telemetry/telemetry_test.go +++ b/coderd/telemetry/telemetry_test.go @@ -1549,3 +1549,303 @@ func TestTelemetry_BoundaryUsageSummary(t *testing.T) { require.Nil(t, snapshot2.BoundaryUsageSummary) }) } + +func TestChatsTelemetry(t *testing.T) { + t.Parallel() + + ctx := testutil.Context(t, testutil.WaitMedium) + db, _ := dbtestutil.NewDB(t) + + user := dbgen.User(t, db, database.User{}) + + // Create chat providers (required FK for model configs). + _, err := db.InsertChatProvider(ctx, database.InsertChatProviderParams{ + Provider: "anthropic", + DisplayName: "Anthropic", + Enabled: true, + CentralApiKeyEnabled: true, + }) + require.NoError(t, err) + _, err = db.InsertChatProvider(ctx, database.InsertChatProviderParams{ + Provider: "openai", + DisplayName: "OpenAI", + Enabled: true, + CentralApiKeyEnabled: true, + }) + require.NoError(t, err) + + // Create a model config. + modelCfg, err := db.InsertChatModelConfig(ctx, database.InsertChatModelConfigParams{ + Provider: "anthropic", + Model: "claude-sonnet-4-20250514", + DisplayName: "Claude Sonnet", + Enabled: true, + IsDefault: true, + ContextLimit: 200000, + CompressionThreshold: 70, + Options: json.RawMessage("{}"), + }) + require.NoError(t, err) + + // Create a second model config to test full dump. + modelCfg2, err := db.InsertChatModelConfig(ctx, database.InsertChatModelConfigParams{ + Provider: "openai", + Model: "gpt-4o", + DisplayName: "GPT-4o", + Enabled: true, + IsDefault: false, + ContextLimit: 128000, + CompressionThreshold: 70, + Options: json.RawMessage("{}"), + }) + require.NoError(t, err) + + // Create a soft-deleted model config — should NOT appear in telemetry. + deletedCfg, err := db.InsertChatModelConfig(ctx, database.InsertChatModelConfigParams{ + Provider: "anthropic", + Model: "claude-deleted", + DisplayName: "Deleted Model", + Enabled: true, + IsDefault: false, + ContextLimit: 100000, + CompressionThreshold: 70, + Options: json.RawMessage("{}"), + }) + require.NoError(t, err) + err = db.DeleteChatModelConfigByID(ctx, deletedCfg.ID) + require.NoError(t, err) + + // Create a root chat with a workspace. + org, err := db.GetDefaultOrganization(ctx) + require.NoError(t, err) + job := dbgen.ProvisionerJob(t, db, nil, database.ProvisionerJob{ + OrganizationID: org.ID, + Type: database.ProvisionerJobTypeTemplateVersionDryRun, + }) + tpl := dbgen.Template(t, db, database.Template{ + OrganizationID: org.ID, + CreatedBy: user.ID, + }) + tv := dbgen.TemplateVersion(t, db, database.TemplateVersion{ + OrganizationID: org.ID, + TemplateID: uuid.NullUUID{UUID: tpl.ID, Valid: true}, + CreatedBy: user.ID, + JobID: job.ID, + }) + ws := dbgen.Workspace(t, db, database.WorkspaceTable{ + OwnerID: user.ID, + OrganizationID: org.ID, + TemplateID: tpl.ID, + }) + _ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{ + Transition: database.WorkspaceTransitionStart, + Reason: database.BuildReasonInitiator, + WorkspaceID: ws.ID, + TemplateVersionID: tv.ID, + JobID: job.ID, + }) + + rootChat, err := db.InsertChat(ctx, database.InsertChatParams{ + OwnerID: user.ID, + LastModelConfigID: modelCfg.ID, + Title: "Root Chat", + Status: database.ChatStatusRunning, + WorkspaceID: uuid.NullUUID{UUID: ws.ID, Valid: true}, + Mode: database.NullChatMode{ChatMode: database.ChatModeComputerUse, Valid: true}, + }) + require.NoError(t, err) + + // Create a child chat (has parent + root). + childChat, err := db.InsertChat(ctx, database.InsertChatParams{ + OwnerID: user.ID, + LastModelConfigID: modelCfg2.ID, + Title: "Child Chat", + Status: database.ChatStatusCompleted, + ParentChatID: uuid.NullUUID{UUID: rootChat.ID, Valid: true}, + RootChatID: uuid.NullUUID{UUID: rootChat.ID, Valid: true}, + }) + require.NoError(t, err) + + // Insert messages for root chat: 2 user, 2 assistant, 1 tool. + _, err = db.InsertChatMessages(ctx, database.InsertChatMessagesParams{ + ChatID: rootChat.ID, + CreatedBy: []uuid.UUID{user.ID, uuid.Nil, user.ID, uuid.Nil, uuid.Nil}, + ModelConfigID: []uuid.UUID{modelCfg.ID, modelCfg.ID, modelCfg.ID, modelCfg.ID, modelCfg.ID}, + Role: []database.ChatMessageRole{database.ChatMessageRoleUser, database.ChatMessageRoleAssistant, database.ChatMessageRoleUser, database.ChatMessageRoleAssistant, database.ChatMessageRoleTool}, + Content: []string{`[{"type":"text","text":"hello"}]`, `[{"type":"text","text":"hi"}]`, `[{"type":"text","text":"help"}]`, `[{"type":"text","text":"sure"}]`, `[{"type":"text","text":"result"}]`}, + ContentVersion: []int16{1, 1, 1, 1, 1}, + Visibility: []database.ChatMessageVisibility{database.ChatMessageVisibilityBoth, database.ChatMessageVisibilityBoth, database.ChatMessageVisibilityBoth, database.ChatMessageVisibilityBoth, database.ChatMessageVisibilityBoth}, + InputTokens: []int64{100, 200, 150, 300, 0}, + OutputTokens: []int64{0, 50, 0, 100, 0}, + TotalTokens: []int64{100, 250, 150, 400, 0}, + ReasoningTokens: []int64{0, 10, 0, 20, 0}, + CacheCreationTokens: []int64{50, 0, 30, 0, 0}, + CacheReadTokens: []int64{0, 25, 0, 40, 0}, + ContextLimit: []int64{200000, 200000, 200000, 200000, 200000}, + Compressed: []bool{false, false, false, false, false}, + TotalCostMicros: []int64{1000, 2000, 1500, 3000, 0}, + RuntimeMs: []int64{0, 500, 0, 800, 100}, + ProviderResponseID: []string{"", "resp-1", "", "resp-2", ""}, + }) + require.NoError(t, err) + + // Insert messages for child chat: 1 user, 1 assistant (compressed). + _, err = db.InsertChatMessages(ctx, database.InsertChatMessagesParams{ + ChatID: childChat.ID, + CreatedBy: []uuid.UUID{user.ID, uuid.Nil}, + ModelConfigID: []uuid.UUID{modelCfg2.ID, modelCfg2.ID}, + Role: []database.ChatMessageRole{database.ChatMessageRoleUser, database.ChatMessageRoleAssistant}, + Content: []string{`[{"type":"text","text":"q"}]`, `[{"type":"text","text":"a"}]`}, + ContentVersion: []int16{1, 1}, + Visibility: []database.ChatMessageVisibility{database.ChatMessageVisibilityBoth, database.ChatMessageVisibilityBoth}, + InputTokens: []int64{500, 600}, + OutputTokens: []int64{0, 200}, + TotalTokens: []int64{500, 800}, + ReasoningTokens: []int64{0, 50}, + CacheCreationTokens: []int64{100, 0}, + CacheReadTokens: []int64{0, 75}, + ContextLimit: []int64{128000, 128000}, + Compressed: []bool{false, true}, + TotalCostMicros: []int64{5000, 8000}, + RuntimeMs: []int64{0, 1200}, + ProviderResponseID: []string{"", "resp-3"}, + }) + require.NoError(t, err) + + // Insert a soft-deleted message on root chat with large token values. + // This acts as "poison" — if the deleted filter is missing, totals + // will be inflated and assertions below will fail. + poisonMsgs, err := db.InsertChatMessages(ctx, database.InsertChatMessagesParams{ + ChatID: rootChat.ID, + CreatedBy: []uuid.UUID{uuid.Nil}, + ModelConfigID: []uuid.UUID{modelCfg.ID}, + Role: []database.ChatMessageRole{database.ChatMessageRoleAssistant}, + Content: []string{`[{"type":"text","text":"poison"}]`}, + ContentVersion: []int16{1}, + Visibility: []database.ChatMessageVisibility{database.ChatMessageVisibilityBoth}, + InputTokens: []int64{999999}, + OutputTokens: []int64{999999}, + TotalTokens: []int64{999999}, + ReasoningTokens: []int64{999999}, + CacheCreationTokens: []int64{999999}, + CacheReadTokens: []int64{999999}, + ContextLimit: []int64{200000}, + Compressed: []bool{false}, + TotalCostMicros: []int64{999999}, + RuntimeMs: []int64{999999}, + ProviderResponseID: []string{""}, + }) + require.NoError(t, err) + err = db.SoftDeleteChatMessageByID(ctx, poisonMsgs[0].ID) + require.NoError(t, err) + + _, snapshot := collectSnapshot(ctx, t, db, nil) + + // --- Assert Chats --- + require.Len(t, snapshot.Chats, 2) + + // Find root and child by HasParent flag. + var foundRoot, foundChild *telemetry.Chat + for i := range snapshot.Chats { + if !snapshot.Chats[i].HasParent { + foundRoot = &snapshot.Chats[i] + } else { + foundChild = &snapshot.Chats[i] + } + } + require.NotNil(t, foundRoot, "expected root chat") + require.NotNil(t, foundChild, "expected child chat") + + // Root chat assertions. + assert.Equal(t, rootChat.ID, foundRoot.ID) + assert.Equal(t, user.ID, foundRoot.OwnerID) + assert.Equal(t, "running", foundRoot.Status) + assert.False(t, foundRoot.HasParent) + assert.Nil(t, foundRoot.RootChatID) + require.NotNil(t, foundRoot.WorkspaceID) + assert.Equal(t, ws.ID, *foundRoot.WorkspaceID) + assert.Equal(t, modelCfg.ID, foundRoot.LastModelConfigID) + require.NotNil(t, foundRoot.Mode) + assert.Equal(t, "computer_use", *foundRoot.Mode) + assert.False(t, foundRoot.Archived) + + // Child chat assertions. + assert.Equal(t, childChat.ID, foundChild.ID) + assert.Equal(t, user.ID, foundChild.OwnerID) + assert.True(t, foundChild.HasParent) + require.NotNil(t, foundChild.RootChatID) + assert.Equal(t, rootChat.ID, *foundChild.RootChatID) + assert.Nil(t, foundChild.WorkspaceID) + assert.Equal(t, "completed", foundChild.Status) + assert.Equal(t, modelCfg2.ID, foundChild.LastModelConfigID) + assert.Nil(t, foundChild.Mode) + assert.False(t, foundChild.Archived) + + // --- Assert ChatMessageSummaries --- + require.Len(t, snapshot.ChatMessageSummaries, 2) + + summaryMap := make(map[uuid.UUID]telemetry.ChatMessageSummary) + for _, s := range snapshot.ChatMessageSummaries { + summaryMap[s.ChatID] = s + } + + // Root chat summary: 2 user + 2 assistant + 1 tool = 5 messages. + rootSummary, ok := summaryMap[rootChat.ID] + require.True(t, ok, "expected summary for root chat") + assert.Equal(t, int64(5), rootSummary.MessageCount) + assert.Equal(t, int64(2), rootSummary.UserMessageCount) + assert.Equal(t, int64(2), rootSummary.AssistantMessageCount) + assert.Equal(t, int64(1), rootSummary.ToolMessageCount) + assert.Equal(t, int64(0), rootSummary.SystemMessageCount) + assert.Equal(t, int64(750), rootSummary.TotalInputTokens) // 100+200+150+300+0 + assert.Equal(t, int64(150), rootSummary.TotalOutputTokens) // 0+50+0+100+0 + assert.Equal(t, int64(30), rootSummary.TotalReasoningTokens) // 0+10+0+20+0 + assert.Equal(t, int64(80), rootSummary.TotalCacheCreationTokens) // 50+0+30+0+0 + assert.Equal(t, int64(65), rootSummary.TotalCacheReadTokens) // 0+25+0+40+0 + assert.Equal(t, int64(7500), rootSummary.TotalCostMicros) // 1000+2000+1500+3000+0 + assert.Equal(t, int64(1400), rootSummary.TotalRuntimeMs) // 0+500+0+800+100 + assert.Equal(t, int64(1), rootSummary.DistinctModelCount) + assert.Equal(t, int64(0), rootSummary.CompressedMessageCount) + + // Child chat summary: 1 user + 1 assistant = 2 messages, 1 compressed. + childSummary, ok := summaryMap[childChat.ID] + require.True(t, ok, "expected summary for child chat") + assert.Equal(t, int64(2), childSummary.MessageCount) + assert.Equal(t, int64(1), childSummary.UserMessageCount) + assert.Equal(t, int64(1), childSummary.AssistantMessageCount) + assert.Equal(t, int64(1100), childSummary.TotalInputTokens) // 500+600 + assert.Equal(t, int64(200), childSummary.TotalOutputTokens) // 0+200 + assert.Equal(t, int64(50), childSummary.TotalReasoningTokens) // 0+50 + assert.Equal(t, int64(0), childSummary.ToolMessageCount) + assert.Equal(t, int64(0), childSummary.SystemMessageCount) + assert.Equal(t, int64(100), childSummary.TotalCacheCreationTokens) // 100+0 + assert.Equal(t, int64(75), childSummary.TotalCacheReadTokens) // 0+75 + assert.Equal(t, int64(13000), childSummary.TotalCostMicros) // 5000+8000 + assert.Equal(t, int64(1200), childSummary.TotalRuntimeMs) // 0+1200 + assert.Equal(t, int64(1), childSummary.DistinctModelCount) + assert.Equal(t, int64(1), childSummary.CompressedMessageCount) + + // --- Assert ChatModelConfigs --- + require.Len(t, snapshot.ChatModelConfigs, 2) + + configMap := make(map[uuid.UUID]telemetry.ChatModelConfig) + for _, c := range snapshot.ChatModelConfigs { + configMap[c.ID] = c + } + + cfg1, ok := configMap[modelCfg.ID] + require.True(t, ok) + assert.Equal(t, "anthropic", cfg1.Provider) + assert.Equal(t, "claude-sonnet-4-20250514", cfg1.Model) + assert.Equal(t, int64(200000), cfg1.ContextLimit) + assert.True(t, cfg1.Enabled) + assert.True(t, cfg1.IsDefault) + + cfg2, ok := configMap[modelCfg2.ID] + require.True(t, ok) + assert.Equal(t, "openai", cfg2.Provider) + assert.Equal(t, "gpt-4o", cfg2.Model) + assert.Equal(t, int64(128000), cfg2.ContextLimit) + assert.True(t, cfg2.Enabled) + assert.False(t, cfg2.IsDefault) +}