From fc3508dc60fd877ebf9618eab40b5fc8e1ac3be4 Mon Sep 17 00:00:00 2001 From: Ethan <39577870+ethanndickson@users.noreply.github.com> Date: Thu, 19 Mar 2026 00:54:32 +1100 Subject: [PATCH] feat: configure acquire chat batch size (#23196) ## Summary - add a hidden deployment config option for chat acquire batch size (`CODER_CHAT_ACQUIRE_BATCH_SIZE` / `chat.acquireBatchSize`) - thread the configured value into chatd startup while preserving the existing default of `10` - clamp the deployment value to the `int32` range before passing it into chatd - regenerate the API/docs/types/testdata artifacts for the new config field ## Why `chatd` currently acquires pending chats in batches of `10` via a compile-time default. This change makes that batch size operator-configurable from deployment config, so we can tune acquisition behavior without another code change. --- cli/testdata/server-config.yaml.golden | 5 +++++ coderd/apidoc/docs.go | 11 ++++++++++ coderd/apidoc/swagger.json | 11 ++++++++++ coderd/chatd/chatd.go | 14 +++++++++--- coderd/coderd.go | 30 +++++++++++++++++--------- codersdk/deployment.go | 22 +++++++++++++++++++ docs/reference/api/general.md | 3 +++ docs/reference/api/schemas.md | 24 +++++++++++++++++++++ site/src/api/typesGenerated.ts | 6 ++++++ 9 files changed, 113 insertions(+), 13 deletions(-) diff --git a/cli/testdata/server-config.yaml.golden b/cli/testdata/server-config.yaml.golden index 974dec3b28..179765bdeb 100644 --- a/cli/testdata/server-config.yaml.golden +++ b/cli/testdata/server-config.yaml.golden @@ -752,6 +752,11 @@ workspace_prebuilds: # limit; disabled when set to zero. # (default: 3, type: int) failure_hard_limit: 3 +# Configure the background chat processing daemon. +chat: + # How many pending chats a worker should acquire per polling cycle. + # (default: 10, type: int) + acquireBatchSize: 10 aibridge: # Whether to start an in-memory aibridged instance. # (default: false, type: bool) diff --git a/coderd/apidoc/docs.go b/coderd/apidoc/docs.go index d441b185b1..236b86dc9b 100644 --- a/coderd/apidoc/docs.go +++ b/coderd/apidoc/docs.go @@ -12714,6 +12714,9 @@ const docTemplate = `{ }, "bridge": { "$ref": "#/definitions/codersdk.AIBridgeConfig" + }, + "chat": { + "$ref": "#/definitions/codersdk.ChatConfig" } } }, @@ -13771,6 +13774,14 @@ const docTemplate = `{ } } }, + "codersdk.ChatConfig": { + "type": "object", + "properties": { + "acquire_batch_size": { + "type": "integer" + } + } + }, "codersdk.ConnectionLatency": { "type": "object", "properties": { diff --git a/coderd/apidoc/swagger.json b/coderd/apidoc/swagger.json index 5bc51ca337..2fd2e29e04 100644 --- a/coderd/apidoc/swagger.json +++ b/coderd/apidoc/swagger.json @@ -11320,6 +11320,9 @@ }, "bridge": { "$ref": "#/definitions/codersdk.AIBridgeConfig" + }, + "chat": { + "$ref": "#/definitions/codersdk.ChatConfig" } } }, @@ -12342,6 +12345,14 @@ } } }, + "codersdk.ChatConfig": { + "type": "object", + "properties": { + "acquire_batch_size": { + "type": "integer" + } + } + }, "codersdk.ConnectionLatency": { "type": "object", "properties": { diff --git a/coderd/chatd/chatd.go b/coderd/chatd/chatd.go index 8738bdd2e1..51874b7fd7 100644 --- a/coderd/chatd/chatd.go +++ b/coderd/chatd/chatd.go @@ -58,11 +58,11 @@ const ( // of 5 means recovery runs at 1/5 of the stale-after duration. staleRecoveryIntervalDivisor = 5 - // maxChatsPerAcquire is the maximum number of chats to + // DefaultMaxChatsPerAcquire is the maximum number of chats to // acquire in a single processOnce call. Batching avoids // waiting a full polling interval between acquisitions // when many chats are pending. - maxChatsPerAcquire int32 = 10 + DefaultMaxChatsPerAcquire int32 = 10 defaultSubagentInstruction = "You are running as a delegated sub-agent chat. Complete the delegated task and provide clear, concise assistant responses for the parent agent." ) @@ -98,6 +98,7 @@ type Server struct { // Configuration pendingChatAcquireInterval time.Duration + maxChatsPerAcquire int32 inFlightChatStaleAfter time.Duration } @@ -1174,6 +1175,7 @@ type Config struct { ReplicaID uuid.UUID SubscribeFn SubscribeFn PendingChatAcquireInterval time.Duration + MaxChatsPerAcquire int32 InFlightChatStaleAfter time.Duration AgentConn AgentConnFunc CreateWorkspace chattool.CreateWorkspaceFn @@ -1199,6 +1201,11 @@ func New(cfg Config) *Server { inFlightChatStaleAfter = DefaultInFlightChatStaleAfter } + maxChatsPerAcquire := cfg.MaxChatsPerAcquire + if maxChatsPerAcquire <= 0 { + maxChatsPerAcquire = DefaultMaxChatsPerAcquire + } + workerID := cfg.ReplicaID if workerID == uuid.Nil { workerID = uuid.New() @@ -1219,6 +1226,7 @@ func New(cfg Config) *Server { providerAPIKeys: cfg.ProviderAPIKeys, instructionCache: make(map[uuid.UUID]cachedInstruction), pendingChatAcquireInterval: pendingChatAcquireInterval, + maxChatsPerAcquire: maxChatsPerAcquire, inFlightChatStaleAfter: inFlightChatStaleAfter, } @@ -1272,7 +1280,7 @@ func (p *Server) processOnce(ctx context.Context) { chats, err := p.db.AcquireChats(acquireCtx, database.AcquireChatsParams{ StartedAt: time.Now(), WorkerID: p.workerID, - NumChats: maxChatsPerAcquire, + NumChats: p.maxChatsPerAcquire, }) acquireCancel() if err != nil { diff --git a/coderd/coderd.go b/coderd/coderd.go index 247fef523f..01856736fb 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -10,6 +10,7 @@ import ( "flag" "fmt" "io" + "math" "net/http" httppprof "net/http/pprof" "net/url" @@ -766,17 +767,26 @@ func New(options *Options) *API { } api.agentProvider = stn + maxChatsPerAcquire := options.DeploymentValues.AI.Chat.AcquireBatchSize.Value() + if maxChatsPerAcquire > math.MaxInt32 { + maxChatsPerAcquire = math.MaxInt32 + } + if maxChatsPerAcquire < math.MinInt32 { + maxChatsPerAcquire = math.MinInt32 + } + api.chatDaemon = chatd.New(chatd.Config{ - Logger: options.Logger.Named("chats"), - Database: options.Database, - ReplicaID: api.ID, - SubscribeFn: options.ChatSubscribeFn, - ProviderAPIKeys: chatProviderAPIKeysFromDeploymentValues(options.DeploymentValues), - AgentConn: api.agentProvider.AgentConn, - CreateWorkspace: api.chatCreateWorkspace, - StartWorkspace: api.chatStartWorkspace, - Pubsub: options.Pubsub, - WebpushDispatcher: options.WebPushDispatcher, + Logger: options.Logger.Named("chats"), + Database: options.Database, + ReplicaID: api.ID, + SubscribeFn: options.ChatSubscribeFn, + MaxChatsPerAcquire: int32(maxChatsPerAcquire), //nolint:gosec // maxChatsPerAcquire is clamped to int32 range above. + ProviderAPIKeys: chatProviderAPIKeysFromDeploymentValues(options.DeploymentValues), + AgentConn: api.agentProvider.AgentConn, + CreateWorkspace: api.chatCreateWorkspace, + StartWorkspace: api.chatStartWorkspace, + Pubsub: options.Pubsub, + WebpushDispatcher: options.WebPushDispatcher, }) gitSyncLogger := options.Logger.Named("gitsync") refresher := gitsync.NewRefresher( diff --git a/codersdk/deployment.go b/codersdk/deployment.go index cbe582f806..296eb33b02 100644 --- a/codersdk/deployment.go +++ b/codersdk/deployment.go @@ -1437,6 +1437,11 @@ func (c *DeploymentValues) Options() serpent.OptionSet { Parent: &deploymentGroupNotifications, YAML: "inbox", } + deploymentGroupChat = serpent.Group{ + Name: "Chat", + YAML: "chat", + Description: "Configure the background chat processing daemon.", + } deploymentGroupAIBridge = serpent.Group{ Name: "AI Bridge", YAML: "aibridge", @@ -3600,6 +3605,18 @@ Write out the current server config as YAML to stdout.`, Group: &deploymentGroupClient, YAML: "hideAITasks", }, + // Chat Options + { + Name: "Chat: Acquire Batch Size", + Description: "How many pending chats a worker should acquire per polling cycle.", + Flag: "chat-acquire-batch-size", + Env: "CODER_CHAT_ACQUIRE_BATCH_SIZE", + Value: &c.AI.Chat.AcquireBatchSize, + Default: "10", + Group: &deploymentGroupChat, + YAML: "acquireBatchSize", + Hidden: true, // Hidden because most operators should not need to modify this. + }, // AI Bridge Options { Name: "AI Bridge Enabled", @@ -4052,9 +4069,14 @@ type AIBridgeProxyConfig struct { UpstreamProxyCA serpent.String `json:"upstream_proxy_ca" typescript:",notnull"` } +type ChatConfig struct { + AcquireBatchSize serpent.Int64 `json:"acquire_batch_size" typescript:",notnull"` +} + type AIConfig struct { BridgeConfig AIBridgeConfig `json:"bridge,omitempty"` BridgeProxyConfig AIBridgeProxyConfig `json:"aibridge_proxy,omitempty"` + Chat ChatConfig `json:"chat,omitempty" typescript:",notnull"` } type SupportConfig struct { diff --git a/docs/reference/api/general.md b/docs/reference/api/general.md index 921aeaef1c..00143a418d 100644 --- a/docs/reference/api/general.md +++ b/docs/reference/api/general.md @@ -204,6 +204,9 @@ curl -X GET http://coder-server:8080/api/v2/deployment/config \ "retention": 0, "send_actor_headers": true, "structured_logging": true + }, + "chat": { + "acquire_batch_size": 0 } }, "allow_workspace_renames": true, diff --git a/docs/reference/api/schemas.md b/docs/reference/api/schemas.md index 3a197078bf..9200650837 100644 --- a/docs/reference/api/schemas.md +++ b/docs/reference/api/schemas.md @@ -786,6 +786,9 @@ "retention": 0, "send_actor_headers": true, "structured_logging": true + }, + "chat": { + "acquire_batch_size": 0 } } ``` @@ -796,6 +799,7 @@ |------------------|--------------------------------------------------------------|----------|--------------|-------------| | `aibridge_proxy` | [codersdk.AIBridgeProxyConfig](#codersdkaibridgeproxyconfig) | false | | | | `bridge` | [codersdk.AIBridgeConfig](#codersdkaibridgeconfig) | false | | | +| `chat` | [codersdk.ChatConfig](#codersdkchatconfig) | false | | | ## codersdk.APIAllowListTarget @@ -1557,6 +1561,20 @@ AuthorizationObject can represent a "set" of objects, such as: all workspaces in | `one_time_passcode` | string | true | | | | `password` | string | true | | | +## codersdk.ChatConfig + +```json +{ + "acquire_batch_size": 0 +} +``` + +### Properties + +| Name | Type | Required | Restrictions | Description | +|----------------------|---------|----------|--------------|-------------| +| `acquire_batch_size` | integer | false | | | + ## codersdk.ConnectionLatency ```json @@ -2720,6 +2738,9 @@ CreateWorkspaceRequest provides options for creating a new workspace. Only one o "retention": 0, "send_actor_headers": true, "structured_logging": true + }, + "chat": { + "acquire_batch_size": 0 } }, "allow_workspace_renames": true, @@ -3292,6 +3313,9 @@ CreateWorkspaceRequest provides options for creating a new workspace. Only one o "retention": 0, "send_actor_headers": true, "structured_logging": true + }, + "chat": { + "acquire_batch_size": 0 } }, "allow_workspace_renames": true, diff --git a/site/src/api/typesGenerated.ts b/site/src/api/typesGenerated.ts index 46113d0a99..f86e9e92a9 100644 --- a/site/src/api/typesGenerated.ts +++ b/site/src/api/typesGenerated.ts @@ -136,6 +136,7 @@ export interface AIBridgeUserPrompt { export interface AIConfig { readonly bridge?: AIBridgeConfig; readonly aibridge_proxy?: AIBridgeProxyConfig; + readonly chat?: ChatConfig; } // From codersdk/allowlist.go @@ -1071,6 +1072,11 @@ export interface Chat { readonly archived: boolean; } +// From codersdk/deployment.go +export interface ChatConfig { + readonly acquire_batch_size: number; +} + // From codersdk/chats.go /** * ChatCostChatBreakdown contains per-root-chat cost aggregation.