From 6acfe6c835e8474d0900c64c12faccaec0d36cd8 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Wed, 27 May 2026 09:45:36 +0100 Subject: [PATCH] fix: classify quota errors as usage_limit instead of auth (#25676) Fixes CODAGT-484. - Removed "quota", "billing", "insufficient_quota", "payment required" from `authStrongPatterns` - Added `usageLimitPatterns` slice with those patterns - Added `usageLimitMatch` signal and rule between overloaded and authStrong in priority - Added terminal/retry messages for `ChatErrorKindUsageLimit` - Simplified auth message (removed billing reference) - Frontend: conditional `!usageLimitStatus.provider` guard on the "View Usage" Alert - Added `TestClassify_UsageLimitBeatsAuth` with 5 cases including real production OpenAI error - Added `ProviderQuotaExceeded` story asserting no "View Usage" link and correct `ChatStatusCallout` rendering > Generated with [Coder Agents](https://coder.com/agents) --- coderd/x/chatd/chaterror/classify.go | 8 ++ coderd/x/chatd/chaterror/classify_test.go | 86 +++++++++++++++++-- coderd/x/chatd/chaterror/message.go | 13 ++- coderd/x/chatd/chaterror/message_test.go | 14 +++ coderd/x/chatd/chaterror/signals.go | 6 +- .../LiveStreamTail.stories.tsx | 34 ++++++++ .../ChatConversation/LiveStreamTail.tsx | 2 +- 7 files changed, 150 insertions(+), 13 deletions(-) diff --git a/coderd/x/chatd/chaterror/classify.go b/coderd/x/chatd/chaterror/classify.go index c02cf71f1b..73e50f083b 100644 --- a/coderd/x/chatd/chaterror/classify.go +++ b/coderd/x/chatd/chaterror/classify.go @@ -197,6 +197,7 @@ func Classify(err error) ClassifiedError { retryableHTTP2StreamReset, hasHTTP2StreamReset := classifyHTTP2StreamReset(err) deadline := errors.Is(err, context.DeadlineExceeded) || strings.Contains(lower, "context deadline exceeded") overloadedMatch := statusCode == 529 || containsAny(lower, overloadedPatterns...) + usageLimitMatch := containsAny(lower, usageLimitPatterns...) authStrong := statusCode == 401 || containsAny(lower, authStrongPatterns...) configMatch := containsAny(lower, configPatterns...) authWeak := statusCode == 403 || containsAny(lower, authWeakPatterns...) @@ -216,6 +217,8 @@ func Classify(err error) ClassifiedError { // transient-looking errors like "503 invalid model" fail fast. // Overloaded stays ahead because 529/overloaded is a dedicated // provider saturation signal, not a common transport wrapper. + // Usage-limit fires before auth so that quota/billing text wins + // over whatever HTTP status code the provider happened to use. // Strong auth still stays above config because bad credentials are // the root cause when both signals appear. rules := []struct { @@ -228,6 +231,11 @@ func Classify(err error) ClassifiedError { kind: codersdk.ChatErrorKindOverloaded, retryable: true, }, + { + match: usageLimitMatch, + kind: codersdk.ChatErrorKindUsageLimit, + retryable: false, + }, { match: authStrong, kind: codersdk.ChatErrorKindAuth, diff --git a/coderd/x/chatd/chaterror/classify_test.go b/coderd/x/chatd/chaterror/classify_test.go index 6fd036cae5..457704bd5f 100644 --- a/coderd/x/chatd/chaterror/classify_test.go +++ b/coderd/x/chatd/chaterror/classify_test.go @@ -79,7 +79,7 @@ func TestClassify(t *testing.T) { name: "AuthBeatsConfig", err: xerrors.New("authentication failed: invalid model"), want: chaterror.ClassifiedError{ - Message: "Authentication with the AI provider failed. Check the API key, permissions, and billing settings.", + Message: "Authentication with the AI provider failed. Check the API key and permissions.", Kind: codersdk.ChatErrorKindAuth, Provider: "", Retryable: false, @@ -101,7 +101,7 @@ func TestClassify(t *testing.T) { name: "BareForbiddenClassifiesAsAuth", err: xerrors.New("forbidden"), want: chaterror.ClassifiedError{ - Message: "Authentication with the AI provider failed. Check the API key, permissions, and billing settings.", + Message: "Authentication with the AI provider failed. Check the API key and permissions.", Kind: codersdk.ChatErrorKindAuth, Provider: "", Retryable: false, @@ -112,7 +112,7 @@ func TestClassify(t *testing.T) { name: "ExplicitStatus401ClassifiesAsAuth", err: xerrors.New("status 401 from upstream"), want: chaterror.ClassifiedError{ - Message: "Authentication with the AI provider failed. Check the API key, permissions, and billing settings.", + Message: "Authentication with the AI provider failed. Check the API key and permissions.", Kind: codersdk.ChatErrorKindAuth, Provider: "", Retryable: false, @@ -123,7 +123,7 @@ func TestClassify(t *testing.T) { name: "ExplicitStatus403ClassifiesAsAuth", err: xerrors.New("status 403 from upstream"), want: chaterror.ClassifiedError{ - Message: "Authentication with the AI provider failed. Check the API key, permissions, and billing settings.", + Message: "Authentication with the AI provider failed. Check the API key and permissions.", Kind: codersdk.ChatErrorKindAuth, Provider: "", Retryable: false, @@ -342,10 +342,10 @@ func TestClassify_PatternCoverage(t *testing.T) { {name: "UnauthorizedLiteral", err: "unauthorized", wantKind: codersdk.ChatErrorKindAuth, wantRetry: false}, {name: "InvalidAPIKeyLiteral", err: "invalid api key", wantKind: codersdk.ChatErrorKindAuth, wantRetry: false}, {name: "InvalidAPIKeyUnderscoreLiteral", err: "invalid_api_key", wantKind: codersdk.ChatErrorKindAuth, wantRetry: false}, - {name: "QuotaLiteral", err: "quota", wantKind: codersdk.ChatErrorKindAuth, wantRetry: false}, - {name: "BillingLiteral", err: "billing", wantKind: codersdk.ChatErrorKindAuth, wantRetry: false}, - {name: "InsufficientQuotaLiteral", err: "insufficient_quota", wantKind: codersdk.ChatErrorKindAuth, wantRetry: false}, - {name: "PaymentRequiredLiteral", err: "payment required", wantKind: codersdk.ChatErrorKindAuth, wantRetry: false}, + {name: "QuotaLiteral", err: "quota", wantKind: codersdk.ChatErrorKindUsageLimit, wantRetry: false}, + {name: "BillingLiteral", err: "billing", wantKind: codersdk.ChatErrorKindUsageLimit, wantRetry: false}, + {name: "InsufficientQuotaLiteral", err: "insufficient_quota", wantKind: codersdk.ChatErrorKindUsageLimit, wantRetry: false}, + {name: "PaymentRequiredLiteral", err: "payment required", wantKind: codersdk.ChatErrorKindUsageLimit, wantRetry: false}, {name: "ForbiddenLiteral", err: "forbidden", wantKind: codersdk.ChatErrorKindAuth, wantRetry: false}, {name: "InvalidModelLiteral", err: "invalid model", wantKind: codersdk.ChatErrorKindConfig, wantRetry: false}, {name: "ModelNotFoundLiteral", err: "model not found", wantKind: codersdk.ChatErrorKindConfig, wantRetry: false}, @@ -719,13 +719,81 @@ func TestClassify_StatusCodeBeatsTypedHTTP2StreamError(t *testing.T) { ) require.Equal(t, chaterror.ClassifiedError{ - Message: "Authentication with the AI provider failed. Check the API key, permissions, and billing settings.", + Message: "Authentication with the AI provider failed. Check the API key and permissions.", Kind: codersdk.ChatErrorKindAuth, Retryable: false, StatusCode: 401, }, chaterror.Classify(err)) } +// TestClassify_UsageLimitBeatsAuth verifies that quota/billing text +// patterns classify as usage_limit even when auth signals are present. +func TestClassify_UsageLimitBeatsAuth(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + err string + wantKind codersdk.ChatErrorKind + wantRetry bool + wantStatus int + wantProvider string + }{ + { + name: "QuotaBeatsAuth", + err: "unauthorized: insufficient_quota", + wantKind: codersdk.ChatErrorKindUsageLimit, + wantRetry: false, + }, + { + name: "QuotaWith429Status", + err: "status 429: insufficient_quota", + wantKind: codersdk.ChatErrorKindUsageLimit, + wantRetry: false, + wantStatus: 429, + }, + { + name: "PureAuthStillWorks", + err: "unauthorized", + wantKind: codersdk.ChatErrorKindAuth, + wantRetry: false, + }, + { + name: "Status401StillAuth", + err: "status 401", + wantKind: codersdk.ChatErrorKindAuth, + wantRetry: false, + wantStatus: 401, + }, + { + // Real production error from OpenAI when quota is exceeded. + name: "OpenAIInsufficientQuotaRealWorld", + err: `stream response: received error while streaming: {"type":"insufficient_quota",` + + `"code":"insufficient_quota","message":"You exceeded your current quota, please check ` + + `your plan and billing details. For more information on this error, read the docs: ` + + `https://platform.openai.com/docs/guides/error-codes/api-errors.","param":null}`, + wantKind: codersdk.ChatErrorKindUsageLimit, + wantRetry: false, + wantProvider: "openai", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + classified := chaterror.Classify(xerrors.New(tt.err)) + require.Equal(t, tt.wantKind, classified.Kind) + require.Equal(t, tt.wantRetry, classified.Retryable) + if tt.wantStatus != 0 { + require.Equal(t, tt.wantStatus, classified.StatusCode) + } + if tt.wantProvider != "" { + require.Equal(t, tt.wantProvider, classified.Provider) + } + }) + } +} + // TestClassify_StatusCodeBeatsHTTP2Transport ensures explicit status // codes still win over the new HTTP/2 patterns. func TestClassify_StatusCodeBeatsHTTP2Transport(t *testing.T) { diff --git a/coderd/x/chatd/chaterror/message.go b/coderd/x/chatd/chaterror/message.go index 3bdb4c1482..a551078349 100644 --- a/coderd/x/chatd/chaterror/message.go +++ b/coderd/x/chatd/chaterror/message.go @@ -32,6 +32,17 @@ func terminalMessage(classified ClassifiedError) string { "%s did not start responding in time.", subject, ) + case codersdk.ChatErrorKindUsageLimit: + displayName := providerDisplayName(classified.Provider) + if displayName == "" { + displayName = "the AI provider" + } + return fmt.Sprintf( + "The usage quota for %s has been exceeded."+ + " Check the billing and quota settings for the provider account.", + displayName, + ) + case codersdk.ChatErrorKindAuth: displayName := providerDisplayName(classified.Provider) if displayName == "" { @@ -39,7 +50,7 @@ func terminalMessage(classified ClassifiedError) string { } return fmt.Sprintf( "Authentication with %s failed."+ - " Check the API key, permissions, and billing settings.", + " Check the API key and permissions.", displayName, ) diff --git a/coderd/x/chatd/chaterror/message_test.go b/coderd/x/chatd/chaterror/message_test.go index 81f1dfff8d..87cb375cbc 100644 --- a/coderd/x/chatd/chaterror/message_test.go +++ b/coderd/x/chatd/chaterror/message_test.go @@ -76,6 +76,20 @@ func TestTerminalMessage(t *testing.T) { retryable: false, want: "The chat request failed unexpectedly.", }, + { + name: "UsageLimit_OpenAI", + kind: codersdk.ChatErrorKindUsageLimit, + provider: "openai", + retryable: false, + want: "The usage quota for OpenAI has been exceeded. Check the billing and quota settings for the provider account.", + }, + { + name: "UsageLimit_UnknownProvider", + kind: codersdk.ChatErrorKindUsageLimit, + provider: "", + retryable: false, + want: "The usage quota for the AI provider has been exceeded. Check the billing and quota settings for the provider account.", + }, } for _, tt := range tests { diff --git a/coderd/x/chatd/chaterror/signals.go b/coderd/x/chatd/chaterror/signals.go index f23d86307e..ebe6ff939b 100644 --- a/coderd/x/chatd/chaterror/signals.go +++ b/coderd/x/chatd/chaterror/signals.go @@ -62,13 +62,15 @@ var ( "unauthorized", "invalid api key", "invalid_api_key", + } + authWeakPatterns = []string{"forbidden"} + usageLimitPatterns = []string{ "quota", "billing", "insufficient_quota", "payment required", } - authWeakPatterns = []string{"forbidden"} - configPatterns = []string{ + configPatterns = []string{ "invalid model", "model not found", "model_not_found", diff --git a/site/src/pages/AgentsPage/components/ChatConversation/LiveStreamTail.stories.tsx b/site/src/pages/AgentsPage/components/ChatConversation/LiveStreamTail.stories.tsx index 13b8b35cc6..a4ddf3bf6f 100644 --- a/site/src/pages/AgentsPage/components/ChatConversation/LiveStreamTail.stories.tsx +++ b/site/src/pages/AgentsPage/components/ChatConversation/LiveStreamTail.stories.tsx @@ -74,6 +74,40 @@ export const UsageLimitExceeded: Story = { }, }; +/** + * Provider quota errors use the standard ChatStatusCallout instead of the + * "View Usage" CTA (which links to Coder's analytics, not the provider's + * billing page). + */ +export const ProviderQuotaExceeded: Story = { + args: { + ...defaultArgs, + liveStatus: buildLiveStatus({ + streamError: { + kind: "usage_limit", + message: + "The usage quota for OpenAI has been exceeded. Check the billing and quota settings for the provider account.", + provider: "openai", + retryable: false, + }, + }), + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + expect( + canvas.getByText(/usage quota for openai has been exceeded/i), + ).toBeVisible(); + // The "View Usage" link must NOT appear for provider-originated quota errors. + expect( + canvas.queryByRole("link", { name: /view usage/i }), + ).not.toBeInTheDocument(); + // Should render ChatStatusCallout instead. + expect( + canvas.getByRole("heading", { name: /usage limit reached/i }), + ).toBeVisible(); + }, +}; + /** Provider failures keep the footer-level terminal callout and status link. */ export const TerminalOverloadedError: Story = { args: { diff --git a/site/src/pages/AgentsPage/components/ChatConversation/LiveStreamTail.tsx b/site/src/pages/AgentsPage/components/ChatConversation/LiveStreamTail.tsx index 137996e4b5..93b2c5ce69 100644 --- a/site/src/pages/AgentsPage/components/ChatConversation/LiveStreamTail.tsx +++ b/site/src/pages/AgentsPage/components/ChatConversation/LiveStreamTail.tsx @@ -90,7 +90,7 @@ export const LiveStreamTailContent = ({ mcpServers={mcpServers} /> )} - {usageLimitStatus ? ( + {usageLimitStatus && !usageLimitStatus.provider ? (