shishantbiswas/coder
1
0
Fork 0
mirror of https://github.com/coder/coder.git synced 2026-06-02 20:48:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: classify quota errors as usage_limit instead of auth (#25676)

Browse Source 
Fixes CODAGT-484.

- Removed "quota", "billing", "insufficient_quota", "payment required"
from `authStrongPatterns`
- Added `usageLimitPatterns` slice with those patterns
- Added `usageLimitMatch` signal and rule between overloaded and
authStrong in priority
- Added terminal/retry messages for `ChatErrorKindUsageLimit`
- Simplified auth message (removed billing reference)
- Frontend: conditional `!usageLimitStatus.provider` guard on the "View
Usage" Alert
- Added `TestClassify_UsageLimitBeatsAuth` with 5 cases including real
production OpenAI error
- Added `ProviderQuotaExceeded` story asserting no "View Usage" link and
correct `ChatStatusCallout` rendering

> Generated with [Coder Agents](https://coder.com/agents)
This commit is contained in:
Cian Johnston
2026-05-27 09:45:36 +01:00
committed by GitHub
parent e32be68687
commit 6acfe6c835
7 changed files with 150 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
coderd/x/chatd/chaterror/classify.go
+8
View File
@@ -197,6 +197,7 @@ func Classify(err error) ClassifiedError {
retryableHTTP2StreamReset, hasHTTP2StreamReset := classifyHTTP2StreamReset(err)
deadline := errors.Is(err, context.DeadlineExceeded) || strings.Contains(lower, "context deadline exceeded")
overloadedMatch := statusCode == 529 || containsAny(lower, overloadedPatterns...)
usageLimitMatch := containsAny(lower, usageLimitPatterns...)
authStrong := statusCode == 401 || containsAny(lower, authStrongPatterns...)
configMatch := containsAny(lower, configPatterns...)
authWeak := statusCode == 403 || containsAny(lower, authWeakPatterns...)
@@ -216,6 +217,8 @@ func Classify(err error) ClassifiedError {
// transient-looking errors like "503 invalid model" fail fast.
// Overloaded stays ahead because 529/overloaded is a dedicated
// provider saturation signal, not a common transport wrapper.
// Usage-limit fires before auth so that quota/billing text wins
// over whatever HTTP status code the provider happened to use.
// Strong auth still stays above config because bad credentials are
// the root cause when both signals appear.
rules := []struct {
@@ -228,6 +231,11 @@ func Classify(err error) ClassifiedError {
kind: codersdk.ChatErrorKindOverloaded,
retryable: true,
},
{
match: usageLimitMatch,
kind: codersdk.ChatErrorKindUsageLimit,
retryable: false,
},
{
match: authStrong,
kind: codersdk.ChatErrorKindAuth,
coderd/x/chatd/chaterror/classify_test.go
+77 -9
View File
@@ -79,7 +79,7 @@ func TestClassify(t *testing.T) {
name: "AuthBeatsConfig",
err: xerrors.New("authentication failed: invalid model"),
want: chaterror.ClassifiedError{
Message: "Authentication with the AI provider failed. Check the API key, permissions, and billing settings.",
Message: "Authentication with the AI provider failed. Check the API key and permissions.",
Kind: codersdk.ChatErrorKindAuth,
Provider: "",
Retryable: false,
@@ -101,7 +101,7 @@ func TestClassify(t *testing.T) {
name: "BareForbiddenClassifiesAsAuth",
err: xerrors.New("forbidden"),
want: chaterror.ClassifiedError{
Message: "Authentication with the AI provider failed. Check the API key, permissions, and billing settings.",
Message: "Authentication with the AI provider failed. Check the API key and permissions.",
Kind: codersdk.ChatErrorKindAuth,
Provider: "",
Retryable: false,
@@ -112,7 +112,7 @@ func TestClassify(t *testing.T) {
name: "ExplicitStatus401ClassifiesAsAuth",
err: xerrors.New("status 401 from upstream"),
want: chaterror.ClassifiedError{
Message: "Authentication with the AI provider failed. Check the API key, permissions, and billing settings.",
Message: "Authentication with the AI provider failed. Check the API key and permissions.",
Kind: codersdk.ChatErrorKindAuth,
Provider: "",
Retryable: false,
@@ -123,7 +123,7 @@ func TestClassify(t *testing.T) {
name: "ExplicitStatus403ClassifiesAsAuth",
err: xerrors.New("status 403 from upstream"),
want: chaterror.ClassifiedError{
Message: "Authentication with the AI provider failed. Check the API key, permissions, and billing settings.",
Message: "Authentication with the AI provider failed. Check the API key and permissions.",
Kind: codersdk.ChatErrorKindAuth,
Provider: "",
Retryable: false,
@@ -342,10 +342,10 @@ func TestClassify_PatternCoverage(t *testing.T) {
{name: "UnauthorizedLiteral", err: "unauthorized", wantKind: codersdk.ChatErrorKindAuth, wantRetry: false},
{name: "InvalidAPIKeyLiteral", err: "invalid api key", wantKind: codersdk.ChatErrorKindAuth, wantRetry: false},
{name: "InvalidAPIKeyUnderscoreLiteral", err: "invalid_api_key", wantKind: codersdk.ChatErrorKindAuth, wantRetry: false},
{name: "QuotaLiteral", err: "quota", wantKind: codersdk.ChatErrorKindAuth, wantRetry: false},
{name: "BillingLiteral", err: "billing", wantKind: codersdk.ChatErrorKindAuth, wantRetry: false},
{name: "InsufficientQuotaLiteral", err: "insufficient_quota", wantKind: codersdk.ChatErrorKindAuth, wantRetry: false},
{name: "PaymentRequiredLiteral", err: "payment required", wantKind: codersdk.ChatErrorKindAuth, wantRetry: false},
{name: "QuotaLiteral", err: "quota", wantKind: codersdk.ChatErrorKindUsageLimit, wantRetry: false},
{name: "BillingLiteral", err: "billing", wantKind: codersdk.ChatErrorKindUsageLimit, wantRetry: false},
{name: "InsufficientQuotaLiteral", err: "insufficient_quota", wantKind: codersdk.ChatErrorKindUsageLimit, wantRetry: false},
{name: "PaymentRequiredLiteral", err: "payment required", wantKind: codersdk.ChatErrorKindUsageLimit, wantRetry: false},
{name: "ForbiddenLiteral", err: "forbidden", wantKind: codersdk.ChatErrorKindAuth, wantRetry: false},
{name: "InvalidModelLiteral", err: "invalid model", wantKind: codersdk.ChatErrorKindConfig, wantRetry: false},
{name: "ModelNotFoundLiteral", err: "model not found", wantKind: codersdk.ChatErrorKindConfig, wantRetry: false},
@@ -719,13 +719,81 @@ func TestClassify_StatusCodeBeatsTypedHTTP2StreamError(t *testing.T) {
)
require.Equal(t, chaterror.ClassifiedError{
Message: "Authentication with the AI provider failed. Check the API key, permissions, and billing settings.",
Message: "Authentication with the AI provider failed. Check the API key and permissions.",
Kind: codersdk.ChatErrorKindAuth,
Retryable: false,
StatusCode: 401,
}, chaterror.Classify(err))
}
// TestClassify_UsageLimitBeatsAuth verifies that quota/billing text
// patterns classify as usage_limit even when auth signals are present.
func TestClassify_UsageLimitBeatsAuth(t *testing.T) {
t.Parallel()
tests := []struct {
name string
err string
wantKind codersdk.ChatErrorKind
wantRetry bool
wantStatus int
wantProvider string
}{
{
name: "QuotaBeatsAuth",
err: "unauthorized: insufficient_quota",
wantKind: codersdk.ChatErrorKindUsageLimit,
wantRetry: false,
},
{
name: "QuotaWith429Status",
err: "status 429: insufficient_quota",
wantKind: codersdk.ChatErrorKindUsageLimit,
wantRetry: false,
wantStatus: 429,
},
{
name: "PureAuthStillWorks",
err: "unauthorized",
wantKind: codersdk.ChatErrorKindAuth,
wantRetry: false,
},
{
name: "Status401StillAuth",
err: "status 401",
wantKind: codersdk.ChatErrorKindAuth,
wantRetry: false,
wantStatus: 401,
},
{
// Real production error from OpenAI when quota is exceeded.
name: "OpenAIInsufficientQuotaRealWorld",
err: `stream response: received error while streaming: {"type":"insufficient_quota",` +
`"code":"insufficient_quota","message":"You exceeded your current quota, please check ` +
`your plan and billing details. For more information on this error, read the docs: ` +
`https://platform.openai.com/docs/guides/error-codes/api-errors.","param":null}`,
wantKind: codersdk.ChatErrorKindUsageLimit,
wantRetry: false,
wantProvider: "openai",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
classified := chaterror.Classify(xerrors.New(tt.err))
require.Equal(t, tt.wantKind, classified.Kind)
require.Equal(t, tt.wantRetry, classified.Retryable)
if tt.wantStatus != 0 {
require.Equal(t, tt.wantStatus, classified.StatusCode)
}
if tt.wantProvider != "" {
require.Equal(t, tt.wantProvider, classified.Provider)
}
})
}
}
// TestClassify_StatusCodeBeatsHTTP2Transport ensures explicit status
// codes still win over the new HTTP/2 patterns.
func TestClassify_StatusCodeBeatsHTTP2Transport(t *testing.T) {
coderd/x/chatd/chaterror/message.go
+12 -1
View File
@@ -32,6 +32,17 @@ func terminalMessage(classified ClassifiedError) string {
"%s did not start responding in time.", subject,
)
case codersdk.ChatErrorKindUsageLimit:
displayName := providerDisplayName(classified.Provider)
if displayName == "" {
displayName = "the AI provider"
}
return fmt.Sprintf(
"The usage quota for %s has been exceeded."+
" Check the billing and quota settings for the provider account.",
displayName,
)
case codersdk.ChatErrorKindAuth:
displayName := providerDisplayName(classified.Provider)
if displayName == "" {
@@ -39,7 +50,7 @@ func terminalMessage(classified ClassifiedError) string {
}
return fmt.Sprintf(
"Authentication with %s failed."+
" Check the API key, permissions, and billing settings.",
" Check the API key and permissions.",
displayName,
)
coderd/x/chatd/chaterror/message_test.go
+14
View File
@@ -76,6 +76,20 @@ func TestTerminalMessage(t *testing.T) {
retryable: false,
want: "The chat request failed unexpectedly.",
},
{
name: "UsageLimit_OpenAI",
kind: codersdk.ChatErrorKindUsageLimit,
provider: "openai",
retryable: false,
want: "The usage quota for OpenAI has been exceeded. Check the billing and quota settings for the provider account.",
},
{
name: "UsageLimit_UnknownProvider",
kind: codersdk.ChatErrorKindUsageLimit,
provider: "",
retryable: false,
want: "The usage quota for the AI provider has been exceeded. Check the billing and quota settings for the provider account.",
},
}
for _, tt := range tests {
coderd/x/chatd/chaterror/signals.go
+4 -2
View File
@@ -62,13 +62,15 @@ var (
"unauthorized",
"invalid api key",
"invalid_api_key",
}
authWeakPatterns = []string{"forbidden"}
usageLimitPatterns = []string{
"quota",
"billing",
"insufficient_quota",
"payment required",
}
authWeakPatterns = []string{"forbidden"}
configPatterns = []string{
configPatterns = []string{
"invalid model",
"model not found",
"model_not_found",
site/src/pages/AgentsPage/components/ChatConversation/LiveStreamTail.stories.tsx
+34
View File
@@ -74,6 +74,40 @@ export const UsageLimitExceeded: Story = {
},
};
/**
* Provider quota errors use the standard ChatStatusCallout instead of the
* "View Usage" CTA (which links to Coder's analytics, not the provider's
* billing page).
*/
export const ProviderQuotaExceeded: Story = {
args: {
...defaultArgs,
liveStatus: buildLiveStatus({
streamError: {
kind: "usage_limit",
message:
"The usage quota for OpenAI has been exceeded. Check the billing and quota settings for the provider account.",
provider: "openai",
retryable: false,
},
}),
},
play: async ({ canvasElement }) => {
const canvas = within(canvasElement);
expect(
canvas.getByText(/usage quota for openai has been exceeded/i),
).toBeVisible();
// The "View Usage" link must NOT appear for provider-originated quota errors.
expect(
canvas.queryByRole("link", { name: /view usage/i }),
).not.toBeInTheDocument();
// Should render ChatStatusCallout instead.
expect(
canvas.getByRole("heading", { name: /usage limit reached/i }),
).toBeVisible();
},
};
/** Provider failures keep the footer-level terminal callout and status link. */
export const TerminalOverloadedError: Story = {
args: {
site/src/pages/AgentsPage/components/ChatConversation/LiveStreamTail.tsx
+1 -1
View File
@@ -90,7 +90,7 @@ export const LiveStreamTailContent = ({
mcpServers={mcpServers}
/>
)}
{usageLimitStatus ? (
{usageLimitStatus && !usageLimitStatus.provider ? (
<Alert
severity="info"
actions={