mirror of
https://github.com/coder/coder.git
synced 2026-06-02 20:48:20 +00:00
fix: classify quota errors as usage_limit instead of auth (#25676)
Fixes CODAGT-484. - Removed "quota", "billing", "insufficient_quota", "payment required" from `authStrongPatterns` - Added `usageLimitPatterns` slice with those patterns - Added `usageLimitMatch` signal and rule between overloaded and authStrong in priority - Added terminal/retry messages for `ChatErrorKindUsageLimit` - Simplified auth message (removed billing reference) - Frontend: conditional `!usageLimitStatus.provider` guard on the "View Usage" Alert - Added `TestClassify_UsageLimitBeatsAuth` with 5 cases including real production OpenAI error - Added `ProviderQuotaExceeded` story asserting no "View Usage" link and correct `ChatStatusCallout` rendering > Generated with [Coder Agents](https://coder.com/agents)
This commit is contained in:
@@ -197,6 +197,7 @@ func Classify(err error) ClassifiedError {
|
||||
retryableHTTP2StreamReset, hasHTTP2StreamReset := classifyHTTP2StreamReset(err)
|
||||
deadline := errors.Is(err, context.DeadlineExceeded) || strings.Contains(lower, "context deadline exceeded")
|
||||
overloadedMatch := statusCode == 529 || containsAny(lower, overloadedPatterns...)
|
||||
usageLimitMatch := containsAny(lower, usageLimitPatterns...)
|
||||
authStrong := statusCode == 401 || containsAny(lower, authStrongPatterns...)
|
||||
configMatch := containsAny(lower, configPatterns...)
|
||||
authWeak := statusCode == 403 || containsAny(lower, authWeakPatterns...)
|
||||
@@ -216,6 +217,8 @@ func Classify(err error) ClassifiedError {
|
||||
// transient-looking errors like "503 invalid model" fail fast.
|
||||
// Overloaded stays ahead because 529/overloaded is a dedicated
|
||||
// provider saturation signal, not a common transport wrapper.
|
||||
// Usage-limit fires before auth so that quota/billing text wins
|
||||
// over whatever HTTP status code the provider happened to use.
|
||||
// Strong auth still stays above config because bad credentials are
|
||||
// the root cause when both signals appear.
|
||||
rules := []struct {
|
||||
@@ -228,6 +231,11 @@ func Classify(err error) ClassifiedError {
|
||||
kind: codersdk.ChatErrorKindOverloaded,
|
||||
retryable: true,
|
||||
},
|
||||
{
|
||||
match: usageLimitMatch,
|
||||
kind: codersdk.ChatErrorKindUsageLimit,
|
||||
retryable: false,
|
||||
},
|
||||
{
|
||||
match: authStrong,
|
||||
kind: codersdk.ChatErrorKindAuth,
|
||||
|
||||
@@ -79,7 +79,7 @@ func TestClassify(t *testing.T) {
|
||||
name: "AuthBeatsConfig",
|
||||
err: xerrors.New("authentication failed: invalid model"),
|
||||
want: chaterror.ClassifiedError{
|
||||
Message: "Authentication with the AI provider failed. Check the API key, permissions, and billing settings.",
|
||||
Message: "Authentication with the AI provider failed. Check the API key and permissions.",
|
||||
Kind: codersdk.ChatErrorKindAuth,
|
||||
Provider: "",
|
||||
Retryable: false,
|
||||
@@ -101,7 +101,7 @@ func TestClassify(t *testing.T) {
|
||||
name: "BareForbiddenClassifiesAsAuth",
|
||||
err: xerrors.New("forbidden"),
|
||||
want: chaterror.ClassifiedError{
|
||||
Message: "Authentication with the AI provider failed. Check the API key, permissions, and billing settings.",
|
||||
Message: "Authentication with the AI provider failed. Check the API key and permissions.",
|
||||
Kind: codersdk.ChatErrorKindAuth,
|
||||
Provider: "",
|
||||
Retryable: false,
|
||||
@@ -112,7 +112,7 @@ func TestClassify(t *testing.T) {
|
||||
name: "ExplicitStatus401ClassifiesAsAuth",
|
||||
err: xerrors.New("status 401 from upstream"),
|
||||
want: chaterror.ClassifiedError{
|
||||
Message: "Authentication with the AI provider failed. Check the API key, permissions, and billing settings.",
|
||||
Message: "Authentication with the AI provider failed. Check the API key and permissions.",
|
||||
Kind: codersdk.ChatErrorKindAuth,
|
||||
Provider: "",
|
||||
Retryable: false,
|
||||
@@ -123,7 +123,7 @@ func TestClassify(t *testing.T) {
|
||||
name: "ExplicitStatus403ClassifiesAsAuth",
|
||||
err: xerrors.New("status 403 from upstream"),
|
||||
want: chaterror.ClassifiedError{
|
||||
Message: "Authentication with the AI provider failed. Check the API key, permissions, and billing settings.",
|
||||
Message: "Authentication with the AI provider failed. Check the API key and permissions.",
|
||||
Kind: codersdk.ChatErrorKindAuth,
|
||||
Provider: "",
|
||||
Retryable: false,
|
||||
@@ -342,10 +342,10 @@ func TestClassify_PatternCoverage(t *testing.T) {
|
||||
{name: "UnauthorizedLiteral", err: "unauthorized", wantKind: codersdk.ChatErrorKindAuth, wantRetry: false},
|
||||
{name: "InvalidAPIKeyLiteral", err: "invalid api key", wantKind: codersdk.ChatErrorKindAuth, wantRetry: false},
|
||||
{name: "InvalidAPIKeyUnderscoreLiteral", err: "invalid_api_key", wantKind: codersdk.ChatErrorKindAuth, wantRetry: false},
|
||||
{name: "QuotaLiteral", err: "quota", wantKind: codersdk.ChatErrorKindAuth, wantRetry: false},
|
||||
{name: "BillingLiteral", err: "billing", wantKind: codersdk.ChatErrorKindAuth, wantRetry: false},
|
||||
{name: "InsufficientQuotaLiteral", err: "insufficient_quota", wantKind: codersdk.ChatErrorKindAuth, wantRetry: false},
|
||||
{name: "PaymentRequiredLiteral", err: "payment required", wantKind: codersdk.ChatErrorKindAuth, wantRetry: false},
|
||||
{name: "QuotaLiteral", err: "quota", wantKind: codersdk.ChatErrorKindUsageLimit, wantRetry: false},
|
||||
{name: "BillingLiteral", err: "billing", wantKind: codersdk.ChatErrorKindUsageLimit, wantRetry: false},
|
||||
{name: "InsufficientQuotaLiteral", err: "insufficient_quota", wantKind: codersdk.ChatErrorKindUsageLimit, wantRetry: false},
|
||||
{name: "PaymentRequiredLiteral", err: "payment required", wantKind: codersdk.ChatErrorKindUsageLimit, wantRetry: false},
|
||||
{name: "ForbiddenLiteral", err: "forbidden", wantKind: codersdk.ChatErrorKindAuth, wantRetry: false},
|
||||
{name: "InvalidModelLiteral", err: "invalid model", wantKind: codersdk.ChatErrorKindConfig, wantRetry: false},
|
||||
{name: "ModelNotFoundLiteral", err: "model not found", wantKind: codersdk.ChatErrorKindConfig, wantRetry: false},
|
||||
@@ -719,13 +719,81 @@ func TestClassify_StatusCodeBeatsTypedHTTP2StreamError(t *testing.T) {
|
||||
)
|
||||
|
||||
require.Equal(t, chaterror.ClassifiedError{
|
||||
Message: "Authentication with the AI provider failed. Check the API key, permissions, and billing settings.",
|
||||
Message: "Authentication with the AI provider failed. Check the API key and permissions.",
|
||||
Kind: codersdk.ChatErrorKindAuth,
|
||||
Retryable: false,
|
||||
StatusCode: 401,
|
||||
}, chaterror.Classify(err))
|
||||
}
|
||||
|
||||
// TestClassify_UsageLimitBeatsAuth verifies that quota/billing text
|
||||
// patterns classify as usage_limit even when auth signals are present.
|
||||
func TestClassify_UsageLimitBeatsAuth(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
err string
|
||||
wantKind codersdk.ChatErrorKind
|
||||
wantRetry bool
|
||||
wantStatus int
|
||||
wantProvider string
|
||||
}{
|
||||
{
|
||||
name: "QuotaBeatsAuth",
|
||||
err: "unauthorized: insufficient_quota",
|
||||
wantKind: codersdk.ChatErrorKindUsageLimit,
|
||||
wantRetry: false,
|
||||
},
|
||||
{
|
||||
name: "QuotaWith429Status",
|
||||
err: "status 429: insufficient_quota",
|
||||
wantKind: codersdk.ChatErrorKindUsageLimit,
|
||||
wantRetry: false,
|
||||
wantStatus: 429,
|
||||
},
|
||||
{
|
||||
name: "PureAuthStillWorks",
|
||||
err: "unauthorized",
|
||||
wantKind: codersdk.ChatErrorKindAuth,
|
||||
wantRetry: false,
|
||||
},
|
||||
{
|
||||
name: "Status401StillAuth",
|
||||
err: "status 401",
|
||||
wantKind: codersdk.ChatErrorKindAuth,
|
||||
wantRetry: false,
|
||||
wantStatus: 401,
|
||||
},
|
||||
{
|
||||
// Real production error from OpenAI when quota is exceeded.
|
||||
name: "OpenAIInsufficientQuotaRealWorld",
|
||||
err: `stream response: received error while streaming: {"type":"insufficient_quota",` +
|
||||
`"code":"insufficient_quota","message":"You exceeded your current quota, please check ` +
|
||||
`your plan and billing details. For more information on this error, read the docs: ` +
|
||||
`https://platform.openai.com/docs/guides/error-codes/api-errors.","param":null}`,
|
||||
wantKind: codersdk.ChatErrorKindUsageLimit,
|
||||
wantRetry: false,
|
||||
wantProvider: "openai",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
classified := chaterror.Classify(xerrors.New(tt.err))
|
||||
require.Equal(t, tt.wantKind, classified.Kind)
|
||||
require.Equal(t, tt.wantRetry, classified.Retryable)
|
||||
if tt.wantStatus != 0 {
|
||||
require.Equal(t, tt.wantStatus, classified.StatusCode)
|
||||
}
|
||||
if tt.wantProvider != "" {
|
||||
require.Equal(t, tt.wantProvider, classified.Provider)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestClassify_StatusCodeBeatsHTTP2Transport ensures explicit status
|
||||
// codes still win over the new HTTP/2 patterns.
|
||||
func TestClassify_StatusCodeBeatsHTTP2Transport(t *testing.T) {
|
||||
|
||||
@@ -32,6 +32,17 @@ func terminalMessage(classified ClassifiedError) string {
|
||||
"%s did not start responding in time.", subject,
|
||||
)
|
||||
|
||||
case codersdk.ChatErrorKindUsageLimit:
|
||||
displayName := providerDisplayName(classified.Provider)
|
||||
if displayName == "" {
|
||||
displayName = "the AI provider"
|
||||
}
|
||||
return fmt.Sprintf(
|
||||
"The usage quota for %s has been exceeded."+
|
||||
" Check the billing and quota settings for the provider account.",
|
||||
displayName,
|
||||
)
|
||||
|
||||
case codersdk.ChatErrorKindAuth:
|
||||
displayName := providerDisplayName(classified.Provider)
|
||||
if displayName == "" {
|
||||
@@ -39,7 +50,7 @@ func terminalMessage(classified ClassifiedError) string {
|
||||
}
|
||||
return fmt.Sprintf(
|
||||
"Authentication with %s failed."+
|
||||
" Check the API key, permissions, and billing settings.",
|
||||
" Check the API key and permissions.",
|
||||
displayName,
|
||||
)
|
||||
|
||||
|
||||
@@ -76,6 +76,20 @@ func TestTerminalMessage(t *testing.T) {
|
||||
retryable: false,
|
||||
want: "The chat request failed unexpectedly.",
|
||||
},
|
||||
{
|
||||
name: "UsageLimit_OpenAI",
|
||||
kind: codersdk.ChatErrorKindUsageLimit,
|
||||
provider: "openai",
|
||||
retryable: false,
|
||||
want: "The usage quota for OpenAI has been exceeded. Check the billing and quota settings for the provider account.",
|
||||
},
|
||||
{
|
||||
name: "UsageLimit_UnknownProvider",
|
||||
kind: codersdk.ChatErrorKindUsageLimit,
|
||||
provider: "",
|
||||
retryable: false,
|
||||
want: "The usage quota for the AI provider has been exceeded. Check the billing and quota settings for the provider account.",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
|
||||
@@ -62,13 +62,15 @@ var (
|
||||
"unauthorized",
|
||||
"invalid api key",
|
||||
"invalid_api_key",
|
||||
}
|
||||
authWeakPatterns = []string{"forbidden"}
|
||||
usageLimitPatterns = []string{
|
||||
"quota",
|
||||
"billing",
|
||||
"insufficient_quota",
|
||||
"payment required",
|
||||
}
|
||||
authWeakPatterns = []string{"forbidden"}
|
||||
configPatterns = []string{
|
||||
configPatterns = []string{
|
||||
"invalid model",
|
||||
"model not found",
|
||||
"model_not_found",
|
||||
|
||||
Reference in New Issue
Block a user