coder/coderd/x/chatd/chaterror/classify_test.go

package chaterror_test

import (
	"context"
	"net/http"
	"strings"
	"testing"
	"time"

	"charm.land/fantasy"
	"github.com/stretchr/testify/require"
	"golang.org/x/xerrors"

	"github.com/coder/coder/v2/coderd/x/chatd/chaterror"
)

func TestClassify(t *testing.T) {
	t.Parallel()

	tests := []struct {
		name string
		err  error
		want chaterror.ClassifiedError
	}{
		{
			name: "AmbiguousOverloadKeepsProviderUnknown",
			err:  xerrors.New("status 529 from upstream"),
			want: chaterror.ClassifiedError{
				Message:    "The AI provider is temporarily overloaded (HTTP 529).",
				Kind:       chaterror.KindOverloaded,
				Provider:   "",
				Retryable:  true,
				StatusCode: 529,
			},
		},
		{
			name: "ExplicitAnthropicOverload",
			err:  xerrors.New("anthropic overloaded_error"),
			want: chaterror.ClassifiedError{
				Message:    "Anthropic is temporarily overloaded.",
				Kind:       chaterror.KindOverloaded,
				Provider:   "anthropic",
				Retryable:  true,
				StatusCode: 0,
			},
		},
		{
			name: "AuthBeatsConfig",
			err:  xerrors.New("authentication failed: invalid model"),
			want: chaterror.ClassifiedError{
				Message:    "Authentication with the AI provider failed. Check the API key, permissions, and billing settings.",
				Kind:       chaterror.KindAuth,
				Provider:   "",
				Retryable:  false,
				StatusCode: 0,
			},
		},
		{
			name: "PureConfig",
			err:  xerrors.New("invalid model"),
			want: chaterror.ClassifiedError{
				Message:    "The AI provider rejected the model configuration. Check the selected model and provider settings.",
				Kind:       chaterror.KindConfig,
				Provider:   "",
				Retryable:  false,
				StatusCode: 0,
			},
		},
		{
			name: "BareForbiddenClassifiesAsAuth",
			err:  xerrors.New("forbidden"),
			want: chaterror.ClassifiedError{
				Message:    "Authentication with the AI provider failed. Check the API key, permissions, and billing settings.",
				Kind:       chaterror.KindAuth,
				Provider:   "",
				Retryable:  false,
				StatusCode: 0,
			},
		},
		{
			name: "ExplicitStatus401ClassifiesAsAuth",
			err:  xerrors.New("status 401 from upstream"),
			want: chaterror.ClassifiedError{
				Message:    "Authentication with the AI provider failed. Check the API key, permissions, and billing settings.",
				Kind:       chaterror.KindAuth,
				Provider:   "",
				Retryable:  false,
				StatusCode: 401,
			},
		},
		{
			name: "ExplicitStatus403ClassifiesAsAuth",
			err:  xerrors.New("status 403 from upstream"),
			want: chaterror.ClassifiedError{
				Message:    "Authentication with the AI provider failed. Check the API key, permissions, and billing settings.",
				Kind:       chaterror.KindAuth,
				Provider:   "",
				Retryable:  false,
				StatusCode: 403,
			},
		},
		{
			name: "ForbiddenContextLengthClassifiesAsConfig",
			err:  xerrors.New("forbidden: context length exceeded"),
			want: chaterror.ClassifiedError{
				Message:    "The AI provider rejected the model configuration. Check the selected model and provider settings.",
				Kind:       chaterror.KindConfig,
				Provider:   "",
				Retryable:  false,
				StatusCode: 0,
			},
		},
		{
			name: "ExplicitStatus429ClassifiesAsRateLimit",
			err:  xerrors.New("status 429 from upstream"),
			want: chaterror.ClassifiedError{
				Message:    "The AI provider is rate limiting requests (HTTP 429).",
				Kind:       chaterror.KindRateLimit,
				Provider:   "",
				Retryable:  true,
				StatusCode: 429,
			},
		},
		{
			name: "RateLimitDoesNotBeatConfig",
			err:  xerrors.New("status 429: invalid model"),
			want: chaterror.ClassifiedError{
				Message:    "The AI provider rejected the model configuration. Check the selected model and provider settings.",
				Kind:       chaterror.KindConfig,
				Provider:   "",
				Retryable:  false,
				StatusCode: 429,
			},
		},
		{
			name: "ServiceUnavailableClassifiesAsRetryableTimeout",
			err:  xerrors.New("service unavailable"),
			want: chaterror.ClassifiedError{
				Message:    "The AI provider is temporarily unavailable.",
				Kind:       chaterror.KindTimeout,
				Provider:   "",
				Retryable:  true,
				StatusCode: 0,
			},
		},
		{
			name: "TimeoutDoesNotBeatConfigViaStatusCode",
			err:  xerrors.New("status 503: invalid model"),
			want: chaterror.ClassifiedError{
				Message:    "The AI provider rejected the model configuration. Check the selected model and provider settings.",
				Kind:       chaterror.KindConfig,
				Provider:   "",
				Retryable:  false,
				StatusCode: 503,
			},
		},
		{
			name: "TimeoutDoesNotBeatConfigViaMessage",
			err:  xerrors.New("service unavailable: model not found"),
			want: chaterror.ClassifiedError{
				Message:    "The AI provider rejected the model configuration. Check the selected model and provider settings.",
				Kind:       chaterror.KindConfig,
				Provider:   "",
				Retryable:  false,
				StatusCode: 0,
			},
		},
		{
			name: "ConnectionRefusedUnsupportedModelClassifiesAsConfig",
			err:  xerrors.New("connection refused: unsupported model"),
			want: chaterror.ClassifiedError{
				Message:    "The AI provider rejected the model configuration. Check the selected model and provider settings.",
				Kind:       chaterror.KindConfig,
				Provider:   "",
				Retryable:  false,
				StatusCode: 0,
			},
		},
		{
			name: "DeadlineExceededStaysNonRetryableTimeout",
			err:  context.DeadlineExceeded,
			want: chaterror.ClassifiedError{
				Message:    "The request timed out before it completed.",
				Kind:       chaterror.KindTimeout,
				Provider:   "",
				Retryable:  false,
				StatusCode: 0,
			},
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			t.Parallel()
			require.Equal(t, tt.want, chaterror.Classify(tt.err))
		})
	}
}

func TestClassify_PatternCoverage(t *testing.T) {
	t.Parallel()

	tests := []struct {
		name      string
		err       string
		wantKind  string
		wantRetry bool
	}{
		{name: "OverloadedLiteral", err: "overloaded", wantKind: chaterror.KindOverloaded, wantRetry: true},
		{name: "RateLimitLiteral", err: "rate limit", wantKind: chaterror.KindRateLimit, wantRetry: true},
		{name: "RateLimitUnderscoreLiteral", err: "rate_limit", wantKind: chaterror.KindRateLimit, wantRetry: true},
		{name: "RateLimitedLiteral", err: "rate limited", wantKind: chaterror.KindRateLimit, wantRetry: true},
		{name: "RateLimitedHyphenLiteral", err: "rate-limited", wantKind: chaterror.KindRateLimit, wantRetry: true},
		{name: "TooManyRequestsLiteral", err: "too many requests", wantKind: chaterror.KindRateLimit, wantRetry: true},
		{name: "TimeoutLiteral", err: "timeout", wantKind: chaterror.KindTimeout, wantRetry: true},
		{name: "TimedOutLiteral", err: "timed out", wantKind: chaterror.KindTimeout, wantRetry: true},
		{name: "ServiceUnavailableLiteral", err: "service unavailable", wantKind: chaterror.KindTimeout, wantRetry: true},
		{name: "UnavailableLiteral", err: "unavailable", wantKind: chaterror.KindTimeout, wantRetry: true},
		{name: "ConnectionResetLiteral", err: "connection reset", wantKind: chaterror.KindTimeout, wantRetry: true},
		{name: "ConnectionRefusedLiteral", err: "connection refused", wantKind: chaterror.KindTimeout, wantRetry: true},
		{name: "EOFLiteral", err: "eof", wantKind: chaterror.KindTimeout, wantRetry: true},
		{name: "BrokenPipeLiteral", err: "broken pipe", wantKind: chaterror.KindTimeout, wantRetry: true},
		{name: "BadGatewayLiteral", err: "bad gateway", wantKind: chaterror.KindTimeout, wantRetry: true},
		{name: "GatewayTimeoutLiteral", err: "gateway timeout", wantKind: chaterror.KindTimeout, wantRetry: true},
		{name: "ClientConnLiteral", err: "client conn", wantKind: chaterror.KindTimeout, wantRetry: true},
		{name: "GOAWAYLiteral", err: "goaway", wantKind: chaterror.KindTimeout, wantRetry: true},
		{name: "HTTP2StreamClosedLiteral", err: "http2: stream closed", wantKind: chaterror.KindTimeout, wantRetry: true},
		{name: "UseOfClosedNetworkConnectionLiteral", err: "use of closed network connection", wantKind: chaterror.KindTimeout, wantRetry: true},
		{name: "AuthenticationLiteral", err: "authentication", wantKind: chaterror.KindAuth, wantRetry: false},
		{name: "UnauthorizedLiteral", err: "unauthorized", wantKind: chaterror.KindAuth, wantRetry: false},
		{name: "InvalidAPIKeyLiteral", err: "invalid api key", wantKind: chaterror.KindAuth, wantRetry: false},
		{name: "InvalidAPIKeyUnderscoreLiteral", err: "invalid_api_key", wantKind: chaterror.KindAuth, wantRetry: false},
		{name: "QuotaLiteral", err: "quota", wantKind: chaterror.KindAuth, wantRetry: false},
		{name: "BillingLiteral", err: "billing", wantKind: chaterror.KindAuth, wantRetry: false},
		{name: "InsufficientQuotaLiteral", err: "insufficient_quota", wantKind: chaterror.KindAuth, wantRetry: false},
		{name: "PaymentRequiredLiteral", err: "payment required", wantKind: chaterror.KindAuth, wantRetry: false},
		{name: "ForbiddenLiteral", err: "forbidden", wantKind: chaterror.KindAuth, wantRetry: false},
		{name: "InvalidModelLiteral", err: "invalid model", wantKind: chaterror.KindConfig, wantRetry: false},
		{name: "ModelNotFoundLiteral", err: "model not found", wantKind: chaterror.KindConfig, wantRetry: false},
		{name: "ModelNotFoundUnderscoreLiteral", err: "model_not_found", wantKind: chaterror.KindConfig, wantRetry: false},
		{name: "UnsupportedModelLiteral", err: "unsupported model", wantKind: chaterror.KindConfig, wantRetry: false},
		{name: "ContextLengthExceededLiteral", err: "context length exceeded", wantKind: chaterror.KindConfig, wantRetry: false},
		{name: "ContextExceededLiteral", err: "context_exceeded", wantKind: chaterror.KindConfig, wantRetry: false},
		{name: "MaximumContextLengthLiteral", err: "maximum context length", wantKind: chaterror.KindConfig, wantRetry: false},
		{name: "MalformedConfigLiteral", err: "malformed config", wantKind: chaterror.KindConfig, wantRetry: false},
		{name: "MalformedConfigurationLiteral", err: "malformed configuration", wantKind: chaterror.KindConfig, wantRetry: false},
		{name: "ServerErrorLiteral", err: "server error", wantKind: chaterror.KindGeneric, wantRetry: true},
		{name: "InternalServerErrorLiteral", err: "internal server error", wantKind: chaterror.KindGeneric, wantRetry: true},
		{name: "ChatInterruptedLiteral", err: "chat interrupted", wantKind: chaterror.KindGeneric, wantRetry: false},
		{name: "RequestInterruptedLiteral", err: "request interrupted", wantKind: chaterror.KindGeneric, wantRetry: false},
		{name: "OperationInterruptedLiteral", err: "operation interrupted", wantKind: chaterror.KindGeneric, wantRetry: false},
		{name: "Status408", err: "status 408", wantKind: chaterror.KindTimeout, wantRetry: true},
		{name: "Status500", err: "status 500", wantKind: chaterror.KindGeneric, wantRetry: true},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			t.Parallel()

			classified := chaterror.Classify(xerrors.New(tt.err))
			require.Equal(t, tt.wantKind, classified.Kind)
			require.Equal(t, tt.wantRetry, classified.Retryable)
		})
	}
}

func TestClassify_TransportFailuresUseBroaderRetryMessage(t *testing.T) {
	t.Parallel()

	tests := []struct {
		name string
		err  string
	}{
		{name: "TimeoutLiteral", err: "timeout"},
		{name: "EOFLiteral", err: "eof"},
		{name: "BrokenPipeLiteral", err: "broken pipe"},
		{name: "ConnectionResetLiteral", err: "connection reset"},
		{name: "ConnectionRefusedLiteral", err: "connection refused"},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			t.Parallel()

			classified := chaterror.Classify(xerrors.New(tt.err))
			require.Equal(t, chaterror.KindTimeout, classified.Kind)
			require.True(t, classified.Retryable)
			require.Equal(
				t,
				"The AI provider is temporarily unavailable.",
				classified.Message,
			)
		})
	}
}

// TestClassify_HTTP2TransportErrors checks HTTP/2 transport errors
// classify as retryable KindTimeout. Split into two sub-tables so a
// bug in transport matching cannot be masked by provider detection
// (and vice versa).
func TestClassify_HTTP2TransportErrors(t *testing.T) {
	t.Parallel()

	// Transport patterns, no provider hint. Provider stays empty and
	// Message uses the generic subject.
	transportOnly := []struct {
		name string
		err  string
	}{
		{
			name: "HTTP2ClientConnForceClosed",
			err:  "http2: client connection force closed via ClientConn.Close",
		},
		{
			name: "HTTP2TransportGOAWAY",
			err:  "http2: Transport received Server's graceful shutdown GOAWAY",
		},
		{
			name: "HTTP2ServerGOAWAY",
			err:  "http2: server sent GOAWAY and closed the connection",
		},
		{
			name: "HTTP2StreamClosed",
			err:  "http2: stream closed",
		},
		{
			name: "UseOfClosedNetworkConnectionOnPOST",
			err:  `Post "https://example.com/v1/messages": use of closed network connection`,
		},
		{
			name: "HTTP2ClientConnIsClosed",
			err:  "http2: client conn is closed",
		},
		{
			name: "HTTP2ClientConnNotUsable",
			err:  "http2: client conn not usable",
		},
		{
			name: "HTTP2ClientConnNotEstablished",
			err:  "http2: client conn could not be established",
		},
		{
			name: "HTTP2ClientConnectionLost",
			err:  "http2: client connection lost",
		},
	}

	for _, tt := range transportOnly {
		t.Run("TransportOnly/"+tt.name, func(t *testing.T) {
			t.Parallel()

			classified := chaterror.Classify(xerrors.New(tt.err))
			require.Equal(t, chaterror.KindTimeout, classified.Kind, "Kind")
			require.True(t, classified.Retryable, "Retryable")
			require.Equal(t, "", classified.Provider, "Provider")
			require.Equal(t,
				"The AI provider is temporarily unavailable.",
				classified.Message,
				"Message",
			)
		})
	}

	// Same transport signature with a provider host in the URL so
	// detectProvider can stamp Provider.
	providerDetection := []struct {
		name        string
		err         string
		provider    string
		wantMessage string
	}{
		{
			name:        "CustomerRegressionAnthropic",
			err:         `stream response: Post "https://api.anthropic.com/v1/messages": http2: client connection force closed via ClientConn.Close`,
			provider:    "anthropic",
			wantMessage: "Anthropic is temporarily unavailable.",
		},
		{
			name:        "OpenAIForceClosed",
			err:         `stream response: Post "https://api.openai.com/v1/chat/completions": http2: client connection force closed via ClientConn.Close`,
			provider:    "openai",
			wantMessage: "OpenAI is temporarily unavailable.",
		},
		{
			name:        "GoogleGOAWAY",
			err:         `stream response: Post "https://generativelanguage.googleapis.com/v1beta/models/gemini-pro:streamGenerateContent": http2: server sent GOAWAY and closed the connection`,
			provider:    "google",
			wantMessage: "Google is temporarily unavailable.",
		},
	}

	for _, tt := range providerDetection {
		t.Run("ProviderDetection/"+tt.name, func(t *testing.T) {
			t.Parallel()

			classified := chaterror.Classify(xerrors.New(tt.err))
			require.Equal(t, chaterror.KindTimeout, classified.Kind, "Kind")
			require.True(t, classified.Retryable, "Retryable")
			require.Equal(t, tt.provider, classified.Provider, "Provider")
			require.Equal(t, tt.wantMessage, classified.Message, "Message")
		})
	}
}

// TestClassify_StatusCodeBeatsHTTP2Transport ensures explicit status
// codes still win over the new HTTP/2 patterns.
func TestClassify_StatusCodeBeatsHTTP2Transport(t *testing.T) {
	t.Parallel()

	tests := []struct {
		name          string
		err           string
		wantKind      string
		wantRetryable bool
		wantStatus    int
	}{
		{
			name:          "HTTP2With429",
			err:           "http2: server error 429 Too Many Requests",
			wantKind:      chaterror.KindRateLimit,
			wantRetryable: true,
			wantStatus:    429,
		},
		{
			name:          "HTTP2With401",
			err:           "http2: 401 unauthorized",
			wantKind:      chaterror.KindAuth,
			wantRetryable: false,
			wantStatus:    401,
		},
		{
			name:          "ClientConnWith429RateLimitWins",
			err:           "http2: client conn is closed: status 429 Too Many Requests",
			wantKind:      chaterror.KindRateLimit,
			wantRetryable: true,
			wantStatus:    429,
		},
		{
			name:          "GOAWAYWith401AuthWins",
			err:           "http2: server sent GOAWAY: status 401 unauthorized",
			wantKind:      chaterror.KindAuth,
			wantRetryable: false,
			wantStatus:    401,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			t.Parallel()

			classified := chaterror.Classify(xerrors.New(tt.err))
			require.Equal(t, tt.wantKind, classified.Kind, "Kind")
			require.Equal(t, tt.wantRetryable, classified.Retryable, "Retryable")
			require.Equal(t, tt.wantStatus, classified.StatusCode, "StatusCode")
		})
	}
}

func TestClassify_StartupTimeoutWrappedClassificationWins(t *testing.T) {
	t.Parallel()

	wrapped := chaterror.WithClassification(
		xerrors.New("context canceled"),
		chaterror.ClassifiedError{
			Kind:      chaterror.KindStartupTimeout,
			Provider:  "openai",
			Retryable: true,
		},
	)

	require.Equal(t, chaterror.ClassifiedError{
		Message:    "OpenAI did not start responding in time.",
		Kind:       chaterror.KindStartupTimeout,
		Provider:   "openai",
		Retryable:  true,
		StatusCode: 0,
	}, chaterror.Classify(wrapped))
}

func TestWithProviderUsesExplicitHint(t *testing.T) {
	t.Parallel()

	classified := chaterror.Classify(xerrors.New("openai received status 429 from upstream"))
	require.Equal(t, "openai", classified.Provider)

	enriched := classified.WithProvider("azure openai")
	require.Equal(t, chaterror.ClassifiedError{
		Message:    "Azure OpenAI is rate limiting requests (HTTP 429).",
		Kind:       chaterror.KindRateLimit,
		Provider:   "azure",
		Retryable:  true,
		StatusCode: 429,
	}, enriched)
}

func TestWithProviderAddsProviderWhenUnknown(t *testing.T) {
	t.Parallel()

	classified := chaterror.Classify(xerrors.New("received status 429 from upstream"))
	require.Empty(t, classified.Provider)

	enriched := classified.WithProvider("openai")
	require.Equal(t, chaterror.ClassifiedError{
		Message:    "OpenAI is rate limiting requests (HTTP 429).",
		Kind:       chaterror.KindRateLimit,
		Provider:   "openai",
		Retryable:  true,
		StatusCode: 429,
	}, enriched)
}

func TestClassify_UsesStructuredProviderStatusAndRetryAfter(t *testing.T) {
	t.Parallel()

	classified := chaterror.Classify(testProviderError(
		"",
		429,
		map[string]string{"Retry-After": "30"},
	))

	require.Equal(t, chaterror.ClassifiedError{
		Message:    "The AI provider is rate limiting requests (HTTP 429).",
		Kind:       chaterror.KindRateLimit,
		Provider:   "",
		Retryable:  true,
		StatusCode: 429,
		RetryAfter: 30 * time.Second,
	}, classified)
}

func TestClassify_PrefersRetryAfterMsOverRetryAfter(t *testing.T) {
	t.Parallel()

	classified := chaterror.Classify(testProviderError(
		"upstream failed",
		429,
		map[string]string{
			"Retry-After":    "30",
			"ReTrY-AfTeR-Ms": "1500",
		},
	))

	require.Equal(t, 429, classified.StatusCode)
	require.Equal(t, 1500*time.Millisecond, classified.RetryAfter)
}

func TestClassify_ParsesRetryAfterHTTPDate(t *testing.T) {
	t.Parallel()

	retryAt := time.Now().Add(3 * time.Second).UTC().Format(http.TimeFormat)
	classified := chaterror.Classify(testProviderError(
		"upstream failed",
		429,
		map[string]string{"Retry-After": retryAt},
	))

	require.Equal(t, 429, classified.StatusCode)
	require.GreaterOrEqual(t, classified.RetryAfter, 2*time.Second)
	require.LessOrEqual(t, classified.RetryAfter, 4*time.Second)
}

func TestClassify_IgnoresInvalidRetryAfter(t *testing.T) {
	t.Parallel()

	classified := chaterror.Classify(testProviderError(
		"upstream failed",
		429,
		map[string]string{"Retry-After": "definitely not a delay"},
	))

	require.Zero(t, classified.RetryAfter)
}

func TestWithProviderPreservesRetryAfter(t *testing.T) {
	t.Parallel()

	classified := chaterror.Classify(testProviderError(
		"",
		429,
		map[string]string{"Retry-After": "30"},
	))

	enriched := classified.WithProvider("openai")
	require.Equal(t, 30*time.Second, enriched.RetryAfter)
	require.Equal(t, chaterror.ClassifiedError{
		Message:    "OpenAI is rate limiting requests (HTTP 429).",
		Kind:       chaterror.KindRateLimit,
		Provider:   "openai",
		Retryable:  true,
		StatusCode: 429,
		RetryAfter: 30 * time.Second,
	}, enriched)
}

func TestClassify_UsesStructuredProviderDetailFromResponseDump(t *testing.T) {
	t.Parallel()

	classified := chaterror.Classify(testProviderError(
		"",
		400,
		nil,
		testProviderResponseDump(`{"error":{"type":"invalid_request_error","message":"Image exceeds 5 MB maximum."}}`),
	))

	require.Equal(t, chaterror.ClassifiedError{
		Message:    "The AI provider returned an unexpected error (HTTP 400).",
		Detail:     "Image exceeds 5 MB maximum.",
		Kind:       chaterror.KindGeneric,
		Provider:   "",
		Retryable:  false,
		StatusCode: 400,
	}, classified)
}

func TestClassify_FallsBackToProviderMessageForDetail(t *testing.T) {
	t.Parallel()

	classified := chaterror.Classify(testProviderError(
		"  image exceeds 5 MB maximum  ",
		400,
		nil,
		testProviderResponseDump("not-json"),
	))

	require.Equal(t, "image exceeds 5 MB maximum", classified.Detail)
}

func TestClassify_TruncatesProviderDetail(t *testing.T) {
	t.Parallel()

	detail := strings.Repeat("x", 510)
	classified := chaterror.Classify(testProviderError(
		"",
		400,
		nil,
		testProviderResponseDump(`{"error":{"message":"`+detail+`"}}`),
	))

	require.Len(t, []rune(classified.Detail), 500)
	require.True(t, strings.HasSuffix(classified.Detail, "…"))
}

func testProviderError(
	message string,
	statusCode int,
	headers map[string]string,
	responseBody ...[]byte,
) error {
	var body []byte
	if len(responseBody) > 0 {
		body = responseBody[0]
	}
	return &fantasy.ProviderError{
		Message:         message,
		StatusCode:      statusCode,
		ResponseHeaders: headers,
		ResponseBody:    body,
	}
}

func testProviderResponseDump(body string) []byte {
	return []byte(`HTTP/1.1 400 Bad Request
Content-Type: application/json

` + body)
}