mirror of
https://github.com/coder/coder.git
synced 2026-06-03 04:58:23 +00:00
f1155ac4d7
## Description Adds automatic key failover for centralized Anthropic provider. When a key pool is configured, each upstream call walks the pool and tries keys in order until one succeeds or the pool is exhausted. Keys are marked **temporary** on 429 (with cooldown from `Retry-After`) and **permanent** on 401/403. Errors that aren't key-specific don't trigger failover. Each agentic-loop iteration gets its own fresh walker, so a tool-call continuation can fail over independently of the initial request. BYOK is unchanged: BYOK requests run as a single attempt with no failover. ## Changes - `config.Anthropic` carries a `KeyPool`. `Key` remains for BYOK X-Api-Key set per interception. - Blocking interceptor: walks the pool, marks keys on key-specific failures, returns on first success or non-failover error. - Streaming interceptor: per-iteration walker. Pre-stream failures fail over to the next key; mid-stream errors are relayed as SSE events. - New `keypool` error types: `TransientExhaustionError` (carries soonest cooldown) and `ErrPermanentExhaustion`. Replace the prior `ErrAllKeysExhausted`. - Error responses now consistently include the outer `"type": "error"` field. ## Related Issues Related to: https://github.com/coder/internal/issues/1446 Related to: https://linear.app/codercom/issue/AIGOV-197/aibridge-automatic-key-failover-for-bridged-and-passthrough-routes ## Follow-up PRs - Bedrock multi-key support. - Refactor provider vs interceptor config separation. - Record the actually-used key in the interception credential hint after failover. > [!NOTE] > Initially generated by Claude Opus 4.7, modified and reviewed by @ssncferreira
111 lines
2.5 KiB
Go
111 lines
2.5 KiB
Go
package keypool_test
|
|
|
|
import (
|
|
"net/http"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
|
|
"github.com/coder/coder/v2/aibridge/keypool"
|
|
)
|
|
|
|
func TestParseRetryAfter(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tests := []struct {
|
|
name string
|
|
headers map[string]string
|
|
nilResponse bool
|
|
expected time.Duration
|
|
}{
|
|
// nil response.
|
|
{
|
|
name: "nil_response",
|
|
nilResponse: true,
|
|
expected: 0,
|
|
},
|
|
// No headers set.
|
|
{
|
|
name: "no_headers",
|
|
headers: nil,
|
|
expected: 0,
|
|
},
|
|
// retry-after-ms (OpenAI, preferred).
|
|
{
|
|
name: "openai_retry_after_ms",
|
|
headers: map[string]string{"retry-after-ms": "2500"},
|
|
expected: 2500 * time.Millisecond,
|
|
},
|
|
{
|
|
name: "whitespace_trimmed_ms",
|
|
headers: map[string]string{"retry-after-ms": " 1500 "},
|
|
expected: 1500 * time.Millisecond,
|
|
},
|
|
{
|
|
name: "negative_ms_returns_zero",
|
|
headers: map[string]string{"retry-after-ms": "-100"},
|
|
expected: 0,
|
|
},
|
|
// Retry-After (standard, seconds).
|
|
{
|
|
name: "standard_retry_after_seconds",
|
|
headers: map[string]string{"Retry-After": "60"},
|
|
expected: 60 * time.Second,
|
|
},
|
|
{
|
|
name: "whitespace_trimmed_seconds",
|
|
headers: map[string]string{"Retry-After": " 30 "},
|
|
expected: 30 * time.Second,
|
|
},
|
|
{
|
|
name: "zero_seconds_returns_zero",
|
|
headers: map[string]string{"Retry-After": "0"},
|
|
expected: 0,
|
|
},
|
|
{
|
|
name: "negative_seconds_returns_zero",
|
|
headers: map[string]string{"Retry-After": "-5"},
|
|
expected: 0,
|
|
},
|
|
// Both headers set: precedence and fallback.
|
|
{
|
|
name: "prefers_retry_after_ms_over_standard",
|
|
headers: map[string]string{
|
|
"retry-after-ms": "1500",
|
|
"Retry-After": "30",
|
|
},
|
|
expected: 1500 * time.Millisecond,
|
|
},
|
|
{
|
|
name: "falls_back_to_standard_when_ms_invalid",
|
|
headers: map[string]string{"retry-after-ms": "invalid", "Retry-After": "10"},
|
|
expected: 10 * time.Second,
|
|
},
|
|
{
|
|
name: "zero_ms_falls_back_to_standard",
|
|
headers: map[string]string{"retry-after-ms": "0", "Retry-After": "5"},
|
|
expected: 5 * time.Second,
|
|
},
|
|
{
|
|
name: "zero_ms_and_zero_seconds_return_zero",
|
|
headers: map[string]string{"retry-after-ms": "0", "Retry-After": "0"},
|
|
expected: 0,
|
|
},
|
|
}
|
|
|
|
for _, tc := range tests {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
var resp *http.Response
|
|
if !tc.nilResponse {
|
|
resp = &http.Response{Header: make(http.Header)}
|
|
for key, val := range tc.headers {
|
|
resp.Header.Set(key, val)
|
|
}
|
|
}
|
|
assert.Equal(t, tc.expected, keypool.ParseRetryAfter(resp))
|
|
})
|
|
}
|
|
}
|