fix: track credential hint across key failover attempts in aibridge (#25735)

## Problem

Centralized requests recorded *the first available key from the pool at
`CreateInterceptor` time* as `credential_hint`, so the interception
could be persisted in the database with a hint that didn't match the key
that actually served the request. The fix consists in storing, at
end-of-interception, the hint of the key that succeeded, or the last
attempted key if all keys are unavailable.

## Changes

- Add `Key.Hint()` and update `credential_hint` on every failover
attempt so it reflects the actually-used key.
- Stop pre-populating `credential_hint` at `CreateInterceptor`.
Centralized starts empty and is updated by the key failover loop.
- Persist the final hint via `RecordInterceptionEnded`; SQL updates
`credential_hint` only when `credential_kind = 'centralized'` so BYOK
keeps its start-time value.
- Log the actually-used hint on interception end/failure; start log uses
a `<keypool-pending>` placeholder for centralized.

> [!NOTE]
> Initially generated by Claude Opus 4.7, modified and reviewed by
@ssncferreira
This commit is contained in:
Susana Ferreira
2026-05-29 12:01:37 +01:00
committed by GitHub
parent a586b7e5e0
commit 7b903cad73
29 changed files with 736 additions and 481 deletions
+24 -8
View File
@@ -20,6 +20,7 @@ import (
"cdr.dev/slog/v3"
"github.com/coder/coder/v2/aibridge/circuitbreaker"
aibcontext "github.com/coder/coder/v2/aibridge/context"
"github.com/coder/coder/v2/aibridge/intercept"
"github.com/coder/coder/v2/aibridge/mcp"
"github.com/coder/coder/v2/aibridge/metrics"
"github.com/coder/coder/v2/aibridge/provider"
@@ -275,11 +276,18 @@ func newInterceptionProcessor(p provider.Provider, cbs *circuitbreaker.ProviderC
slog.F("user_agent", r.UserAgent()),
slog.F("streaming", interceptor.Streaming()),
slog.F("credential_kind", string(cred.Kind)),
)
// Log BYOK credentials. Centralized credentials are set by
// the key failover loop.
credLogFields := []slog.Field{}
if cred.Kind == intercept.CredentialKindBYOK {
credLogFields = append(credLogFields,
slog.F("credential_hint", cred.Hint),
slog.F("credential_length", cred.Length),
)
log.Debug(ctx, "interception started")
}
log.Debug(ctx, "interception started", credLogFields...)
if m != nil {
m.InterceptionsInflight.WithLabelValues(p.Name(), interceptor.Model(), route).Add(1)
defer func() {
@@ -288,22 +296,30 @@ func newInterceptionProcessor(p provider.Provider, cbs *circuitbreaker.ProviderC
}
// Process request with circuit breaker protection if configured
if err := cbs.Execute(route, interceptor.Model(), w, func(rw http.ResponseWriter) error {
execErr := cbs.Execute(route, interceptor.Model(), w, func(rw http.ResponseWriter) error {
return interceptor.ProcessRequest(rw, r)
}); err != nil {
})
// For centralized, the hint now reflects the last attempted
// key from the failover loop.
credHint := interceptor.Credential().Hint
credLen := interceptor.Credential().Length
if execErr != nil {
if m != nil {
m.InterceptionCount.WithLabelValues(p.Name(), interceptor.Model(), metrics.InterceptionCountStatusFailed, route, r.Method, actor.ID, string(client)).Add(1)
}
span.SetStatus(codes.Error, fmt.Sprintf("interception failed: %v", err))
log.Warn(ctx, "interception failed", slog.Error(err))
span.SetStatus(codes.Error, fmt.Sprintf("interception failed: %v", execErr))
log.Warn(ctx, "interception failed", slog.Error(execErr), slog.F("credential_hint", credHint), slog.F("credential_length", credLen))
} else {
if m != nil {
m.InterceptionCount.WithLabelValues(p.Name(), interceptor.Model(), metrics.InterceptionCountStatusCompleted, route, r.Method, actor.ID, string(client)).Add(1)
}
log.Debug(ctx, "interception ended")
log.Debug(ctx, "interception ended", slog.F("credential_hint", credHint), slog.F("credential_length", credLen))
}
_ = asyncRecorder.RecordInterceptionEnded(ctx, &recorder.InterceptionRecordEnded{ID: interceptor.ID().String()})
_ = asyncRecorder.RecordInterceptionEnded(ctx, &recorder.InterceptionRecordEnded{
ID: interceptor.ID().String(),
CredentialHint: credHint,
})
// Ensure all recording have completed before completing request.
asyncRecorder.Wait()
@@ -291,15 +291,16 @@ func (i *BlockingInterception) newChatCompletionWithKey(ctx context.Context, svc
// 401/403. Errors that aren't key-specific don't trigger
// failover and are returned to the caller.
func (i *BlockingInterception) newChatCompletionWithKeyFailover(ctx context.Context, svc openai.ChatCompletionService, opts []option.RequestOption) (*openai.ChatCompletion, error) {
// TODO(ssncferreira): update the interception's credential
// hint with the actually-used key (the successful key on
// success, the last tried key on failure) in the upstack PR.
walker := i.cfg.KeyPool.Walker()
for {
key, keyPoolErr := walker.Next()
if keyPoolErr != nil {
return nil, keyPoolErr
}
// Record the key in use so the hint reflects the last attempted key.
i.credential = intercept.NewCredentialInfo(intercept.CredentialKindCentralized, key.Value())
i.logger.Debug(ctx, "using centralized api key",
slog.F("credential_hint", i.Credential().Hint), slog.F("credential_length", i.Credential().Length))
requestOpts := append([]option.RequestOption{}, opts...)
requestOpts = append(requestOpts,
@@ -72,31 +72,35 @@ func TestBlockingInterception_KeyFailover(t *testing.T) {
expectedRetryAfter string
// Expected key states after the request, by index in keys.
expectedKeyStates []keypool.KeyState
// Expected credential hint after ProcessRequest: last
// attempted key for centralized, user key from initial request for BYOK.
expectedCredentialHint string
}{
{
// Given: 1 valid key returning 200.
// Then: 1 request, 200 response, key remains valid.
name: "single_valid_key",
keys: []string{"k0"},
keys: []string{"k0-long-key"},
responses: map[string]upstreamResponse{
"k0": {statusCode: http.StatusOK, body: successBody},
"k0-long-key": {statusCode: http.StatusOK, body: successBody},
},
expectedRequestCount: 1,
expectedStatusCode: http.StatusOK,
expectedKeyStates: []keypool.KeyState{keypool.KeyStateValid},
expectedCredentialHint: utils.MaskSecret("k0-long-key"),
},
{
// Given: 2 keys; key-0 returns 429, key-1 returns 200.
// Then: 2 requests, 200 response, key-0 temporary, key-1 valid.
name: "failover_after_429",
keys: []string{"k0", "k1"},
keys: []string{"k0-long-key", "k1-long-key"},
responses: map[string]upstreamResponse{
"k0": {
"k0-long-key": {
statusCode: http.StatusTooManyRequests,
headers: map[string]string{"Retry-After": "5"},
body: rateLimitBody,
},
"k1": {statusCode: http.StatusOK, body: successBody},
"k1-long-key": {statusCode: http.StatusOK, body: successBody},
},
expectedRequestCount: 2,
expectedStatusCode: http.StatusOK,
@@ -104,15 +108,16 @@ func TestBlockingInterception_KeyFailover(t *testing.T) {
keypool.KeyStateTemporary,
keypool.KeyStateValid,
},
expectedCredentialHint: utils.MaskSecret("k1-long-key"),
},
{
// Given: 2 keys; key-0 returns 401, key-1 returns 200.
// Then: 2 requests, 200 response, key-0 permanent, key-1 valid.
name: "failover_after_401",
keys: []string{"k0", "k1"},
keys: []string{"k0-long-key", "k1-long-key"},
responses: map[string]upstreamResponse{
"k0": {statusCode: http.StatusUnauthorized, body: authErrorBody},
"k1": {statusCode: http.StatusOK, body: successBody},
"k0-long-key": {statusCode: http.StatusUnauthorized, body: authErrorBody},
"k1-long-key": {statusCode: http.StatusOK, body: successBody},
},
expectedRequestCount: 2,
expectedStatusCode: http.StatusOK,
@@ -120,15 +125,16 @@ func TestBlockingInterception_KeyFailover(t *testing.T) {
keypool.KeyStatePermanent,
keypool.KeyStateValid,
},
expectedCredentialHint: utils.MaskSecret("k1-long-key"),
},
{
// Given: 2 keys; key-0 returns 403, key-1 returns 200.
// Then: 2 requests, 200 response, key-0 permanent, key-1 valid.
name: "failover_after_403",
keys: []string{"k0", "k1"},
keys: []string{"k0-long-key", "k1-long-key"},
responses: map[string]upstreamResponse{
"k0": {statusCode: http.StatusForbidden, body: authErrorBody},
"k1": {statusCode: http.StatusOK, body: successBody},
"k0-long-key": {statusCode: http.StatusForbidden, body: authErrorBody},
"k1-long-key": {statusCode: http.StatusOK, body: successBody},
},
expectedRequestCount: 2,
expectedStatusCode: http.StatusOK,
@@ -136,25 +142,26 @@ func TestBlockingInterception_KeyFailover(t *testing.T) {
keypool.KeyStatePermanent,
keypool.KeyStateValid,
},
expectedCredentialHint: utils.MaskSecret("k1-long-key"),
},
{
// Given: 3 keys; all return 429 with cooldowns 5s, 3s, 10s.
// Then: 3 requests, 429 response with smallest Retry-After,
// all keys temporary.
name: "all_keys_rate_limited",
keys: []string{"k0", "k1", "k2"},
keys: []string{"k0-long-key", "k1-long-key", "k2-long-key"},
responses: map[string]upstreamResponse{
"k0": {
"k0-long-key": {
statusCode: http.StatusTooManyRequests,
headers: map[string]string{"Retry-After": "5"},
body: rateLimitBody,
},
"k1": {
"k1-long-key": {
statusCode: http.StatusTooManyRequests,
headers: map[string]string{"Retry-After": "3"},
body: rateLimitBody,
},
"k2": {
"k2-long-key": {
statusCode: http.StatusTooManyRequests,
headers: map[string]string{"Retry-After": "10"},
body: rateLimitBody,
@@ -168,15 +175,16 @@ func TestBlockingInterception_KeyFailover(t *testing.T) {
keypool.KeyStateTemporary,
keypool.KeyStateTemporary,
},
expectedCredentialHint: utils.MaskSecret("k2-long-key"),
},
{
// Given: 2 keys; both return 401.
// Then: 2 requests, 502 api_error response, both keys permanent.
name: "all_keys_unauthorized",
keys: []string{"k0", "k1"},
keys: []string{"k0-long-key", "k1-long-key"},
responses: map[string]upstreamResponse{
"k0": {statusCode: http.StatusUnauthorized, body: authErrorBody},
"k1": {statusCode: http.StatusUnauthorized, body: authErrorBody},
"k0-long-key": {statusCode: http.StatusUnauthorized, body: authErrorBody},
"k1-long-key": {statusCode: http.StatusUnauthorized, body: authErrorBody},
},
expectedRequestCount: 2,
expectedStatusCode: http.StatusBadGateway,
@@ -184,14 +192,15 @@ func TestBlockingInterception_KeyFailover(t *testing.T) {
keypool.KeyStatePermanent,
keypool.KeyStatePermanent,
},
expectedCredentialHint: utils.MaskSecret("k1-long-key"),
},
{
// Given: 2 keys; key-0 returns 500.
// Then: 1 request, 500 response, both keys remain valid.
name: "server_error_no_failover",
keys: []string{"k0", "k1"},
keys: []string{"k0-long-key", "k1-long-key"},
responses: map[string]upstreamResponse{
"k0": {statusCode: http.StatusInternalServerError, body: serverErrorBody},
"k0-long-key": {statusCode: http.StatusInternalServerError, body: serverErrorBody},
},
expectedRequestCount: 1,
expectedStatusCode: http.StatusInternalServerError,
@@ -199,6 +208,7 @@ func TestBlockingInterception_KeyFailover(t *testing.T) {
keypool.KeyStateValid,
keypool.KeyStateValid,
},
expectedCredentialHint: utils.MaskSecret("k0-long-key"),
},
{
// Given: BYOK with a single key returning 429.
@@ -222,6 +232,7 @@ func TestBlockingInterception_KeyFailover(t *testing.T) {
expectedRequestCount: 1,
expectedStatusCode: http.StatusTooManyRequests,
expectedRetryAfter: "5",
expectedCredentialHint: utils.MaskSecret("user-byok"),
},
}
@@ -252,6 +263,7 @@ func TestBlockingInterception_KeyFailover(t *testing.T) {
cfg := config.OpenAI{BaseURL: upstream.URL + "/"}
var pool *keypool.Pool
credInfo := intercept.NewCredentialInfo(intercept.CredentialKindCentralized, "")
if len(tc.keys) > 0 {
var err error
pool, err = keypool.New(tc.keys, quartz.NewMock(t))
@@ -259,6 +271,7 @@ func TestBlockingInterception_KeyFailover(t *testing.T) {
cfg.KeyPool = pool
} else if tc.byokKey != "" {
cfg.Key = tc.byokKey
credInfo = intercept.NewCredentialInfo(intercept.CredentialKindBYOK, tc.byokKey)
}
interceptor := NewBlockingInterceptor(
@@ -269,7 +282,7 @@ func TestBlockingInterception_KeyFailover(t *testing.T) {
http.Header{},
"Authorization",
otel.Tracer("blocking_test"),
intercept.NewCredentialInfo(intercept.CredentialKindCentralized, ""),
credInfo,
)
interceptor.Setup(slog.Make(), &testutil.MockRecorder{}, nil)
@@ -288,6 +301,7 @@ func TestBlockingInterception_KeyFailover(t *testing.T) {
if pool != nil {
assert.Equal(t, tc.expectedKeyStates, pool.PoolState(), "key states")
}
assert.Equal(t, tc.expectedCredentialHint, interceptor.Credential().Hint, "credential hint")
})
}
}
@@ -309,6 +323,9 @@ func TestBlockingInterception_AgenticLoopFailover(t *testing.T) {
expectedSeenKeys []string
expectedStatusCode int
expectedKeyStates []keypool.KeyState
// Expected credential hint after ProcessRequest: hint of the
// last attempted key across all agentic-loop iterations.
expectedCredentialHint string
}{
{
// Given: 2 keys; both upstream calls succeed on key-0.
@@ -319,12 +336,13 @@ func TestBlockingInterception_AgenticLoopFailover(t *testing.T) {
{statusCode: http.StatusOK, body: textCompleteBody},
},
expectedRequestCount: 2,
expectedSeenKeys: []string{"k0", "k0"},
expectedSeenKeys: []string{"k0-long-key", "k0-long-key"},
expectedStatusCode: http.StatusOK,
expectedKeyStates: []keypool.KeyState{
keypool.KeyStateValid,
keypool.KeyStateValid,
},
expectedCredentialHint: utils.MaskSecret("k0-long-key"),
},
{
// Given: 2 keys; key-0 succeeds initially, then 429s
@@ -342,12 +360,13 @@ func TestBlockingInterception_AgenticLoopFailover(t *testing.T) {
{statusCode: http.StatusOK, body: textCompleteBody},
},
expectedRequestCount: 3,
expectedSeenKeys: []string{"k0", "k0", "k1"},
expectedSeenKeys: []string{"k0-long-key", "k0-long-key", "k1-long-key"},
expectedStatusCode: http.StatusOK,
expectedKeyStates: []keypool.KeyState{
keypool.KeyStateTemporary,
keypool.KeyStateValid,
},
expectedCredentialHint: utils.MaskSecret("k1-long-key"),
},
{
// Given: 2 keys; key-0 succeeds initially, then both
@@ -369,12 +388,13 @@ func TestBlockingInterception_AgenticLoopFailover(t *testing.T) {
},
},
expectedRequestCount: 3,
expectedSeenKeys: []string{"k0", "k0", "k1"},
expectedSeenKeys: []string{"k0-long-key", "k0-long-key", "k1-long-key"},
expectedStatusCode: http.StatusTooManyRequests,
expectedKeyStates: []keypool.KeyState{
keypool.KeyStateTemporary,
keypool.KeyStateTemporary,
},
expectedCredentialHint: utils.MaskSecret("k1-long-key"),
},
}
@@ -409,7 +429,7 @@ func TestBlockingInterception_AgenticLoopFailover(t *testing.T) {
}))
t.Cleanup(upstream.Close)
pool, err := keypool.New([]string{"k0", "k1"}, quartz.NewMock(t))
pool, err := keypool.New([]string{"k0-long-key", "k1-long-key"}, quartz.NewMock(t))
require.NoError(t, err)
cfg := config.OpenAI{
@@ -459,6 +479,7 @@ func TestBlockingInterception_AgenticLoopFailover(t *testing.T) {
defer seenKeysMu.Unlock()
assert.Equal(t, tc.expectedSeenKeys, seenKeys, "seen keys")
assert.Equal(t, tc.expectedKeyStates, pool.PoolState(), "key states")
assert.Equal(t, tc.expectedCredentialHint, interceptor.Credential().Hint, "credential hint")
})
}
}
@@ -164,6 +164,11 @@ func (i *StreamingInterception) ProcessRequest(w http.ResponseWriter, r *http.Re
break
}
currentKey = key
// Record the key in use so the hint reflects the last attempted key.
i.credential = intercept.NewCredentialInfo(intercept.CredentialKindCentralized, key.Value())
logger.Debug(ctx, "using centralized api key",
slog.F("credential_hint", i.Credential().Hint), slog.F("credential_length", i.Credential().Length))
opts = append(opts,
option.WithAPIKey(key.Value()),
// Disable SDK retries because the failover
@@ -144,14 +144,17 @@ func TestStreamingInterception_KeyFailover(t *testing.T) {
expectedRetryAfter string
// Expected key states after the request, by index in keys.
expectedKeyStates []keypool.KeyState
// Expected credential hint after ProcessRequest: last
// attempted key for centralized, user key from initial request for BYOK.
expectedCredentialHint string
}{
{
// Given: 1 valid key returning a successful stream.
// Then: 1 request, 200 response, key remains valid.
name: "single_valid_key",
keys: []string{"k0"},
keys: []string{"k0-long-key"},
responses: map[string]upstreamResponse{
"k0": {
"k0-long-key": {
statusCode: http.StatusOK,
headers: map[string]string{"Content-Type": "text/event-stream"},
body: streamingSuccessBody,
@@ -160,20 +163,21 @@ func TestStreamingInterception_KeyFailover(t *testing.T) {
expectedRequestCount: 1,
expectedStatusCode: http.StatusOK,
expectedKeyStates: []keypool.KeyState{keypool.KeyStateValid},
expectedCredentialHint: utils.MaskSecret("k0-long-key"),
},
{
// Given: 2 keys; key-0 returns 429 pre-stream, key-1
// streams successfully.
// Then: 2 requests, 200 response, key-0 temporary, key-1 valid.
name: "failover_after_429",
keys: []string{"k0", "k1"},
keys: []string{"k0-long-key", "k1-long-key"},
responses: map[string]upstreamResponse{
"k0": {
"k0-long-key": {
statusCode: http.StatusTooManyRequests,
headers: map[string]string{"Retry-After": "5"},
body: rateLimitBody,
},
"k1": {
"k1-long-key": {
statusCode: http.StatusOK,
headers: map[string]string{"Content-Type": "text/event-stream"},
body: streamingSuccessBody,
@@ -185,16 +189,17 @@ func TestStreamingInterception_KeyFailover(t *testing.T) {
keypool.KeyStateTemporary,
keypool.KeyStateValid,
},
expectedCredentialHint: utils.MaskSecret("k1-long-key"),
},
{
// Given: 2 keys; key-0 returns 401 pre-stream, key-1
// streams successfully.
// Then: 2 requests, 200 response, key-0 permanent, key-1 valid.
name: "failover_after_401",
keys: []string{"k0", "k1"},
keys: []string{"k0-long-key", "k1-long-key"},
responses: map[string]upstreamResponse{
"k0": {statusCode: http.StatusUnauthorized, body: authErrorBody},
"k1": {
"k0-long-key": {statusCode: http.StatusUnauthorized, body: authErrorBody},
"k1-long-key": {
statusCode: http.StatusOK,
headers: map[string]string{"Content-Type": "text/event-stream"},
body: streamingSuccessBody,
@@ -206,15 +211,16 @@ func TestStreamingInterception_KeyFailover(t *testing.T) {
keypool.KeyStatePermanent,
keypool.KeyStateValid,
},
expectedCredentialHint: utils.MaskSecret("k1-long-key"),
},
{
// Given: 2 keys; key-0 returns 403 pre-stream, key-1 streams.
// Then: 2 requests, 200 response, key-0 permanent, key-1 valid.
name: "failover_after_403",
keys: []string{"k0", "k1"},
keys: []string{"k0-long-key", "k1-long-key"},
responses: map[string]upstreamResponse{
"k0": {statusCode: http.StatusForbidden, body: authErrorBody},
"k1": {
"k0-long-key": {statusCode: http.StatusForbidden, body: authErrorBody},
"k1-long-key": {
statusCode: http.StatusOK,
headers: map[string]string{"Content-Type": "text/event-stream"},
body: streamingSuccessBody,
@@ -226,6 +232,7 @@ func TestStreamingInterception_KeyFailover(t *testing.T) {
keypool.KeyStatePermanent,
keypool.KeyStateValid,
},
expectedCredentialHint: utils.MaskSecret("k1-long-key"),
},
{
// Given: 3 keys; all return 429 pre-stream with
@@ -233,19 +240,19 @@ func TestStreamingInterception_KeyFailover(t *testing.T) {
// Then: 3 requests, 429 response with smallest
// Retry-After, all keys temporary.
name: "all_keys_rate_limited",
keys: []string{"k0", "k1", "k2"},
keys: []string{"k0-long-key", "k1-long-key", "k2-long-key"},
responses: map[string]upstreamResponse{
"k0": {
"k0-long-key": {
statusCode: http.StatusTooManyRequests,
headers: map[string]string{"Retry-After": "5"},
body: rateLimitBody,
},
"k1": {
"k1-long-key": {
statusCode: http.StatusTooManyRequests,
headers: map[string]string{"Retry-After": "3"},
body: rateLimitBody,
},
"k2": {
"k2-long-key": {
statusCode: http.StatusTooManyRequests,
headers: map[string]string{"Retry-After": "10"},
body: rateLimitBody,
@@ -259,15 +266,16 @@ func TestStreamingInterception_KeyFailover(t *testing.T) {
keypool.KeyStateTemporary,
keypool.KeyStateTemporary,
},
expectedCredentialHint: utils.MaskSecret("k2-long-key"),
},
{
// Given: 2 keys; both return 401 pre-stream.
// Then: 2 requests, 502 api_error response, both keys permanent.
name: "all_keys_unauthorized",
keys: []string{"k0", "k1"},
keys: []string{"k0-long-key", "k1-long-key"},
responses: map[string]upstreamResponse{
"k0": {statusCode: http.StatusUnauthorized, body: authErrorBody},
"k1": {statusCode: http.StatusUnauthorized, body: authErrorBody},
"k0-long-key": {statusCode: http.StatusUnauthorized, body: authErrorBody},
"k1-long-key": {statusCode: http.StatusUnauthorized, body: authErrorBody},
},
expectedRequestCount: 2,
expectedStatusCode: http.StatusBadGateway,
@@ -275,14 +283,15 @@ func TestStreamingInterception_KeyFailover(t *testing.T) {
keypool.KeyStatePermanent,
keypool.KeyStatePermanent,
},
expectedCredentialHint: utils.MaskSecret("k1-long-key"),
},
{
// Given: 2 keys; key-0 returns 500 pre-stream.
// Then: 1 request, 500 response, both keys remain valid.
name: "server_error_no_failover",
keys: []string{"k0", "k1"},
keys: []string{"k0-long-key", "k1-long-key"},
responses: map[string]upstreamResponse{
"k0": {statusCode: http.StatusInternalServerError, body: serverErrorBody},
"k0-long-key": {statusCode: http.StatusInternalServerError, body: serverErrorBody},
},
expectedRequestCount: 1,
expectedStatusCode: http.StatusInternalServerError,
@@ -290,6 +299,7 @@ func TestStreamingInterception_KeyFailover(t *testing.T) {
keypool.KeyStateValid,
keypool.KeyStateValid,
},
expectedCredentialHint: utils.MaskSecret("k0-long-key"),
},
{
// Given: BYOK with a single key returning 429.
@@ -313,6 +323,7 @@ func TestStreamingInterception_KeyFailover(t *testing.T) {
expectedRequestCount: 1,
expectedStatusCode: http.StatusTooManyRequests,
expectedRetryAfter: "5",
expectedCredentialHint: utils.MaskSecret("user-byok"),
},
}
@@ -342,6 +353,7 @@ func TestStreamingInterception_KeyFailover(t *testing.T) {
cfg := config.OpenAI{BaseURL: upstream.URL + "/"}
var pool *keypool.Pool
credInfo := intercept.NewCredentialInfo(intercept.CredentialKindCentralized, "")
if len(tc.keys) > 0 {
var err error
pool, err = keypool.New(tc.keys, quartz.NewMock(t))
@@ -349,6 +361,7 @@ func TestStreamingInterception_KeyFailover(t *testing.T) {
cfg.KeyPool = pool
} else if tc.byokKey != "" {
cfg.Key = tc.byokKey
credInfo = intercept.NewCredentialInfo(intercept.CredentialKindBYOK, tc.byokKey)
}
interceptor := NewStreamingInterceptor(
@@ -359,7 +372,7 @@ func TestStreamingInterception_KeyFailover(t *testing.T) {
http.Header{},
"Authorization",
otel.Tracer("streaming_test"),
intercept.NewCredentialInfo(intercept.CredentialKindCentralized, ""),
credInfo,
)
interceptor.Setup(slog.Make(), &testutil.MockRecorder{}, nil)
@@ -378,6 +391,7 @@ func TestStreamingInterception_KeyFailover(t *testing.T) {
if pool != nil {
assert.Equal(t, tc.expectedKeyStates, pool.PoolState(), "key states")
}
assert.Equal(t, tc.expectedCredentialHint, interceptor.Credential().Hint, "credential hint")
})
}
}
@@ -435,6 +449,9 @@ func TestStreamingInterception_AgenticLoopFailover(t *testing.T) {
// error (e.g. all keys exhausted).
expectedErr bool
expectedKeyStates []keypool.KeyState
// Expected credential hint after ProcessRequest: hint of the
// last attempted key across all agentic-loop iterations.
expectedCredentialHint string
}{
{
// Given: 2 keys; both upstream calls succeed on key-0.
@@ -445,13 +462,14 @@ func TestStreamingInterception_AgenticLoopFailover(t *testing.T) {
{statusCode: http.StatusOK, headers: sseHeaders, body: textStreamBody},
},
expectedRequestCount: 2,
expectedSeenKeys: []string{"k0", "k0"},
expectedSeenKeys: []string{"k0-long-key", "k0-long-key"},
expectedBodyContains: "done",
expectErrorAsSSEEvent: false,
expectedKeyStates: []keypool.KeyState{
keypool.KeyStateValid,
keypool.KeyStateValid,
},
expectedCredentialHint: utils.MaskSecret("k0-long-key"),
},
{
// Given: 2 keys; key-0 succeeds initially, then 429s
@@ -469,13 +487,14 @@ func TestStreamingInterception_AgenticLoopFailover(t *testing.T) {
{statusCode: http.StatusOK, headers: sseHeaders, body: textStreamBody},
},
expectedRequestCount: 3,
expectedSeenKeys: []string{"k0", "k0", "k1"},
expectedSeenKeys: []string{"k0-long-key", "k0-long-key", "k1-long-key"},
expectedBodyContains: "done",
expectErrorAsSSEEvent: false,
expectedKeyStates: []keypool.KeyState{
keypool.KeyStateTemporary,
keypool.KeyStateValid,
},
expectedCredentialHint: utils.MaskSecret("k1-long-key"),
},
{
// Given: 2 keys; key-0 succeeds initially, then both
@@ -497,7 +516,7 @@ func TestStreamingInterception_AgenticLoopFailover(t *testing.T) {
},
},
expectedRequestCount: 3,
expectedSeenKeys: []string{"k0", "k0", "k1"},
expectedSeenKeys: []string{"k0-long-key", "k0-long-key", "k1-long-key"},
expectedBodyContains: "all configured keys are rate-limited",
expectErrorAsSSEEvent: true,
expectedErr: true,
@@ -505,6 +524,7 @@ func TestStreamingInterception_AgenticLoopFailover(t *testing.T) {
keypool.KeyStateTemporary,
keypool.KeyStateTemporary,
},
expectedCredentialHint: utils.MaskSecret("k1-long-key"),
},
}
@@ -538,7 +558,7 @@ func TestStreamingInterception_AgenticLoopFailover(t *testing.T) {
}))
t.Cleanup(upstream.Close)
pool, err := keypool.New([]string{"k0", "k1"}, quartz.NewMock(t))
pool, err := keypool.New([]string{"k0-long-key", "k1-long-key"}, quartz.NewMock(t))
require.NoError(t, err)
cfg := config.OpenAI{
@@ -596,6 +616,7 @@ func TestStreamingInterception_AgenticLoopFailover(t *testing.T) {
defer seenKeysMu.Unlock()
assert.Equal(t, tc.expectedSeenKeys, seenKeys, "seen keys")
assert.Equal(t, tc.expectedKeyStates, pool.PoolState(), "key states")
assert.Equal(t, tc.expectedCredentialHint, interceptor.Credential().Hint, "credential hint")
})
}
}
+4 -3
View File
@@ -367,15 +367,16 @@ func (i *BlockingInterception) newMessageWithKey(ctx context.Context, svc anthro
// Errors that aren't key-specific don't trigger failover and
// are returned to the caller.
func (i *BlockingInterception) newMessageWithKeyFailover(ctx context.Context, svc anthropic.MessageService) (*anthropic.Message, error) {
// TODO(ssncferreira): update the interception's credential
// hint with the actually-used key (the successful key on
// success, the last tried key on failure) in the upstack PR.
walker := i.cfg.KeyPool.Walker()
for {
key, keyPoolErr := walker.Next()
if keyPoolErr != nil {
return nil, keyPoolErr
}
// Record the key in use so the hint reflects the last attempted key.
i.credential = intercept.NewCredentialInfo(intercept.CredentialKindCentralized, key.Value())
i.logger.Debug(ctx, "using centralized api key",
slog.F("credential_hint", i.Credential().Hint), slog.F("credential_length", i.Credential().Length))
msg, err := i.newMessageWithKey(ctx, svc,
option.WithAPIKey(key.Value()),
@@ -19,6 +19,7 @@ import (
"github.com/coder/coder/v2/aibridge/internal/testutil"
"github.com/coder/coder/v2/aibridge/keypool"
"github.com/coder/coder/v2/aibridge/mcp"
"github.com/coder/coder/v2/aibridge/utils"
"github.com/coder/quartz"
)
@@ -54,31 +55,35 @@ func TestBlockingInterception_KeyFailover(t *testing.T) {
expectedRetryAfter string
// Expected key states after the request, by index in keys.
expectedKeyStates []keypool.KeyState
// Expected credential hint after ProcessRequest: last
// attempted key for centralized, user key from initial request for BYOK.
expectedCredentialHint string
}{
{
// Given: 1 valid key returning 200.
// Then: 1 request, 200 response, key remains valid.
name: "single_valid_key",
keys: []string{"k0"},
keys: []string{"k0-long-key"},
responses: map[string]upstreamResponse{
"k0": {statusCode: http.StatusOK, body: successBody},
"k0-long-key": {statusCode: http.StatusOK, body: successBody},
},
expectedRequestCount: 1,
expectedStatusCode: http.StatusOK,
expectedKeyStates: []keypool.KeyState{keypool.KeyStateValid},
expectedCredentialHint: utils.MaskSecret("k0-long-key"),
},
{
// Given: 2 keys; key-0 returns 429, key-1 returns 200.
// Then: 2 requests, 200 response, key-0 temporary, key-1 valid.
name: "failover_after_429",
keys: []string{"k0", "k1"},
keys: []string{"k0-long-key", "k1-long-key"},
responses: map[string]upstreamResponse{
"k0": {
"k0-long-key": {
statusCode: http.StatusTooManyRequests,
headers: map[string]string{"Retry-After": "5"},
body: rateLimitBody,
},
"k1": {statusCode: http.StatusOK, body: successBody},
"k1-long-key": {statusCode: http.StatusOK, body: successBody},
},
expectedRequestCount: 2,
expectedStatusCode: http.StatusOK,
@@ -86,15 +91,16 @@ func TestBlockingInterception_KeyFailover(t *testing.T) {
keypool.KeyStateTemporary,
keypool.KeyStateValid,
},
expectedCredentialHint: utils.MaskSecret("k1-long-key"),
},
{
// Given: 2 keys; key-0 returns 401, key-1 returns 200.
// Then: 2 requests, 200 response, key-0 permanent, key-1 valid.
name: "failover_after_401",
keys: []string{"k0", "k1"},
keys: []string{"k0-long-key", "k1-long-key"},
responses: map[string]upstreamResponse{
"k0": {statusCode: http.StatusUnauthorized, body: authErrorBody},
"k1": {statusCode: http.StatusOK, body: successBody},
"k0-long-key": {statusCode: http.StatusUnauthorized, body: authErrorBody},
"k1-long-key": {statusCode: http.StatusOK, body: successBody},
},
expectedRequestCount: 2,
expectedStatusCode: http.StatusOK,
@@ -102,15 +108,16 @@ func TestBlockingInterception_KeyFailover(t *testing.T) {
keypool.KeyStatePermanent,
keypool.KeyStateValid,
},
expectedCredentialHint: utils.MaskSecret("k1-long-key"),
},
{
// Given: 2 keys; key-0 returns 403, key-1 returns 200.
// Then: 2 requests, 200 response, key-0 permanent, key-1 valid.
name: "failover_after_403",
keys: []string{"k0", "k1"},
keys: []string{"k0-long-key", "k1-long-key"},
responses: map[string]upstreamResponse{
"k0": {statusCode: http.StatusForbidden, body: authErrorBody},
"k1": {statusCode: http.StatusOK, body: successBody},
"k0-long-key": {statusCode: http.StatusForbidden, body: authErrorBody},
"k1-long-key": {statusCode: http.StatusOK, body: successBody},
},
expectedRequestCount: 2,
expectedStatusCode: http.StatusOK,
@@ -118,25 +125,26 @@ func TestBlockingInterception_KeyFailover(t *testing.T) {
keypool.KeyStatePermanent,
keypool.KeyStateValid,
},
expectedCredentialHint: utils.MaskSecret("k1-long-key"),
},
{
// Given: 3 keys; all return 429 with cooldowns 5s, 3s, 10s.
// Then: 3 requests, 429 response with smallest Retry-After,
// all keys temporary.
name: "all_keys_rate_limited",
keys: []string{"k0", "k1", "k2"},
keys: []string{"k0-long-key", "k1-long-key", "k2-long-key"},
responses: map[string]upstreamResponse{
"k0": {
"k0-long-key": {
statusCode: http.StatusTooManyRequests,
headers: map[string]string{"Retry-After": "5"},
body: rateLimitBody,
},
"k1": {
"k1-long-key": {
statusCode: http.StatusTooManyRequests,
headers: map[string]string{"Retry-After": "3"},
body: rateLimitBody,
},
"k2": {
"k2-long-key": {
statusCode: http.StatusTooManyRequests,
headers: map[string]string{"Retry-After": "10"},
body: rateLimitBody,
@@ -150,15 +158,16 @@ func TestBlockingInterception_KeyFailover(t *testing.T) {
keypool.KeyStateTemporary,
keypool.KeyStateTemporary,
},
expectedCredentialHint: utils.MaskSecret("k2-long-key"),
},
{
// Given: 2 keys; both return 401.
// Then: 2 requests, 502 api_error response, both keys permanent.
name: "all_keys_unauthorized",
keys: []string{"k0", "k1"},
keys: []string{"k0-long-key", "k1-long-key"},
responses: map[string]upstreamResponse{
"k0": {statusCode: http.StatusUnauthorized, body: authErrorBody},
"k1": {statusCode: http.StatusUnauthorized, body: authErrorBody},
"k0-long-key": {statusCode: http.StatusUnauthorized, body: authErrorBody},
"k1-long-key": {statusCode: http.StatusUnauthorized, body: authErrorBody},
},
expectedRequestCount: 2,
expectedStatusCode: http.StatusBadGateway,
@@ -166,14 +175,15 @@ func TestBlockingInterception_KeyFailover(t *testing.T) {
keypool.KeyStatePermanent,
keypool.KeyStatePermanent,
},
expectedCredentialHint: utils.MaskSecret("k1-long-key"),
},
{
// Given: 2 keys; key-0 returns 500.
// Then: 1 request, 500 response, both keys remain valid.
name: "server_error_no_failover",
keys: []string{"k0", "k1"},
keys: []string{"k0-long-key", "k1-long-key"},
responses: map[string]upstreamResponse{
"k0": {statusCode: http.StatusInternalServerError, body: serverErrorBody},
"k0-long-key": {statusCode: http.StatusInternalServerError, body: serverErrorBody},
},
expectedRequestCount: 1,
expectedStatusCode: http.StatusInternalServerError,
@@ -181,6 +191,7 @@ func TestBlockingInterception_KeyFailover(t *testing.T) {
keypool.KeyStateValid,
keypool.KeyStateValid,
},
expectedCredentialHint: utils.MaskSecret("k0-long-key"),
},
{
// Given: BYOK with a single key returning 429.
@@ -204,6 +215,7 @@ func TestBlockingInterception_KeyFailover(t *testing.T) {
expectedRequestCount: 1,
expectedStatusCode: http.StatusTooManyRequests,
expectedRetryAfter: "5",
expectedCredentialHint: utils.MaskSecret("user-byok"),
},
}
@@ -234,6 +246,7 @@ func TestBlockingInterception_KeyFailover(t *testing.T) {
cfg := config.Anthropic{BaseURL: upstream.URL + "/"}
var pool *keypool.Pool
credInfo := intercept.NewCredentialInfo(intercept.CredentialKindCentralized, "")
if len(tc.keys) > 0 {
var err error
pool, err = keypool.New(tc.keys, quartz.NewMock(t))
@@ -241,6 +254,7 @@ func TestBlockingInterception_KeyFailover(t *testing.T) {
cfg.KeyPool = pool
} else if tc.byokKey != "" {
cfg.Key = tc.byokKey
credInfo = intercept.NewCredentialInfo(intercept.CredentialKindBYOK, tc.byokKey)
}
payload, err := NewRequestPayload([]byte(requestBody))
@@ -255,7 +269,7 @@ func TestBlockingInterception_KeyFailover(t *testing.T) {
http.Header{},
"X-Api-Key",
otel.Tracer("blocking_test"),
intercept.NewCredentialInfo(intercept.CredentialKindCentralized, ""),
credInfo,
)
interceptor.Setup(slog.Make(), &testutil.MockRecorder{}, nil)
@@ -271,6 +285,7 @@ func TestBlockingInterception_KeyFailover(t *testing.T) {
assert.Equal(t, tc.expectedRequestCount, requestCount.Load(), "upstream request count")
assert.Equal(t, tc.expectedStatusCode, w.Code, "response status code")
assert.Equal(t, tc.expectedRetryAfter, w.Header().Get("Retry-After"), "Retry-After header")
assert.Equal(t, tc.expectedCredentialHint, interceptor.Credential().Hint, "credential hint")
if pool != nil {
assert.Equal(t, tc.expectedKeyStates, pool.PoolState(), "key states")
}
@@ -296,6 +311,9 @@ func TestBlockingInterception_AgenticLoopFailover(t *testing.T) {
expectedStatusCode int
expectedRetryAfter string
expectedKeyStates []keypool.KeyState
// Expected credential hint after ProcessRequest: hint of the
// last attempted key across all agentic-loop iterations.
expectedCredentialHint string
}{
{
// Given: 2 keys; both upstream calls succeed on key-0.
@@ -306,12 +324,13 @@ func TestBlockingInterception_AgenticLoopFailover(t *testing.T) {
{statusCode: http.StatusOK, body: successBody},
},
expectedRequestCount: 2,
expectedSeenKeys: []string{"k0", "k0"},
expectedSeenKeys: []string{"k0-long-key", "k0-long-key"},
expectedStatusCode: http.StatusOK,
expectedKeyStates: []keypool.KeyState{
keypool.KeyStateValid,
keypool.KeyStateValid,
},
expectedCredentialHint: utils.MaskSecret("k0-long-key"),
},
{
// Given: 2 keys; key-0 succeeds initially, then 429s
@@ -329,12 +348,13 @@ func TestBlockingInterception_AgenticLoopFailover(t *testing.T) {
{statusCode: http.StatusOK, body: successBody},
},
expectedRequestCount: 3,
expectedSeenKeys: []string{"k0", "k0", "k1"},
expectedSeenKeys: []string{"k0-long-key", "k0-long-key", "k1-long-key"},
expectedStatusCode: http.StatusOK,
expectedKeyStates: []keypool.KeyState{
keypool.KeyStateTemporary,
keypool.KeyStateValid,
},
expectedCredentialHint: utils.MaskSecret("k1-long-key"),
},
{
// Given: 2 keys; key-0 succeeds initially, then both
@@ -356,13 +376,14 @@ func TestBlockingInterception_AgenticLoopFailover(t *testing.T) {
},
},
expectedRequestCount: 3,
expectedSeenKeys: []string{"k0", "k0", "k1"},
expectedSeenKeys: []string{"k0-long-key", "k0-long-key", "k1-long-key"},
expectedStatusCode: http.StatusTooManyRequests,
expectedRetryAfter: "3",
expectedKeyStates: []keypool.KeyState{
keypool.KeyStateTemporary,
keypool.KeyStateTemporary,
},
expectedCredentialHint: utils.MaskSecret("k1-long-key"),
},
}
@@ -397,7 +418,7 @@ func TestBlockingInterception_AgenticLoopFailover(t *testing.T) {
}))
t.Cleanup(upstream.Close)
pool, err := keypool.New([]string{"k0", "k1"}, quartz.NewMock(t))
pool, err := keypool.New([]string{"k0-long-key", "k1-long-key"}, quartz.NewMock(t))
require.NoError(t, err)
cfg := config.Anthropic{
@@ -447,6 +468,7 @@ func TestBlockingInterception_AgenticLoopFailover(t *testing.T) {
assert.Equal(t, tc.expectedRequestCount, requestCount.Load(), "upstream request count")
assert.Equal(t, tc.expectedStatusCode, w.Code, "response status code")
assert.Equal(t, tc.expectedRetryAfter, w.Header().Get("Retry-After"), "Retry-After header")
assert.Equal(t, tc.expectedCredentialHint, interceptor.Credential().Hint, "credential hint")
seenKeysMu.Lock()
defer seenKeysMu.Unlock()
+5
View File
@@ -195,6 +195,11 @@ newStream:
break
}
currentKey = key
// Record the key in use so the hint reflects the last attempted key.
i.credential = intercept.NewCredentialInfo(intercept.CredentialKindCentralized, key.Value())
logger.Debug(ctx, "using centralized api key",
slog.F("credential_hint", i.Credential().Hint), slog.F("credential_length", i.Credential().Length))
streamOpts = append(streamOpts,
option.WithAPIKey(key.Value()),
// Disable SDK retries because the failover
@@ -21,6 +21,7 @@ import (
"github.com/coder/coder/v2/aibridge/internal/testutil"
"github.com/coder/coder/v2/aibridge/keypool"
"github.com/coder/coder/v2/aibridge/mcp"
"github.com/coder/coder/v2/aibridge/utils"
"github.com/coder/quartz"
)
@@ -60,14 +61,17 @@ func TestStreamingInterception_KeyFailover(t *testing.T) {
expectedRetryAfter string
// Expected key states after the request, by index in keys.
expectedKeyStates []keypool.KeyState
// Expected credential hint after ProcessRequest: last
// attempted key for centralized, user key from initial request for BYOK.
expectedCredentialHint string
}{
{
// Given: 1 valid key returning a successful stream.
// Then: 1 request, 200 response, key remains valid.
name: "single_valid_key",
keys: []string{"k0"},
keys: []string{"k0-long-key"},
responses: map[string]upstreamResponse{
"k0": {
"k0-long-key": {
statusCode: http.StatusOK,
headers: map[string]string{"Content-Type": "text/event-stream"},
body: streamingSuccessBody,
@@ -76,20 +80,21 @@ func TestStreamingInterception_KeyFailover(t *testing.T) {
expectedRequestCount: 1,
expectedStatusCode: http.StatusOK,
expectedKeyStates: []keypool.KeyState{keypool.KeyStateValid},
expectedCredentialHint: utils.MaskSecret("k0-long-key"),
},
{
// Given: 2 keys; key-0 returns 429 pre-stream, key-1
// streams successfully.
// Then: 2 requests, 200 response, key-0 temporary, key-1 valid.
name: "failover_after_429",
keys: []string{"k0", "k1"},
keys: []string{"k0-long-key", "k1-long-key"},
responses: map[string]upstreamResponse{
"k0": {
"k0-long-key": {
statusCode: http.StatusTooManyRequests,
headers: map[string]string{"Retry-After": "5"},
body: rateLimitBody,
},
"k1": {
"k1-long-key": {
statusCode: http.StatusOK,
headers: map[string]string{"Content-Type": "text/event-stream"},
body: streamingSuccessBody,
@@ -101,16 +106,17 @@ func TestStreamingInterception_KeyFailover(t *testing.T) {
keypool.KeyStateTemporary,
keypool.KeyStateValid,
},
expectedCredentialHint: utils.MaskSecret("k1-long-key"),
},
{
// Given: 2 keys; key-0 returns 401 pre-stream, key-1
// streams successfully.
// Then: 2 requests, 200 response, key-0 permanent, key-1 valid.
name: "failover_after_401",
keys: []string{"k0", "k1"},
keys: []string{"k0-long-key", "k1-long-key"},
responses: map[string]upstreamResponse{
"k0": {statusCode: http.StatusUnauthorized, body: authErrorBody},
"k1": {
"k0-long-key": {statusCode: http.StatusUnauthorized, body: authErrorBody},
"k1-long-key": {
statusCode: http.StatusOK,
headers: map[string]string{"Content-Type": "text/event-stream"},
body: streamingSuccessBody,
@@ -122,15 +128,16 @@ func TestStreamingInterception_KeyFailover(t *testing.T) {
keypool.KeyStatePermanent,
keypool.KeyStateValid,
},
expectedCredentialHint: utils.MaskSecret("k1-long-key"),
},
{
// Given: 2 keys; key-0 returns 403 pre-stream, key-1 streams.
// Then: 2 requests, 200 response, key-0 permanent, key-1 valid.
name: "failover_after_403",
keys: []string{"k0", "k1"},
keys: []string{"k0-long-key", "k1-long-key"},
responses: map[string]upstreamResponse{
"k0": {statusCode: http.StatusForbidden, body: authErrorBody},
"k1": {
"k0-long-key": {statusCode: http.StatusForbidden, body: authErrorBody},
"k1-long-key": {
statusCode: http.StatusOK,
headers: map[string]string{"Content-Type": "text/event-stream"},
body: streamingSuccessBody,
@@ -142,6 +149,7 @@ func TestStreamingInterception_KeyFailover(t *testing.T) {
keypool.KeyStatePermanent,
keypool.KeyStateValid,
},
expectedCredentialHint: utils.MaskSecret("k1-long-key"),
},
{
// Given: 3 keys; all return 429 pre-stream with
@@ -149,19 +157,19 @@ func TestStreamingInterception_KeyFailover(t *testing.T) {
// Then: 3 requests, 429 response with smallest
// Retry-After, all keys temporary.
name: "all_keys_rate_limited",
keys: []string{"k0", "k1", "k2"},
keys: []string{"k0-long-key", "k1-long-key", "k2-long-key"},
responses: map[string]upstreamResponse{
"k0": {
"k0-long-key": {
statusCode: http.StatusTooManyRequests,
headers: map[string]string{"Retry-After": "5"},
body: rateLimitBody,
},
"k1": {
"k1-long-key": {
statusCode: http.StatusTooManyRequests,
headers: map[string]string{"Retry-After": "3"},
body: rateLimitBody,
},
"k2": {
"k2-long-key": {
statusCode: http.StatusTooManyRequests,
headers: map[string]string{"Retry-After": "10"},
body: rateLimitBody,
@@ -175,15 +183,16 @@ func TestStreamingInterception_KeyFailover(t *testing.T) {
keypool.KeyStateTemporary,
keypool.KeyStateTemporary,
},
expectedCredentialHint: utils.MaskSecret("k2-long-key"),
},
{
// Given: 2 keys; both return 401 pre-stream.
// Then: 2 requests, 502 api_error response, both keys permanent.
name: "all_keys_unauthorized",
keys: []string{"k0", "k1"},
keys: []string{"k0-long-key", "k1-long-key"},
responses: map[string]upstreamResponse{
"k0": {statusCode: http.StatusUnauthorized, body: authErrorBody},
"k1": {statusCode: http.StatusUnauthorized, body: authErrorBody},
"k0-long-key": {statusCode: http.StatusUnauthorized, body: authErrorBody},
"k1-long-key": {statusCode: http.StatusUnauthorized, body: authErrorBody},
},
expectedRequestCount: 2,
expectedStatusCode: http.StatusBadGateway,
@@ -191,14 +200,15 @@ func TestStreamingInterception_KeyFailover(t *testing.T) {
keypool.KeyStatePermanent,
keypool.KeyStatePermanent,
},
expectedCredentialHint: utils.MaskSecret("k1-long-key"),
},
{
// Given: 2 keys; key-0 returns 500 pre-stream.
// Then: 1 request, 500 response, both keys remain valid.
name: "server_error_no_failover",
keys: []string{"k0", "k1"},
keys: []string{"k0-long-key", "k1-long-key"},
responses: map[string]upstreamResponse{
"k0": {statusCode: http.StatusInternalServerError, body: serverErrorBody},
"k0-long-key": {statusCode: http.StatusInternalServerError, body: serverErrorBody},
},
expectedRequestCount: 1,
expectedStatusCode: http.StatusInternalServerError,
@@ -206,6 +216,7 @@ func TestStreamingInterception_KeyFailover(t *testing.T) {
keypool.KeyStateValid,
keypool.KeyStateValid,
},
expectedCredentialHint: utils.MaskSecret("k0-long-key"),
},
{
// Given: BYOK with a single key returning 429.
@@ -229,6 +240,7 @@ func TestStreamingInterception_KeyFailover(t *testing.T) {
expectedRequestCount: 1,
expectedStatusCode: http.StatusTooManyRequests,
expectedRetryAfter: "5",
expectedCredentialHint: utils.MaskSecret("user-byok"),
},
}
@@ -258,6 +270,7 @@ func TestStreamingInterception_KeyFailover(t *testing.T) {
cfg := config.Anthropic{BaseURL: upstream.URL + "/"}
var pool *keypool.Pool
credInfo := intercept.NewCredentialInfo(intercept.CredentialKindCentralized, "")
if len(tc.keys) > 0 {
var err error
pool, err = keypool.New(tc.keys, quartz.NewMock(t))
@@ -265,6 +278,7 @@ func TestStreamingInterception_KeyFailover(t *testing.T) {
cfg.KeyPool = pool
} else if tc.byokKey != "" {
cfg.Key = tc.byokKey
credInfo = intercept.NewCredentialInfo(intercept.CredentialKindBYOK, tc.byokKey)
}
payload, err := NewRequestPayload([]byte(requestBody))
@@ -279,7 +293,7 @@ func TestStreamingInterception_KeyFailover(t *testing.T) {
http.Header{},
"X-Api-Key",
otel.Tracer("streaming_test"),
intercept.NewCredentialInfo(intercept.CredentialKindCentralized, ""),
credInfo,
)
interceptor.Setup(slog.Make(), &testutil.MockRecorder{}, nil)
@@ -301,6 +315,7 @@ func TestStreamingInterception_KeyFailover(t *testing.T) {
if pool != nil {
assert.Equal(t, tc.expectedKeyStates, pool.PoolState(), "key states")
}
assert.Equal(t, tc.expectedCredentialHint, interceptor.Credential().Hint, "credential hint")
})
}
}
@@ -387,6 +402,9 @@ func TestStreamingInterception_AgenticLoopFailover(t *testing.T) {
// error (e.g. all keys exhausted).
expectedErr bool
expectedKeyStates []keypool.KeyState
// Expected credential hint after ProcessRequest: hint of the
// last attempted key across all agentic-loop iterations.
expectedCredentialHint string
}{
{
// Given: 2 keys; both upstream calls succeed on key-0.
@@ -397,13 +415,14 @@ func TestStreamingInterception_AgenticLoopFailover(t *testing.T) {
{statusCode: http.StatusOK, headers: sseHeaders, body: textStreamBody},
},
expectedRequestCount: 2,
expectedSeenKeys: []string{"k0", "k0"},
expectedSeenKeys: []string{"k0-long-key", "k0-long-key"},
expectedBodyContains: "done",
expectErrorAsSSEEvent: false,
expectedKeyStates: []keypool.KeyState{
keypool.KeyStateValid,
keypool.KeyStateValid,
},
expectedCredentialHint: utils.MaskSecret("k0-long-key"),
},
{
// Given: 2 keys; key-0 succeeds initially, then 429s
@@ -421,13 +440,14 @@ func TestStreamingInterception_AgenticLoopFailover(t *testing.T) {
{statusCode: http.StatusOK, headers: sseHeaders, body: textStreamBody},
},
expectedRequestCount: 3,
expectedSeenKeys: []string{"k0", "k0", "k1"},
expectedSeenKeys: []string{"k0-long-key", "k0-long-key", "k1-long-key"},
expectedBodyContains: "done",
expectErrorAsSSEEvent: false,
expectedKeyStates: []keypool.KeyState{
keypool.KeyStateTemporary,
keypool.KeyStateValid,
},
expectedCredentialHint: utils.MaskSecret("k1-long-key"),
},
{
// Given: 2 keys; key-0 succeeds initially, then both
@@ -453,7 +473,7 @@ func TestStreamingInterception_AgenticLoopFailover(t *testing.T) {
},
},
expectedRequestCount: 3,
expectedSeenKeys: []string{"k0", "k0", "k1"},
expectedSeenKeys: []string{"k0-long-key", "k0-long-key", "k1-long-key"},
expectedBodyContains: "all configured keys are rate-limited",
expectErrorAsSSEEvent: true,
expectedErr: true,
@@ -461,6 +481,7 @@ func TestStreamingInterception_AgenticLoopFailover(t *testing.T) {
keypool.KeyStateTemporary,
keypool.KeyStateTemporary,
},
expectedCredentialHint: utils.MaskSecret("k1-long-key"),
},
}
@@ -494,7 +515,7 @@ func TestStreamingInterception_AgenticLoopFailover(t *testing.T) {
}))
t.Cleanup(upstream.Close)
pool, err := keypool.New([]string{"k0", "k1"}, quartz.NewMock(t))
pool, err := keypool.New([]string{"k0-long-key", "k1-long-key"}, quartz.NewMock(t))
require.NoError(t, err)
cfg := config.Anthropic{
@@ -553,6 +574,7 @@ func TestStreamingInterception_AgenticLoopFailover(t *testing.T) {
defer seenKeysMu.Unlock()
assert.Equal(t, tc.expectedSeenKeys, seenKeys, "seen keys")
assert.Equal(t, tc.expectedKeyStates, pool.PoolState(), "key states")
assert.Equal(t, tc.expectedCredentialHint, interceptor.Credential().Hint, "credential hint")
})
}
}
+4 -3
View File
@@ -171,15 +171,16 @@ func (i *BlockingResponsesInterceptor) newResponseWithKey(ctx context.Context, s
// Errors that aren't key-specific don't trigger failover and
// are returned to the caller.
func (i *BlockingResponsesInterceptor) newResponseWithKeyFailover(ctx context.Context, srv responses.ResponseService, opts []option.RequestOption) (*responses.Response, error) {
// TODO(ssncferreira): update the interception's credential
// hint with the actually-used key (the successful key on
// success, the last tried key on failure) in the upstack PR.
walker := i.cfg.KeyPool.Walker()
for {
key, keyPoolErr := walker.Next()
if keyPoolErr != nil {
return nil, keyPoolErr
}
// Record the key in use so the hint reflects the last attempted key.
i.credential = intercept.NewCredentialInfo(intercept.CredentialKindCentralized, key.Value())
i.logger.Debug(ctx, "using centralized api key",
slog.F("credential_hint", i.Credential().Hint), slog.F("credential_length", i.Credential().Length))
requestOpts := append([]option.RequestOption{}, opts...)
requestOpts = append(requestOpts,
@@ -58,31 +58,35 @@ func TestBlockingResponsesInterceptor_KeyFailover(t *testing.T) {
expectedRetryAfter string
// Expected key states after the request, by index in keys.
expectedKeyStates []keypool.KeyState
// Expected credential hint after ProcessRequest: last
// attempted key for centralized, user key from initial request for BYOK.
expectedCredentialHint string
}{
{
// Given: 1 valid key returning 200.
// Then: 1 request, 200 response, key remains valid.
name: "single_valid_key",
keys: []string{"k0"},
keys: []string{"k0-long-key"},
responses: map[string]upstreamResponse{
"k0": {statusCode: http.StatusOK, body: successBody},
"k0-long-key": {statusCode: http.StatusOK, body: successBody},
},
expectedRequestCount: 1,
expectedStatusCode: http.StatusOK,
expectedKeyStates: []keypool.KeyState{keypool.KeyStateValid},
expectedCredentialHint: utils.MaskSecret("k0-long-key"),
},
{
// Given: 2 keys; key-0 returns 429, key-1 returns 200.
// Then: 2 requests, 200 response, key-0 temporary, key-1 valid.
name: "failover_after_429",
keys: []string{"k0", "k1"},
keys: []string{"k0-long-key", "k1-long-key"},
responses: map[string]upstreamResponse{
"k0": {
"k0-long-key": {
statusCode: http.StatusTooManyRequests,
headers: map[string]string{"Retry-After": "5"},
body: rateLimitBody,
},
"k1": {statusCode: http.StatusOK, body: successBody},
"k1-long-key": {statusCode: http.StatusOK, body: successBody},
},
expectedRequestCount: 2,
expectedStatusCode: http.StatusOK,
@@ -90,15 +94,16 @@ func TestBlockingResponsesInterceptor_KeyFailover(t *testing.T) {
keypool.KeyStateTemporary,
keypool.KeyStateValid,
},
expectedCredentialHint: utils.MaskSecret("k1-long-key"),
},
{
// Given: 2 keys; key-0 returns 401, key-1 returns 200.
// Then: 2 requests, 200 response, key-0 permanent, key-1 valid.
name: "failover_after_401",
keys: []string{"k0", "k1"},
keys: []string{"k0-long-key", "k1-long-key"},
responses: map[string]upstreamResponse{
"k0": {statusCode: http.StatusUnauthorized, body: authErrorBody},
"k1": {statusCode: http.StatusOK, body: successBody},
"k0-long-key": {statusCode: http.StatusUnauthorized, body: authErrorBody},
"k1-long-key": {statusCode: http.StatusOK, body: successBody},
},
expectedRequestCount: 2,
expectedStatusCode: http.StatusOK,
@@ -106,15 +111,16 @@ func TestBlockingResponsesInterceptor_KeyFailover(t *testing.T) {
keypool.KeyStatePermanent,
keypool.KeyStateValid,
},
expectedCredentialHint: utils.MaskSecret("k1-long-key"),
},
{
// Given: 2 keys; key-0 returns 403, key-1 returns 200.
// Then: 2 requests, 200 response, key-0 permanent, key-1 valid.
name: "failover_after_403",
keys: []string{"k0", "k1"},
keys: []string{"k0-long-key", "k1-long-key"},
responses: map[string]upstreamResponse{
"k0": {statusCode: http.StatusForbidden, body: authErrorBody},
"k1": {statusCode: http.StatusOK, body: successBody},
"k0-long-key": {statusCode: http.StatusForbidden, body: authErrorBody},
"k1-long-key": {statusCode: http.StatusOK, body: successBody},
},
expectedRequestCount: 2,
expectedStatusCode: http.StatusOK,
@@ -122,25 +128,26 @@ func TestBlockingResponsesInterceptor_KeyFailover(t *testing.T) {
keypool.KeyStatePermanent,
keypool.KeyStateValid,
},
expectedCredentialHint: utils.MaskSecret("k1-long-key"),
},
{
// Given: 3 keys; all return 429 with cooldowns 5s, 3s, 10s.
// Then: 3 requests, 429 response with smallest Retry-After,
// all keys temporary.
name: "all_keys_rate_limited",
keys: []string{"k0", "k1", "k2"},
keys: []string{"k0-long-key", "k1-long-key", "k2-long-key"},
responses: map[string]upstreamResponse{
"k0": {
"k0-long-key": {
statusCode: http.StatusTooManyRequests,
headers: map[string]string{"Retry-After": "5"},
body: rateLimitBody,
},
"k1": {
"k1-long-key": {
statusCode: http.StatusTooManyRequests,
headers: map[string]string{"Retry-After": "3"},
body: rateLimitBody,
},
"k2": {
"k2-long-key": {
statusCode: http.StatusTooManyRequests,
headers: map[string]string{"Retry-After": "10"},
body: rateLimitBody,
@@ -154,15 +161,16 @@ func TestBlockingResponsesInterceptor_KeyFailover(t *testing.T) {
keypool.KeyStateTemporary,
keypool.KeyStateTemporary,
},
expectedCredentialHint: utils.MaskSecret("k2-long-key"),
},
{
// Given: 2 keys; both return 401.
// Then: 2 requests, 502 api_error response, both keys permanent.
name: "all_keys_unauthorized",
keys: []string{"k0", "k1"},
keys: []string{"k0-long-key", "k1-long-key"},
responses: map[string]upstreamResponse{
"k0": {statusCode: http.StatusUnauthorized, body: authErrorBody},
"k1": {statusCode: http.StatusUnauthorized, body: authErrorBody},
"k0-long-key": {statusCode: http.StatusUnauthorized, body: authErrorBody},
"k1-long-key": {statusCode: http.StatusUnauthorized, body: authErrorBody},
},
expectedRequestCount: 2,
expectedStatusCode: http.StatusBadGateway,
@@ -170,14 +178,15 @@ func TestBlockingResponsesInterceptor_KeyFailover(t *testing.T) {
keypool.KeyStatePermanent,
keypool.KeyStatePermanent,
},
expectedCredentialHint: utils.MaskSecret("k1-long-key"),
},
{
// Given: 2 keys; key-0 returns 500.
// Then: 1 request, 500 response, both keys remain valid.
name: "server_error_no_failover",
keys: []string{"k0", "k1"},
keys: []string{"k0-long-key", "k1-long-key"},
responses: map[string]upstreamResponse{
"k0": {statusCode: http.StatusInternalServerError, body: serverErrorBody},
"k0-long-key": {statusCode: http.StatusInternalServerError, body: serverErrorBody},
},
expectedRequestCount: 1,
expectedStatusCode: http.StatusInternalServerError,
@@ -185,6 +194,7 @@ func TestBlockingResponsesInterceptor_KeyFailover(t *testing.T) {
keypool.KeyStateValid,
keypool.KeyStateValid,
},
expectedCredentialHint: utils.MaskSecret("k0-long-key"),
},
{
// Given: BYOK with a single key returning 429.
@@ -206,6 +216,7 @@ func TestBlockingResponsesInterceptor_KeyFailover(t *testing.T) {
},
expectedRequestCount: 1,
expectedStatusCode: http.StatusTooManyRequests,
expectedCredentialHint: utils.MaskSecret("user-byok"),
},
}
@@ -235,6 +246,7 @@ func TestBlockingResponsesInterceptor_KeyFailover(t *testing.T) {
t.Cleanup(upstream.Close)
cfg := config.OpenAI{BaseURL: upstream.URL + "/"}
credInfo := intercept.NewCredentialInfo(intercept.CredentialKindCentralized, "")
var pool *keypool.Pool
if len(tc.keys) > 0 {
var err error
@@ -243,6 +255,7 @@ func TestBlockingResponsesInterceptor_KeyFailover(t *testing.T) {
cfg.KeyPool = pool
} else if tc.byokKey != "" {
cfg.Key = tc.byokKey
credInfo = intercept.NewCredentialInfo(intercept.CredentialKindBYOK, tc.byokKey)
}
payload, err := NewRequestPayload([]byte(requestBody))
@@ -256,7 +269,7 @@ func TestBlockingResponsesInterceptor_KeyFailover(t *testing.T) {
http.Header{},
"Authorization",
otel.Tracer("blocking_test"),
intercept.NewCredentialInfo(intercept.CredentialKindCentralized, ""),
credInfo,
)
interceptor.Setup(slog.Make(), &testutil.MockRecorder{}, nil)
@@ -272,6 +285,7 @@ func TestBlockingResponsesInterceptor_KeyFailover(t *testing.T) {
assert.Equal(t, tc.expectedRequestCount, requestCount.Load(), "upstream request count")
assert.Equal(t, tc.expectedStatusCode, w.Code, "response status code")
assert.Equal(t, tc.expectedRetryAfter, w.Header().Get("Retry-After"), "Retry-After header")
assert.Equal(t, tc.expectedCredentialHint, interceptor.Credential().Hint, "credential hint")
if pool != nil {
assert.Equal(t, tc.expectedKeyStates, pool.PoolState(), "key states")
}
@@ -296,6 +310,9 @@ func TestBlockingResponsesInterceptor_AgenticLoopFailover(t *testing.T) {
expectedSeenKeys []string
expectedStatusCode int
expectedKeyStates []keypool.KeyState
// Expected credential hint after ProcessRequest: hint of the
// last attempted key across all agentic-loop iterations.
expectedCredentialHint string
}{
{
// Given: 2 keys; both upstream calls succeed on key-0.
@@ -306,12 +323,13 @@ func TestBlockingResponsesInterceptor_AgenticLoopFailover(t *testing.T) {
{statusCode: http.StatusOK, body: textCompleteBody},
},
expectedRequestCount: 2,
expectedSeenKeys: []string{"k0", "k0"},
expectedSeenKeys: []string{"k0-long-key", "k0-long-key"},
expectedStatusCode: http.StatusOK,
expectedKeyStates: []keypool.KeyState{
keypool.KeyStateValid,
keypool.KeyStateValid,
},
expectedCredentialHint: utils.MaskSecret("k0-long-key"),
},
{
// Given: 2 keys; key-0 succeeds initially, then 429s
@@ -329,12 +347,13 @@ func TestBlockingResponsesInterceptor_AgenticLoopFailover(t *testing.T) {
{statusCode: http.StatusOK, body: textCompleteBody},
},
expectedRequestCount: 3,
expectedSeenKeys: []string{"k0", "k0", "k1"},
expectedSeenKeys: []string{"k0-long-key", "k0-long-key", "k1-long-key"},
expectedStatusCode: http.StatusOK,
expectedKeyStates: []keypool.KeyState{
keypool.KeyStateTemporary,
keypool.KeyStateValid,
},
expectedCredentialHint: utils.MaskSecret("k1-long-key"),
},
{
// Given: 2 keys; key-0 succeeds initially, then both
@@ -356,12 +375,13 @@ func TestBlockingResponsesInterceptor_AgenticLoopFailover(t *testing.T) {
},
},
expectedRequestCount: 3,
expectedSeenKeys: []string{"k0", "k0", "k1"},
expectedSeenKeys: []string{"k0-long-key", "k0-long-key", "k1-long-key"},
expectedStatusCode: http.StatusTooManyRequests,
expectedKeyStates: []keypool.KeyState{
keypool.KeyStateTemporary,
keypool.KeyStateTemporary,
},
expectedCredentialHint: utils.MaskSecret("k1-long-key"),
},
}
@@ -396,7 +416,7 @@ func TestBlockingResponsesInterceptor_AgenticLoopFailover(t *testing.T) {
}))
t.Cleanup(upstream.Close)
pool, err := keypool.New([]string{"k0", "k1"}, quartz.NewMock(t))
pool, err := keypool.New([]string{"k0-long-key", "k1-long-key"}, quartz.NewMock(t))
require.NoError(t, err)
cfg := config.OpenAI{
@@ -444,6 +464,7 @@ func TestBlockingResponsesInterceptor_AgenticLoopFailover(t *testing.T) {
assert.Equal(t, tc.expectedRequestCount, requestCount.Load(), "upstream request count")
assert.Equal(t, tc.expectedStatusCode, w.Code, "response status code")
assert.Equal(t, tc.expectedCredentialHint, interceptor.Credential().Hint, "credential hint")
seenKeysMu.Lock()
defer seenKeysMu.Unlock()
@@ -144,6 +144,11 @@ func (i *StreamingResponsesInterceptor) ProcessRequest(w http.ResponseWriter, r
return xerrors.Errorf("key pool exhausted: %w", keyPoolErr)
}
currentKey = key
// Record the key in use so the hint reflects the last attempted key.
i.credential = intercept.NewCredentialInfo(intercept.CredentialKindCentralized, key.Value())
i.logger.Debug(ctx, "using centralized api key",
slog.F("credential_hint", i.Credential().Hint), slog.F("credential_length", i.Credential().Length))
opts = append(opts,
option.WithAPIKey(key.Value()),
// Disable SDK retries because the failover
@@ -51,14 +51,17 @@ func TestStreamingResponsesInterceptor_KeyFailover(t *testing.T) {
expectedRetryAfter string
// Expected key states after the request, by index in keys.
expectedKeyStates []keypool.KeyState
// Expected credential hint after ProcessRequest: last
// attempted key for centralized, user key from initial request for BYOK.
expectedCredentialHint string
}{
{
// Given: 1 valid key returning a successful stream.
// Then: 1 request, 200 response, key remains valid.
name: "single_valid_key",
keys: []string{"k0"},
keys: []string{"k0-long-key"},
responses: map[string]upstreamResponse{
"k0": {
"k0-long-key": {
statusCode: http.StatusOK,
headers: map[string]string{"Content-Type": "text/event-stream"},
body: streamingSuccessBody,
@@ -67,20 +70,21 @@ func TestStreamingResponsesInterceptor_KeyFailover(t *testing.T) {
expectedRequestCount: 1,
expectedStatusCode: http.StatusOK,
expectedKeyStates: []keypool.KeyState{keypool.KeyStateValid},
expectedCredentialHint: utils.MaskSecret("k0-long-key"),
},
{
// Given: 2 keys; key-0 returns 429 pre-stream, key-1
// streams successfully.
// Then: 2 requests, 200 response, key-0 temporary, key-1 valid.
name: "failover_after_429",
keys: []string{"k0", "k1"},
keys: []string{"k0-long-key", "k1-long-key"},
responses: map[string]upstreamResponse{
"k0": {
"k0-long-key": {
statusCode: http.StatusTooManyRequests,
headers: map[string]string{"Retry-After": "5"},
body: rateLimitBody,
},
"k1": {
"k1-long-key": {
statusCode: http.StatusOK,
headers: map[string]string{"Content-Type": "text/event-stream"},
body: streamingSuccessBody,
@@ -92,16 +96,17 @@ func TestStreamingResponsesInterceptor_KeyFailover(t *testing.T) {
keypool.KeyStateTemporary,
keypool.KeyStateValid,
},
expectedCredentialHint: utils.MaskSecret("k1-long-key"),
},
{
// Given: 2 keys; key-0 returns 401 pre-stream, key-1
// streams successfully.
// Then: 2 requests, 200 response, key-0 permanent, key-1 valid.
name: "failover_after_401",
keys: []string{"k0", "k1"},
keys: []string{"k0-long-key", "k1-long-key"},
responses: map[string]upstreamResponse{
"k0": {statusCode: http.StatusUnauthorized, body: authErrorBody},
"k1": {
"k0-long-key": {statusCode: http.StatusUnauthorized, body: authErrorBody},
"k1-long-key": {
statusCode: http.StatusOK,
headers: map[string]string{"Content-Type": "text/event-stream"},
body: streamingSuccessBody,
@@ -113,15 +118,16 @@ func TestStreamingResponsesInterceptor_KeyFailover(t *testing.T) {
keypool.KeyStatePermanent,
keypool.KeyStateValid,
},
expectedCredentialHint: utils.MaskSecret("k1-long-key"),
},
{
// Given: 2 keys; key-0 returns 403 pre-stream, key-1 streams.
// Then: 2 requests, 200 response, key-0 permanent, key-1 valid.
name: "failover_after_403",
keys: []string{"k0", "k1"},
keys: []string{"k0-long-key", "k1-long-key"},
responses: map[string]upstreamResponse{
"k0": {statusCode: http.StatusForbidden, body: authErrorBody},
"k1": {
"k0-long-key": {statusCode: http.StatusForbidden, body: authErrorBody},
"k1-long-key": {
statusCode: http.StatusOK,
headers: map[string]string{"Content-Type": "text/event-stream"},
body: streamingSuccessBody,
@@ -133,6 +139,7 @@ func TestStreamingResponsesInterceptor_KeyFailover(t *testing.T) {
keypool.KeyStatePermanent,
keypool.KeyStateValid,
},
expectedCredentialHint: utils.MaskSecret("k1-long-key"),
},
{
// Given: 3 keys; all return 429 pre-stream with
@@ -140,19 +147,19 @@ func TestStreamingResponsesInterceptor_KeyFailover(t *testing.T) {
// Then: 3 requests, 429 response with smallest
// Retry-After, all keys temporary.
name: "all_keys_rate_limited",
keys: []string{"k0", "k1", "k2"},
keys: []string{"k0-long-key", "k1-long-key", "k2-long-key"},
responses: map[string]upstreamResponse{
"k0": {
"k0-long-key": {
statusCode: http.StatusTooManyRequests,
headers: map[string]string{"Retry-After": "5"},
body: rateLimitBody,
},
"k1": {
"k1-long-key": {
statusCode: http.StatusTooManyRequests,
headers: map[string]string{"Retry-After": "3"},
body: rateLimitBody,
},
"k2": {
"k2-long-key": {
statusCode: http.StatusTooManyRequests,
headers: map[string]string{"Retry-After": "10"},
body: rateLimitBody,
@@ -166,15 +173,16 @@ func TestStreamingResponsesInterceptor_KeyFailover(t *testing.T) {
keypool.KeyStateTemporary,
keypool.KeyStateTemporary,
},
expectedCredentialHint: utils.MaskSecret("k2-long-key"),
},
{
// Given: 2 keys; both return 401 pre-stream.
// Then: 2 requests, 502 api_error response, both keys permanent.
name: "all_keys_unauthorized",
keys: []string{"k0", "k1"},
keys: []string{"k0-long-key", "k1-long-key"},
responses: map[string]upstreamResponse{
"k0": {statusCode: http.StatusUnauthorized, body: authErrorBody},
"k1": {statusCode: http.StatusUnauthorized, body: authErrorBody},
"k0-long-key": {statusCode: http.StatusUnauthorized, body: authErrorBody},
"k1-long-key": {statusCode: http.StatusUnauthorized, body: authErrorBody},
},
expectedRequestCount: 2,
expectedStatusCode: http.StatusBadGateway,
@@ -182,14 +190,15 @@ func TestStreamingResponsesInterceptor_KeyFailover(t *testing.T) {
keypool.KeyStatePermanent,
keypool.KeyStatePermanent,
},
expectedCredentialHint: utils.MaskSecret("k1-long-key"),
},
{
// Given: 2 keys; key-0 returns 500 pre-stream.
// Then: 1 request, 500 response, both keys remain valid.
name: "server_error_no_failover",
keys: []string{"k0", "k1"},
keys: []string{"k0-long-key", "k1-long-key"},
responses: map[string]upstreamResponse{
"k0": {statusCode: http.StatusInternalServerError, body: serverErrorBody},
"k0-long-key": {statusCode: http.StatusInternalServerError, body: serverErrorBody},
},
expectedRequestCount: 1,
expectedStatusCode: http.StatusInternalServerError,
@@ -197,6 +206,7 @@ func TestStreamingResponsesInterceptor_KeyFailover(t *testing.T) {
keypool.KeyStateValid,
keypool.KeyStateValid,
},
expectedCredentialHint: utils.MaskSecret("k0-long-key"),
},
{
// Given: BYOK with a single key returning 429.
@@ -218,6 +228,7 @@ func TestStreamingResponsesInterceptor_KeyFailover(t *testing.T) {
},
expectedRequestCount: 1,
expectedStatusCode: http.StatusTooManyRequests,
expectedCredentialHint: utils.MaskSecret("user-byok"),
},
}
@@ -246,6 +257,7 @@ func TestStreamingResponsesInterceptor_KeyFailover(t *testing.T) {
t.Cleanup(upstream.Close)
cfg := config.OpenAI{BaseURL: upstream.URL + "/"}
credInfo := intercept.NewCredentialInfo(intercept.CredentialKindCentralized, "")
var pool *keypool.Pool
if len(tc.keys) > 0 {
var err error
@@ -254,6 +266,7 @@ func TestStreamingResponsesInterceptor_KeyFailover(t *testing.T) {
cfg.KeyPool = pool
} else if tc.byokKey != "" {
cfg.Key = tc.byokKey
credInfo = intercept.NewCredentialInfo(intercept.CredentialKindBYOK, tc.byokKey)
}
payload, err := NewRequestPayload([]byte(streamingRequestBody))
@@ -267,7 +280,7 @@ func TestStreamingResponsesInterceptor_KeyFailover(t *testing.T) {
http.Header{},
"Authorization",
otel.Tracer("streaming_test"),
intercept.NewCredentialInfo(intercept.CredentialKindCentralized, ""),
credInfo,
)
interceptor.Setup(slog.Make(), &testutil.MockRecorder{}, nil)
@@ -283,6 +296,7 @@ func TestStreamingResponsesInterceptor_KeyFailover(t *testing.T) {
assert.Equal(t, tc.expectedRequestCount, requestCount.Load(), "upstream request count")
assert.Equal(t, tc.expectedStatusCode, w.Code, "response status code")
assert.Equal(t, tc.expectedRetryAfter, w.Header().Get("Retry-After"), "Retry-After header")
assert.Equal(t, tc.expectedCredentialHint, interceptor.Credential().Hint, "credential hint")
if pool != nil {
assert.Equal(t, tc.expectedKeyStates, pool.PoolState(), "key states")
}
@@ -339,6 +353,9 @@ func TestStreamingResponsesInterceptor_AgenticLoopFailover(t *testing.T) {
// error (e.g. all keys exhausted).
expectedErr bool
expectedKeyStates []keypool.KeyState
// Expected credential hint after ProcessRequest: hint of the
// last attempted key across all agentic-loop iterations.
expectedCredentialHint string
}{
{
// Given: 2 keys; both upstream calls succeed on key-0.
@@ -349,12 +366,13 @@ func TestStreamingResponsesInterceptor_AgenticLoopFailover(t *testing.T) {
{statusCode: http.StatusOK, headers: sseHeaders, body: textStreamBody},
},
expectedRequestCount: 2,
expectedSeenKeys: []string{"k0", "k0"},
expectedSeenKeys: []string{"k0-long-key", "k0-long-key"},
expectedBodyContains: "done",
expectedKeyStates: []keypool.KeyState{
keypool.KeyStateValid,
keypool.KeyStateValid,
},
expectedCredentialHint: utils.MaskSecret("k0-long-key"),
},
{
// Given: 2 keys; key-0 succeeds initially, then 429s
@@ -372,12 +390,13 @@ func TestStreamingResponsesInterceptor_AgenticLoopFailover(t *testing.T) {
{statusCode: http.StatusOK, headers: sseHeaders, body: textStreamBody},
},
expectedRequestCount: 3,
expectedSeenKeys: []string{"k0", "k0", "k1"},
expectedSeenKeys: []string{"k0-long-key", "k0-long-key", "k1-long-key"},
expectedBodyContains: "done",
expectedKeyStates: []keypool.KeyState{
keypool.KeyStateTemporary,
keypool.KeyStateValid,
},
expectedCredentialHint: utils.MaskSecret("k1-long-key"),
},
{
// Given: 2 keys; key-0 succeeds initially, then both
@@ -399,13 +418,14 @@ func TestStreamingResponsesInterceptor_AgenticLoopFailover(t *testing.T) {
},
},
expectedRequestCount: 3,
expectedSeenKeys: []string{"k0", "k0", "k1"},
expectedSeenKeys: []string{"k0-long-key", "k0-long-key", "k1-long-key"},
expectedBodyContains: "all configured keys are rate-limited",
expectedErr: true,
expectedKeyStates: []keypool.KeyState{
keypool.KeyStateTemporary,
keypool.KeyStateTemporary,
},
expectedCredentialHint: utils.MaskSecret("k1-long-key"),
},
}
@@ -439,7 +459,7 @@ func TestStreamingResponsesInterceptor_AgenticLoopFailover(t *testing.T) {
}))
t.Cleanup(upstream.Close)
pool, err := keypool.New([]string{"k0", "k1"}, quartz.NewMock(t))
pool, err := keypool.New([]string{"k0-long-key", "k1-long-key"}, quartz.NewMock(t))
require.NoError(t, err)
cfg := config.OpenAI{
@@ -489,6 +509,7 @@ func TestStreamingResponsesInterceptor_AgenticLoopFailover(t *testing.T) {
assert.Equal(t, tc.expectedRequestCount, requestCount.Load(), "upstream request count")
body := w.Body.String()
assert.Contains(t, body, tc.expectedBodyContains, "response body")
assert.Equal(t, tc.expectedCredentialHint, interceptor.Credential().Hint, "credential hint")
seenKeysMu.Lock()
defer seenKeysMu.Unlock()
+2 -3
View File
@@ -5,7 +5,6 @@ import (
"net/http"
"cdr.dev/slog/v3"
"github.com/coder/coder/v2/aibridge/utils"
)
// MarkKeyOnStatus marks key based on a key-specific HTTP
@@ -32,7 +31,7 @@ func MarkKeyOnStatus(
if key.MarkTemporary(cooldown) {
logger.Info(ctx, "key marked temporary",
slog.F("provider", providerName),
slog.F("api_key_hint", utils.MaskSecret(key.Value())),
slog.F("api_key_hint", key.Hint()),
slog.F("status", statusCode),
slog.F("cooldown", cooldown))
}
@@ -41,7 +40,7 @@ func MarkKeyOnStatus(
if key.MarkPermanent() {
logger.Warn(ctx, "key marked permanent",
slog.F("provider", providerName),
slog.F("api_key_hint", utils.MaskSecret(key.Value())),
slog.F("api_key_hint", key.Hint()),
slog.F("status", statusCode))
}
return true
+7
View File
@@ -7,6 +7,7 @@ import (
"golang.org/x/xerrors"
"github.com/coder/coder/v2/aibridge/utils"
"github.com/coder/quartz"
)
@@ -116,6 +117,12 @@ func (k *Key) Value() string {
return k.value
}
// Hint returns a masked, identifiable fragment of the key, suitable
// for logs and persisted records.
func (k *Key) Hint() string {
return utils.MaskSecret(k.value)
}
// State returns the current state of the key, derived from its
// permanent flag and cooldown deadline.
func (k *Key) State() KeyState {
+3 -8
View File
@@ -170,15 +170,10 @@ func (p *Anthropic) CreateInterceptor(_ http.ResponseWriter, r *http.Request, tr
authHeaderName = "Authorization"
credKind = intercept.CredentialKindBYOK
credSecret = token
} else if cfg.KeyPool != nil {
// Centralized: use the first key as a placeholder hint.
// TODO(ssncferreira): record the actually-used key in
// the interception record to reflect failover.
if key, keyPoolErr := cfg.KeyPool.Walker().Next(); keyPoolErr == nil {
credSecret = key.Value()
}
}
// Centralized leaves credSecret empty: the hint is set by the
// failover loop on each key attempt and persisted at
// end-of-interception.
cred := intercept.NewCredentialInfo(credKind, credSecret)
var interceptor intercept.Interceptor
+3 -1
View File
@@ -257,7 +257,9 @@ func TestAnthropic_CreateInterceptor_BYOK(t *testing.T) {
setHeaders: map[string]string{},
wantXApiKey: "test-key",
wantCredentialKind: intercept.CredentialKindCentralized,
wantCredentialHint: "t...y",
// Centralized hint is empty at CreateInterceptor; set
// by the key failover loop during ProcessRequest.
wantCredentialHint: "",
},
{
name: "Messages_BYOK_BearerToken_And_APIKey",
+3 -7
View File
@@ -143,14 +143,10 @@ func (p *OpenAI) CreateInterceptor(_ http.ResponseWriter, r *http.Request, trace
cfg.KeyPool = nil
credKind = intercept.CredentialKindBYOK
credSecret = token
} else if cfg.KeyPool != nil {
// Centralized: use the first key as a placeholder hint.
// TODO(ssncferreira): record the actually-used key in
// the interception record to reflect failover.
if key, keyPoolErr := cfg.KeyPool.Walker().Next(); keyPoolErr == nil {
credSecret = key.Value()
}
}
// Centralized leaves credSecret empty: the hint is set by the
// failover loop on each key attempt and persisted at
// end-of-interception.
cred := intercept.NewCredentialInfo(credKind, credSecret)
path := strings.TrimPrefix(r.URL.Path, p.RoutePrefix())
+6 -2
View File
@@ -229,7 +229,9 @@ func TestOpenAI_CreateInterceptor(t *testing.T) {
setHeaders: map[string]string{},
wantAuthorization: "Bearer centralized-key",
wantCredentialKind: intercept.CredentialKindCentralized,
wantCredentialHint: "ce...ey",
// Centralized hint is empty at CreateInterceptor; set
// by the key failover loop during ProcessRequest.
wantCredentialHint: "",
},
{
name: "Responses_BYOK",
@@ -249,7 +251,9 @@ func TestOpenAI_CreateInterceptor(t *testing.T) {
setHeaders: map[string]string{},
wantAuthorization: "Bearer centralized-key",
wantCredentialKind: intercept.CredentialKindCentralized,
wantCredentialHint: "ce...ey",
// Centralized hint is empty at CreateInterceptor; set
// by the key failover loop during ProcessRequest.
wantCredentialHint: "",
},
// X-Api-Key should not appear in production since clients use Authorization,
// but ensure it is stripped if it does arrive.
+7
View File
@@ -39,13 +39,20 @@ type InterceptionRecord struct {
Client string
UserAgent string
CorrelatingToolCallID *string
// CredentialKind is always set: either BYOK or centralized.
CredentialKind string
// CredentialHint is only set for BYOK, where the key is known
// from the request. Centralized uses key failover, so the hint
// can only be determined at end-of-interception.
CredentialHint string
}
type InterceptionRecordEnded struct {
ID string
EndedAt time.Time
// CredentialHint is the hint observed at end-of-interception.
// Only applied to the DB row for centralized; ignored for BYOK.
CredentialHint string
}
type TokenUsageRecord struct {
+237 -226
View File
@@ -218,6 +218,7 @@ type RecordInterceptionEndedRequest struct {
Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` // UUID.
EndedAt *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=ended_at,json=endedAt,proto3" json:"ended_at,omitempty"`
CredentialHint string `protobuf:"bytes,3,opt,name=credential_hint,json=credentialHint,proto3" json:"credential_hint,omitempty"`
}
func (x *RecordInterceptionEndedRequest) Reset() {
@@ -266,6 +267,13 @@ func (x *RecordInterceptionEndedRequest) GetEndedAt() *timestamppb.Timestamp {
return nil
}
func (x *RecordInterceptionEndedRequest) GetCredentialHint() string {
if x != nil {
return x.CredentialHint
}
return ""
}
type RecordInterceptionEndedResponse struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
@@ -1295,249 +1303,252 @@ var file_coderd_aibridged_proto_aibridged_proto_rawDesc = []byte{
0x42, 0x14, 0x0a, 0x12, 0x5f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x73, 0x65, 0x73, 0x73,
0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x22, 0x1c, 0x0a, 0x1a, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64,
0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70,
0x6f, 0x6e, 0x73, 0x65, 0x22, 0x67, 0x0a, 0x1e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x49, 0x6e,
0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x64, 0x65, 0x64, 0x52,
0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01,
0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x35, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x65, 0x64, 0x5f,
0x61, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c,
0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73,
0x74, 0x61, 0x6d, 0x70, 0x52, 0x07, 0x65, 0x6e, 0x64, 0x65, 0x64, 0x41, 0x74, 0x22, 0x21, 0x0a,
0x1f, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74,
0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x64, 0x65, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
0x22, 0xe9, 0x03, 0x0a, 0x17, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x54, 0x6f, 0x6b, 0x65, 0x6e,
0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x27, 0x0a, 0x0f,
0x69, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18,
0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74,
0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x15, 0x0a, 0x06, 0x6d, 0x73, 0x67, 0x5f, 0x69, 0x64, 0x18,
0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6d, 0x73, 0x67, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c,
0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01,
0x28, 0x03, 0x52, 0x0b, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x12,
0x23, 0x0a, 0x0d, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73,
0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x54, 0x6f,
0x6b, 0x65, 0x6e, 0x73, 0x12, 0x48, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61,
0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52,
0x6f, 0x6e, 0x73, 0x65, 0x22, 0x90, 0x01, 0x0a, 0x1e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x49,
0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x64, 0x65, 0x64,
0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20,
0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x35, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x65, 0x64,
0x5f, 0x61, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67,
0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65,
0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x07, 0x65, 0x6e, 0x64, 0x65, 0x64, 0x41, 0x74, 0x12, 0x27,
0x0a, 0x0f, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x5f, 0x68, 0x69, 0x6e,
0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74,
0x69, 0x61, 0x6c, 0x48, 0x69, 0x6e, 0x74, 0x22, 0x21, 0x0a, 0x1f, 0x52, 0x65, 0x63, 0x6f, 0x72,
0x64, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x64,
0x65, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0xe9, 0x03, 0x0a, 0x17, 0x52,
0x65, 0x63, 0x6f, 0x72, 0x64, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52,
0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45,
0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x39,
0x0a, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x06, 0x20, 0x01,
0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74,
0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09,
0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x35, 0x0a, 0x17, 0x63, 0x61, 0x63,
0x68, 0x65, 0x5f, 0x72, 0x65, 0x61, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x74, 0x6f,
0x6b, 0x65, 0x6e, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x52, 0x14, 0x63, 0x61, 0x63, 0x68,
0x65, 0x52, 0x65, 0x61, 0x64, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73,
0x12, 0x37, 0x0a, 0x18, 0x63, 0x61, 0x63, 0x68, 0x65, 0x5f, 0x77, 0x72, 0x69, 0x74, 0x65, 0x5f,
0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x18, 0x08, 0x20, 0x01,
0x28, 0x03, 0x52, 0x15, 0x63, 0x61, 0x63, 0x68, 0x65, 0x57, 0x72, 0x69, 0x74, 0x65, 0x49, 0x6e,
0x70, 0x75, 0x74, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x1a, 0x51, 0x0a, 0x0d, 0x4d, 0x65, 0x74,
0x61, 0x64, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65,
0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2a, 0x0a, 0x05,
0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x67, 0x6f,
0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x41, 0x6e,
0x79, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x1a, 0x0a, 0x18,
0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x55, 0x73, 0x61, 0x67, 0x65,
0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0xcb, 0x02, 0x0a, 0x18, 0x52, 0x65, 0x63,
0x6f, 0x72, 0x64, 0x50, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65,
0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x27, 0x0a, 0x0f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65,
0x70, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e,
0x69, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x15,
0x0a, 0x06, 0x6d, 0x73, 0x67, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05,
0x6d, 0x73, 0x67, 0x49, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x18,
0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x12, 0x49, 0x0a,
0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32,
0x2d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x50, 0x72,
0x6f, 0x6d, 0x70, 0x74, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x27, 0x0a, 0x0f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x63,
0x65, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52,
0x0e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12,
0x15, 0x0a, 0x06, 0x6d, 0x73, 0x67, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52,
0x05, 0x6d, 0x73, 0x67, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f,
0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x69, 0x6e,
0x70, 0x75, 0x74, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x6f, 0x75, 0x74,
0x70, 0x75, 0x74, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03,
0x52, 0x0c, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x12, 0x48,
0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b,
0x32, 0x2c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x54,
0x6f, 0x6b, 0x65, 0x6e, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08,
0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x39, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x61,
0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67,
0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67,
0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54,
0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65,
0x64, 0x41, 0x74, 0x1a, 0x51, 0x0a, 0x0d, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45,
0x64, 0x41, 0x74, 0x12, 0x35, 0x0a, 0x17, 0x63, 0x61, 0x63, 0x68, 0x65, 0x5f, 0x72, 0x65, 0x61,
0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x18, 0x07,
0x20, 0x01, 0x28, 0x03, 0x52, 0x14, 0x63, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x61, 0x64, 0x49,
0x6e, 0x70, 0x75, 0x74, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x12, 0x37, 0x0a, 0x18, 0x63, 0x61,
0x63, 0x68, 0x65, 0x5f, 0x77, 0x72, 0x69, 0x74, 0x65, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f,
0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x03, 0x52, 0x15, 0x63, 0x61,
0x63, 0x68, 0x65, 0x57, 0x72, 0x69, 0x74, 0x65, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x54, 0x6f, 0x6b,
0x65, 0x6e, 0x73, 0x1a, 0x51, 0x0a, 0x0d, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45,
0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28,
0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2a, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18,
0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70,
0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x41, 0x6e, 0x79, 0x52, 0x05, 0x76, 0x61, 0x6c,
0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x1b, 0x0a, 0x19, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64,
0x50, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f,
0x6e, 0x73, 0x65, 0x22, 0x8f, 0x04, 0x0a, 0x16, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x54, 0x6f,
0x6f, 0x6c, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x27,
0x0a, 0x0f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69,
0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65,
0x70, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x15, 0x0a, 0x06, 0x6d, 0x73, 0x67, 0x5f, 0x69,
0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6d, 0x73, 0x67, 0x49, 0x64, 0x12, 0x22,
0x0a, 0x0a, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x03, 0x20, 0x01,
0x28, 0x09, 0x48, 0x00, 0x52, 0x09, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x55, 0x72, 0x6c, 0x88,
0x01, 0x01, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x6f, 0x6f, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09,
0x52, 0x04, 0x74, 0x6f, 0x6f, 0x6c, 0x12, 0x14, 0x0a, 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x18,
0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x12, 0x1a, 0x0a, 0x08,
0x69, 0x6e, 0x6a, 0x65, 0x63, 0x74, 0x65, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08,
0x69, 0x6e, 0x6a, 0x65, 0x63, 0x74, 0x65, 0x64, 0x12, 0x2e, 0x0a, 0x10, 0x69, 0x6e, 0x76, 0x6f,
0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x07, 0x20, 0x01,
0x28, 0x09, 0x48, 0x01, 0x52, 0x0f, 0x69, 0x6e, 0x76, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e,
0x45, 0x72, 0x72, 0x6f, 0x72, 0x88, 0x01, 0x01, 0x12, 0x47, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61,
0x64, 0x61, 0x74, 0x61, 0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x70, 0x72, 0x6f,
0x74, 0x6f, 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x54, 0x6f, 0x6f, 0x6c, 0x55, 0x73, 0x61,
0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61,
0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74,
0x61, 0x12, 0x39, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18,
0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70,
0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d,
0x70, 0x52, 0x09, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x20, 0x0a, 0x0c,
0x74, 0x6f, 0x6f, 0x6c, 0x5f, 0x63, 0x61, 0x6c, 0x6c, 0x5f, 0x69, 0x64, 0x18, 0x0a, 0x20, 0x01,
0x28, 0x09, 0x52, 0x0a, 0x74, 0x6f, 0x6f, 0x6c, 0x43, 0x61, 0x6c, 0x6c, 0x49, 0x64, 0x1a, 0x51,
0x0a, 0x0d, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12,
0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65,
0x79, 0x12, 0x2a, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b,
0x32, 0x14, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62,
0x75, 0x66, 0x2e, 0x41, 0x6e, 0x79, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38,
0x01, 0x42, 0x0d, 0x0a, 0x0b, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x75, 0x72, 0x6c,
0x42, 0x13, 0x0a, 0x11, 0x5f, 0x69, 0x6e, 0x76, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f,
0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x19, 0x0a, 0x17, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x54,
0x6f, 0x6f, 0x6c, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
0x22, 0xb8, 0x02, 0x0a, 0x19, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x6c,
0x54, 0x68, 0x6f, 0x75, 0x67, 0x68, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x27,
0x0a, 0x0f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69,
0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65,
0x70, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65,
0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e,
0x74, 0x12, 0x4a, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x03, 0x20,
0x03, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x65, 0x63, 0x6f,
0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x54, 0x68, 0x6f, 0x75, 0x67, 0x68, 0x74, 0x52, 0x65,
0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45, 0x6e,
0x74, 0x72, 0x79, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x39, 0x0a,
0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28,
0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x63,
0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x1a, 0x51, 0x0a, 0x0d, 0x4d, 0x65, 0x74, 0x61,
0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x1a, 0x0a, 0x18, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64,
0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
0x73, 0x65, 0x22, 0xcb, 0x02, 0x0a, 0x18, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x50, 0x72, 0x6f,
0x6d, 0x70, 0x74, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12,
0x27, 0x0a, 0x0f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x5f,
0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x63,
0x65, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x15, 0x0a, 0x06, 0x6d, 0x73, 0x67, 0x5f,
0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6d, 0x73, 0x67, 0x49, 0x64, 0x12,
0x16, 0x0a, 0x06, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52,
0x06, 0x70, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x12, 0x49, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64,
0x61, 0x74, 0x61, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x70, 0x72, 0x6f, 0x74,
0x6f, 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x50, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x55, 0x73,
0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64,
0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61,
0x74, 0x61, 0x12, 0x39, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74,
0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e,
0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61,
0x6d, 0x70, 0x52, 0x09, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x1a, 0x51, 0x0a,
0x0d, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10,
0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79,
0x12, 0x2a, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32,
0x14, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75,
0x66, 0x2e, 0x41, 0x6e, 0x79, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01,
0x22, 0x1b, 0x0a, 0x19, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x50, 0x72, 0x6f, 0x6d, 0x70, 0x74,
0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x8f, 0x04,
0x0a, 0x16, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x54, 0x6f, 0x6f, 0x6c, 0x55, 0x73, 0x61, 0x67,
0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x27, 0x0a, 0x0f, 0x69, 0x6e, 0x74, 0x65,
0x72, 0x63, 0x65, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28,
0x09, 0x52, 0x0e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x49,
0x64, 0x12, 0x15, 0x0a, 0x06, 0x6d, 0x73, 0x67, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28,
0x09, 0x52, 0x05, 0x6d, 0x73, 0x67, 0x49, 0x64, 0x12, 0x22, 0x0a, 0x0a, 0x73, 0x65, 0x72, 0x76,
0x65, 0x72, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x09,
0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x55, 0x72, 0x6c, 0x88, 0x01, 0x01, 0x12, 0x12, 0x0a, 0x04,
0x74, 0x6f, 0x6f, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x6f, 0x6f, 0x6c,
0x12, 0x14, 0x0a, 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52,
0x05, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x69, 0x6e, 0x6a, 0x65, 0x63, 0x74,
0x65, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x69, 0x6e, 0x6a, 0x65, 0x63, 0x74,
0x65, 0x64, 0x12, 0x2e, 0x0a, 0x10, 0x69, 0x6e, 0x76, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e,
0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x48, 0x01, 0x52, 0x0f,
0x69, 0x6e, 0x76, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x88,
0x01, 0x01, 0x12, 0x47, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x08,
0x20, 0x03, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x65, 0x63,
0x6f, 0x72, 0x64, 0x54, 0x6f, 0x6f, 0x6c, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75,
0x65, 0x73, 0x74, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72,
0x79, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x39, 0x0a, 0x0a, 0x63,
0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32,
0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75,
0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x63, 0x72, 0x65,
0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x20, 0x0a, 0x0c, 0x74, 0x6f, 0x6f, 0x6c, 0x5f, 0x63,
0x61, 0x6c, 0x6c, 0x5f, 0x69, 0x64, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x74, 0x6f,
0x6f, 0x6c, 0x43, 0x61, 0x6c, 0x6c, 0x49, 0x64, 0x1a, 0x51, 0x0a, 0x0d, 0x4d, 0x65, 0x74, 0x61,
0x64, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79,
0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2a, 0x0a, 0x05, 0x76,
0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x67, 0x6f, 0x6f,
0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x41, 0x6e, 0x79,
0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x1c, 0x0a, 0x1a, 0x52,
0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x42, 0x0d, 0x0a, 0x0b, 0x5f,
0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x75, 0x72, 0x6c, 0x42, 0x13, 0x0a, 0x11, 0x5f, 0x69,
0x6e, 0x76, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22,
0x19, 0x0a, 0x17, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x54, 0x6f, 0x6f, 0x6c, 0x55, 0x73, 0x61,
0x67, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0xb8, 0x02, 0x0a, 0x19, 0x52,
0x65, 0x63, 0x6f, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x54, 0x68, 0x6f, 0x75, 0x67, 0x68,
0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x35, 0x0a, 0x1a, 0x47, 0x65, 0x74,
0x4d, 0x43, 0x50, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73,
0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x75, 0x73, 0x65, 0x72, 0x5f,
0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x75, 0x73, 0x65, 0x72, 0x49, 0x64,
0x22, 0xb2, 0x01, 0x0a, 0x1b, 0x47, 0x65, 0x74, 0x4d, 0x43, 0x50, 0x53, 0x65, 0x72, 0x76, 0x65,
0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
0x12, 0x40, 0x0a, 0x10, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x5f, 0x6d, 0x63, 0x70, 0x5f, 0x63, 0x6f,
0x6e, 0x66, 0x69, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x70, 0x72, 0x6f,
0x74, 0x6f, 0x2e, 0x4d, 0x43, 0x50, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66,
0x69, 0x67, 0x52, 0x0e, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x4d, 0x63, 0x70, 0x43, 0x6f, 0x6e, 0x66,
0x69, 0x67, 0x12, 0x51, 0x0a, 0x19, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x61,
0x75, 0x74, 0x68, 0x5f, 0x6d, 0x63, 0x70, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x18,
0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x4d, 0x43,
0x50, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x16, 0x65,
0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x41, 0x75, 0x74, 0x68, 0x4d, 0x63, 0x70, 0x43, 0x6f,
0x6e, 0x66, 0x69, 0x67, 0x73, 0x22, 0x85, 0x01, 0x0a, 0x0f, 0x4d, 0x43, 0x50, 0x53, 0x65, 0x72,
0x76, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18,
0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c,
0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x12, 0x28, 0x0a, 0x10, 0x74,
0x6f, 0x6f, 0x6c, 0x5f, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x5f, 0x72, 0x65, 0x67, 0x65, 0x78, 0x18,
0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x74, 0x6f, 0x6f, 0x6c, 0x41, 0x6c, 0x6c, 0x6f, 0x77,
0x52, 0x65, 0x67, 0x65, 0x78, 0x12, 0x26, 0x0a, 0x0f, 0x74, 0x6f, 0x6f, 0x6c, 0x5f, 0x64, 0x65,
0x6e, 0x79, 0x5f, 0x72, 0x65, 0x67, 0x65, 0x78, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d,
0x74, 0x6f, 0x6f, 0x6c, 0x44, 0x65, 0x6e, 0x79, 0x52, 0x65, 0x67, 0x65, 0x78, 0x22, 0x72, 0x0a,
0x24, 0x47, 0x65, 0x74, 0x4d, 0x43, 0x50, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x41, 0x63, 0x63,
0x65, 0x73, 0x73, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x42, 0x61, 0x74, 0x63, 0x68, 0x52, 0x65,
0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x69, 0x64,
0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x75, 0x73, 0x65, 0x72, 0x49, 0x64, 0x12, 0x31,
0x0a, 0x15, 0x6d, 0x63, 0x70, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x63, 0x6f, 0x6e,
0x66, 0x69, 0x67, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x12, 0x6d,
0x63, 0x70, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x49, 0x64,
0x73, 0x22, 0xda, 0x02, 0x0a, 0x25, 0x47, 0x65, 0x74, 0x4d, 0x43, 0x50, 0x53, 0x65, 0x72, 0x76,
0x65, 0x72, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x42, 0x61,
0x74, 0x63, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x63, 0x0a, 0x0d, 0x61,
0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03,
0x28, 0x0b, 0x32, 0x3e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x4d, 0x43,
0x50, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x54, 0x6f, 0x6b,
0x65, 0x6e, 0x73, 0x42, 0x61, 0x74, 0x63, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
0x2e, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x45, 0x6e, 0x74,
0x72, 0x79, 0x52, 0x0c, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73,
0x12, 0x50, 0x0a, 0x06, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b,
0x32, 0x38, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x4d, 0x43, 0x50, 0x53,
0x65, 0x72, 0x76, 0x65, 0x72, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x54, 0x6f, 0x6b, 0x65, 0x6e,
0x73, 0x42, 0x61, 0x74, 0x63, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x45,
0x72, 0x72, 0x6f, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x65, 0x72, 0x72, 0x6f,
0x72, 0x73, 0x1a, 0x3f, 0x0a, 0x11, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x54, 0x6f, 0x6b, 0x65,
0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01,
0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c,
0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a,
0x02, 0x38, 0x01, 0x1a, 0x39, 0x0a, 0x0b, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x45, 0x6e, 0x74,
0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52,
0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20,
0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x3e,
0x0a, 0x13, 0x49, 0x73, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x52, 0x65,
0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01,
0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x15, 0x0a, 0x06, 0x6b, 0x65, 0x79, 0x5f, 0x69,
0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6b, 0x65, 0x79, 0x49, 0x64, 0x22, 0x6b,
0x0a, 0x14, 0x49, 0x73, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x52, 0x65,
0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x5f,
0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x49,
0x64, 0x12, 0x1c, 0x0a, 0x0a, 0x61, 0x70, 0x69, 0x5f, 0x6b, 0x65, 0x79, 0x5f, 0x69, 0x64, 0x18,
0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x61, 0x70, 0x69, 0x4b, 0x65, 0x79, 0x49, 0x64, 0x12,
0x1a, 0x0a, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28,
0x09, 0x52, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x32, 0xa9, 0x04, 0x0a, 0x08,
0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x65, 0x72, 0x12, 0x59, 0x0a, 0x12, 0x52, 0x65, 0x63, 0x6f,
0x72, 0x64, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x20,
0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x49, 0x6e, 0x74,
0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
0x1a, 0x21, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x49,
0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f,
0x6e, 0x73, 0x65, 0x12, 0x68, 0x0a, 0x17, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x49, 0x6e, 0x74,
0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x64, 0x65, 0x64, 0x12, 0x25,
0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x49, 0x6e, 0x74,
0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x64, 0x65, 0x64, 0x52, 0x65,
0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x65,
0x63, 0x6f, 0x72, 0x64, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x69, 0x6f, 0x6e,
0x45, 0x6e, 0x64, 0x65, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x53, 0x0a,
0x10, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x55, 0x73, 0x61, 0x67,
0x65, 0x12, 0x1e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64,
0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
0x74, 0x1a, 0x1f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64,
0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
0x73, 0x65, 0x12, 0x56, 0x0a, 0x11, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x50, 0x72, 0x6f, 0x6d,
0x70, 0x74, 0x55, 0x73, 0x61, 0x67, 0x65, 0x12, 0x1f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e,
0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x50, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x55, 0x73, 0x61, 0x67,
0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
0x2e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x50, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x55, 0x73, 0x61,
0x67, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x50, 0x0a, 0x0f, 0x52, 0x65,
0x63, 0x6f, 0x72, 0x64, 0x54, 0x6f, 0x6f, 0x6c, 0x55, 0x73, 0x61, 0x67, 0x65, 0x12, 0x1d, 0x2e,
0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x54, 0x6f, 0x6f, 0x6c,
0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x70,
0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x54, 0x6f, 0x6f, 0x6c, 0x55,
0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x59, 0x0a, 0x12,
0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x54, 0x68, 0x6f, 0x75, 0x67,
0x68, 0x74, 0x12, 0x20, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x72,
0x64, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x54, 0x68, 0x6f, 0x75, 0x67, 0x68, 0x74, 0x52, 0x65, 0x71,
0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x65, 0x63,
0x6f, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x54, 0x68, 0x6f, 0x75, 0x67, 0x68, 0x74, 0x52,
0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0xeb, 0x01, 0x0a, 0x0f, 0x4d, 0x43, 0x50, 0x43,
0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x12, 0x5c, 0x0a, 0x13, 0x47,
0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x27, 0x0a, 0x0f, 0x69, 0x6e, 0x74, 0x65,
0x72, 0x63, 0x65, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28,
0x09, 0x52, 0x0e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x49,
0x64, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01,
0x28, 0x09, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x12, 0x4a, 0x0a, 0x08, 0x6d,
0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2e, 0x2e,
0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65,
0x6c, 0x54, 0x68, 0x6f, 0x75, 0x67, 0x68, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e,
0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x6d,
0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x39, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74,
0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f,
0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69,
0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64,
0x41, 0x74, 0x1a, 0x51, 0x0a, 0x0d, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45, 0x6e,
0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2a, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02,
0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72,
0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x41, 0x6e, 0x79, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75,
0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x1c, 0x0a, 0x1a, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x4d,
0x6f, 0x64, 0x65, 0x6c, 0x54, 0x68, 0x6f, 0x75, 0x67, 0x68, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f,
0x6e, 0x73, 0x65, 0x22, 0x35, 0x0a, 0x1a, 0x47, 0x65, 0x74, 0x4d, 0x43, 0x50, 0x53, 0x65, 0x72,
0x76, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
0x74, 0x12, 0x17, 0x0a, 0x07, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01,
0x28, 0x09, 0x52, 0x06, 0x75, 0x73, 0x65, 0x72, 0x49, 0x64, 0x22, 0xb2, 0x01, 0x0a, 0x1b, 0x47,
0x65, 0x74, 0x4d, 0x43, 0x50, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69,
0x67, 0x73, 0x12, 0x21, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x4d, 0x43,
0x50, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x52, 0x65,
0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65,
0x74, 0x4d, 0x43, 0x50, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67,
0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x7a, 0x0a, 0x1d, 0x47, 0x65, 0x74,
0x4d, 0x43, 0x50, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x54,
0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x42, 0x61, 0x74, 0x63, 0x68, 0x12, 0x2b, 0x2e, 0x70, 0x72, 0x6f,
0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x4d, 0x43, 0x50, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x41,
0x63, 0x63, 0x65, 0x73, 0x73, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x42, 0x61, 0x74, 0x63, 0x68,
0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e,
0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x40, 0x0a, 0x10, 0x63, 0x6f,
0x64, 0x65, 0x72, 0x5f, 0x6d, 0x63, 0x70, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x01,
0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x4d, 0x43, 0x50,
0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0e, 0x63, 0x6f,
0x64, 0x65, 0x72, 0x4d, 0x63, 0x70, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x51, 0x0a, 0x19,
0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x5f, 0x6d, 0x63,
0x70, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32,
0x16, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x4d, 0x43, 0x50, 0x53, 0x65, 0x72, 0x76, 0x65,
0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x16, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61,
0x6c, 0x41, 0x75, 0x74, 0x68, 0x4d, 0x63, 0x70, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x22,
0x85, 0x01, 0x0a, 0x0f, 0x4d, 0x43, 0x50, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x43, 0x6f, 0x6e,
0x66, 0x69, 0x67, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52,
0x02, 0x69, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09,
0x52, 0x03, 0x75, 0x72, 0x6c, 0x12, 0x28, 0x0a, 0x10, 0x74, 0x6f, 0x6f, 0x6c, 0x5f, 0x61, 0x6c,
0x6c, 0x6f, 0x77, 0x5f, 0x72, 0x65, 0x67, 0x65, 0x78, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52,
0x0e, 0x74, 0x6f, 0x6f, 0x6c, 0x41, 0x6c, 0x6c, 0x6f, 0x77, 0x52, 0x65, 0x67, 0x65, 0x78, 0x12,
0x26, 0x0a, 0x0f, 0x74, 0x6f, 0x6f, 0x6c, 0x5f, 0x64, 0x65, 0x6e, 0x79, 0x5f, 0x72, 0x65, 0x67,
0x65, 0x78, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x74, 0x6f, 0x6f, 0x6c, 0x44, 0x65,
0x6e, 0x79, 0x52, 0x65, 0x67, 0x65, 0x78, 0x22, 0x72, 0x0a, 0x24, 0x47, 0x65, 0x74, 0x4d, 0x43,
0x50, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x54, 0x6f, 0x6b,
0x65, 0x6e, 0x73, 0x42, 0x61, 0x74, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12,
0x17, 0x0a, 0x07, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
0x52, 0x06, 0x75, 0x73, 0x65, 0x72, 0x49, 0x64, 0x12, 0x31, 0x0a, 0x15, 0x6d, 0x63, 0x70, 0x5f,
0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x69, 0x64,
0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x12, 0x6d, 0x63, 0x70, 0x53, 0x65, 0x72, 0x76,
0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x49, 0x64, 0x73, 0x22, 0xda, 0x02, 0x0a, 0x25,
0x47, 0x65, 0x74, 0x4d, 0x43, 0x50, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x41, 0x63, 0x63, 0x65,
0x73, 0x73, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x42, 0x61, 0x74, 0x63, 0x68, 0x52, 0x65, 0x73,
0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0x55, 0x0a, 0x0a, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69,
0x7a, 0x65, 0x72, 0x12, 0x47, 0x0a, 0x0c, 0x49, 0x73, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69,
0x7a, 0x65, 0x64, 0x12, 0x1a, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x49, 0x73, 0x41, 0x75,
0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
0x1b, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x49, 0x73, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72,
0x69, 0x7a, 0x65, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x32, 0x5a, 0x30,
0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x6f, 0x64, 0x65, 0x72,
0x2f, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x2f, 0x76, 0x32, 0x2f, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x64,
0x2f, 0x61, 0x69, 0x62, 0x72, 0x69, 0x64, 0x67, 0x65, 0x64, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f,
0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x63, 0x0a, 0x0d, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f,
0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3e, 0x2e, 0x70,
0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x4d, 0x43, 0x50, 0x53, 0x65, 0x72, 0x76, 0x65,
0x72, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x42, 0x61, 0x74,
0x63, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x41, 0x63, 0x63, 0x65, 0x73,
0x73, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0c, 0x61, 0x63,
0x63, 0x65, 0x73, 0x73, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x12, 0x50, 0x0a, 0x06, 0x65, 0x72,
0x72, 0x6f, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x38, 0x2e, 0x70, 0x72, 0x6f,
0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x4d, 0x43, 0x50, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x41,
0x63, 0x63, 0x65, 0x73, 0x73, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x42, 0x61, 0x74, 0x63, 0x68,
0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x45,
0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x1a, 0x3f, 0x0a, 0x11,
0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72,
0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03,
0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01,
0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x39, 0x0a,
0x0b, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03,
0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14,
0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76,
0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x3e, 0x0a, 0x13, 0x49, 0x73, 0x41, 0x75,
0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12,
0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65,
0x79, 0x12, 0x15, 0x0a, 0x06, 0x6b, 0x65, 0x79, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28,
0x09, 0x52, 0x05, 0x6b, 0x65, 0x79, 0x49, 0x64, 0x22, 0x6b, 0x0a, 0x14, 0x49, 0x73, 0x41, 0x75,
0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
0x12, 0x19, 0x0a, 0x08, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01,
0x28, 0x09, 0x52, 0x07, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x12, 0x1c, 0x0a, 0x0a, 0x61,
0x70, 0x69, 0x5f, 0x6b, 0x65, 0x79, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52,
0x08, 0x61, 0x70, 0x69, 0x4b, 0x65, 0x79, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x75, 0x73, 0x65,
0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x75, 0x73, 0x65,
0x72, 0x6e, 0x61, 0x6d, 0x65, 0x32, 0xa9, 0x04, 0x0a, 0x08, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64,
0x65, 0x72, 0x12, 0x59, 0x0a, 0x12, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x49, 0x6e, 0x74, 0x65,
0x72, 0x63, 0x65, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x20, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
0x2e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74,
0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x70, 0x72, 0x6f,
0x74, 0x6f, 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65,
0x70, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x68, 0x0a,
0x17, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74,
0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x64, 0x65, 0x64, 0x12, 0x25, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
0x2e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74,
0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x64, 0x65, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
0x26, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x49, 0x6e,
0x74, 0x65, 0x72, 0x63, 0x65, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x64, 0x65, 0x64, 0x52,
0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x53, 0x0a, 0x10, 0x52, 0x65, 0x63, 0x6f, 0x72,
0x64, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x55, 0x73, 0x61, 0x67, 0x65, 0x12, 0x1e, 0x2e, 0x70, 0x72,
0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x55,
0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x70, 0x72,
0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x55,
0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x56, 0x0a, 0x11,
0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x50, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x55, 0x73, 0x61, 0x67,
0x65, 0x12, 0x1f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64,
0x50, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65,
0x73, 0x74, 0x1a, 0x20, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x72,
0x64, 0x50, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x73, 0x70,
0x6f, 0x6e, 0x73, 0x65, 0x12, 0x50, 0x0a, 0x0f, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x54, 0x6f,
0x6f, 0x6c, 0x55, 0x73, 0x61, 0x67, 0x65, 0x12, 0x1d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e,
0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x54, 0x6f, 0x6f, 0x6c, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52,
0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52,
0x65, 0x63, 0x6f, 0x72, 0x64, 0x54, 0x6f, 0x6f, 0x6c, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65,
0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x59, 0x0a, 0x12, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64,
0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x54, 0x68, 0x6f, 0x75, 0x67, 0x68, 0x74, 0x12, 0x20, 0x2e, 0x70,
0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x6c,
0x54, 0x68, 0x6f, 0x75, 0x67, 0x68, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21,
0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x4d, 0x6f, 0x64,
0x65, 0x6c, 0x54, 0x68, 0x6f, 0x75, 0x67, 0x68, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
0x65, 0x32, 0xeb, 0x01, 0x0a, 0x0f, 0x4d, 0x43, 0x50, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75,
0x72, 0x61, 0x74, 0x6f, 0x72, 0x12, 0x5c, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x4d, 0x43, 0x50, 0x53,
0x65, 0x72, 0x76, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x12, 0x21, 0x2e, 0x70,
0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x4d, 0x43, 0x50, 0x53, 0x65, 0x72, 0x76, 0x65,
0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
0x22, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x4d, 0x43, 0x50, 0x53, 0x65,
0x72, 0x76, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f,
0x6e, 0x73, 0x65, 0x12, 0x7a, 0x0a, 0x1d, 0x47, 0x65, 0x74, 0x4d, 0x43, 0x50, 0x53, 0x65, 0x72,
0x76, 0x65, 0x72, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x42,
0x61, 0x74, 0x63, 0x68, 0x12, 0x2b, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74,
0x4d, 0x43, 0x50, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x54,
0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x42, 0x61, 0x74, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
0x74, 0x1a, 0x2c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x74, 0x4d, 0x43, 0x50,
0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x54, 0x6f, 0x6b, 0x65,
0x6e, 0x73, 0x42, 0x61, 0x74, 0x63, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32,
0x55, 0x0a, 0x0a, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x72, 0x12, 0x47, 0x0a,
0x0c, 0x49, 0x73, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x12, 0x1a, 0x2e,
0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x49, 0x73, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a,
0x65, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x70, 0x72, 0x6f, 0x74,
0x6f, 0x2e, 0x49, 0x73, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x52, 0x65,
0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x32, 0x5a, 0x30, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62,
0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x2f, 0x63, 0x6f, 0x64, 0x65, 0x72,
0x2f, 0x76, 0x32, 0x2f, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x64, 0x2f, 0x61, 0x69, 0x62, 0x72, 0x69,
0x64, 0x67, 0x65, 0x64, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74,
0x6f, 0x33,
}
var (
+1
View File
@@ -58,6 +58,7 @@ message RecordInterceptionResponse {}
message RecordInterceptionEndedRequest {
string id = 1; // UUID.
google.protobuf.Timestamp ended_at = 2;
string credential_hint = 3;
}
message RecordInterceptionEndedResponse {}
+1
View File
@@ -47,6 +47,7 @@ func (t *recorderTranslation) RecordInterceptionEnded(ctx context.Context, req *
_, err := t.client.RecordInterceptionEnded(ctx, &proto.RecordInterceptionEndedRequest{
Id: req.ID,
EndedAt: timestamppb.New(req.EndedAt),
CredentialHint: req.CredentialHint,
})
return err
}
@@ -224,6 +224,7 @@ func (s *Server) RecordInterceptionEnded(ctx context.Context, in *proto.RecordIn
_, err = s.store.UpdateAIBridgeInterceptionEnded(ctx, database.UpdateAIBridgeInterceptionEndedParams{
ID: intcID,
EndedAt: in.EndedAt.AsTime(),
CredentialHint: in.CredentialHint,
})
if err != nil {
return nil, xerrors.Errorf("end interception: %w", err)
@@ -946,6 +946,7 @@ func TestRecordInterceptionEnded(t *testing.T) {
request: &proto.RecordInterceptionEndedRequest{
Id: uuid.UUID{1}.String(),
EndedAt: timestamppb.Now(),
CredentialHint: "sk-a...efgh",
},
setupMocks: func(t *testing.T, db *dbmock.MockStore, req *proto.RecordInterceptionEndedRequest) {
interceptionID, err := uuid.Parse(req.GetId())
@@ -954,6 +955,7 @@ func TestRecordInterceptionEnded(t *testing.T) {
db.EXPECT().UpdateAIBridgeInterceptionEnded(gomock.Any(), database.UpdateAIBridgeInterceptionEndedParams{
ID: interceptionID,
EndedAt: req.EndedAt.AsTime(),
CredentialHint: req.CredentialHint,
}).Return(database.AIBridgeInterception{
ID: interceptionID,
InitiatorID: uuid.UUID{2},
@@ -961,6 +963,7 @@ func TestRecordInterceptionEnded(t *testing.T) {
Model: "mod",
StartedAt: time.Now(),
EndedAt: sql.NullTime{Time: req.EndedAt.AsTime(), Valid: true},
CredentialHint: req.CredentialHint,
}, nil)
},
},
+1
View File
@@ -2004,6 +2004,7 @@ func AIBridgeInterception(t testing.TB, db database.Store, seed database.InsertA
interception, err = db.UpdateAIBridgeInterceptionEnded(genCtx, database.UpdateAIBridgeInterceptionEndedParams{
ID: interception.ID,
EndedAt: *endedAt,
CredentialHint: takeFirst(seed.CredentialHint, ""),
})
require.NoError(t, err, "insert aibridge interception")
}
+50
View File
@@ -9923,6 +9923,7 @@ func TestUpdateAIBridgeInterceptionEnded(t *testing.T) {
got, err := db.UpdateAIBridgeInterceptionEnded(ctx, database.UpdateAIBridgeInterceptionEndedParams{
ID: uuid.New(),
EndedAt: time.Now(),
CredentialHint: "sk-a...efgh",
})
require.ErrorContains(t, err, "no rows in result set")
require.EqualValues(t, database.AIBridgeInterception{}, got)
@@ -9959,16 +9960,19 @@ func TestUpdateAIBridgeInterceptionEnded(t *testing.T) {
updated, err := db.UpdateAIBridgeInterceptionEnded(ctx, database.UpdateAIBridgeInterceptionEndedParams{
ID: intc0.ID,
EndedAt: endedAt,
CredentialHint: "sk-a...efgh",
})
require.NoError(t, err)
require.EqualValues(t, updated.ID, intc0.ID)
require.True(t, updated.EndedAt.Valid)
require.WithinDuration(t, endedAt, updated.EndedAt.Time, 5*time.Second)
require.Equal(t, "sk-a...efgh", updated.CredentialHint)
// Updating first interception again should fail
updated, err = db.UpdateAIBridgeInterceptionEnded(ctx, database.UpdateAIBridgeInterceptionEndedParams{
ID: intc0.ID,
EndedAt: endedAt.Add(time.Hour),
CredentialHint: "sk-a...efgh",
})
require.ErrorIs(t, err, sql.ErrNoRows)
@@ -9979,6 +9983,52 @@ func TestUpdateAIBridgeInterceptionEnded(t *testing.T) {
require.False(t, got.EndedAt.Valid)
}
})
t.Run("CentralizedHintUpdated", func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitLong)
user := dbgen.User(t, db, database.User{})
intc, err := db.InsertAIBridgeInterception(ctx, database.InsertAIBridgeInterceptionParams{
ID: uuid.New(),
InitiatorID: user.ID,
Metadata: json.RawMessage("{}"),
CredentialKind: database.CredentialKindCentralized,
CredentialHint: "",
})
require.NoError(t, err)
updated, err := db.UpdateAIBridgeInterceptionEnded(ctx, database.UpdateAIBridgeInterceptionEndedParams{
ID: intc.ID,
EndedAt: time.Now(),
CredentialHint: "sk-a...efgh",
})
require.NoError(t, err)
require.Equal(t, "sk-a...efgh", updated.CredentialHint)
})
t.Run("BYOKHintPreserved", func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitLong)
user := dbgen.User(t, db, database.User{})
intc, err := db.InsertAIBridgeInterception(ctx, database.InsertAIBridgeInterceptionParams{
ID: uuid.New(),
InitiatorID: user.ID,
Metadata: json.RawMessage("{}"),
CredentialKind: database.CredentialKindByok,
CredentialHint: "sk-u...byok",
})
require.NoError(t, err)
updated, err := db.UpdateAIBridgeInterceptionEnded(ctx, database.UpdateAIBridgeInterceptionEndedParams{
ID: intc.ID,
EndedAt: time.Now(),
CredentialHint: "sk-a...efgh",
})
require.NoError(t, err)
require.Equal(t, "sk-u...byok", updated.CredentialHint)
})
}
func TestDeleteExpiredAPIKeys(t *testing.T) {
+11 -3
View File
@@ -2389,20 +2389,28 @@ func (q *sqlQuerier) ListAIBridgeUserPromptsByInterceptionIDs(ctx context.Contex
const updateAIBridgeInterceptionEnded = `-- name: UpdateAIBridgeInterceptionEnded :one
UPDATE aibridge_interceptions
SET ended_at = $1::timestamptz
SET ended_at = $1::timestamptz,
-- BYOK records its hint at the start of the interception.
-- Centralized uses key failover, so its hint is only known
-- at end-of-interception.
credential_hint = CASE
WHEN credential_kind = 'centralized' THEN $2::text
ELSE credential_hint
END
WHERE
id = $2::uuid
id = $3::uuid
AND ended_at IS NULL
RETURNING id, initiator_id, provider, model, started_at, metadata, ended_at, api_key_id, client, thread_parent_id, thread_root_id, client_session_id, session_id, provider_name, credential_kind, credential_hint
`
type UpdateAIBridgeInterceptionEndedParams struct {
EndedAt time.Time `db:"ended_at" json:"ended_at"`
CredentialHint string `db:"credential_hint" json:"credential_hint"`
ID uuid.UUID `db:"id" json:"id"`
}
func (q *sqlQuerier) UpdateAIBridgeInterceptionEnded(ctx context.Context, arg UpdateAIBridgeInterceptionEndedParams) (AIBridgeInterception, error) {
row := q.db.QueryRowContext(ctx, updateAIBridgeInterceptionEnded, arg.EndedAt, arg.ID)
row := q.db.QueryRowContext(ctx, updateAIBridgeInterceptionEnded, arg.EndedAt, arg.CredentialHint, arg.ID)
var i AIBridgeInterception
err := row.Scan(
&i.ID,
+8 -1
View File
@@ -8,7 +8,14 @@ RETURNING *;
-- name: UpdateAIBridgeInterceptionEnded :one
UPDATE aibridge_interceptions
SET ended_at = @ended_at::timestamptz
SET ended_at = @ended_at::timestamptz,
-- BYOK records its hint at the start of the interception.
-- Centralized uses key failover, so its hint is only known
-- at end-of-interception.
credential_hint = CASE
WHEN credential_kind = 'centralized' THEN @credential_hint::text
ELSE credential_hint
END
WHERE
id = @id::uuid
AND ended_at IS NULL