shishantbiswas/coder
1
0
Fork 0
mirror of https://github.com/coder/coder.git synced 2026-06-02 20:48:20 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
main
coder/aibridge/internal/integrationtest/metrics_internal_test.go
T
Ethan c650aabbef chore: standardize on *_internal_test.go for white-box tests (#25601) 
My agent added `//nolint:testpackage` to a test file on one of my PRs.
Again. This PR cleans it up across the entire repo and updates the
in-repo conventions so future agents stop doing it.

The repo already has a precedent for white-box tests that need to touch
unexported symbols: `*_internal_test.go` (145+ existing files). The
`testpackage` linter's default `skip-regexp` exempts that filename
suffix, so the `//nolint:testpackage` directive is unnecessary in every
case where someone reached for it. This PR renames 51 such files to
`*_internal_test.go` via `git mv` so blame and history follow, and
strips the dead directive from 2 files that were already correctly named
(`coderd/oauth2provider/authorize_internal_test.go`,
`coderd/x/chatd/advisor_internal_test.go`).

`.claude/docs/TESTING.md` now documents the rule explicitly under *Test
Package Naming*, which is imported into the root `AGENTS.md` via
`@.claude/docs/TESTING.md`. The rule: prefer `package foo_test`; if you
need internal access, rename the file to `*_internal_test.go` rather
than adding a nolint directive.
2026-05-22 20:24:38 +10:00

447 lines
14 KiB
Go
Raw Permalink Blame History

package integrationtest
import (
"bytes"
"context"
"io"
"net/http"
"net/http/httptest"
"testing"
"github.com/prometheus/client_golang/prometheus"
promtest "github.com/prometheus/client_golang/prometheus/testutil"
"github.com/stretchr/testify/require"
"github.com/tidwall/sjson"
"github.com/coder/coder/v2/aibridge"
"github.com/coder/coder/v2/aibridge/config"
"github.com/coder/coder/v2/aibridge/fixtures"
"github.com/coder/coder/v2/aibridge/internal/testutil"
"github.com/coder/coder/v2/aibridge/metrics"
)
func TestMetrics_Interception(t *testing.T) {
t.Parallel()
cases := []struct {
name string
fixture []byte
path string
headers http.Header
expectStatus string
expectModel string
expectRoute string
expectProvider string
expectClient aibridge.Client
allowOverflow bool // error fixtures may cause retries
}{
{
name: "ant_simple",
fixture: fixtures.AntSimple,
path: pathAnthropicMessages,
expectStatus: metrics.InterceptionCountStatusCompleted,
expectModel: "claude-sonnet-4-0",
expectRoute: "/v1/messages",
expectProvider: config.ProviderAnthropic,
expectClient: aibridge.ClientUnknown,
},
{
name: "ant_error",
fixture: fixtures.AntNonStreamError,
path: pathAnthropicMessages,
headers: http.Header{"User-Agent": []string{"kilo-code/1.2.3"}},
expectStatus: metrics.InterceptionCountStatusFailed,
expectModel: "claude-sonnet-4-0",
expectRoute: "/v1/messages",
expectProvider: config.ProviderAnthropic,
expectClient: aibridge.ClientKilo,
allowOverflow: true,
},
{
name: "ant_simple_claude_code",
fixture: fixtures.AntSimple,
path: pathAnthropicMessages,
headers: http.Header{"User-Agent": []string{"claude-code/1.0.0"}},
expectStatus: metrics.InterceptionCountStatusCompleted,
expectModel: "claude-sonnet-4-0",
expectRoute: "/v1/messages",
expectProvider: config.ProviderAnthropic,
expectClient: aibridge.ClientClaudeCode,
},
{
name: "oai_chat_simple",
fixture: fixtures.OaiChatSimple,
path: pathOpenAIChatCompletions,
headers: http.Header{"User-Agent": []string{"copilot/1.0.0"}},
expectStatus: metrics.InterceptionCountStatusCompleted,
expectModel: "gpt-4.1",
expectRoute: "/v1/chat/completions",
expectProvider: config.ProviderOpenAI,
expectClient: aibridge.ClientCopilotCLI,
},
{
name: "oai_chat_error",
fixture: fixtures.OaiChatNonStreamError,
path: pathOpenAIChatCompletions,
headers: http.Header{"User-Agent": []string{"githubcopilotchat/0.30.0"}},
expectStatus: metrics.InterceptionCountStatusFailed,
expectModel: "gpt-4.1",
expectRoute: "/v1/chat/completions",
expectProvider: config.ProviderOpenAI,
expectClient: aibridge.ClientCopilotVSC,
allowOverflow: true,
},
{
name: "oai_responses_blocking_simple",
fixture: fixtures.OaiResponsesBlockingSimple,
path: pathOpenAIResponses,
headers: http.Header{"X-Cursor-Client-Version": []string{"0.50.0"}},
expectStatus: metrics.InterceptionCountStatusCompleted,
expectModel: "gpt-4o-mini",
expectRoute: "/v1/responses",
expectProvider: config.ProviderOpenAI,
expectClient: aibridge.ClientCursor,
},
{
name: "oai_responses_blocking_error",
fixture: fixtures.OaiResponsesBlockingHTTPErr,
path: pathOpenAIResponses,
headers: http.Header{"User-Agent": []string{"codex/1.0.0"}},
expectStatus: metrics.InterceptionCountStatusFailed,
expectModel: "gpt-4o-mini",
expectRoute: "/v1/responses",
expectProvider: config.ProviderOpenAI,
expectClient: aibridge.ClientCodex,
allowOverflow: true,
},
{
name: "oai_responses_streaming_simple",
fixture: fixtures.OaiResponsesStreamingSimple,
path: pathOpenAIResponses,
headers: http.Header{"User-Agent": []string{"zed/0.200.0"}},
expectStatus: metrics.InterceptionCountStatusCompleted,
expectModel: "gpt-4o-mini",
expectRoute: "/v1/responses",
expectProvider: config.ProviderOpenAI,
expectClient: aibridge.ClientZed,
},
{
name: "oai_responses_streaming_error",
fixture: fixtures.OaiResponsesStreamingHTTPErr,
path: pathOpenAIResponses,
headers: http.Header{"Originator": []string{"roo-code"}},
expectStatus: metrics.InterceptionCountStatusFailed,
expectModel: "gpt-4o-mini",
expectRoute: "/v1/responses",
expectProvider: config.ProviderOpenAI,
expectClient: aibridge.ClientRoo,
allowOverflow: true,
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
ctx, cancel := context.WithTimeout(t.Context(), testutil.WaitLong)
t.Cleanup(cancel)
fix := fixtures.Parse(t, tc.fixture)
upstream := newMockUpstream(ctx, t, newFixtureResponse(fix))
upstream.AllowOverflow = tc.allowOverflow
m := aibridge.NewMetrics(prometheus.NewRegistry())
bridgeServer := newBridgeTestServer(ctx, t, upstream.URL,
withMetrics(m),
)
resp, err := bridgeServer.makeRequest(t, http.MethodPost, tc.path, fix.Request(), tc.headers)
require.NoError(t, err)
defer resp.Body.Close()
_, err = io.ReadAll(resp.Body)
require.NoError(t, err)
count := promtest.ToFloat64(m.InterceptionCount.WithLabelValues(
tc.expectProvider, tc.expectModel, tc.expectStatus, tc.expectRoute, "POST", defaultActorID, string(tc.expectClient)))
require.Equal(t, 1.0, count)
require.Equal(t, 1, promtest.CollectAndCount(m.InterceptionDuration))
require.Equal(t, 1, promtest.CollectAndCount(m.InterceptionCount))
})
}
}
func TestMetrics_InterceptionsInflight(t *testing.T) {
t.Parallel()
fix := fixtures.Parse(t, fixtures.AntSimple)
ctx, cancel := context.WithTimeout(t.Context(), testutil.WaitLong)
t.Cleanup(cancel)
blockCh := make(chan struct{})
// Setup a mock HTTP server which blocks until the request is marked as inflight then proceeds.
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
<-blockCh
}))
t.Cleanup(srv.Close)
m := aibridge.NewMetrics(prometheus.NewRegistry())
bridgeServer := newBridgeTestServer(ctx, t, srv.URL,
withMetrics(m),
)
// Make request in background.
doneCh := make(chan struct{})
go func() {
defer close(doneCh)
req, _ := http.NewRequestWithContext(ctx, http.MethodPost, bridgeServer.URL+pathAnthropicMessages, bytes.NewReader(fix.Request()))
req.Header.Set("Content-Type", "application/json")
resp, err := http.DefaultClient.Do(req)
if err == nil {
defer resp.Body.Close()
_, err = io.ReadAll(resp.Body)
require.NoError(t, err)
}
}()
// Wait until request is detected as inflight.
require.Eventually(t, func() bool {
return promtest.ToFloat64(
m.InterceptionsInflight.WithLabelValues(config.ProviderAnthropic, "claude-sonnet-4-0", "/v1/messages"),
) == 1
}, testutil.WaitMedium, testutil.IntervalFast)
// Unblock request, await completion.
close(blockCh)
select {
case <-doneCh:
case <-ctx.Done():
t.Fatal(ctx.Err())
}
// Metric is not updated immediately after request completes, so wait until it is.
require.Eventually(t, func() bool {
return promtest.ToFloat64(
m.InterceptionsInflight.WithLabelValues(config.ProviderAnthropic, "claude-sonnet-4-0", "/v1/messages"),
) == 0
}, testutil.WaitMedium, testutil.IntervalFast)
}
func TestMetrics_PassthroughCount(t *testing.T) {
t.Parallel()
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
t.Cleanup(upstream.Close)
m := aibridge.NewMetrics(prometheus.NewRegistry())
bridgeServer := newBridgeTestServer(t.Context(), t, upstream.URL,
withMetrics(m),
)
resp, err := bridgeServer.makeRequest(t, http.MethodGet, "/openai/v1/models", nil)
require.NoError(t, err)
defer resp.Body.Close()
require.Equal(t, http.StatusOK, resp.StatusCode)
count := promtest.ToFloat64(m.PassthroughCount.WithLabelValues(
config.ProviderOpenAI, "/models", "GET"))
require.Equal(t, 1.0, count)
}
func TestMetrics_PromptCount(t *testing.T) {
t.Parallel()
ctx, cancel := context.WithTimeout(t.Context(), testutil.WaitLong)
t.Cleanup(cancel)
fix := fixtures.Parse(t, fixtures.OaiChatSimple)
upstream := newMockUpstream(ctx, t, newFixtureResponse(fix))
m := aibridge.NewMetrics(prometheus.NewRegistry())
bridgeServer := newBridgeTestServer(ctx, t, upstream.URL,
withMetrics(m),
)
resp, err := bridgeServer.makeRequest(t, http.MethodPost, pathOpenAIChatCompletions, fix.Request(), http.Header{"User-Agent": []string{"claude-code/1.0.0"}})
require.NoError(t, err)
defer resp.Body.Close()
require.Equal(t, http.StatusOK, resp.StatusCode)
_, err = io.ReadAll(resp.Body)
require.NoError(t, err)
prompts := promtest.ToFloat64(m.PromptCount.WithLabelValues(
config.ProviderOpenAI, "gpt-4.1", defaultActorID, string(aibridge.ClientClaudeCode)))
require.Equal(t, 1.0, prompts)
}
func TestMetrics_TokenUseCount(t *testing.T) {
t.Parallel()
cases := []struct {
name string
fixture []byte
reqPath string
streaming bool
expectProvider string
expectModel string
expectedLabels map[string]float64
}{
{
name: "openai_responses",
fixture: fixtures.OaiResponsesBlockingCachedInputTokens,
reqPath: pathOpenAIResponses,
expectProvider: config.ProviderOpenAI,
expectModel: "gpt-4.1",
expectedLabels: map[string]float64{
"input": 129, // 12033 - 11904 cached
"output": 44,
"cache_read_input_tokens": 11904,
"cache_write_input_tokens": 0,
"output_reasoning": 0,
"total_tokens": 12077,
},
},
{
name: "anthropic_messages_streaming",
fixture: fixtures.AntSingleBuiltinTool,
reqPath: pathAnthropicMessages,
streaming: true,
expectProvider: config.ProviderAnthropic,
expectModel: "claude-sonnet-4-20250514",
expectedLabels: map[string]float64{
"input": 2,
"output": 66,
"cache_read_input_tokens": 13993,
"cache_write_input_tokens": 22,
},
},
{
name: "openai_chat_completions",
fixture: fixtures.OaiChatSimple,
reqPath: pathOpenAIChatCompletions,
expectProvider: config.ProviderOpenAI,
expectModel: "gpt-4.1",
expectedLabels: map[string]float64{
"input": 19,
"output": 200,
"cache_read_input_tokens": 0,
"cache_write_input_tokens": 0,
"completion_reasoning": 0,
"completion_accepted_prediction": 0,
"completion_rejected_prediction": 0,
},
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
ctx, cancel := context.WithTimeout(t.Context(), testutil.WaitLong)
t.Cleanup(cancel)
fix := fixtures.Parse(t, tc.fixture)
upstream := newMockUpstream(ctx, t, newFixtureResponse(fix))
m := aibridge.NewMetrics(prometheus.NewRegistry())
bridgeServer := newBridgeTestServer(ctx, t, upstream.URL,
withMetrics(m),
)
reqBody := fix.Request()
if tc.streaming {
var err error
reqBody, err = sjson.SetBytes(reqBody, "stream", true)
require.NoError(t, err)
}
resp, err := bridgeServer.makeRequest(t, http.MethodPost, tc.reqPath, reqBody, nil)
require.NoError(t, err)
defer resp.Body.Close()
require.Equal(t, http.StatusOK, resp.StatusCode)
_, _ = io.ReadAll(resp.Body)
// metrics are updated asynchronously
require.Eventually(t, func() bool {
return promtest.ToFloat64(m.TokenUseCount.WithLabelValues(
tc.expectProvider, tc.expectModel, "input", defaultActorID, string(aibridge.ClientUnknown))) > 0
}, testutil.WaitMedium, testutil.IntervalFast)
for label, expected := range tc.expectedLabels {
require.Equal(t, expected, promtest.ToFloat64(m.TokenUseCount.WithLabelValues(
tc.expectProvider, tc.expectModel, label, defaultActorID, string(aibridge.ClientUnknown),
)), "metric label %q mismatch", label)
}
})
}
}
func TestMetrics_NonInjectedToolUseCount(t *testing.T) {
t.Parallel()
ctx, cancel := context.WithTimeout(t.Context(), testutil.WaitLong)
t.Cleanup(cancel)
fix := fixtures.Parse(t, fixtures.OaiChatSingleBuiltinTool)
upstream := newMockUpstream(ctx, t, newFixtureResponse(fix))
m := aibridge.NewMetrics(prometheus.NewRegistry())
bridgeServer := newBridgeTestServer(ctx, t, upstream.URL,
withMetrics(m),
)
resp, err := bridgeServer.makeRequest(t, http.MethodPost, pathOpenAIChatCompletions, fix.Request())
require.NoError(t, err)
defer resp.Body.Close()
require.Equal(t, http.StatusOK, resp.StatusCode)
_, err = io.ReadAll(resp.Body)
require.NoError(t, err)
count := promtest.ToFloat64(m.NonInjectedToolUseCount.WithLabelValues(
config.ProviderOpenAI, "gpt-4.1", "read_file"))
require.Equal(t, 1.0, count)
}
func TestMetrics_InjectedToolUseCount(t *testing.T) {
t.Parallel()
ctx, cancel := context.WithTimeout(t.Context(), testutil.WaitLong)
t.Cleanup(cancel)
// First request returns the tool invocation, the second returns the mocked response to the tool result.
fix := fixtures.Parse(t, fixtures.AntSingleInjectedTool)
upstream := newMockUpstream(ctx, t, newFixtureResponse(fix), newFixtureToolResponse(fix))
m := aibridge.NewMetrics(prometheus.NewRegistry())
// Setup mocked MCP server & tools.
mockMCP := setupMCPForTest(t, defaultTracer)
bridgeServer := newBridgeTestServer(ctx, t, upstream.URL,
withMetrics(m),
withMCP(mockMCP),
)
resp, err := bridgeServer.makeRequest(t, http.MethodPost, pathAnthropicMessages, fix.Request())
require.NoError(t, err)
defer resp.Body.Close()
require.Equal(t, http.StatusOK, resp.StatusCode)
_, err = io.ReadAll(resp.Body)
require.NoError(t, err)
// Wait until full roundtrip has completed.
require.Eventually(t, func() bool {
return upstream.Calls.Load() == 2
}, testutil.WaitMedium, testutil.IntervalFast)
recorder := bridgeServer.Recorder
require.Len(t, recorder.ToolUsages(), 1)
require.True(t, recorder.ToolUsages()[0].Injected)
require.NotNil(t, recorder.ToolUsages()[0].ServerURL)
actualServerURL := *recorder.ToolUsages()[0].ServerURL
count := promtest.ToFloat64(m.InjectedToolUseCount.WithLabelValues(
config.ProviderAnthropic, "claude-sonnet-4-20250514", actualServerURL, mockToolName))
require.Equal(t, 1.0, count)
}
Reference in New Issue View Git Blame Copy Permalink