mirror of
https://github.com/coder/coder.git
synced 2026-06-04 21:48:22 +00:00
Thomas Kosiewski
e56381eb61
Stream advisor output into the advisor tool card while the nested advisor call is still running. This keeps the advisor implementation intentionally advisor-specific: the parent model still receives the same final structured tool result, while the frontend receives transient `tool-result.result_delta` parts to render partial advisor text in the expanded card. The final persisted chat history remains unchanged. Refs CODAGT-322. Generated by Coder Agents. <details> <summary>Implementation plan</summary> - Publish advisor text deltas from the nested `chatloop.Run` via `RunAdvisorOptions.OnAdviceDelta`. - Forward those deltas through `chatadvisor.Tool` with the parent advisor tool call ID. - Emit transient `ChatMessagePartTypeToolResult` websocket parts with `ResultDelta` from `chatd`. - Add `result_delta` to the generated tool-result TypeScript variant. - Accumulate tool result deltas in frontend stream state and keep the tool running until the final result arrives. - Render streamed advisor advice in the existing advisor card using streaming markdown mode, while retaining the updated advisor UI. </details>
387 lines
12 KiB
Go
387 lines
12 KiB
Go
package chatadvisor_test
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"strings"
|
|
"testing"
|
|
|
|
"charm.land/fantasy"
|
|
"github.com/stretchr/testify/require"
|
|
"golang.org/x/xerrors"
|
|
|
|
"github.com/coder/coder/v2/coderd/x/chatd/chatadvisor"
|
|
"github.com/coder/coder/v2/coderd/x/chatd/chattest"
|
|
)
|
|
|
|
func TestAdvisorToolSuccess(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
runtime, err := chatadvisor.NewRuntime(chatadvisor.RuntimeConfig{
|
|
Model: &chattest.FakeModel{
|
|
ProviderName: "test-provider",
|
|
ModelName: "test-model",
|
|
StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) {
|
|
return streamFromParts([]fantasy.StreamPart{
|
|
{Type: fantasy.StreamPartTypeTextStart, ID: "text-1"},
|
|
{Type: fantasy.StreamPartTypeTextDelta, ID: "text-1", Delta: "Use the smaller diff."},
|
|
{Type: fantasy.StreamPartTypeTextEnd, ID: "text-1"},
|
|
{Type: fantasy.StreamPartTypeFinish, FinishReason: fantasy.FinishReasonStop},
|
|
}), nil
|
|
},
|
|
},
|
|
MaxUsesPerRun: 2,
|
|
MaxOutputTokens: 128,
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
tool := chatadvisor.Tool(chatadvisor.ToolOptions{
|
|
Runtime: runtime,
|
|
GetConversationSnapshot: func() []fantasy.Message {
|
|
return []fantasy.Message{{
|
|
Role: fantasy.MessageRoleUser,
|
|
Content: []fantasy.MessagePart{
|
|
fantasy.TextPart{Text: "We need a safe fix."},
|
|
},
|
|
}}
|
|
},
|
|
})
|
|
|
|
resp := runAdvisorTool(t, tool, chatadvisor.AdvisorArgs{Question: "What's the safest next step?"})
|
|
require.False(t, resp.IsError)
|
|
|
|
var result chatadvisor.AdvisorResult
|
|
require.NoError(t, json.Unmarshal([]byte(resp.Content), &result))
|
|
require.Equal(t, chatadvisor.ResultTypeAdvice, result.Type)
|
|
require.Equal(t, "Use the smaller diff.", result.Advice)
|
|
require.Equal(t, "test-provider/test-model", result.AdvisorModel)
|
|
require.Equal(t, 1, result.RemainingUses)
|
|
}
|
|
|
|
func TestAdvisorToolPublishesAdviceDeltasWithToolCallID(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
type publishedDelta struct {
|
|
toolCallID string
|
|
delta string
|
|
}
|
|
var published []publishedDelta
|
|
|
|
runtime, err := chatadvisor.NewRuntime(chatadvisor.RuntimeConfig{
|
|
Model: &chattest.FakeModel{
|
|
ProviderName: "test-provider",
|
|
ModelName: "test-model",
|
|
StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) {
|
|
return streamFromParts([]fantasy.StreamPart{
|
|
{Type: fantasy.StreamPartTypeTextStart, ID: "text-1"},
|
|
{Type: fantasy.StreamPartTypeTextDelta, ID: "text-1", Delta: "Prefer "},
|
|
{Type: fantasy.StreamPartTypeTextDelta, ID: "text-1", Delta: "the small diff."},
|
|
{Type: fantasy.StreamPartTypeTextEnd, ID: "text-1"},
|
|
{Type: fantasy.StreamPartTypeFinish, FinishReason: fantasy.FinishReasonStop},
|
|
}), nil
|
|
},
|
|
},
|
|
MaxUsesPerRun: 2,
|
|
MaxOutputTokens: 128,
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
tool := chatadvisor.Tool(chatadvisor.ToolOptions{
|
|
Runtime: runtime,
|
|
GetConversationSnapshot: func() []fantasy.Message { return nil },
|
|
PublishAdviceDelta: func(toolCallID string, delta string) {
|
|
published = append(published, publishedDelta{toolCallID: toolCallID, delta: delta})
|
|
},
|
|
})
|
|
|
|
resp := runAdvisorTool(t, tool, chatadvisor.AdvisorArgs{Question: "What's safest?"})
|
|
require.False(t, resp.IsError)
|
|
require.Equal(t, []publishedDelta{
|
|
{toolCallID: "call-1", delta: "Prefer "},
|
|
{toolCallID: "call-1", delta: "the small diff."},
|
|
}, published)
|
|
|
|
var result chatadvisor.AdvisorResult
|
|
require.NoError(t, json.Unmarshal([]byte(resp.Content), &result))
|
|
require.Equal(t, chatadvisor.ResultTypeAdvice, result.Type)
|
|
require.Equal(t, "Prefer the small diff.", result.Advice)
|
|
}
|
|
|
|
func TestAdvisorToolPublishesAdviceResetWithToolCallID(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
type publishedEvent struct {
|
|
kind string
|
|
toolCallID string
|
|
delta string
|
|
}
|
|
var (
|
|
calls int
|
|
published []publishedEvent
|
|
)
|
|
|
|
runtime, err := chatadvisor.NewRuntime(chatadvisor.RuntimeConfig{
|
|
Model: &chattest.FakeModel{
|
|
ProviderName: "test-provider",
|
|
ModelName: "test-model",
|
|
StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) {
|
|
calls++
|
|
if calls == 1 {
|
|
return streamFromParts([]fantasy.StreamPart{
|
|
{Type: fantasy.StreamPartTypeTextStart, ID: "text-1"},
|
|
{Type: fantasy.StreamPartTypeTextDelta, ID: "text-1", Delta: "stale "},
|
|
{Type: fantasy.StreamPartTypeError, Error: xerrors.New("received status 429 from upstream")},
|
|
}), nil
|
|
}
|
|
return streamFromParts([]fantasy.StreamPart{
|
|
{Type: fantasy.StreamPartTypeTextStart, ID: "text-1"},
|
|
{Type: fantasy.StreamPartTypeTextDelta, ID: "text-1", Delta: "fresh advice"},
|
|
{Type: fantasy.StreamPartTypeTextEnd, ID: "text-1"},
|
|
{Type: fantasy.StreamPartTypeFinish, FinishReason: fantasy.FinishReasonStop},
|
|
}), nil
|
|
},
|
|
},
|
|
MaxUsesPerRun: 2,
|
|
MaxOutputTokens: 128,
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
tool := chatadvisor.Tool(chatadvisor.ToolOptions{
|
|
Runtime: runtime,
|
|
GetConversationSnapshot: func() []fantasy.Message { return nil },
|
|
PublishAdviceDelta: func(toolCallID string, delta string) {
|
|
published = append(published, publishedEvent{
|
|
kind: "delta",
|
|
toolCallID: toolCallID,
|
|
delta: delta,
|
|
})
|
|
},
|
|
PublishAdviceReset: func(toolCallID string) {
|
|
published = append(published, publishedEvent{
|
|
kind: "reset",
|
|
toolCallID: toolCallID,
|
|
})
|
|
},
|
|
})
|
|
|
|
resp := runAdvisorTool(t, tool, chatadvisor.AdvisorArgs{Question: "What's safest?"})
|
|
require.False(t, resp.IsError)
|
|
require.Equal(t, []publishedEvent{
|
|
{kind: "delta", toolCallID: "call-1", delta: "stale "},
|
|
{kind: "reset", toolCallID: "call-1"},
|
|
{kind: "delta", toolCallID: "call-1", delta: "fresh advice"},
|
|
}, published)
|
|
|
|
var result chatadvisor.AdvisorResult
|
|
require.NoError(t, json.Unmarshal([]byte(resp.Content), &result))
|
|
require.Equal(t, chatadvisor.ResultTypeAdvice, result.Type)
|
|
require.Equal(t, "fresh advice", result.Advice)
|
|
}
|
|
|
|
func TestAdvisorToolRejectsEmptyQuestion(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tool := chatadvisor.Tool(chatadvisor.ToolOptions{
|
|
Runtime: mustAdvisorRuntime(t),
|
|
GetConversationSnapshot: func() []fantasy.Message {
|
|
return nil
|
|
},
|
|
})
|
|
|
|
resp := runAdvisorTool(t, tool, chatadvisor.AdvisorArgs{Question: " \t\n "})
|
|
require.True(t, resp.IsError)
|
|
require.Contains(t, resp.Content, "question is required")
|
|
}
|
|
|
|
func TestAdvisorToolRejectsLongQuestion(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tool := chatadvisor.Tool(chatadvisor.ToolOptions{
|
|
Runtime: mustAdvisorRuntime(t),
|
|
GetConversationSnapshot: func() []fantasy.Message {
|
|
return nil
|
|
},
|
|
})
|
|
|
|
resp := runAdvisorTool(t, tool, chatadvisor.AdvisorArgs{Question: strings.Repeat("x", 2001)})
|
|
require.True(t, resp.IsError)
|
|
require.Contains(t, resp.Content, "2000 runes or fewer")
|
|
}
|
|
|
|
func TestAdvisorToolRejectsMissingRuntime(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tool := chatadvisor.Tool(chatadvisor.ToolOptions{
|
|
GetConversationSnapshot: func() []fantasy.Message {
|
|
return nil
|
|
},
|
|
})
|
|
|
|
resp := runAdvisorTool(t, tool, chatadvisor.AdvisorArgs{Question: "Need advice"})
|
|
require.True(t, resp.IsError)
|
|
require.Contains(t, resp.Content, "advisor runtime is not configured")
|
|
}
|
|
|
|
func TestAdvisorToolRejectsMissingSnapshotFunc(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tool := chatadvisor.Tool(chatadvisor.ToolOptions{Runtime: mustAdvisorRuntime(t)})
|
|
|
|
resp := runAdvisorTool(t, tool, chatadvisor.AdvisorArgs{Question: "Need advice"})
|
|
require.True(t, resp.IsError)
|
|
require.Contains(t, resp.Content, "conversation snapshot provider is not configured")
|
|
}
|
|
|
|
func TestAdvisorToolReportsNestedError(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
runtime, err := chatadvisor.NewRuntime(chatadvisor.RuntimeConfig{
|
|
Model: &chattest.FakeModel{
|
|
ProviderName: "test-provider",
|
|
ModelName: "test-model",
|
|
StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) {
|
|
return nil, xerrors.New("boom")
|
|
},
|
|
},
|
|
MaxUsesPerRun: 1,
|
|
MaxOutputTokens: 64,
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
tool := chatadvisor.Tool(chatadvisor.ToolOptions{
|
|
Runtime: runtime,
|
|
GetConversationSnapshot: func() []fantasy.Message { return nil },
|
|
})
|
|
|
|
resp := runAdvisorTool(t, tool, chatadvisor.AdvisorArgs{Question: "why?"})
|
|
require.False(t, resp.IsError)
|
|
|
|
var result chatadvisor.AdvisorResult
|
|
require.NoError(t, json.Unmarshal([]byte(resp.Content), &result))
|
|
require.Equal(t, chatadvisor.ResultTypeError, result.Type)
|
|
require.Contains(t, result.Error, "boom")
|
|
require.Empty(t, result.Advice)
|
|
require.Empty(t, result.AdvisorModel)
|
|
// A failed nested run does not consume the per-run quota.
|
|
require.Equal(t, 1, result.RemainingUses)
|
|
}
|
|
|
|
func TestAdvisorToolReportsLimitReached(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
runtime, err := chatadvisor.NewRuntime(chatadvisor.RuntimeConfig{
|
|
Model: &chattest.FakeModel{
|
|
ProviderName: "test-provider",
|
|
ModelName: "test-model",
|
|
StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) {
|
|
return streamFromParts([]fantasy.StreamPart{
|
|
{Type: fantasy.StreamPartTypeTextStart, ID: "text-1"},
|
|
{Type: fantasy.StreamPartTypeTextDelta, ID: "text-1", Delta: "first"},
|
|
{Type: fantasy.StreamPartTypeTextEnd, ID: "text-1"},
|
|
{Type: fantasy.StreamPartTypeFinish, FinishReason: fantasy.FinishReasonStop},
|
|
}), nil
|
|
},
|
|
},
|
|
MaxUsesPerRun: 1,
|
|
MaxOutputTokens: 64,
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
tool := chatadvisor.Tool(chatadvisor.ToolOptions{
|
|
Runtime: runtime,
|
|
GetConversationSnapshot: func() []fantasy.Message { return nil },
|
|
})
|
|
|
|
first := runAdvisorTool(t, tool, chatadvisor.AdvisorArgs{Question: "first?"})
|
|
require.False(t, first.IsError)
|
|
|
|
second := runAdvisorTool(t, tool, chatadvisor.AdvisorArgs{Question: "second?"})
|
|
require.False(t, second.IsError)
|
|
|
|
var result chatadvisor.AdvisorResult
|
|
require.NoError(t, json.Unmarshal([]byte(second.Content), &result))
|
|
require.Equal(t, chatadvisor.ResultTypeLimitReached, result.Type)
|
|
require.Equal(t, 0, result.RemainingUses)
|
|
require.Empty(t, result.Advice)
|
|
require.Empty(t, result.Error)
|
|
require.Empty(t, result.AdvisorModel)
|
|
}
|
|
|
|
func TestAdvisorToolReportsEmptyModelOutput(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
runtime, err := chatadvisor.NewRuntime(chatadvisor.RuntimeConfig{
|
|
Model: &chattest.FakeModel{
|
|
ProviderName: "test-provider",
|
|
ModelName: "test-model",
|
|
StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) {
|
|
return streamFromParts([]fantasy.StreamPart{
|
|
{Type: fantasy.StreamPartTypeFinish, FinishReason: fantasy.FinishReasonStop},
|
|
}), nil
|
|
},
|
|
},
|
|
MaxUsesPerRun: 1,
|
|
MaxOutputTokens: 64,
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
tool := chatadvisor.Tool(chatadvisor.ToolOptions{
|
|
Runtime: runtime,
|
|
GetConversationSnapshot: func() []fantasy.Message { return nil },
|
|
})
|
|
|
|
resp := runAdvisorTool(t, tool, chatadvisor.AdvisorArgs{Question: "anything?"})
|
|
require.False(t, resp.IsError)
|
|
|
|
var result chatadvisor.AdvisorResult
|
|
require.NoError(t, json.Unmarshal([]byte(resp.Content), &result))
|
|
require.Equal(t, chatadvisor.ResultTypeError, result.Type)
|
|
require.Contains(t, result.Error, "no text output")
|
|
require.Empty(t, result.Advice)
|
|
// An advisor call that produces no advice does not count as a
|
|
// successful use, so the quota must still be available.
|
|
require.Equal(t, 1, result.RemainingUses)
|
|
}
|
|
|
|
func mustAdvisorRuntime(t *testing.T) *chatadvisor.Runtime {
|
|
t.Helper()
|
|
|
|
runtime, err := chatadvisor.NewRuntime(chatadvisor.RuntimeConfig{
|
|
Model: &chattest.FakeModel{
|
|
ProviderName: "test-provider",
|
|
ModelName: "test-model",
|
|
StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) {
|
|
return streamFromParts([]fantasy.StreamPart{
|
|
{Type: fantasy.StreamPartTypeTextStart, ID: "text-1"},
|
|
{Type: fantasy.StreamPartTypeTextDelta, ID: "text-1", Delta: "fallback advice"},
|
|
{Type: fantasy.StreamPartTypeTextEnd, ID: "text-1"},
|
|
{Type: fantasy.StreamPartTypeFinish, FinishReason: fantasy.FinishReasonStop},
|
|
}), nil
|
|
},
|
|
},
|
|
MaxUsesPerRun: 2,
|
|
MaxOutputTokens: 64,
|
|
})
|
|
require.NoError(t, err)
|
|
return runtime
|
|
}
|
|
|
|
func runAdvisorTool(
|
|
t *testing.T,
|
|
tool fantasy.AgentTool,
|
|
args chatadvisor.AdvisorArgs,
|
|
) fantasy.ToolResponse {
|
|
t.Helper()
|
|
|
|
data, err := json.Marshal(args)
|
|
require.NoError(t, err)
|
|
|
|
resp, err := tool.Run(t.Context(), fantasy.ToolCall{
|
|
ID: "call-1",
|
|
Name: "advisor",
|
|
Input: string(data),
|
|
})
|
|
require.NoError(t, err)
|
|
return resp
|
|
}
|