mirror of
https://github.com/coder/coder.git
synced 2026-06-03 04:58:23 +00:00
Kyle Carberry
b969d66978
Adds client-executed dynamic tools to the chat API. Dynamic tools are
declared by the client at chat creation time, presented to the LLM
alongside built-in tools, but executed by the client rather than chatd.
This enables external systems (Slack bots, IDE extensions, Discord bots,
CI/CD integrations) to plug custom tools into the LLM chat loop without
modifying chatd's built-in tool set.
Modeled after OpenAI's Assistants API: the chat pauses with
`requires_action` status when the LLM calls a dynamic tool, the client
POSTs results back via `POST /chats/{id}/tool-results`, and the chat
resumes.
See [this example](https://github.com/coder/coder-slackbot-poc) as a
reference for how this is used. It's highly-configurable, which would
enable creating chats from webhooks, periodically polling, or running as
a Slackbot.
<details>
<summary>Design context</summary>
### Architecture
The chatloop **exits** when it encounters dynamic tools and
**re-enters** when results arrive. No blocking channels, no pubsub for
tool results, no in-memory registry. The DB is the only coordination
mechanism.
```
Phase 1 (chatloop):
LLM response → execute built-in tools only →
Persist(assistant + built-in results) →
status = requires_action → chatloop exits
Phase 2 (POST /tool-results):
Persist(dynamic tool results) →
status = pending → wakeCh → chatloop re-enters
```
### Validation (POST /tool-results)
1. Chat status must be `requires_action` (409 if not)
2. Read chat's `dynamic_tools` → set of dynamic tool names
3. Read last assistant message → extract tool-call parts matching
dynamic tool names
4. Submitted tool_call_ids must match exactly (400 for missing/extra)
5. Persist tool-result message parts, set status to `pending`, signal
wake
### Idempotency
Tool call IDs scoped per LLM step. State machine (`requires_action` →
`pending`) is the guard. First POST wins, subsequent get 409.
### Mixed tool calls
When the LLM calls both built-in and dynamic tools in one step, built-in
tools execute immediately. Their results are persisted in phase 1.
Dynamic tool results arrive via POST in phase 2. The LLM sees all
results when the chatloop resumes.
</details>
> 🤖 Generated by Coder Agents
115 lines
3.1 KiB
Go
115 lines
3.1 KiB
Go
package chatd
|
|
|
|
import (
|
|
"encoding/json"
|
|
"testing"
|
|
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"cdr.dev/slog/v3/sloggers/slogtest"
|
|
"github.com/coder/coder/v2/codersdk"
|
|
)
|
|
|
|
func TestDynamicToolsFromSDK(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
t.Run("EmptySlice", func(t *testing.T) {
|
|
t.Parallel()
|
|
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
|
|
result := dynamicToolsFromSDK(logger, nil)
|
|
require.Nil(t, result)
|
|
})
|
|
|
|
t.Run("ValidToolWithSchema", func(t *testing.T) {
|
|
t.Parallel()
|
|
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
|
|
tools := []codersdk.DynamicTool{
|
|
{
|
|
Name: "my_tool",
|
|
Description: "A useful tool",
|
|
InputSchema: json.RawMessage(`{"type":"object","properties":{"input":{"type":"string"}},"required":["input"]}`),
|
|
},
|
|
}
|
|
result := dynamicToolsFromSDK(logger, tools)
|
|
require.Len(t, result, 1)
|
|
|
|
info := result[0].Info()
|
|
require.Equal(t, "my_tool", info.Name)
|
|
require.Equal(t, "A useful tool", info.Description)
|
|
require.NotNil(t, info.Parameters)
|
|
require.Contains(t, info.Parameters, "input")
|
|
require.Equal(t, []string{"input"}, info.Required)
|
|
})
|
|
|
|
t.Run("ToolWithoutSchema", func(t *testing.T) {
|
|
t.Parallel()
|
|
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
|
|
tools := []codersdk.DynamicTool{
|
|
{
|
|
Name: "no_schema",
|
|
Description: "Tool with no schema",
|
|
},
|
|
}
|
|
result := dynamicToolsFromSDK(logger, tools)
|
|
require.Len(t, result, 1)
|
|
|
|
info := result[0].Info()
|
|
require.Equal(t, "no_schema", info.Name)
|
|
require.Nil(t, info.Parameters)
|
|
require.Nil(t, info.Required)
|
|
})
|
|
|
|
t.Run("MalformedSchema", func(t *testing.T) {
|
|
t.Parallel()
|
|
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
|
|
tools := []codersdk.DynamicTool{
|
|
{
|
|
Name: "bad_schema",
|
|
Description: "Tool with malformed schema",
|
|
InputSchema: json.RawMessage("not-json"),
|
|
},
|
|
}
|
|
result := dynamicToolsFromSDK(logger, tools)
|
|
require.Len(t, result, 1)
|
|
|
|
info := result[0].Info()
|
|
require.Equal(t, "bad_schema", info.Name)
|
|
require.Nil(t, info.Parameters)
|
|
require.Nil(t, info.Required)
|
|
})
|
|
|
|
t.Run("MultipleTools", func(t *testing.T) {
|
|
t.Parallel()
|
|
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
|
|
tools := []codersdk.DynamicTool{
|
|
{Name: "first", Description: "First tool"},
|
|
{Name: "second", Description: "Second tool"},
|
|
{Name: "third", Description: "Third tool"},
|
|
}
|
|
result := dynamicToolsFromSDK(logger, tools)
|
|
require.Len(t, result, 3)
|
|
require.Equal(t, "first", result[0].Info().Name)
|
|
require.Equal(t, "second", result[1].Info().Name)
|
|
require.Equal(t, "third", result[2].Info().Name)
|
|
})
|
|
|
|
t.Run("SchemaWithoutProperties", func(t *testing.T) {
|
|
t.Parallel()
|
|
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
|
|
tools := []codersdk.DynamicTool{
|
|
{
|
|
Name: "bare_schema",
|
|
Description: "Schema with no properties",
|
|
InputSchema: json.RawMessage(`{"type":"object"}`),
|
|
},
|
|
}
|
|
result := dynamicToolsFromSDK(logger, tools)
|
|
require.Len(t, result, 1)
|
|
|
|
info := result[0].Info()
|
|
require.Equal(t, "bare_schema", info.Name)
|
|
require.Nil(t, info.Parameters)
|
|
require.Nil(t, info.Required)
|
|
})
|
|
}
|