mirror of
https://github.com/coder/coder.git
synced 2026-06-02 20:48:20 +00:00
0bb09935bc
Adds a deployment-wide setting to select the computer-use provider (Anthropic or OpenAI) for AI agents, plus the OpenAI computer-use runner needed to honor that selection. The setting is stored in `site_configs` under `agents_computer_use_provider`, defaults to Anthropic when unset, and is exposed via experimental GET/PUT endpoints under `/api/experimental/chats/config/computer-use-provider`. The chatd computer-use tool now dispatches to either `runAnthropicComputerUse` or `runOpenAIComputerUse` based on the resolved provider, with provider-specific result metadata for OpenAI screenshots. Frontend adds a provider dropdown to the Agents Experiments settings page nested under the virtual desktop toggle, with disabled state handling while virtual desktop is off and skeleton loaders while config queries are in flight. Hugo and Codex review follow-up: - Uses shared provider validation and clearer computer-use constant names. - Removes stale OpenAI pending-safety-checks commentary. - Documents why provider result metadata is needed for OpenAI screenshots. - Keeps the computer-use subagent visible when provider credentials are missing, then returns a clear spawn-time configuration error. - Uses OpenAI's recommended 1600x900 screenshot geometry to preserve the native 16:9 aspect ratio. - Moves OpenAI-specific computer-use helpers into `coderd/x/chatd/chatopenai/computeruse` after rebasing onto the provider package refactor in `main`. - Converts OpenAI pixel scroll deltas to Coder desktop wheel-click amounts. - Preserves OpenAI pointer modifiers with key down/up desktop actions and rejects unsupported non-left double-click buttons explicitly. - Maps OpenAI back/forward side-button clicks to browser navigation key actions. - Defaults omitted OpenAI click buttons to left-click. - Retries mouse release cleanup if the final OpenAI drag release fails. - Keeps computer-use subagent availability messages stable when provider config cannot be loaded, while logging the backend error. - Releases remaining OpenAI modifier keys if a synthetic key-up cleanup action fails. - Updates Storybook interaction stories so provider snapshots show the selected final provider. > Mux updated this PR description on behalf of Mike.
227 lines
4.6 KiB
Go
227 lines
4.6 KiB
Go
package workspacesdk_test
|
|
|
|
import (
|
|
"testing"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
|
|
"github.com/coder/coder/v2/codersdk/workspacesdk"
|
|
)
|
|
|
|
func TestNewDesktopGeometry(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tests := []struct {
|
|
name string
|
|
nativeWidth int
|
|
nativeHeight int
|
|
declaredWidth int
|
|
declaredHeight int
|
|
}{
|
|
{
|
|
name: "1366x768_keeps_native_geometry",
|
|
nativeWidth: 1366,
|
|
nativeHeight: 768,
|
|
declaredWidth: 1366,
|
|
declaredHeight: 768,
|
|
},
|
|
{
|
|
name: "1920x1080_prefers_1280x720",
|
|
nativeWidth: 1920,
|
|
nativeHeight: 1080,
|
|
declaredWidth: 1280,
|
|
declaredHeight: 720,
|
|
},
|
|
{
|
|
name: "1920x1200_prefers_1280x800",
|
|
nativeWidth: 1920,
|
|
nativeHeight: 1200,
|
|
declaredWidth: 1280,
|
|
declaredHeight: 800,
|
|
},
|
|
{
|
|
name: "2048x1536_prefers_1024x768",
|
|
nativeWidth: 2048,
|
|
nativeHeight: 1536,
|
|
declaredWidth: 1024,
|
|
declaredHeight: 768,
|
|
},
|
|
{
|
|
name: "3840x2160_prefers_1280x720",
|
|
nativeWidth: 3840,
|
|
nativeHeight: 2160,
|
|
declaredWidth: 1280,
|
|
declaredHeight: 720,
|
|
},
|
|
{
|
|
name: "1568x1000_prefers_1280x816",
|
|
nativeWidth: 1568,
|
|
nativeHeight: 1000,
|
|
declaredWidth: 1280,
|
|
declaredHeight: 816,
|
|
},
|
|
{
|
|
name: "portrait_falls_back_to_generic_scaling",
|
|
nativeWidth: 1000,
|
|
nativeHeight: 2000,
|
|
declaredWidth: 758,
|
|
declaredHeight: 1516,
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
geometry := workspacesdk.NewDesktopGeometry(
|
|
tt.nativeWidth,
|
|
tt.nativeHeight,
|
|
)
|
|
|
|
assert.Equal(t, tt.nativeWidth, geometry.NativeWidth)
|
|
assert.Equal(t, tt.nativeHeight, geometry.NativeHeight)
|
|
assert.Equal(t, tt.declaredWidth, geometry.DeclaredWidth)
|
|
assert.Equal(t, tt.declaredHeight, geometry.DeclaredHeight)
|
|
assert.LessOrEqual(t, max(geometry.DeclaredWidth, geometry.DeclaredHeight), 1568)
|
|
assert.LessOrEqual(t, geometry.DeclaredWidth*geometry.DeclaredHeight, 1_150_000)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestDefaultDesktopGeometry(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
geometry := workspacesdk.DefaultDesktopGeometry()
|
|
|
|
assert.Equal(t, workspacesdk.DesktopNativeWidth, geometry.NativeWidth)
|
|
assert.Equal(t, workspacesdk.DesktopNativeHeight, geometry.NativeHeight)
|
|
assert.Equal(t, 1280, geometry.DeclaredWidth)
|
|
assert.Equal(t, 720, geometry.DeclaredHeight)
|
|
}
|
|
|
|
// TestDefaultOpenAIComputerUseDesktopGeometry pins the model-facing coordinate
|
|
// system for OpenAI computer use so future geometry changes are intentional.
|
|
func TestDefaultOpenAIComputerUseDesktopGeometry(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
geometry := workspacesdk.DefaultOpenAIComputerUseDesktopGeometry()
|
|
|
|
assert.Equal(t, 1920, geometry.NativeWidth)
|
|
assert.Equal(t, 1080, geometry.NativeHeight)
|
|
assert.Equal(t, 1600, geometry.DeclaredWidth)
|
|
assert.Equal(t, 900, geometry.DeclaredHeight)
|
|
}
|
|
|
|
func TestDesktopGeometryDeclaredPointToNative(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
geometry := workspacesdk.NewDesktopGeometryWithDeclared(1920, 1080, 1280, 720)
|
|
|
|
tests := []struct {
|
|
name string
|
|
x int
|
|
y int
|
|
wantX int
|
|
wantY int
|
|
}{
|
|
{
|
|
name: "origin",
|
|
x: 0,
|
|
y: 0,
|
|
wantX: 0,
|
|
wantY: 0,
|
|
},
|
|
{
|
|
name: "center",
|
|
x: 640,
|
|
y: 360,
|
|
wantX: 960,
|
|
wantY: 540,
|
|
},
|
|
{
|
|
name: "max_coordinate_maps_to_last_native_pixel",
|
|
x: 1279,
|
|
y: 719,
|
|
wantX: 1919,
|
|
wantY: 1079,
|
|
},
|
|
{
|
|
name: "out_of_bounds_values_are_clamped",
|
|
x: 5000,
|
|
y: -5,
|
|
wantX: 1919,
|
|
wantY: 0,
|
|
},
|
|
{
|
|
name: "rounding_applies",
|
|
x: 853,
|
|
y: 402,
|
|
wantX: 1280,
|
|
wantY: 603,
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
gotX, gotY := geometry.DeclaredPointToNative(tt.x, tt.y)
|
|
assert.Equal(t, tt.wantX, gotX)
|
|
assert.Equal(t, tt.wantY, gotY)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestDesktopGeometryNativePointToDeclared(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
geometry := workspacesdk.NewDesktopGeometryWithDeclared(1920, 1080, 1366, 768)
|
|
|
|
tests := []struct {
|
|
name string
|
|
x int
|
|
y int
|
|
wantX int
|
|
wantY int
|
|
}{
|
|
{
|
|
name: "origin",
|
|
x: 0,
|
|
y: 0,
|
|
wantX: 0,
|
|
wantY: 0,
|
|
},
|
|
{
|
|
name: "center",
|
|
x: 960,
|
|
y: 540,
|
|
wantX: 683,
|
|
wantY: 384,
|
|
},
|
|
{
|
|
name: "bottom_right_maps_to_last_pixel",
|
|
x: 1919,
|
|
y: 1079,
|
|
wantX: 1365,
|
|
wantY: 767,
|
|
},
|
|
{
|
|
name: "out_of_bounds_values_are_clamped",
|
|
x: -10,
|
|
y: 5000,
|
|
wantX: 0,
|
|
wantY: 767,
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
gotX, gotY := geometry.NativePointToDeclared(tt.x, tt.y)
|
|
assert.Equal(t, tt.wantX, gotX)
|
|
assert.Equal(t, tt.wantY, gotY)
|
|
})
|
|
}
|
|
}
|