toki
0110ca1af8
OpenCode의 일반 Chat 요청을 GPT provider가 거부한 뒤 재시도 가능한 오류로 왜곡해 벤치가 장시간 정체됐다. 선택된 protocol profile에 맞춰 출력 토큰 필드를 정규화하고 upstream 400을 비재시도 validation 오류로 유지한다.
535 lines
25 KiB
Go
535 lines
25 KiB
Go
package openai
|
|
|
|
import (
|
|
"encoding/json"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
|
|
"iop/packages/go/config"
|
|
)
|
|
|
|
func TestWorkspaceToolBindingContract(t *testing.T) {
|
|
structured := workspaceAlternative("structured", "write_file", false, true)
|
|
command := workspaceAlternative("command", "run_workspace", true, false)
|
|
openAITools := []any{openAIChatTool("write_file", structuredSchema()), unrelatedTool()}
|
|
anthropicTools := []any{anthropicWorkspaceTool("write_file", structuredSchema()), unrelatedTool()}
|
|
|
|
t.Run("normalizes actual OpenAI and Anthropic definitions", func(t *testing.T) {
|
|
openAIBinding, err := compileWorkspaceBinding([]config.ExecutionWorkspaceToolAlternative{structured}, openAITools)
|
|
if err != nil {
|
|
t.Fatalf("compile OpenAI tool: %v", err)
|
|
}
|
|
anthropicBinding, err := compileWorkspaceBinding([]config.ExecutionWorkspaceToolAlternative{structured}, anthropicTools)
|
|
if err != nil {
|
|
t.Fatalf("compile Anthropic tool: %v", err)
|
|
}
|
|
if openAIBinding.alternativeName != "structured" || anthropicBinding.alternativeName != "structured" {
|
|
t.Fatalf("unexpected selected alternatives: %q, %q", openAIBinding.alternativeName, anthropicBinding.alternativeName)
|
|
}
|
|
if openAIBinding.fingerprint != anthropicBinding.fingerprint {
|
|
t.Fatalf("normalized endpoint shapes must fingerprint identically: %s != %s", openAIBinding.fingerprint, anthropicBinding.fingerprint)
|
|
}
|
|
})
|
|
|
|
t.Run("normalizes native decoded Anthropic tool", func(t *testing.T) {
|
|
rawSchema, err := json.Marshal(structuredSchema())
|
|
if err != nil {
|
|
t.Fatalf("marshal native schema: %v", err)
|
|
}
|
|
nativeBinding, err := compileWorkspaceBinding([]config.ExecutionWorkspaceToolAlternative{structured}, []anthropicTool{
|
|
anthropicTool{Name: "write_file", Description: "workspace tool", InputSchema: rawSchema},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("compile native Anthropic tool: %v", err)
|
|
}
|
|
openAIBinding := mustBinding(t, structured, []any{openAIChatTool("write_file", structuredSchema())})
|
|
if nativeBinding.fingerprint != openAIBinding.fingerprint {
|
|
t.Fatalf("native Anthropic fingerprint = %s, want %s", nativeBinding.fingerprint, openAIBinding.fingerprint)
|
|
}
|
|
})
|
|
|
|
t.Run("rejects unsupported command placeholders and missing write content", func(t *testing.T) {
|
|
for name, argv := range map[string][]any{
|
|
"missing content": {"write", "{path}"},
|
|
"duplicate content": {"write", "{path}", "{content}", "{content}"},
|
|
"embedded placeholder": {"write", "--path={path}", "{content}"},
|
|
"unknown placeholder": {"write", "{path}", "{unsupported}", "{content}"},
|
|
"missing path placeholder": {"write", "{content}"},
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
invalid := workspaceAlternative("invalid-command", "run_workspace", true, true)
|
|
invalid.Operations["write"] = config.ExecutionWorkspaceOperation{
|
|
ToolName: "run_workspace", SchemaMatcher: map[string]any{"type": "object"},
|
|
ArgumentMap: map[string]any{"path": "path", "content": "content", "command": "command", "argv": argv},
|
|
ResultMatcher: successMatcher(), CreatesParents: true,
|
|
}
|
|
if _, err := compileWorkspaceBinding([]config.ExecutionWorkspaceToolAlternative{invalid}, []any{openAIChatTool("run_workspace", commandSchema())}); err == nil {
|
|
t.Fatal("invalid command template unexpectedly compiled")
|
|
}
|
|
})
|
|
}
|
|
})
|
|
|
|
t.Run("uses configured order and rejects name heuristics", func(t *testing.T) {
|
|
fallback, err := compileWorkspaceBinding([]config.ExecutionWorkspaceToolAlternative{command, structured}, openAITools)
|
|
if err != nil {
|
|
t.Fatalf("compile fallback: %v", err)
|
|
}
|
|
if fallback.alternativeName != "structured" {
|
|
t.Fatalf("expected configured second alternative, got %q", fallback.alternativeName)
|
|
}
|
|
if _, err := compileWorkspaceBinding([]config.ExecutionWorkspaceToolAlternative{structured}, []any{unrelatedTool()}); err == nil {
|
|
t.Fatal("unrelated get_weather tool must not bind by lexical role inference")
|
|
}
|
|
})
|
|
|
|
t.Run("rejects missing tools, schema mismatch, and incomplete parent contract", func(t *testing.T) {
|
|
missing := workspaceAlternative("missing", "absent_tool", true, true)
|
|
if _, err := compileWorkspaceBinding([]config.ExecutionWorkspaceToolAlternative{missing}, openAITools); err == nil {
|
|
t.Fatal("missing configured tool unexpectedly bound")
|
|
}
|
|
mismatched := workspaceAlternative("mismatched", "write_file", true, true)
|
|
mismatched.Operations["write"] = config.ExecutionWorkspaceOperation{
|
|
ToolName: "write_file",
|
|
SchemaMatcher: map[string]any{"type": "object", "properties": map[string]any{"bytes": map[string]any{"type": "number"}}},
|
|
ArgumentMap: map[string]any{"path": "path", "content": "content"},
|
|
ResultMatcher: successMatcher(),
|
|
CreatesParents: true,
|
|
}
|
|
if _, err := compileWorkspaceBinding([]config.ExecutionWorkspaceToolAlternative{mismatched}, openAITools); err == nil {
|
|
t.Fatal("schema-mismatched configured tool unexpectedly bound")
|
|
}
|
|
noPrepare := workspaceAlternative("no-prepare", "write_file", false, false)
|
|
delete(noPrepare.Operations, "prepare")
|
|
if _, err := compileWorkspaceBinding([]config.ExecutionWorkspaceToolAlternative{noPrepare}, openAITools); err == nil {
|
|
t.Fatal("write without parent capability or prepare unexpectedly bound")
|
|
}
|
|
})
|
|
|
|
t.Run("copies full contract into a stable fingerprint", func(t *testing.T) {
|
|
binding, err := compileWorkspaceBinding([]config.ExecutionWorkspaceToolAlternative{structured}, openAITools)
|
|
if err != nil {
|
|
t.Fatalf("compile binding: %v", err)
|
|
}
|
|
before := binding.fingerprint
|
|
structured.Operations["write"] = config.ExecutionWorkspaceOperation{ToolName: "changed"}
|
|
if binding.fingerprint != before || binding.operation(opKindWrite).toolName != "write_file" {
|
|
t.Fatal("binding retained mutable config state")
|
|
}
|
|
openAITools[0].(map[string]any)["function"].(map[string]any)["parameters"].(map[string]any)["properties"].(map[string]any)["path"] = map[string]any{"type": "number"}
|
|
if binding.operation(opKindWrite).normalizedSchema.properties["path"].(map[string]any)["type"] != "string" {
|
|
t.Fatal("binding retained mutable request tool schema")
|
|
}
|
|
withDifferentReceipt := workspaceAlternative("structured", "write_file", false, true)
|
|
withDifferentReceipt.Operations["write"] = config.ExecutionWorkspaceOperation{
|
|
ToolName: "write_file", SchemaMatcher: map[string]any{"type": "object"},
|
|
ArgumentMap: map[string]any{"path": "path", "content": "content"},
|
|
ResultMatcher: map[string]any{"status": "success", "result": map[string]any{"saved": true}}, CreatesParents: true,
|
|
}
|
|
changed, err := compileWorkspaceBinding([]config.ExecutionWorkspaceToolAlternative{withDifferentReceipt}, openAITools)
|
|
if err != nil {
|
|
t.Fatalf("compile changed receipt binding: %v", err)
|
|
}
|
|
if before == changed.fingerprint {
|
|
t.Fatal("fingerprint omitted configured result contract")
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestWorkspaceContainmentGuard(t *testing.T) {
|
|
writeCall := func(path string) normalizedToolCall {
|
|
return normalizedToolCall{ID: "guard-call", Name: "write_file", Arguments: map[string]any{"path": path, "content": "x"}}
|
|
}
|
|
|
|
t.Run("parent-capable write admits fresh nested parents", func(t *testing.T) {
|
|
binding := mustBinding(t, workspaceAlternative("parents", "write_file", false, true), []any{openAIChatTool("write_file", structuredSchema())})
|
|
payload, err := encodeWorkspaceCall(binding, opKindWrite, writeCall(".iop/job/request-1/plan.md"))
|
|
if err != nil {
|
|
t.Fatalf("encode workspace call: %v", err)
|
|
}
|
|
if err := evaluateContainmentGuard(t.TempDir(), payload.containmentGuard); err != nil {
|
|
t.Fatalf("parent-capable guard rejected a fresh nested path: %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("write without parent capability requires immediate parent", func(t *testing.T) {
|
|
binding := mustBinding(t, workspaceAlternative("prepare-required", "write_file", false, false), []any{openAIChatTool("write_file", structuredSchema())})
|
|
payload, err := encodeWorkspaceCall(binding, opKindWrite, writeCall(".iop/job/request-2/plan.md"))
|
|
if err != nil {
|
|
t.Fatalf("encode workspace call: %v", err)
|
|
}
|
|
if err := evaluateContainmentGuard(t.TempDir(), payload.containmentGuard); err == nil {
|
|
t.Fatal("non-parent-capable guard accepted a missing immediate parent")
|
|
}
|
|
})
|
|
|
|
t.Run("root workspace admits existing relative target", func(t *testing.T) {
|
|
binding := mustBinding(t, workspaceAlternative("parents", "write_file", false, true), []any{openAIChatTool("write_file", structuredSchema())})
|
|
payload, err := encodeWorkspaceCall(binding, opKindWrite, writeCall("tmp"))
|
|
if err != nil {
|
|
t.Fatalf("encode workspace call: %v", err)
|
|
}
|
|
if err := evaluateContainmentGuard("/", payload.containmentGuard); err != nil {
|
|
t.Fatalf("root workspace guard rejected existing relative target: %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("root workspace admits non-parent-capable target with existing immediate parent", func(t *testing.T) {
|
|
binding := mustBinding(t, workspaceAlternative("prepare-required", "write_file", false, false), []any{openAIChatTool("write_file", structuredSchema())})
|
|
payload, err := encodeWorkspaceCall(binding, opKindWrite, writeCall("tmp/iop_root_test_file.txt"))
|
|
if err != nil {
|
|
t.Fatalf("encode workspace call: %v", err)
|
|
}
|
|
if err := evaluateContainmentGuard("/", payload.containmentGuard); err != nil {
|
|
t.Fatalf("root workspace guard rejected non-parent-capable target with existing parent: %v", err)
|
|
}
|
|
})
|
|
|
|
for name, setup := range map[string]func(t *testing.T, root, outside string){
|
|
"final symlink": func(t *testing.T, root, outside string) {
|
|
t.Helper()
|
|
if err := os.MkdirAll(filepath.Join(root, ".iop", "job", "request-3"), 0o755); err != nil {
|
|
t.Fatalf("create workspace path: %v", err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(outside, "target.md"), []byte("outside"), 0o600); err != nil {
|
|
t.Fatalf("create outside target: %v", err)
|
|
}
|
|
if err := os.Symlink(filepath.Join(outside, "target.md"), filepath.Join(root, ".iop", "job", "request-3", "plan.md")); err != nil {
|
|
t.Fatalf("create final symlink: %v", err)
|
|
}
|
|
},
|
|
"ancestor symlink": func(t *testing.T, root, outside string) {
|
|
t.Helper()
|
|
if err := os.Symlink(outside, filepath.Join(root, ".iop")); err != nil {
|
|
t.Fatalf("create ancestor symlink: %v", err)
|
|
}
|
|
},
|
|
} {
|
|
t.Run(name+" escapes workspace", func(t *testing.T) {
|
|
root := t.TempDir()
|
|
outside := t.TempDir()
|
|
setup(t, root, outside)
|
|
binding := mustBinding(t, workspaceAlternative("parents", "write_file", false, true), []any{openAIChatTool("write_file", structuredSchema())})
|
|
payload, err := encodeWorkspaceCall(binding, opKindWrite, writeCall(".iop/job/request-3/plan.md"))
|
|
if err != nil {
|
|
t.Fatalf("encode workspace call: %v", err)
|
|
}
|
|
if err := evaluateContainmentGuard(root, payload.containmentGuard); err == nil {
|
|
t.Fatal("symlink escape was accepted")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// evaluateContainmentGuard executes only the generated guard against a
|
|
// temporary workspace fixture. It never invokes a caller workspace command.
|
|
func evaluateContainmentGuard(root, guard string) error {
|
|
cmd := exec.Command("sh", "-c", guard)
|
|
cmd.Env = append(os.Environ(), "IOP_WORKSPACE_CWD="+root)
|
|
return cmd.Run()
|
|
}
|
|
|
|
func TestWorkspaceCommandEncodingAndGuards(t *testing.T) {
|
|
structured := workspaceAlternative("structured", "write_file", false, true)
|
|
command := workspaceAlternative("command", "run_workspace", true, false)
|
|
|
|
t.Run("structured payload preserves typed values and identities", func(t *testing.T) {
|
|
binding := mustBinding(t, structured, []any{openAIChatTool("write_file", structuredSchema())})
|
|
content := map[string]any{"lines": []any{"first", 2, true}, "nested": map[string]any{"raw": "' $HOME"}}
|
|
payload, err := encodeWorkspaceCall(binding, opKindWrite, normalizedToolCall{
|
|
ID: "public-1", ProviderCallID: "provider-1", Name: "write_file",
|
|
Arguments: map[string]any{"path": ".iop/job/r1/plan.md", "content": content, "ignored": "must not pass"},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("encode structured call: %v", err)
|
|
}
|
|
if payload.publicCallID != "public-1" || payload.providerCallID != "provider-1" {
|
|
t.Fatalf("call identities lost: %#v", payload)
|
|
}
|
|
if !reflect.DeepEqual(payload.structuredArgs["content"], content) {
|
|
t.Fatalf("structured content changed: %#v", payload.structuredArgs["content"])
|
|
}
|
|
if _, present := payload.structuredArgs["ignored"]; present {
|
|
t.Fatal("unmapped structured argument escaped the configured contract")
|
|
}
|
|
})
|
|
|
|
t.Run("command mapping has fixed positions and shell-safe output", func(t *testing.T) {
|
|
binding := mustBinding(t, command, []any{openAIChatTool("run_workspace", commandSchema())})
|
|
call := normalizedToolCall{ID: "public-2", Name: "run_workspace", Arguments: map[string]any{"path": ".iop/job/r2/review.md", "content": "hello 'world'"}}
|
|
first, err := encodeWorkspaceCall(binding, opKindWrite, call)
|
|
if err != nil {
|
|
t.Fatalf("encode command call: %v", err)
|
|
}
|
|
second, err := encodeWorkspaceCall(binding, opKindWrite, call)
|
|
if err != nil || first.commandString != second.commandString {
|
|
t.Fatalf("command encoding is not deterministic: %q / %q (%v)", first.commandString, second.commandString, err)
|
|
}
|
|
wantArgv := []string{"write", ".iop/job/r2/review.md", "hello 'world'"}
|
|
if !reflect.DeepEqual(first.commandArgv, wantArgv) {
|
|
t.Fatalf("command argv = %#v, want %#v", first.commandArgv, wantArgv)
|
|
}
|
|
if !strings.Contains(first.commandString, "'\\''") {
|
|
t.Fatalf("command does not safely quote apostrophe: %q", first.commandString)
|
|
}
|
|
if !strings.HasPrefix(first.commandString, first.containmentGuard+" && ") {
|
|
t.Fatalf("caller command does not execute its containment guard: %q", first.commandString)
|
|
}
|
|
})
|
|
|
|
t.Run("no-escape guard is concrete and unsafe paths fail before caller execution", func(t *testing.T) {
|
|
binding := mustBinding(t, structured, []any{openAIChatTool("write_file", structuredSchema())})
|
|
for _, path := range []string{"../escape", "/etc/passwd", ".iop/job/r3/../../escape", "bad;rm"} {
|
|
if _, err := encodeWorkspaceCall(binding, opKindWrite, normalizedToolCall{ID: "public-3", Name: "write_file", Arguments: map[string]any{"path": path, "content": "x"}}); err == nil {
|
|
t.Fatalf("unsafe path %q was accepted", path)
|
|
}
|
|
}
|
|
payload, err := encodeWorkspaceCall(binding, opKindWrite, normalizedToolCall{ID: "public-4", Name: "write_file", Arguments: map[string]any{"path": ".iop/job/r4/plan.md", "content": "x"}})
|
|
if err != nil {
|
|
t.Fatalf("encode safe path: %v", err)
|
|
}
|
|
for _, required := range []string{"IOP_WS_ROOT=", "IOP_WORKSPACE_CWD", `realpath "`, "IOP_WS_CANDIDATE=", "path escapes workspace root"} {
|
|
if !strings.Contains(payload.containmentGuard, required) {
|
|
t.Fatalf("guard missing %q: %s", required, payload.containmentGuard)
|
|
}
|
|
}
|
|
if strings.Contains(payload.containmentGuard, "realpath -e") {
|
|
t.Fatalf("guard contains GNU-only realpath option: %s", payload.containmentGuard)
|
|
}
|
|
if !strings.Contains(payload.containmentGuard, `IOP_WS_CANDIDATE="$IOP_WS_ROOT/.iop/job/r4/plan.md"`) {
|
|
t.Fatalf("guard does not retain exact candidate path: %s", payload.containmentGuard)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestWorkspaceBindingReceipts(t *testing.T) {
|
|
binding := mustBinding(t, workspaceAlternative("structured", "write_file", false, true), []any{openAIChatTool("write_file", structuredSchema())})
|
|
payload, err := encodeWorkspaceCall(binding, opKindWrite, normalizedToolCall{
|
|
ID: "public-receipt", ProviderCallID: "provider-receipt", Name: "write_file",
|
|
Arguments: map[string]any{"path": ".iop/job/r5/plan.md", "content": "plan"},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("encode payload: %v", err)
|
|
}
|
|
|
|
t.Run("configured exact receipt accepts either issued identity", func(t *testing.T) {
|
|
for _, id := range []string{"public-receipt", "provider-receipt"} {
|
|
receipt := matchResultReceipt(binding, payload, workspaceResult{callID: id, status: "success", body: []byte(`{"written":true}`)})
|
|
if !receipt.matched || receipt.fingerprint != binding.fingerprint || receipt.path != payload.safePath {
|
|
t.Fatalf("valid receipt did not correlate: %#v", receipt)
|
|
}
|
|
}
|
|
})
|
|
|
|
for name, result := range map[string]workspaceResult{
|
|
"opaque": {callID: "public-receipt", status: "success"},
|
|
"error": {callID: "public-receipt", status: "error", body: []byte(`{"written":true}`)},
|
|
"embedded error": {callID: "public-receipt", status: "success", body: []byte(`{"written":true,"error":{"message":"nope"}}`)},
|
|
"trailing JSON": {callID: "public-receipt", status: "success", body: []byte(`{"written":true} {"error":"nope"}`)},
|
|
"wrong id": {callID: "other", status: "success", body: []byte(`{"written":true}`)},
|
|
"wrong body": {callID: "public-receipt", status: "success", body: []byte(`{"written":false}`)},
|
|
"arbitrary JSON": {callID: "public-receipt", status: "success", body: []byte(`{"anything":"else"}`)},
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
if receipt := matchResultReceipt(binding, payload, result); receipt.matched {
|
|
t.Fatalf("mismatched receipt was accepted: %#v", receipt)
|
|
}
|
|
})
|
|
}
|
|
|
|
t.Run("rejects every issued payload mutation", func(t *testing.T) {
|
|
mutations := map[string]func(*workspaceEncodedPayload){
|
|
"operation": func(p *workspaceEncodedPayload) { p.operation = opKindPrepare },
|
|
"path": func(p *workspaceEncodedPayload) { p.safePath = ".iop/job/r5/review.md" },
|
|
"arguments": func(p *workspaceEncodedPayload) { p.structuredArgs["content"] = "mutated" },
|
|
"guard": func(p *workspaceEncodedPayload) { p.containmentGuard = "mutated" },
|
|
}
|
|
for name, mutate := range mutations {
|
|
t.Run(name, func(t *testing.T) {
|
|
copy := cloneWorkspacePayload(payload)
|
|
mutate(copy)
|
|
if receipt := matchResultReceipt(binding, copy, workspaceResult{callID: "public-receipt", status: "success", body: []byte(`{"written":true}`)}); receipt.matched {
|
|
t.Fatalf("mutated payload unexpectedly matched: %#v", receipt)
|
|
}
|
|
})
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestWorkspaceResultExactness(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
result workspaceResult
|
|
wantExact bool
|
|
}{
|
|
{
|
|
name: "empty success body is opaque",
|
|
result: workspaceResult{status: "success", body: nil},
|
|
wantExact: false,
|
|
},
|
|
{
|
|
name: "whitespace success body is opaque",
|
|
result: workspaceResult{status: "success", body: []byte(" \n\t ")},
|
|
wantExact: false,
|
|
},
|
|
{
|
|
name: "empty explicit error is exact",
|
|
result: workspaceResult{status: "error", body: nil},
|
|
wantExact: true,
|
|
},
|
|
{
|
|
name: "non-empty matcher failure is exact",
|
|
result: workspaceResult{status: "success", body: []byte(`{"written":false}`)},
|
|
wantExact: true,
|
|
},
|
|
{
|
|
name: "malformed json body is opaque",
|
|
result: workspaceResult{status: "success", body: []byte(`not-json`)},
|
|
wantExact: false,
|
|
},
|
|
{
|
|
name: "trailing json body is opaque",
|
|
result: workspaceResult{status: "success", body: []byte(`{"written":true} {"error":"nope"}`)},
|
|
wantExact: false,
|
|
},
|
|
{
|
|
name: "valid success receipt body is exact",
|
|
result: workspaceResult{status: "success", body: []byte(`{"written":true}`)},
|
|
wantExact: true,
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
if got := workspaceResultIsExact(tt.result); got != tt.wantExact {
|
|
t.Fatalf("workspaceResultIsExact() = %v, want %v", got, tt.wantExact)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestWorkspaceOperationMatrix(t *testing.T) {
|
|
nativeSchema, err := json.Marshal(structuredSchema())
|
|
if err != nil {
|
|
t.Fatalf("marshal native schema: %v", err)
|
|
}
|
|
for _, tc := range []struct {
|
|
name string
|
|
command bool
|
|
tools any
|
|
}{
|
|
{name: "structured", tools: []any{openAIChatTool("workspace", structuredSchema())}},
|
|
{name: "command", command: true, tools: []any{openAIChatTool("workspace", commandSchema())}},
|
|
{name: "native Anthropic", tools: []anthropicTool{{Name: "workspace", Description: "workspace tool", InputSchema: nativeSchema}}},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
binding := mustBinding(t, fullWorkspaceAlternative(tc.name, "workspace", tc.command), tc.tools)
|
|
for _, operation := range canonicalOperationOrder {
|
|
args := map[string]any{"path": ".iop/job/r6/" + string(operation) + ".md"}
|
|
if operation == opKindWrite {
|
|
args["content"] = "content"
|
|
}
|
|
payload, err := encodeWorkspaceCall(binding, operation, normalizedToolCall{ID: "call-" + string(operation), Name: "workspace", Arguments: args})
|
|
if err != nil {
|
|
t.Fatalf("encode %s: %v", operation, err)
|
|
}
|
|
if payload.operation != operation || payload.correlationDigest == "" {
|
|
t.Fatalf("payload for %s is not sealed: %#v", operation, payload)
|
|
}
|
|
if receipt := matchResultReceipt(binding, payload, workspaceResult{callID: payload.publicCallID, status: "success", body: []byte(`{"written":true}`)}); !receipt.matched {
|
|
t.Fatalf("valid %s receipt did not match: %#v", operation, receipt)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
t.Run("ordered complete alternatives and missing tools", func(t *testing.T) {
|
|
first := fullWorkspaceAlternative("first", "first_workspace", false)
|
|
second := fullWorkspaceAlternative("second", "second_workspace", false)
|
|
tools := []any{openAIChatTool("second_workspace", structuredSchema()), openAIChatTool("first_workspace", structuredSchema())}
|
|
binding, err := compileWorkspaceBinding([]config.ExecutionWorkspaceToolAlternative{second, first}, tools)
|
|
if err != nil || binding.alternativeName != "second" {
|
|
t.Fatalf("configured first complete alternative was not selected: binding=%#v err=%v", binding, err)
|
|
}
|
|
if _, err := compileWorkspaceBinding([]config.ExecutionWorkspaceToolAlternative{first}, []any{openAIChatTool("first_workspace", structuredSchema()), unrelatedTool()}); err != nil {
|
|
t.Fatalf("extra unrelated tool must not invalidate a complete alternative: %v", err)
|
|
}
|
|
if _, err := compileWorkspaceBinding([]config.ExecutionWorkspaceToolAlternative{first}, []any{unrelatedTool()}); err == nil {
|
|
t.Fatal("missing complete operation tool unexpectedly bound")
|
|
}
|
|
})
|
|
}
|
|
|
|
func mustBinding(t *testing.T, alternative config.ExecutionWorkspaceToolAlternative, tools any) *workspaceBinding {
|
|
t.Helper()
|
|
binding, err := compileWorkspaceBinding([]config.ExecutionWorkspaceToolAlternative{alternative}, tools)
|
|
if err != nil {
|
|
t.Fatalf("compile binding: %v", err)
|
|
}
|
|
return binding
|
|
}
|
|
|
|
func workspaceAlternative(name, toolName string, command, createsParents bool) config.ExecutionWorkspaceToolAlternative {
|
|
argumentMap := map[string]any{"path": "path", "content": "content"}
|
|
prepareArgumentMap := map[string]any{"path": "path"}
|
|
if command {
|
|
argumentMap = map[string]any{"path": "path", "content": "content", "command": "command", "argv": []any{"write", "{path}", "{content}"}}
|
|
prepareArgumentMap = map[string]any{"path": "path", "command": "command", "argv": []any{"mkdir", "{path}"}}
|
|
}
|
|
return config.ExecutionWorkspaceToolAlternative{
|
|
Name: name,
|
|
Operations: map[string]config.ExecutionWorkspaceOperation{
|
|
"prepare": {ToolName: toolName, SchemaMatcher: map[string]any{"type": "object"}, ArgumentMap: prepareArgumentMap, ResultMatcher: successMatcher(), CreatesParents: true},
|
|
"write": {ToolName: toolName, SchemaMatcher: map[string]any{"type": "object"}, ArgumentMap: argumentMap, ResultMatcher: successMatcher(), CreatesParents: createsParents},
|
|
},
|
|
}
|
|
}
|
|
|
|
func fullWorkspaceAlternative(name, toolName string, command bool) config.ExecutionWorkspaceToolAlternative {
|
|
alternative := workspaceAlternative(name, toolName, command, true)
|
|
for _, operation := range []workspaceOperationKind{opKindRead, opKindDelete} {
|
|
argumentMap := map[string]any{"path": "path"}
|
|
if command {
|
|
argumentMap = map[string]any{"path": "path", "command": "command", "argv": []any{string(operation), "{path}"}}
|
|
}
|
|
alternative.Operations[string(operation)] = config.ExecutionWorkspaceOperation{
|
|
ToolName: toolName, SchemaMatcher: map[string]any{"type": "object"}, ArgumentMap: argumentMap, ResultMatcher: successMatcher(), CreatesParents: true,
|
|
}
|
|
}
|
|
return alternative
|
|
}
|
|
|
|
func cloneWorkspacePayload(payload *workspaceEncodedPayload) *workspaceEncodedPayload {
|
|
copy := *payload
|
|
copy.structuredArgs = cloneAnyMap(payload.structuredArgs)
|
|
copy.commandArgv = append([]string(nil), payload.commandArgv...)
|
|
return ©
|
|
}
|
|
|
|
func successMatcher() map[string]any {
|
|
return map[string]any{"status": "success", "result": map[string]any{"written": true}}
|
|
}
|
|
|
|
func structuredSchema() map[string]any {
|
|
return map[string]any{"type": "object", "properties": map[string]any{"path": map[string]any{"type": "string"}, "content": map[string]any{}}, "required": []any{"path", "content"}}
|
|
}
|
|
|
|
func commandSchema() map[string]any {
|
|
return map[string]any{"type": "object", "properties": map[string]any{"command": map[string]any{"type": "string"}}}
|
|
}
|
|
|
|
func openAIChatTool(name string, schema map[string]any) map[string]any {
|
|
return map[string]any{"type": "function", "function": map[string]any{"name": name, "description": "workspace tool", "parameters": schema}}
|
|
}
|
|
|
|
func anthropicWorkspaceTool(name string, schema map[string]any) map[string]any {
|
|
return map[string]any{"name": name, "description": "workspace tool", "input_schema": schema}
|
|
}
|
|
|
|
func unrelatedTool() map[string]any {
|
|
return openAIChatTool("get_weather", map[string]any{"type": "object", "properties": map[string]any{"city": map[string]any{"type": "string"}}})
|
|
}
|