문서 완성도 판정을 여러 런타임 계층에서 반복해 부분 Review가 Reviewer에 도달하지 못하던 실패를 없애기 위해 의미 검증 책임을 Reviewer로 모은다.
963 lines
47 KiB
Go
963 lines
47 KiB
Go
package openai
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
|
|
edgeservice "iop/apps/edge/internal/service"
|
|
"iop/packages/go/config"
|
|
"iop/packages/go/singlerequesttemplate"
|
|
)
|
|
|
|
const scriptedAbsoluteWorkspaceTask = "system: Working directory: /workspace\nuser: immutable user task"
|
|
|
|
func TestHotPathLightLocalTransition(t *testing.T) {
|
|
for _, endpoint := range []string{"openai", "anthropic"} {
|
|
endpoint := endpoint
|
|
t.Run(endpoint, func(t *testing.T) {
|
|
fixture := newScriptedLightFixture(t, endpoint, false)
|
|
final := fixture.run()
|
|
if final.Code != http.StatusOK || !strings.Contains(final.Body.String(), "review-resolution-visible") {
|
|
t.Fatalf("final response: status=%d body=%s", final.Code, final.Body.String())
|
|
}
|
|
history, _ := json.Marshal(fixture.history)
|
|
if !strings.Contains(string(history), "worker-handoff-visible") {
|
|
t.Fatalf("worker-authored review handoff was not issued: history=%s", history)
|
|
}
|
|
fixture.assertCleanupCommitted(7)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestHotPathStageCanonicalReadMapsToCallerCommandTool(t *testing.T) {
|
|
binding := mustBinding(t, fullWorkspaceAlternative("command", "bash", true), []any{openAIChatTool("bash", commandSchema())})
|
|
record := &hotPathLightRecord{
|
|
requestID: "req_stage_map", phase: hotPathPhaseLocalActive, binding: binding,
|
|
}
|
|
coordinator := newLogicalRequestCoordinator(logicalRequestCoordinatorOptions{IDSource: func() (string, error) { return "call_public", nil }})
|
|
paths := newReservedPaths(record.requestID)
|
|
mapped, pending, err := mapHotPathStageCalls(record, normalizedStageOutput{ToolCalls: []normalizedToolCall{{
|
|
ID: "provider_read", Name: "read", Arguments: map[string]any{"filePath": paths.PlanPath},
|
|
}}}, hotPathPendingLocalTools, coordinator, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(mapped.ToolCalls) != 1 || mapped.ToolCalls[0].Name != "bash" || mapped.ToolCalls[0].Arguments["command"] == nil {
|
|
t.Fatalf("mapped caller command=%+v", mapped.ToolCalls)
|
|
}
|
|
if pending[mapped.ToolCalls[0].ID].payload == nil {
|
|
t.Fatalf("reserved read pending payload=%+v", pending)
|
|
}
|
|
}
|
|
|
|
func TestHotPathStageOrdinaryWorkspacePathPassesThrough(t *testing.T) {
|
|
callerTools := []any{openAIChatTool("bash", commandSchema())}
|
|
record := &hotPathLightRecord{
|
|
requestID: "req_stage_ordinary", phase: hotPathPhaseLocalActive, tools: callerTools,
|
|
}
|
|
coordinator := newLogicalRequestCoordinator(logicalRequestCoordinatorOptions{IDSource: func() (string, error) { return "call_public", nil }})
|
|
call := normalizedToolCall{
|
|
ID: "provider_work", Name: "bash",
|
|
Arguments: map[string]any{"command": "inspect .iop/job/ordinary-workspace-path and continue"},
|
|
}
|
|
mapped, pending, err := mapHotPathStageCalls(record, normalizedStageOutput{ToolCalls: []normalizedToolCall{call}}, hotPathPendingLocalTools, coordinator, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(mapped.ToolCalls) != 1 || mapped.ToolCalls[0].Name != "bash" {
|
|
t.Fatalf("ordinary caller tool changed: %+v", mapped.ToolCalls)
|
|
}
|
|
if pending[mapped.ToolCalls[0].ID].payload != nil {
|
|
t.Fatalf("ordinary workspace path became an artifact operation: %+v", pending)
|
|
}
|
|
}
|
|
|
|
func TestWorkerReviewReadIsNotClassifiedAsHandoffWrite(t *testing.T) {
|
|
binding := mustBinding(t, fullWorkspaceAlternative("workspace", "workspace", false), []any{openAIChatTool("workspace", structuredSchema())})
|
|
paths := newReservedPaths("req_review_read")
|
|
read := normalizedToolCall{Name: "workspace", Arguments: map[string]any{"path": paths.ReviewPath}}
|
|
if isWorkerReviewHandoffCall(binding, paths, read) {
|
|
t.Fatal("review read was classified as a handoff write")
|
|
}
|
|
write := normalizedToolCall{Name: "workspace", Arguments: map[string]any{"path": paths.ReviewPath, "content": testCompletedReviewText()}}
|
|
if !isWorkerReviewHandoffCall(binding, paths, write) {
|
|
t.Fatal("review write was not classified as a handoff write")
|
|
}
|
|
}
|
|
|
|
func TestWorkerProviderReceivesOnlyCallerTools(t *testing.T) {
|
|
callerTools := []any{openAIChatTool("run_command", commandSchema())}
|
|
tools := hotPathStageProviderTools(hotPathDispatchSnapshot{Phase: hotPathPhaseLocalActive, Tools: callerTools})
|
|
schemas, err := normalizeToolSchemas(tools)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if schemas["run_command"] == nil || schemas["iop_write_worker_review"] != nil || len(schemas) != 1 {
|
|
t.Fatalf("worker provider tools=%+v", schemas)
|
|
}
|
|
}
|
|
|
|
func TestInitialReviewRequiresToolUseBeforeTerminal(t *testing.T) {
|
|
initial := hotPathDispatchSnapshot{Phase: hotPathPhaseReviewActive}
|
|
if got := hotPathChatStageToolChoice(initial); got != "required" {
|
|
t.Fatalf("initial review tool choice = %q, want required", got)
|
|
}
|
|
retry := initial
|
|
retry.Transcript = []hotPathStageExchange{{}}
|
|
if got := hotPathChatStageToolChoice(retry); got != "auto" {
|
|
t.Fatalf("continued review tool choice = %q, want auto", got)
|
|
}
|
|
if got := hotPathChatStageToolChoice(hotPathDispatchSnapshot{Phase: hotPathPhaseLocalActive}); got != "auto" {
|
|
t.Fatalf("worker tool choice = %q, want auto", got)
|
|
}
|
|
}
|
|
|
|
func assertLightFailureAfterCleanup(t *testing.T, fixture *scriptedLightFixture, message string) {
|
|
t.Helper()
|
|
cleanup := fixture.request()
|
|
if cleanup.Code != http.StatusOK || !strings.Contains(cleanup.Body.String(), "delete_file") {
|
|
t.Fatalf("failure cleanup response: status=%d body=%s", cleanup.Code, cleanup.Body.String())
|
|
}
|
|
fixture.consumeToolResponse(cleanup, []string{`{"written":true}`})
|
|
final := fixture.request()
|
|
if final.Code != http.StatusBadRequest || !strings.Contains(final.Body.String(), message) {
|
|
t.Fatalf("failure terminal response: status=%d body=%s", final.Code, final.Body.String())
|
|
}
|
|
}
|
|
|
|
func driveScriptedLightToLocalAfterPlanRead(t *testing.T, fixture *scriptedLightFixture) {
|
|
t.Helper()
|
|
prepare := fixture.request()
|
|
fixture.consumeToolResponse(prepare, []string{`{"written":true}`})
|
|
pair := fixture.request()
|
|
fixture.consumeToolResponse(pair, []string{`{"written":true}`, `{"written":true}`})
|
|
localRead := fixture.request()
|
|
fixture.consumeToolResponse(localRead, []string{`{"written":true}`})
|
|
}
|
|
|
|
func TestHotPathStageInputIsolation(t *testing.T) {
|
|
paths := newReservedPaths("req_stage_isolation")
|
|
selector := hotPathStageCorrelation{StageID: "stg_selector", ResponseID: "provider:selector.actual/1", RunID: "run-selector", ProviderID: "provider.actual", Terminal: "stop,done\"quoted\""}
|
|
local := hotPathStageCorrelation{StageID: "stg_local", ResponseID: "provider:local.actual/2", RunID: "run-local", ProviderID: "provider.actual", Terminal: "tool_calls,stop"}
|
|
localInput := buildLocalStageInput(scriptedAbsoluteWorkspaceTask, paths, selector)
|
|
reviewInput := buildReviewStageInput(scriptedAbsoluteWorkspaceTask, paths, selector, local)
|
|
|
|
for _, input := range []hotPathStageInput{localInput, reviewInput} {
|
|
phase := hotPathPhaseLocalActive
|
|
if input.Role == "review" {
|
|
phase = hotPathPhaseReviewActive
|
|
}
|
|
prompt, err := input.prompt(phase)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, forbidden := range []string{"PLAN_FILE_SECRET", "credential-secret", "previous internal prompt", "provider-target.internal"} {
|
|
if strings.Contains(prompt, forbidden) {
|
|
t.Fatalf("stage prompt leaked %q: %s", forbidden, prompt)
|
|
}
|
|
}
|
|
if input.Role == "local" {
|
|
want := "/workspace/" + paths.PlanPath + "\nRead it and complete the task."
|
|
if prompt != want {
|
|
t.Fatalf("local prompt got=%q want=%q", prompt, want)
|
|
}
|
|
for _, forbidden := range []string{"immutable user task", paths.ReviewPath, "Committed selector stage success:", "Committed local stage success:", selector.StageID, local.StageID} {
|
|
if strings.Contains(prompt, forbidden) {
|
|
t.Fatalf("local prompt leaked %q: %s", forbidden, prompt)
|
|
}
|
|
}
|
|
continue
|
|
}
|
|
if !strings.Contains(prompt, "immutable user task") || !strings.Contains(prompt, paths.PlanPath) || !strings.Contains(prompt, paths.ReviewPath) {
|
|
t.Fatalf("stage prompt omitted immutable input: %s", prompt)
|
|
}
|
|
|
|
// Exact committed selector correlation must be present for both roles.
|
|
if !strings.Contains(prompt, "Committed selector stage success:") {
|
|
t.Fatalf("prompt missing committed selector correlation: %s", prompt)
|
|
}
|
|
if !strings.Contains(prompt, selector.StageID) || !strings.Contains(prompt, selector.RunID) {
|
|
t.Fatalf("prompt omitted exact selector correlation fields: %s", prompt)
|
|
}
|
|
|
|
// Verify serialized JSON block decoding and single-line format
|
|
selHeaderIdx := strings.Index(prompt, "Committed selector stage success:\n")
|
|
if selHeaderIdx == -1 {
|
|
t.Fatalf("prompt missing selector header format")
|
|
}
|
|
selJSONLine := prompt[selHeaderIdx+len("Committed selector stage success:\n"):]
|
|
if newlineIdx := strings.IndexByte(selJSONLine, '\n'); newlineIdx != -1 {
|
|
selJSONLine = selJSONLine[:newlineIdx]
|
|
}
|
|
var selDecoded correlationPromptValue
|
|
if err := json.Unmarshal([]byte(selJSONLine), &selDecoded); err != nil {
|
|
t.Fatalf("failed to decode selector correlation JSON line %q: %v", selJSONLine, err)
|
|
}
|
|
if selDecoded.StageID != selector.StageID || selDecoded.ResponseID != selector.ResponseID || selDecoded.RunID != selector.RunID || selDecoded.ProviderID != selector.ProviderID || selDecoded.Terminal != selector.Terminal {
|
|
t.Fatalf("decoded selector correlation mismatch: got %#v want %#v", selDecoded, selector)
|
|
}
|
|
|
|
// Review stage must carry both selector and local correlations.
|
|
if input.Role == "review" {
|
|
if !strings.Contains(prompt, "Committed local stage success:") {
|
|
t.Fatalf("review prompt missing committed local correlation: %s", prompt)
|
|
}
|
|
if !strings.Contains(prompt, local.StageID) || !strings.Contains(prompt, local.RunID) {
|
|
t.Fatalf("review prompt omitted exact local correlation fields: %s", prompt)
|
|
}
|
|
|
|
locHeaderIdx := strings.Index(prompt, "Committed local stage success:\n")
|
|
if locHeaderIdx == -1 {
|
|
t.Fatalf("prompt missing local header format")
|
|
}
|
|
locJSONLine := prompt[locHeaderIdx+len("Committed local stage success:\n"):]
|
|
if newlineIdx := strings.IndexByte(locJSONLine, '\n'); newlineIdx != -1 {
|
|
locJSONLine = locJSONLine[:newlineIdx]
|
|
}
|
|
var locDecoded correlationPromptValue
|
|
if err := json.Unmarshal([]byte(locJSONLine), &locDecoded); err != nil {
|
|
t.Fatalf("failed to decode local correlation JSON line %q: %v", locJSONLine, err)
|
|
}
|
|
if locDecoded.StageID != local.StageID || locDecoded.ResponseID != local.ResponseID || locDecoded.RunID != local.RunID || locDecoded.ProviderID != local.ProviderID || locDecoded.Terminal != local.Terminal {
|
|
t.Fatalf("decoded local correlation mismatch: got %#v want %#v", locDecoded, local)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Test invalid correlation field values fail closed for opaque fields.
|
|
invalidOpaqueValues := []string{
|
|
"",
|
|
"invalid\nvalue",
|
|
"invalid\rvalue",
|
|
"invalid\tvalue",
|
|
strings.Repeat("a", 257),
|
|
}
|
|
|
|
for _, invalid := range invalidOpaqueValues {
|
|
// Mutate Selector ResponseID
|
|
selBadResponse := selector
|
|
selBadResponse.ResponseID = invalid
|
|
inputBadSelResponse := buildLocalStageInput("immutable user task", paths, selBadResponse)
|
|
if p, err := inputBadSelResponse.prompt(hotPathPhaseLocalActive); err == nil || p != "" {
|
|
t.Fatalf("selector ResponseID %q accepted: prompt=%q, err=%v", invalid, p, err)
|
|
}
|
|
|
|
// Mutate Selector ProviderID
|
|
selBadProvider := selector
|
|
selBadProvider.ProviderID = invalid
|
|
inputBadSelProvider := buildLocalStageInput("immutable user task", paths, selBadProvider)
|
|
if p, err := inputBadSelProvider.prompt(hotPathPhaseLocalActive); err == nil || p != "" {
|
|
t.Fatalf("selector ProviderID %q accepted: prompt=%q, err=%v", invalid, p, err)
|
|
}
|
|
|
|
// Mutate Selector Terminal
|
|
selBadTerminal := selector
|
|
selBadTerminal.Terminal = invalid
|
|
inputBadSelTerminal := buildLocalStageInput("immutable user task", paths, selBadTerminal)
|
|
if p, err := inputBadSelTerminal.prompt(hotPathPhaseLocalActive); err == nil || p != "" {
|
|
t.Fatalf("selector Terminal %q accepted: prompt=%q, err=%v", invalid, p, err)
|
|
}
|
|
|
|
// Mutate Local ResponseID in review stage
|
|
localBadResponse := local
|
|
localBadResponse.ResponseID = invalid
|
|
inputBadLocalResponse := buildReviewStageInput("immutable user task", paths, selector, localBadResponse)
|
|
if p, err := inputBadLocalResponse.prompt(hotPathPhaseReviewActive); err == nil || p != "" {
|
|
t.Fatalf("local ResponseID %q accepted in review stage: prompt=%q, err=%v", invalid, p, err)
|
|
}
|
|
|
|
// Mutate Local ProviderID in review stage
|
|
localBadProvider := local
|
|
localBadProvider.ProviderID = invalid
|
|
inputBadLocalProvider := buildReviewStageInput("immutable user task", paths, selector, localBadProvider)
|
|
if p, err := inputBadLocalProvider.prompt(hotPathPhaseReviewActive); err == nil || p != "" {
|
|
t.Fatalf("local ProviderID %q accepted in review stage: prompt=%q, err=%v", invalid, p, err)
|
|
}
|
|
|
|
// Mutate Local Terminal in review stage
|
|
localBadTerminal := local
|
|
localBadTerminal.Terminal = invalid
|
|
inputBadLocalTerminal := buildReviewStageInput("immutable user task", paths, selector, localBadTerminal)
|
|
if p, err := inputBadLocalTerminal.prompt(hotPathPhaseReviewActive); err == nil || p != "" {
|
|
t.Fatalf("local Terminal %q accepted in review stage: prompt=%q, err=%v", invalid, p, err)
|
|
}
|
|
}
|
|
|
|
// Test invalid IOP-owned ID field values fail closed.
|
|
invalidLogicalIDs := []string{
|
|
"",
|
|
"invalid:value",
|
|
"invalid,value",
|
|
"invalid.value",
|
|
"invalid\nvalue",
|
|
strings.Repeat("a", 257),
|
|
}
|
|
|
|
for _, invalid := range invalidLogicalIDs {
|
|
selBadStage := selector
|
|
selBadStage.StageID = invalid
|
|
if p, err := buildLocalStageInput(scriptedAbsoluteWorkspaceTask, paths, selBadStage).prompt(hotPathPhaseLocalActive); err == nil || p != "" {
|
|
t.Fatalf("selector StageID %q accepted: prompt=%q, err=%v", invalid, p, err)
|
|
}
|
|
|
|
selBadRun := selector
|
|
selBadRun.RunID = invalid
|
|
if p, err := buildLocalStageInput(scriptedAbsoluteWorkspaceTask, paths, selBadRun).prompt(hotPathPhaseLocalActive); err == nil || p != "" {
|
|
t.Fatalf("selector RunID %q accepted: prompt=%q, err=%v", invalid, p, err)
|
|
}
|
|
}
|
|
|
|
pinned := routeDispatch{
|
|
Managed: true, PrincipalRef: "principal", ModelGroupKey: "local-model", RouteID: "route-local",
|
|
CredentialSlotRef: "slot-local", ProfileID: "profile", UpstreamModel: "served-local",
|
|
ResourceSelector: "resource", RouteRevision: 7, CredentialRevision: 11, ProjectionGeneration: 13,
|
|
}
|
|
changed := pinned
|
|
changed.CredentialRevision++
|
|
if samePinnedHotPathRoute(pinned, changed) {
|
|
t.Fatal("credential revision drift was accepted")
|
|
}
|
|
changed = pinned
|
|
changed.RouteRevision++
|
|
if samePinnedHotPathRoute(pinned, changed) {
|
|
t.Fatal("route revision drift was accepted")
|
|
}
|
|
}
|
|
|
|
type scriptedLightPoolService struct {
|
|
providerFakeRunService
|
|
mu sync.Mutex
|
|
endpoint string
|
|
candidate edgeservice.ProviderPoolCandidate
|
|
responses []func(string) string
|
|
requests []edgeservice.ProviderPoolDispatchRequest
|
|
}
|
|
|
|
func (s *scriptedLightPoolService) SubmitProviderPool(ctx context.Context, req edgeservice.ProviderPoolDispatchRequest) (*edgeservice.ProviderPoolDispatchResult, error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
s.mu.Lock()
|
|
index := len(s.requests)
|
|
s.requests = append(s.requests, req)
|
|
if index >= len(s.responses) {
|
|
s.mu.Unlock()
|
|
return nil, fmt.Errorf("unexpected light stage dispatch %d", index+1)
|
|
}
|
|
response := s.responses[index]
|
|
candidate := s.candidate
|
|
endpoint := s.endpoint
|
|
s.mu.Unlock()
|
|
|
|
providerBody, operation, err := materializeScriptedProviderBody(req, candidate)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var requestID string
|
|
if strings.Contains(string(providerBody), "IOP caller-workspace selector instruction") {
|
|
requestID, err = scriptedSelectorRequestID(providerBody, operation)
|
|
} else {
|
|
requestID, err = scriptedRequestIDFromProviderBody(providerBody)
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
body := response(requestID)
|
|
dispatch := edgeservice.RunDispatch{
|
|
RunID: fmt.Sprintf("run-light-%d", index+1), NodeID: "node-light", ModelGroupKey: req.Run.ModelGroupKey,
|
|
ProviderID: candidate.ProviderID, ExecutionPath: string(edgeservice.ProviderPoolPathTunnel),
|
|
ProfileID: candidate.ProfileID, ProfileDriver: candidate.ProfileDriver,
|
|
ProfileCapabilities: append([]string(nil), candidate.ProfileCapabilities...),
|
|
}
|
|
frames := staticProviderTunnelFrames(body)
|
|
if endpoint == "anthropic" {
|
|
frames = anthropicTunnelFrames(http.StatusOK, "application/json", []byte(body))
|
|
}
|
|
return &edgeservice.ProviderPoolDispatchResult{
|
|
Path: edgeservice.ProviderPoolPathTunnel, Tunnel: &fakeTunnelHandle{dispatch: dispatch, frames: frames}, DispatchInfo: dispatch,
|
|
}, nil
|
|
}
|
|
|
|
func (s *scriptedLightPoolService) snapshots() []edgeservice.ProviderPoolDispatchRequest {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
return append([]edgeservice.ProviderPoolDispatchRequest(nil), s.requests...)
|
|
}
|
|
|
|
type scriptedLightFixture struct {
|
|
t *testing.T
|
|
endpoint string
|
|
server *Server
|
|
service *scriptedLightPoolService
|
|
tools []any
|
|
history []any
|
|
repair bool
|
|
}
|
|
|
|
func newScriptedLightFixture(t *testing.T, endpoint string, repair bool) *scriptedLightFixture {
|
|
t.Helper()
|
|
candidate := anthropicTestCandidate(t, map[string]string{"openai": "openai", "anthropic": "anthropic"}[endpoint])
|
|
service := &scriptedLightPoolService{endpoint: endpoint, candidate: candidate}
|
|
service.responses = []func(string) string{
|
|
func(requestID string) string { return scriptedArtifactPrepare(endpoint, requestID) },
|
|
func(requestID string) string { return scriptedArtifactPair(endpoint, requestID) },
|
|
func(requestID string) string { return scriptedArtifactLocalRead(endpoint, requestID) },
|
|
func(string) string { return scriptedLocalWorkTool(endpoint) },
|
|
func(requestID string) string { return scriptedReviewWrite(endpoint, requestID) },
|
|
func(requestID string) string { return scriptedReviewReadsAndInspection(endpoint, requestID) },
|
|
}
|
|
if repair {
|
|
service.responses = append(service.responses,
|
|
func(string) string { return scriptedRepairTool(endpoint) },
|
|
func(string) string { return scriptedLightCompletion(endpoint, "repair-complete-visible") },
|
|
)
|
|
} else {
|
|
service.responses = append(service.responses, func(string) string {
|
|
return scriptedLightCompletion(endpoint, "review-resolution-visible PASS and DEFECT prose")
|
|
})
|
|
}
|
|
|
|
preset := hotPathSelectorPreset([]string{config.ModeDirect, config.ModeLight})
|
|
preset.WorkspaceTools = []config.ExecutionWorkspaceToolAlternative{scriptedLightWorkspaceAlternative()}
|
|
server := NewServer(config.EdgeOpenAIConf{}, service, nil)
|
|
server.SetEdgeID("edge-scripted-light")
|
|
server.SetExecutionPresets([]config.ExecutionPreset{preset})
|
|
server.SetModelCatalog([]config.ModelCatalogEntry{
|
|
{ID: "virtual-model", ExecutionPreset: preset.ID},
|
|
{ID: "selector-model", Providers: map[string]string{candidate.ProviderID: "served-selector"}},
|
|
{ID: "local-model", Providers: map[string]string{candidate.ProviderID: "served-local"}},
|
|
{ID: "review-model", Providers: map[string]string{candidate.ProviderID: "served-review"}},
|
|
})
|
|
tools := scriptedLightTools(endpoint)
|
|
return &scriptedLightFixture{
|
|
t: t, endpoint: endpoint, server: server, service: service, tools: tools,
|
|
history: []any{map[string]any{"role": "user", "content": "Working directory: /workspace\nimmutable user task"}}, repair: repair,
|
|
}
|
|
}
|
|
|
|
func scriptedLightWorkspaceAlternative() config.ExecutionWorkspaceToolAlternative {
|
|
matcher := successMatcher()
|
|
return config.ExecutionWorkspaceToolAlternative{
|
|
Name: "scripted-light-tools",
|
|
Operations: map[string]config.ExecutionWorkspaceOperation{
|
|
"prepare": {ToolName: "mkdir_p", SchemaMatcher: map[string]any{"type": "object"}, ArgumentMap: map[string]any{"path": "path"}, ResultMatcher: matcher, CreatesParents: true},
|
|
"read": {ToolName: "read_file", SchemaMatcher: map[string]any{"type": "object"}, ArgumentMap: map[string]any{"path": "path"}, ResultMatcher: matcher},
|
|
"write": {ToolName: "write_file", SchemaMatcher: map[string]any{"type": "object"}, ArgumentMap: map[string]any{"path": "path", "content": "content"}, ResultMatcher: matcher, CreatesParents: false},
|
|
"delete": {ToolName: "delete_file", SchemaMatcher: map[string]any{"type": "object"}, ArgumentMap: map[string]any{"path": "path"}, ResultMatcher: matcher},
|
|
},
|
|
}
|
|
}
|
|
|
|
func scriptedLightTools(endpoint string) []any {
|
|
tools := scriptedArtifactTools(endpoint)
|
|
schema := map[string]any{"type": "object", "properties": map[string]any{"command": map[string]any{"type": "string"}}, "required": []any{"command"}}
|
|
if endpoint == "anthropic" {
|
|
return append(tools, anthropicWorkspaceTool("run_command", schema))
|
|
}
|
|
return append(tools, openAIChatTool("run_command", schema))
|
|
}
|
|
|
|
func (f *scriptedLightFixture) run() *httptest.ResponseRecorder {
|
|
f.t.Helper()
|
|
cleanup := f.runToCleanup()
|
|
f.consumeToolResponse(cleanup, []string{`{"written":true}`})
|
|
return f.request()
|
|
}
|
|
|
|
func (f *scriptedLightFixture) runToCleanup() *httptest.ResponseRecorder {
|
|
f.t.Helper()
|
|
prepare := f.request()
|
|
f.consumeToolResponse(prepare, []string{`{"written":true}`})
|
|
pair := f.request()
|
|
f.consumeToolResponse(pair, []string{`{"written":true}`, `{"written":true}`})
|
|
localRead := f.request()
|
|
f.consumeToolResponse(localRead, []string{`{"written":true}`})
|
|
localWork := f.request()
|
|
f.consumeToolResponse(localWork, []string{`{"ok":true}`})
|
|
workerReview := f.request()
|
|
f.consumeToolResponse(workerReview, []string{`{"written":true}`})
|
|
reviewInspection := f.request()
|
|
f.consumeToolResponse(reviewInspection, []string{`{"written":true}`, `{"written":true}`, `{"ok":true}`})
|
|
resolution := f.request()
|
|
if !f.repair {
|
|
return resolution
|
|
}
|
|
f.consumeToolResponse(resolution, []string{`{"ok":true}`})
|
|
return f.request()
|
|
}
|
|
|
|
func (f *scriptedLightFixture) request() *httptest.ResponseRecorder {
|
|
return f.requestWithOptions(0, false)
|
|
}
|
|
|
|
func (f *scriptedLightFixture) requestWithOptions(outputCap int, stream bool) *httptest.ResponseRecorder {
|
|
f.t.Helper()
|
|
body := scriptedArtifactRequestBodyWithOptions(f.t, f.endpoint, f.tools, f.history, outputCap, stream)
|
|
return serveScriptedArtifactRequest(f.t, f.server, f.endpoint, body)
|
|
}
|
|
|
|
func (f *scriptedLightFixture) requestWithContext(ctx context.Context, outputCap int) *httptest.ResponseRecorder {
|
|
f.t.Helper()
|
|
body := scriptedArtifactRequestBodyWithOptions(f.t, f.endpoint, f.tools, f.history, outputCap, false)
|
|
return serveScriptedArtifactRequestContext(f.t, f.server, f.endpoint, body, ctx)
|
|
}
|
|
|
|
func (f *scriptedLightFixture) consumeToolResponse(response *httptest.ResponseRecorder, results []string) {
|
|
f.t.Helper()
|
|
if response.Code != http.StatusOK {
|
|
f.t.Fatalf("tool response status=%d body=%s", response.Code, response.Body.String())
|
|
}
|
|
assistant, ids, err := artifactAssistantFromResponse(f.endpoint, response.Body.Bytes())
|
|
if err != nil || len(ids) != len(results) {
|
|
f.t.Fatalf("decode tool response: ids=%v results=%v err=%v body=%s", ids, results, err, response.Body.String())
|
|
}
|
|
f.history = append(f.history, assistant)
|
|
f.history = scriptedArtifactAppendResults(f.endpoint, f.history, ids, results)
|
|
}
|
|
|
|
func (f *scriptedLightFixture) assertCleanupCommitted(wantCalls int) {
|
|
f.t.Helper()
|
|
requests := f.service.snapshots()
|
|
if len(requests) != wantCalls {
|
|
f.t.Fatalf("provider calls=%d, want %d", len(requests), wantCalls)
|
|
}
|
|
if requests[0].Run.ModelGroupKey != "selector-model" || requests[1].Run.ModelGroupKey != "selector-model" {
|
|
f.t.Fatalf("selector model groups changed: %q %q", requests[0].Run.ModelGroupKey, requests[0].Run.ModelGroupKey)
|
|
}
|
|
if requests[2].Run.ModelGroupKey != "local-model" || requests[3].Run.ModelGroupKey != "local-model" {
|
|
f.t.Fatalf("local model group changed: %q %q", requests[2].Run.ModelGroupKey, requests[3].Run.ModelGroupKey)
|
|
}
|
|
localStage := requests[2].Run.Metadata["iop_stage_id"]
|
|
if localStage == "" || requests[3].Run.Metadata["iop_stage_id"] != localStage {
|
|
f.t.Fatalf("local stage was not resumed: %#v %#v", requests[2].Run.Metadata, requests[3].Run.Metadata)
|
|
}
|
|
reviewStage := requests[5].Run.Metadata["iop_stage_id"]
|
|
if reviewStage == "" || reviewStage == localStage {
|
|
f.t.Fatalf("review stage identity is not fixed and distinct: local=%q review=%q", localStage, reviewStage)
|
|
}
|
|
for index := 5; index < len(requests); index++ {
|
|
if requests[index].Run.ModelGroupKey != "review-model" || requests[index].Run.Metadata["iop_stage_id"] != reviewStage {
|
|
f.t.Fatalf("review dispatch %d changed binding: group=%q metadata=%#v", index, requests[index].Run.ModelGroupKey, requests[index].Run.Metadata)
|
|
}
|
|
}
|
|
|
|
selectorStage := requests[1].Run.Metadata["iop_stage_id"]
|
|
selectorResponse := "chatcmpl-scripted-pair"
|
|
if f.endpoint == "anthropic" {
|
|
selectorResponse = "msg-scripted-pair"
|
|
}
|
|
|
|
localResponse := "chatcmpl-review-write"
|
|
if f.endpoint == "anthropic" {
|
|
localResponse = "msg-review-write"
|
|
}
|
|
|
|
// Regression: local stage must carry selector correlation and must NOT
|
|
// carry local correlation in both normalized Run.Input and tunnel body.
|
|
for index := 2; index <= 4; index++ {
|
|
assertLocalCorrelationRegression(f.t, requests[index], f.service.candidate, selectorStage, selectorResponse)
|
|
}
|
|
|
|
// Regression: review stage must carry both selector and local correlations
|
|
// in both normalized Run.Input and tunnel body.
|
|
for index := 5; index < len(requests); index++ {
|
|
assertReviewCorrelationRegression(f.t, requests[index], f.service.candidate, selectorStage, selectorResponse, localStage, localResponse)
|
|
}
|
|
|
|
// Regression: forbidden data must not appear in any provider-visible payload.
|
|
for index, req := range requests {
|
|
for _, forbidden := range []string{"PLAN_FILE_SECRET", "credential-secret", "previous internal prompt", "provider-target.internal"} {
|
|
if strings.Contains(req.Run.Prompt, forbidden) {
|
|
f.t.Fatalf("request %d Run.Prompt leaked %q", index, forbidden)
|
|
}
|
|
if body, ok := req.Run.Input["prompt"]; ok {
|
|
if strings.Contains(fmt.Sprint(body), forbidden) {
|
|
f.t.Fatalf("request %d Run.Input[\"prompt\"] leaked %q", index, forbidden)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
f.assertCleanupStoresRemoved()
|
|
}
|
|
|
|
func (f *scriptedLightFixture) assertCleanupStoresRemoved() {
|
|
f.t.Helper()
|
|
f.server.lightFlows.mu.Lock()
|
|
lightCount := len(f.server.lightFlows.records)
|
|
f.server.lightFlows.mu.Unlock()
|
|
if lightCount != 0 {
|
|
f.t.Fatalf("light records=%d, want 0 after cleanup commit", lightCount)
|
|
}
|
|
f.server.artifactFrontiers.mu.Lock()
|
|
artifactCount := len(f.server.artifactFrontiers.records)
|
|
f.server.artifactFrontiers.mu.Unlock()
|
|
if artifactCount != 0 {
|
|
f.t.Fatalf("artifact records=%d, want 0 after cleanup commit", artifactCount)
|
|
}
|
|
f.server.requestCoordinator.mu.Lock()
|
|
coordinatorCount := len(f.server.requestCoordinator.requests)
|
|
f.server.requestCoordinator.mu.Unlock()
|
|
if coordinatorCount != 0 {
|
|
f.t.Fatalf("coordinator records=%d, want 0 after cleanup commit", coordinatorCount)
|
|
}
|
|
}
|
|
|
|
// assertLocalCorrelationRegression verifies that a captured local-stage request
|
|
// contains only the issued PLAN path and the fixed short worker instruction.
|
|
func assertLocalCorrelationRegression(t *testing.T, req edgeservice.ProviderPoolDispatchRequest, selected edgeservice.ProviderPoolCandidate, selectorStage, selectorResponse string) {
|
|
t.Helper()
|
|
prompt := req.Run.Prompt
|
|
if prompt == "" {
|
|
t.Fatalf("local request prompt is empty")
|
|
}
|
|
input, ok := req.Run.Input["prompt"]
|
|
if !ok || input == nil {
|
|
t.Fatalf("local Run.Input[\"prompt\"] is missing")
|
|
}
|
|
inputStr := fmt.Sprint(input)
|
|
|
|
want := "/workspace/" + newReservedPaths(req.Run.Metadata["iop_logical_request_id"]).PlanPath + "\nRead it and complete the task."
|
|
if prompt != want || inputStr != want {
|
|
t.Fatalf("local prompt got=%q input=%q want=%q", prompt, inputStr, want)
|
|
}
|
|
for _, forbidden := range []string{selectorStage, selectorResponse, "Committed selector stage success:", "Committed local stage success:", "User task:", "REVIEW"} {
|
|
if forbidden != "" && (strings.Contains(prompt, forbidden) || strings.Contains(inputStr, forbidden)) {
|
|
t.Fatalf("local prompt leaked %q: prompt=%q input=%q", forbidden, prompt, inputStr)
|
|
}
|
|
}
|
|
|
|
// Mandatory: decode and verify selected protocol tunnel prompt.
|
|
_, tunnelPrompt, err := decodeSelectedTunnelPrompt(req, selected)
|
|
if err != nil {
|
|
t.Fatalf("local tunnel decode error: %v", err)
|
|
}
|
|
if tunnelPrompt != prompt {
|
|
t.Fatalf("local decoded tunnel prompt mismatch: got %q want %q", tunnelPrompt, prompt)
|
|
}
|
|
if tunnelPrompt != want {
|
|
t.Fatalf("local tunnel prompt got=%q want=%q", tunnelPrompt, want)
|
|
}
|
|
}
|
|
|
|
// assertReviewCorrelationRegression verifies that a captured review-stage request
|
|
// carries both committed selector and local correlations in Run.Prompt,
|
|
// Run.Input["prompt"], and the decoded tunnel body.
|
|
func assertReviewCorrelationRegression(t *testing.T, req edgeservice.ProviderPoolDispatchRequest, selected edgeservice.ProviderPoolCandidate, selectorStage, selectorResponse, localStage, localResponse string) {
|
|
t.Helper()
|
|
prompt := req.Run.Prompt
|
|
if prompt == "" {
|
|
t.Fatalf("review request prompt is empty")
|
|
}
|
|
input, ok := req.Run.Input["prompt"]
|
|
if !ok || input == nil {
|
|
t.Fatalf("review Run.Input[\"prompt\"] is missing")
|
|
}
|
|
inputStr := fmt.Sprint(input)
|
|
if got := fmt.Sprint(req.Run.Input["system"]); got != hotPathReviewSystemPrompt {
|
|
t.Fatalf("review Run.Input system prompt mismatch: %q", got)
|
|
}
|
|
|
|
if !strings.Contains(prompt, "Committed selector stage success:") {
|
|
t.Fatalf("review Run.Prompt missing selector correlation: %s", prompt)
|
|
}
|
|
if !strings.Contains(prompt, "Committed local stage success:") {
|
|
t.Fatalf("review Run.Prompt missing local correlation: %s", prompt)
|
|
}
|
|
if !strings.Contains(prompt, selectorStage) || !strings.Contains(prompt, selectorResponse) {
|
|
t.Fatalf("review Run.Prompt missing exact selector stage/response %q/%q: %s", selectorStage, selectorResponse, prompt)
|
|
}
|
|
if !strings.Contains(prompt, localStage) || !strings.Contains(prompt, localResponse) {
|
|
t.Fatalf("review Run.Prompt missing exact local stage/response %q/%q: %s", localStage, localResponse, prompt)
|
|
}
|
|
|
|
if !strings.Contains(inputStr, "Committed selector stage success:") {
|
|
t.Fatalf("review Run.Input[\"prompt\"] missing selector correlation: %v", input)
|
|
}
|
|
if !strings.Contains(inputStr, "Committed local stage success:") {
|
|
t.Fatalf("review Run.Input[\"prompt\"] missing local correlation: %v", input)
|
|
}
|
|
if !strings.Contains(inputStr, selectorStage) || !strings.Contains(inputStr, selectorResponse) {
|
|
t.Fatalf("review Run.Input[\"prompt\"] missing exact selector stage/response %q/%q: %v", selectorStage, selectorResponse, input)
|
|
}
|
|
if !strings.Contains(inputStr, localStage) || !strings.Contains(inputStr, localResponse) {
|
|
t.Fatalf("review Run.Input[\"prompt\"] missing exact local stage/response %q/%q: %v", localStage, localResponse, input)
|
|
}
|
|
|
|
// Mandatory: decode and verify selected protocol tunnel prompt.
|
|
prepared, tunnelPrompt, err := decodeSelectedTunnelPrompt(req, selected)
|
|
if err != nil {
|
|
t.Fatalf("review tunnel decode error: %v", err)
|
|
}
|
|
if tunnelPrompt != prompt {
|
|
t.Fatalf("review decoded tunnel prompt mismatch: got %q want %q", tunnelPrompt, prompt)
|
|
}
|
|
body, err := prepared.BuildBody("target-model")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if selected.ProfileDriver == string(config.ProtocolDriverAnthropicMessages) {
|
|
var payload struct {
|
|
System string `json:"system"`
|
|
}
|
|
if err := json.Unmarshal(body, &payload); err != nil || payload.System != hotPathReviewSystemPrompt {
|
|
t.Fatalf("review Messages system prompt mismatch: system=%q err=%v body=%s", payload.System, err, body)
|
|
}
|
|
} else {
|
|
var payload struct {
|
|
Messages []struct {
|
|
Role string `json:"role"`
|
|
Content string `json:"content"`
|
|
} `json:"messages"`
|
|
}
|
|
if err := json.Unmarshal(body, &payload); err != nil || len(payload.Messages) < 2 || payload.Messages[0].Role != "system" || payload.Messages[0].Content != hotPathReviewSystemPrompt {
|
|
t.Fatalf("review Chat system prompt mismatch: messages=%+v err=%v body=%s", payload.Messages, err, body)
|
|
}
|
|
}
|
|
if !strings.Contains(tunnelPrompt, "Committed selector stage success:") {
|
|
t.Fatalf("review tunnel body missing selector correlation: %s", tunnelPrompt)
|
|
}
|
|
if !strings.Contains(tunnelPrompt, "Committed local stage success:") {
|
|
t.Fatalf("review tunnel body missing local correlation: %s", tunnelPrompt)
|
|
}
|
|
if !strings.Contains(tunnelPrompt, selectorStage) || !strings.Contains(tunnelPrompt, selectorResponse) {
|
|
t.Fatalf("review tunnel body missing exact selector stage/response %q/%q: %s", selectorStage, selectorResponse, tunnelPrompt)
|
|
}
|
|
if !strings.Contains(tunnelPrompt, localStage) || !strings.Contains(tunnelPrompt, localResponse) {
|
|
t.Fatalf("review tunnel body missing exact local stage/response %q/%q: %s", localStage, localResponse, tunnelPrompt)
|
|
}
|
|
}
|
|
|
|
// decodeSelectedTunnelPrompt invokes PrepareProtocolTunnel unconditionally, builds the protocol
|
|
// body, checks expected path/op for OpenAI vs Anthropic, and extracts the first user message content string.
|
|
func decodeSelectedTunnelPrompt(req edgeservice.ProviderPoolDispatchRequest, selected edgeservice.ProviderPoolCandidate) (edgeservice.SubmitProviderTunnelRequest, string, error) {
|
|
if req.PrepareProtocolTunnel == nil {
|
|
return edgeservice.SubmitProviderTunnelRequest{}, "", fmt.Errorf("PrepareProtocolTunnel is not set")
|
|
}
|
|
prepared, err := req.PrepareProtocolTunnel(req.Tunnel, selected)
|
|
if err != nil {
|
|
return prepared, "", fmt.Errorf("PrepareProtocolTunnel error: %w", err)
|
|
}
|
|
if prepared.BuildBody == nil {
|
|
return prepared, "", fmt.Errorf("BuildBody is not set after PrepareProtocolTunnel")
|
|
}
|
|
bodyBytes, err := prepared.BuildBody("target-model")
|
|
if err != nil {
|
|
return prepared, "", fmt.Errorf("BuildBody error: %w", err)
|
|
}
|
|
|
|
if selected.ProfileDriver == string(config.ProtocolDriverAnthropicMessages) {
|
|
if prepared.Path != "/v1/messages" || prepared.Operation != string(config.OperationMessages) {
|
|
return prepared, "", fmt.Errorf("anthropic tunnel path/op mismatch: path=%q op=%q", prepared.Path, prepared.Operation)
|
|
}
|
|
var payload struct {
|
|
Messages []struct {
|
|
Role string `json:"role"`
|
|
Content any `json:"content"`
|
|
} `json:"messages"`
|
|
}
|
|
if err := json.Unmarshal(bodyBytes, &payload); err != nil {
|
|
return prepared, "", fmt.Errorf("unmarshal anthropic payload: %w (body=%s)", err, string(bodyBytes))
|
|
}
|
|
for _, message := range payload.Messages {
|
|
if message.Role == "user" {
|
|
return prepared, extractMessageContentString(message.Content), nil
|
|
}
|
|
}
|
|
return prepared, "", fmt.Errorf("anthropic body missing user message: %s", string(bodyBytes))
|
|
} else {
|
|
if prepared.Path != "/v1/chat/completions" || prepared.Operation != string(config.OperationChatCompletions) {
|
|
return prepared, "", fmt.Errorf("openai tunnel path/op mismatch: path=%q op=%q", prepared.Path, prepared.Operation)
|
|
}
|
|
var payload struct {
|
|
Messages []struct {
|
|
Role string `json:"role"`
|
|
Content any `json:"content"`
|
|
} `json:"messages"`
|
|
}
|
|
if err := json.Unmarshal(bodyBytes, &payload); err != nil {
|
|
return prepared, "", fmt.Errorf("unmarshal openai payload: %w (body=%s)", err, string(bodyBytes))
|
|
}
|
|
for _, message := range payload.Messages {
|
|
if message.Role == "user" {
|
|
return prepared, extractMessageContentString(message.Content), nil
|
|
}
|
|
}
|
|
return prepared, "", fmt.Errorf("openai body missing user message: %s", string(bodyBytes))
|
|
}
|
|
}
|
|
|
|
func extractMessageContentString(content any) string {
|
|
switch v := content.(type) {
|
|
case string:
|
|
return v
|
|
case []any:
|
|
var parts []string
|
|
for _, item := range v {
|
|
if m, ok := item.(map[string]any); ok {
|
|
if text, ok := m["text"].(string); ok {
|
|
parts = append(parts, text)
|
|
}
|
|
}
|
|
}
|
|
return strings.Join(parts, "")
|
|
default:
|
|
return fmt.Sprint(content)
|
|
}
|
|
}
|
|
|
|
func scriptedLightCompletion(endpoint, content string) string {
|
|
if endpoint == "anthropic" {
|
|
return fmt.Sprintf(`{"id":"msg-light-complete","type":"message","role":"assistant","content":[{"type":"text","text":%q}],"stop_reason":"end_turn","usage":{"input_tokens":1,"output_tokens":1}}`, content)
|
|
}
|
|
raw, _ := json.Marshal(content)
|
|
return fmt.Sprintf(`{"id":"chatcmpl-light-complete","created":9,"choices":[{"message":{"role":"assistant","content":%s},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2}}`, raw)
|
|
}
|
|
|
|
func scriptedLightCompletionWithUsage(endpoint, content, reasoning string, inputTokens, outputTokens int) string {
|
|
if endpoint == "anthropic" {
|
|
return fmt.Sprintf(`{"id":"msg-light-complete","type":"message","role":"assistant","content":[{"type":"thinking","thinking":%q,"signature":"sig-local"},{"type":"text","text":%q}],"stop_reason":"end_turn","usage":{"input_tokens":%d,"output_tokens":%d}}`, reasoning, content, inputTokens, outputTokens)
|
|
}
|
|
contentRaw, _ := json.Marshal(content)
|
|
reasoningRaw, _ := json.Marshal(reasoning)
|
|
return fmt.Sprintf(`{"id":"chatcmpl-light-complete","created":9,"choices":[{"message":{"role":"assistant","content":%s,"reasoning_content":%s},"finish_reason":"stop"}],"usage":{"prompt_tokens":%d,"completion_tokens":%d,"total_tokens":%d}}`, contentRaw, reasoningRaw, inputTokens, outputTokens, inputTokens+outputTokens)
|
|
}
|
|
|
|
func scriptedReviewWrite(endpoint, requestID string) string {
|
|
path := newReservedPaths(requestID).ReviewPath
|
|
review := testCompletedReviewText()
|
|
if endpoint == "anthropic" {
|
|
return fmt.Sprintf(`{"id":"msg-review-write","type":"message","role":"assistant","content":[{"type":"text","text":"worker-handoff-visible"},{"type":"tool_use","id":"provider-review-write","name":"write_file","input":{"path":%q,"content":%q}}],"stop_reason":"tool_use"}`, path, review)
|
|
}
|
|
args, _ := json.Marshal(map[string]string{"path": path, "content": review})
|
|
return fmt.Sprintf(`{"id":"chatcmpl-review-write","created":5,"choices":[{"message":{"role":"assistant","content":"worker-handoff-visible","tool_calls":[{"id":"provider-review-write","type":"function","function":{"name":"write_file","arguments":%q}}]},"finish_reason":"tool_calls"}]}`, string(args))
|
|
}
|
|
|
|
func testCompletedReviewText() string {
|
|
review, _ := pendingCompletedReview()
|
|
return string(review)
|
|
}
|
|
|
|
func pendingCompletedReview() ([]byte, error) {
|
|
return singlerequesttemplate.RenderReview(singlerequesttemplate.DefaultReviewTemplate, singlerequesttemplate.ReviewFields{
|
|
ItemStatus: "- P1: completed\n- P2: completed", Changes: "Implemented the requested caller workspace result.",
|
|
Verification: "Verified the requested result with caller tools.", Deviations: "None.",
|
|
}, singlerequesttemplate.MaxTemplateBytes)
|
|
}
|
|
|
|
func scriptedLocalWorkTool(endpoint string) string {
|
|
return scriptedOrdinaryTool(endpoint, "provider-local-work", "implement requested result")
|
|
}
|
|
|
|
func scriptedLocalWorkAndHandoff(endpoint, requestID string) string {
|
|
path := newReservedPaths(requestID).ReviewPath
|
|
review := testCompletedReviewText()
|
|
if endpoint == "anthropic" {
|
|
return fmt.Sprintf(`{"id":"msg-worker-handoff","type":"message","role":"assistant","content":[{"type":"tool_use","id":"provider-local-work","name":"run_command","input":{"command":"implement requested result"}},{"type":"tool_use","id":"provider-review-write","name":"write_file","input":{"path":%q,"content":%q}}],"stop_reason":"tool_use"}`, path, review)
|
|
}
|
|
workArgs, _ := json.Marshal(map[string]string{"command": "implement requested result"})
|
|
reviewArgs, _ := json.Marshal(map[string]string{"path": path, "content": review})
|
|
return fmt.Sprintf(`{"id":"chatcmpl-worker-handoff","created":4,"choices":[{"message":{"role":"assistant","tool_calls":[{"id":"provider-local-work","type":"function","function":{"name":"run_command","arguments":%q}},{"id":"provider-review-write","type":"function","function":{"name":"write_file","arguments":%q}}]},"finish_reason":"tool_calls"}]}`, string(workArgs), string(reviewArgs))
|
|
}
|
|
|
|
func scriptedMalformedWorkerHandoff(endpoint, requestID string) string {
|
|
path := newReservedPaths(requestID).ReviewPath
|
|
if endpoint == "anthropic" {
|
|
return fmt.Sprintf(`{"id":"msg-worker-handoff-bad","type":"message","role":"assistant","content":[{"type":"tool_use","id":"provider-review-write-bad","name":"write_file","input":{"path":%q,"content":"not a review handoff"}}],"stop_reason":"tool_use"}`, path)
|
|
}
|
|
args, _ := json.Marshal(map[string]string{"path": path, "content": "not a review handoff"})
|
|
return fmt.Sprintf(`{"id":"chatcmpl-worker-handoff-bad","created":4,"choices":[{"message":{"role":"assistant","tool_calls":[{"id":"provider-review-write-bad","type":"function","function":{"name":"write_file","arguments":%q}}]},"finish_reason":"tool_calls"}]}`, string(args))
|
|
}
|
|
|
|
func scriptedReviewInspection(endpoint string) string {
|
|
return scriptedOrdinaryTool(endpoint, "provider-review-inspect", "inspect caller result")
|
|
}
|
|
|
|
func scriptedOrdinaryTool(endpoint, id, command string) string {
|
|
if endpoint == "anthropic" {
|
|
return fmt.Sprintf(`{"id":"msg-ordinary","type":"message","role":"assistant","content":[{"type":"tool_use","id":%q,"name":"run_command","input":{"command":%q}}],"stop_reason":"tool_use"}`, id, command)
|
|
}
|
|
args, _ := json.Marshal(map[string]string{"command": command})
|
|
return fmt.Sprintf(`{"id":"chatcmpl-ordinary","created":4,"choices":[{"message":{"role":"assistant","tool_calls":[{"id":%q,"type":"function","function":{"name":"run_command","arguments":%q}}]},"finish_reason":"tool_calls"}]}`, id, string(args))
|
|
}
|
|
|
|
func scriptedReviewArtifactReads(endpoint, requestID string) string {
|
|
paths := newReservedPaths(requestID)
|
|
if endpoint == "anthropic" {
|
|
return fmt.Sprintf(`{"id":"msg-review-reads","type":"message","role":"assistant","content":[{"type":"tool_use","id":"provider-review-plan-read","name":"read_file","input":{"path":%q}},{"type":"tool_use","id":"provider-review-handoff-read","name":"read_file","input":{"path":%q}}],"stop_reason":"tool_use"}`, paths.PlanPath, paths.ReviewPath)
|
|
}
|
|
planArgs, _ := json.Marshal(map[string]string{"path": paths.PlanPath})
|
|
reviewArgs, _ := json.Marshal(map[string]string{"path": paths.ReviewPath})
|
|
return fmt.Sprintf(`{"id":"chatcmpl-review-reads","created":6,"choices":[{"message":{"role":"assistant","tool_calls":[{"id":"provider-review-plan-read","type":"function","function":{"name":"read_file","arguments":%q}},{"id":"provider-review-handoff-read","type":"function","function":{"name":"read_file","arguments":%q}}]},"finish_reason":"tool_calls"}]}`, string(planArgs), string(reviewArgs))
|
|
}
|
|
|
|
func scriptedReviewReadsAndInspection(endpoint, requestID string) string {
|
|
paths := newReservedPaths(requestID)
|
|
if endpoint == "anthropic" {
|
|
return fmt.Sprintf(`{"id":"msg-review-inspection","type":"message","role":"assistant","content":[{"type":"tool_use","id":"provider-review-plan-read","name":"read_file","input":{"path":%q}},{"type":"tool_use","id":"provider-review-handoff-read","name":"read_file","input":{"path":%q}},{"type":"tool_use","id":"provider-review-inspect","name":"run_command","input":{"command":"inspect caller result"}}],"stop_reason":"tool_use"}`, paths.PlanPath, paths.ReviewPath)
|
|
}
|
|
planArgs, _ := json.Marshal(map[string]string{"path": paths.PlanPath})
|
|
reviewArgs, _ := json.Marshal(map[string]string{"path": paths.ReviewPath})
|
|
inspectArgs, _ := json.Marshal(map[string]string{"command": "inspect caller result"})
|
|
return fmt.Sprintf(`{"id":"chatcmpl-review-inspection","created":6,"choices":[{"message":{"role":"assistant","tool_calls":[{"id":"provider-review-plan-read","type":"function","function":{"name":"read_file","arguments":%q}},{"id":"provider-review-handoff-read","type":"function","function":{"name":"read_file","arguments":%q}},{"id":"provider-review-inspect","type":"function","function":{"name":"run_command","arguments":%q}}]},"finish_reason":"tool_calls"}]}`, string(planArgs), string(reviewArgs), string(inspectArgs))
|
|
}
|
|
|
|
func scriptedReviewReadsAndInspectionWithUsage(requestID string, inputTokens, outputTokens int) string {
|
|
paths := newReservedPaths(requestID)
|
|
return fmt.Sprintf(`{"id":"msg-review-inspection","type":"message","role":"assistant","content":[{"type":"thinking","thinking":"review-reason","signature":"sig-review"},{"type":"text","text":"review-visible"},{"type":"tool_use","id":"provider-review-plan-read","name":"read_file","input":{"path":%q}},{"type":"tool_use","id":"provider-review-handoff-read","name":"read_file","input":{"path":%q}},{"type":"tool_use","id":"provider-review-inspect","name":"run_command","input":{"command":"inspect caller result"}}],"stop_reason":"tool_use","usage":{"input_tokens":%d,"output_tokens":%d}}`, paths.PlanPath, paths.ReviewPath, inputTokens, outputTokens)
|
|
}
|
|
|
|
func scriptedReviewWriteWithUsage(endpoint, requestID string, inputTokens, outputTokens int) string {
|
|
path := newReservedPaths(requestID).ReviewPath
|
|
review := testCompletedReviewText()
|
|
if endpoint == "anthropic" {
|
|
return fmt.Sprintf(`{"id":"msg-review-write","type":"message","role":"assistant","content":[{"type":"thinking","thinking":"review-reason","signature":"sig-review"},{"type":"text","text":"review-visible"},{"type":"tool_use","id":"provider-review-write","name":"write_file","input":{"path":%q,"content":%q}}],"stop_reason":"tool_use","usage":{"input_tokens":%d,"output_tokens":%d}}`, path, review, inputTokens, outputTokens)
|
|
}
|
|
args, _ := json.Marshal(map[string]string{"path": path, "content": review})
|
|
return fmt.Sprintf(`{"id":"chatcmpl-review-write","created":5,"choices":[{"message":{"role":"assistant","content":"review-visible","reasoning_content":"review-reason","tool_calls":[{"id":"provider-review-write","type":"function","function":{"name":"write_file","arguments":%q}}]},"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":%d,"completion_tokens":%d,"total_tokens":%d}}`, string(args), inputTokens, outputTokens, inputTokens+outputTokens)
|
|
}
|
|
|
|
func assertCapturedHotPathBudget(t *testing.T, req edgeservice.ProviderPoolDispatchRequest, candidate edgeservice.ProviderPoolCandidate, want int) {
|
|
t.Helper()
|
|
options, ok := req.Run.Input["options"].(map[string]any)
|
|
if !ok || options["max_tokens"] != want {
|
|
t.Fatalf("normalized remaining cap = %#v, want %d", req.Run.Input["options"], want)
|
|
}
|
|
prepared, _, err := decodeSelectedTunnelPrompt(req, candidate)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
body, err := prepared.BuildBody("served-stage")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var tunnel map[string]any
|
|
if err := json.Unmarshal(body, &tunnel); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if tunnel["max_tokens"] != float64(want) {
|
|
t.Fatalf("tunnel remaining cap = %#v, want %d; body=%s", tunnel["max_tokens"], want, body)
|
|
}
|
|
}
|
|
|
|
func scriptedReviewRead(endpoint, requestID string) string {
|
|
path := newReservedPaths(requestID).ReviewPath
|
|
if endpoint == "anthropic" {
|
|
return fmt.Sprintf(`{"id":"msg-review-read","type":"message","role":"assistant","content":[{"type":"text","text":"review-read-visible"},{"type":"tool_use","id":"provider-review-read","name":"read_file","input":{"path":%q}}],"stop_reason":"tool_use"}`, path)
|
|
}
|
|
args, _ := json.Marshal(map[string]string{"path": path})
|
|
return fmt.Sprintf(`{"id":"chatcmpl-review-read","created":6,"choices":[{"message":{"role":"assistant","content":"review-read-visible","tool_calls":[{"id":"provider-review-read","type":"function","function":{"name":"read_file","arguments":%q}}]},"finish_reason":"tool_calls"}]}`, string(args))
|
|
}
|
|
|
|
func scriptedRepairTool(endpoint string) string {
|
|
if endpoint == "anthropic" {
|
|
return `{"id":"msg-repair","type":"message","role":"assistant","content":[{"type":"text","text":"PASS prose but repair tool decides"},{"type":"tool_use","id":"provider-repair","name":"run_command","input":{"command":"go test ./..."}}],"stop_reason":"tool_use"}`
|
|
}
|
|
return `{"id":"chatcmpl-repair","created":7,"choices":[{"message":{"role":"assistant","content":"PASS prose but repair tool decides","tool_calls":[{"id":"provider-repair","type":"function","function":{"name":"run_command","arguments":"{\"command\":\"go test ./...\"}"}}]},"finish_reason":"tool_calls"}]}`
|
|
}
|