package openai import ( "encoding/json" "fmt" "net/http" "net/http/httptest" "sort" "strings" "sync" "sync/atomic" "testing" "iop/packages/go/config" ) func TestArtifactPairFrontierMatrix(t *testing.T) { for _, endpoint := range []string{"openai", "anthropic"} { endpoint := endpoint t.Run(endpoint, func(t *testing.T) { t.Run("parent-capable reversed pair becomes locally eligible once", func(t *testing.T) { fixture := newArtifactPairFixture(t, endpoint, true) publicIDs := fixture.issuePair() fixture.assertPendingPayloads(publicIDs, []string{fixture.paths.PlanPath, fixture.paths.ReviewPath}) ingress, _, body, err := fixture.continueWithResult([]artifactTestResult{ {id: publicIDs[1], body: `{"written":true}`}, {id: publicIDs[0], body: `{"written":true}`}, }, nil) if err != nil { t.Fatalf("consume reversed pair: %v", err) } if ingress.Artifact.Kind != artifactDispositionLocalEligible { t.Fatalf("local eligibility disposition = %#v", ingress.Artifact) } fixture.assertPhase(artifactPhaseLocalEligible) if _, _, err := fixture.continueRaw(body); err == nil || !strings.Contains(err.Error(), "replay") { t.Fatalf("replayed pair error = %v, want replay rejection", err) } fixture.assertPhase(artifactPhaseLocalEligible) }) t.Run("prepare resumes the exact selector stage before pair", func(t *testing.T) { fixture := newArtifactPairFixture(t, endpoint, false) prepareIDs := fixture.issuePrepare() if len(prepareIDs) != 1 { t.Fatalf("prepare ids = %#v", prepareIDs) } fixture.assertPendingPayloads(prepareIDs, []string{fixture.paths.JobDir}) ingress, metadata, _, err := fixture.continueWithResult([]artifactTestResult{{id: prepareIDs[0], body: `{"written":true}`}}, nil) if err != nil { t.Fatalf("consume prepare: %v", err) } if metadata["iop_stage_id"] != fixture.stageID || ingress.Artifact.Kind != artifactDispositionResumeSelector { t.Fatalf("prepare disposition = %#v, original stage = %q", metadata, fixture.stageID) } fixture.assertPhase(artifactPhasePairReady) pairIDs := fixture.issuePair() ingress, metadata, _, err = fixture.continueWithResult([]artifactTestResult{ {id: pairIDs[1], body: `{"written":true}`}, {id: pairIDs[0], body: `{"written":true}`}, }, nil) if err != nil { t.Fatalf("consume pair after prepare: %v", err) } if ingress.Artifact.Kind != artifactDispositionLocalEligible { t.Fatalf("pair disposition = %#v", ingress.Artifact) } fixture.assertPhase(artifactPhaseLocalEligible) }) t.Run("pair-ready selector cannot downgrade to direct", func(t *testing.T) { fixture := newArtifactPairFixture(t, endpoint, false) prepareIDs := fixture.issuePrepare() _, metadata, _, err := fixture.continueWithResult([]artifactTestResult{{id: prepareIDs[0], body: `{"written":true}`}}, nil) if err != nil { t.Fatalf("consume prepare: %v", err) } fixture.assertPhase(artifactPhasePairReady) recorder := httptest.NewRecorder() err = fixture.server.dispatchPresetTurn( recorder, httptest.NewRequest(http.MethodPost, "/", nil), fixture.dispatch, fixture.endpoint, false, metadata, normalizedStageOutput{ResponseID: "provider_direct", Content: "must not escape pair frontier"}, hotPathTestGate(fixture.dispatch.Preset), ) if err == nil || recorder.Code != http.StatusBadRequest { t.Fatalf("pair-ready direct downgrade = err %v, status %d", err, recorder.Code) } }) t.Run("general tool continuation bypasses artifact hook", func(t *testing.T) { fixture := newArtifactPairFixture(t, endpoint, true) publicID := fixture.issueGeneralTool() metadata, _, err := fixture.continueWith([]artifactTestResult{{id: publicID, body: "general result"}}, nil) if err != nil { t.Fatalf("consume general continuation: %v", err) } if metadata["iop_stage_id"] == "" || metadata["iop_stage_id"] == fixture.stageID { t.Fatalf("general continuation did not activate a fresh stage: %#v", metadata) } fixture.assertPhase(artifactPhasePinned) }) for _, rejection := range []struct { name string results func([]string) []artifactTestResult mutate func(any) }{ {name: "missing", results: func(ids []string) []artifactTestResult { return []artifactTestResult{{id: ids[0], body: `{"written":true}`}} }}, {name: "extra", results: func(ids []string) []artifactTestResult { return []artifactTestResult{{id: ids[0], body: `{"written":true}`}, {id: ids[1], body: `{"written":true}`}, {id: "call_extra", body: `{"written":true}`}} }}, {name: "duplicate", results: func(ids []string) []artifactTestResult { return []artifactTestResult{{id: ids[0], body: `{"written":true}`}, {id: ids[0], body: `{"written":true}`}} }}, {name: "opaque", results: func(ids []string) []artifactTestResult { return []artifactTestResult{{id: ids[0], body: "opaque"}, {id: ids[1], body: `{"written":true}`}} }}, {name: "alternate public ids", results: func(ids []string) []artifactTestResult { return []artifactTestResult{{id: "call_alternate_plan", body: `{"written":true}`}, {id: "call_alternate_review", body: `{"written":true}`}} }, mutate: mutateArtifactAssistantIDs}, } { rejection := rejection t.Run("reject "+rejection.name, func(t *testing.T) { fixture := newArtifactPairFixture(t, endpoint, true) ids := fixture.issuePair() before := fixture.stateSignature() if _, _, err := fixture.continueWith(rejection.results(ids), rejection.mutate); err == nil { t.Fatalf("%s continuation unexpectedly succeeded", rejection.name) } if after := fixture.stateSignature(); after != before { t.Fatalf("%s advanced state: before=%s after=%s", rejection.name, before, after) } }) } for _, emission := range []struct { name string planPath string }{ {name: "traversal path", planPath: ".iop/job/../escape/plan.md"}, {name: "alternate request path", planPath: ".iop/job/other-request/plan.md"}, } { emission := emission t.Run("reject "+emission.name, func(t *testing.T) { fixture := newArtifactPairFixture(t, endpoint, true) if _, err := fixture.issue([]normalizedToolCall{ artifactProviderWrite("provider_plan", emission.planPath, "plan"), artifactProviderWrite("provider_review", fixture.paths.ReviewPath, "review"), }); err == nil { t.Fatalf("%s emission unexpectedly succeeded", emission.name) } }) } t.Run("concurrent duplicate consumption advances once", func(t *testing.T) { fixture := newArtifactPairFixture(t, endpoint, true) ids := fixture.issuePair() body := fixture.continuationBody([]artifactTestResult{ {id: ids[1], body: `{"written":true}`}, {id: ids[0], body: `{"written":true}`}, }, nil) var successes atomic.Int32 var wg sync.WaitGroup for range 2 { wg.Add(1) go func() { defer wg.Done() if _, _, err := fixture.continueRaw(body); err == nil { successes.Add(1) } }() } wg.Wait() if got := successes.Load(); got != 1 { t.Fatalf("concurrent successes = %d, want 1", got) } fixture.assertPhase(artifactPhaseLocalEligible) }) }) } } type artifactPairFixture struct { t *testing.T endpoint string server *Server dispatch routeDispatch requestID string stageID string ownerEdgeID string principalRef string paths reservedPaths tools []any history []any lastAssistant any } type artifactTestResult struct { id string body string failed bool } func newArtifactPairFixture(t *testing.T, endpoint string, createsParents bool) *artifactPairFixture { t.Helper() var sequence atomic.Int64 idSource := func() (string, error) { return fmt.Sprintf("artifact_%03d", sequence.Add(1)), nil } coordinator := newLogicalRequestCoordinator(logicalRequestCoordinatorOptions{IDSource: idSource}) server := NewServer(config.EdgeOpenAIConf{}, nil, nil) server.requestCoordinator = coordinator server.artifactFrontiers = newArtifactFrontierStore(32) server.SetEdgeID("edge-artifact") alternative := workspaceAlternative("artifact-structured", "workspace", false, createsParents) preset := config.ExecutionPreset{ ID: "artifact-preset", Selector: config.ExecutionModelBinding{Model: "selector-model"}, AllowedModes: []string{modeLight}, WorkspaceTools: []config.ExecutionWorkspaceToolAlternative{alternative}, } dispatch := routeDispatch{IsPreset: true, PresetID: preset.ID, Preset: preset, ExternalModelID: "virtual-artifact"} schema := map[string]any{ "type": "object", "properties": map[string]any{"path": map[string]any{"type": "string"}, "content": map[string]any{}}, } tools := []any{openAIChatTool("workspace", schema)} if endpoint == "anthropic" { tools = []any{anthropicWorkspaceTool("workspace", schema)} } history := []any{map[string]any{"role": "user", "content": "task"}} body := artifactRequestBody(t, endpoint, tools, history) metadata := map[string]string{principalMetaRef: "principal-artifact"} var err error if endpoint == "anthropic" { _, err = server.joinPresetAnthropicIngress(nil, dispatch, body, metadata) } else { _, err = server.joinPresetChatIngress(nil, dispatch, body, metadata) } if err != nil { t.Fatalf("join initial %s artifact request: %v", endpoint, err) } requestID := metadata["iop_logical_request_id"] stageID := metadata["iop_stage_id"] if requestID == "" || stageID == "" { t.Fatalf("initial metadata = %#v", metadata) } return &artifactPairFixture{ t: t, endpoint: endpoint, server: server, dispatch: dispatch, requestID: requestID, stageID: stageID, ownerEdgeID: "edge-artifact", principalRef: "principal-artifact", paths: newReservedPaths(requestID), tools: tools, history: history, } } func (f *artifactPairFixture) issuePrepare() []string { f.t.Helper() ids, err := f.issue([]normalizedToolCall{{ ID: "provider_prepare", Name: "workspace", Arguments: map[string]any{"path": f.paths.JobDir}, }}) if err != nil { f.t.Fatalf("issue prepare: %v", err) } return ids } func (f *artifactPairFixture) issuePair() []string { f.t.Helper() ids, err := f.issue([]normalizedToolCall{ artifactProviderWrite("provider_plan", f.paths.PlanPath, "plan"), artifactProviderWrite("provider_review", f.paths.ReviewPath, "review"), }) if err != nil { f.t.Fatalf("issue pair: %v", err) } return ids } func (f *artifactPairFixture) issueGeneralTool() string { f.t.Helper() recorder := httptest.NewRecorder() turn := &hotPathTurn{ RequestID: f.requestID, StageID: f.stageID, CallID: "http_call", OwnerEdgeID: f.ownerEdgeID, PrincipalRef: f.principalRef, Preset: f.dispatch.Preset, Dispatch: f.dispatch, Protocol: f.endpoint, PublicModelID: f.dispatch.ExternalModelID, Writer: recorder, Request: httptest.NewRequest(http.MethodPost, "/", nil), } output := normalizedStageOutput{ ResponseID: "provider_response", Created: 123, ToolCalls: []normalizedToolCall{{ID: "call_general", ProviderCallID: "provider_general", Name: "search", Arguments: map[string]any{"query": "status"}}}, } if err := f.server.runDirectTurn(turn.Request.Context(), turn, output); err != nil { f.t.Fatalf("issue general tool: %v", err) } assistant, ids, err := artifactAssistantFromResponse(f.endpoint, recorder.Body.Bytes()) if err != nil || len(ids) != 1 { f.t.Fatalf("decode general tool response: ids=%#v err=%v", ids, err) } f.history = append(f.history, assistant) f.lastAssistant = assistant return ids[0] } func artifactProviderWrite(id, path, content string) normalizedToolCall { return normalizedToolCall{ID: id, Name: "workspace", Arguments: map[string]any{"path": path, "content": content}} } func (f *artifactPairFixture) issue(calls []normalizedToolCall) ([]string, error) { f.t.Helper() recorder := httptest.NewRecorder() turn := &hotPathTurn{ RequestID: f.requestID, StageID: f.stageID, CallID: "http_call", OwnerEdgeID: f.ownerEdgeID, PrincipalRef: f.principalRef, Preset: f.dispatch.Preset, Dispatch: f.dispatch, Protocol: f.endpoint, PublicModelID: f.dispatch.ExternalModelID, Writer: recorder, Request: httptest.NewRequest(http.MethodPost, "/", nil), } err := f.server.runArtifactPairTurn(turn, normalizedStageOutput{ ResponseID: "provider_response", Created: 123, ToolCalls: calls, }, hotPathTestGate(turn.Preset)) if err != nil { return nil, err } if recorder.Code != http.StatusOK { return nil, fmt.Errorf("artifact response status %d: %s", recorder.Code, recorder.Body.String()) } assistant, ids, err := artifactAssistantFromResponse(f.endpoint, recorder.Body.Bytes()) if err != nil { return nil, err } f.history = append(f.history, assistant) f.lastAssistant = assistant return ids, nil } func artifactAssistantFromResponse(endpoint string, body []byte) (any, []string, error) { if endpoint == "anthropic" { var response struct { Content []map[string]any `json:"content"` } if err := json.Unmarshal(body, &response); err != nil { return nil, nil, err } ids := make([]string, 0, len(response.Content)) for _, block := range response.Content { if block["type"] == "tool_use" { ids = append(ids, block["id"].(string)) } } return map[string]any{"role": "assistant", "content": response.Content}, ids, nil } var response struct { Choices []struct { Message map[string]any `json:"message"` } `json:"choices"` } if err := json.Unmarshal(body, &response); err != nil || len(response.Choices) != 1 { return nil, nil, fmt.Errorf("decode Chat artifact response: %v", err) } toolCalls, _ := response.Choices[0].Message["tool_calls"].([]any) ids := make([]string, 0, len(toolCalls)) for _, value := range toolCalls { call, _ := value.(map[string]any) ids = append(ids, call["id"].(string)) } return response.Choices[0].Message, ids, nil } func (f *artifactPairFixture) continueWith(results []artifactTestResult, mutate func(any)) (map[string]string, []byte, error) { _, metadata, body, err := f.continueWithResult(results, mutate) return metadata, body, err } func (f *artifactPairFixture) continueWithResult(results []artifactTestResult, mutate func(any)) (presetIngressResult, map[string]string, []byte, error) { f.t.Helper() body := f.continuationBody(results, mutate) ingress, metadata, _, err := f.continueRawResult(body) if err == nil { f.history = artifactMessagesFromBody(f.t, body) } return ingress, metadata, body, err } func (f *artifactPairFixture) continueRaw(body []byte) (map[string]string, []byte, error) { _, metadata, rawBody, err := f.continueRawResult(body) return metadata, rawBody, err } func (f *artifactPairFixture) continueRawResult(body []byte) (presetIngressResult, map[string]string, []byte, error) { metadata := map[string]string{principalMetaRef: f.principalRef} var ingress presetIngressResult var err error if f.endpoint == "anthropic" { ingress, err = f.server.joinPresetAnthropicIngress(nil, f.dispatch, body, metadata) } else { ingress, err = f.server.joinPresetChatIngress(nil, f.dispatch, body, metadata) } return ingress, metadata, body, err } func (f *artifactPairFixture) continuationBody(results []artifactTestResult, mutate func(any)) []byte { f.t.Helper() history := cloneArtifactJSON[[]any](f.t, f.history) if mutate != nil { mutate(history[len(history)-1]) } if f.endpoint == "anthropic" { blocks := make([]any, 0, len(results)) for _, result := range results { block := map[string]any{"type": "tool_result", "tool_use_id": result.id, "content": result.body} if result.failed { block["is_error"] = true } blocks = append(blocks, block) } history = append(history, map[string]any{"role": "user", "content": blocks}) } else { for _, result := range results { content := result.body if result.failed { content = `{"error":{"message":"failed"}}` } history = append(history, map[string]any{"role": "tool", "tool_call_id": result.id, "content": content}) } } return artifactRequestBody(f.t, f.endpoint, f.tools, history) } func mutateArtifactAssistantIDs(assistant any) { message, _ := assistant.(map[string]any) if blocks, ok := message["content"].([]any); ok { index := 0 for _, value := range blocks { block, _ := value.(map[string]any) if block["type"] == "tool_use" { if index == 0 { block["id"] = "call_alternate_plan" } else { block["id"] = "call_alternate_review" } index++ } } return } toolCalls, _ := message["tool_calls"].([]any) for index, value := range toolCalls { call, _ := value.(map[string]any) if index == 0 { call["id"] = "call_alternate_plan" } else { call["id"] = "call_alternate_review" } } } func artifactRequestBody(t *testing.T, endpoint string, tools, history []any) []byte { t.Helper() envelope := map[string]any{"model": "virtual-artifact", "messages": history, "tools": tools} if endpoint == "anthropic" { envelope["max_tokens"] = 64 } body, err := json.Marshal(envelope) if err != nil { t.Fatalf("marshal artifact request: %v", err) } return body } func artifactMessagesFromBody(t *testing.T, body []byte) []any { t.Helper() var envelope struct { Messages []any `json:"messages"` } if err := json.Unmarshal(body, &envelope); err != nil { t.Fatalf("decode artifact messages: %v", err) } return envelope.Messages } func cloneArtifactJSON[T any](t *testing.T, value any) T { t.Helper() raw, err := json.Marshal(value) if err != nil { t.Fatalf("marshal cloned artifact JSON: %v", err) } var out T if err := json.Unmarshal(raw, &out); err != nil { t.Fatalf("unmarshal cloned artifact JSON: %v", err) } return out } func (f *artifactPairFixture) assertPendingPayloads(ids, wantPaths []string) { f.t.Helper() f.server.artifactFrontiers.mu.Lock() defer f.server.artifactFrontiers.mu.Unlock() record := f.server.artifactFrontiers.records[f.requestID] if record == nil || len(record.pending) != len(ids) { f.t.Fatalf("pending frontier = %#v", record) } for index, id := range ids { payload := record.pending[id] if payload == nil || payload.safePath != wantPaths[index] { f.t.Fatalf("payload[%q] = %#v, want path %q", id, payload, wantPaths[index]) } if payload.publicCallID != id || payload.providerCallID == "" || payload.providerCallID == id { f.t.Fatalf("payload identities are not public/provider correlated: %#v", payload) } if payload.fingerprint != record.binding.bindingFingerprint() || payload.correlationDigest == "" { f.t.Fatalf("payload is not sealed to pinned binding: %#v", payload) } } } func (f *artifactPairFixture) assertPhase(want artifactFrontierPhase) { f.t.Helper() f.server.artifactFrontiers.mu.Lock() defer f.server.artifactFrontiers.mu.Unlock() record := f.server.artifactFrontiers.records[f.requestID] if record == nil || record.phase != want { f.t.Fatalf("artifact phase = %#v, want %q", record, want) } } func (f *artifactPairFixture) stateSignature() string { f.t.Helper() snap, err := f.server.requestCoordinator.snapshot(f.requestID) if err != nil { f.t.Fatalf("snapshot artifact coordinator: %v", err) } f.server.artifactFrontiers.mu.Lock() defer f.server.artifactFrontiers.mu.Unlock() record := f.server.artifactFrontiers.records[f.requestID] if record == nil { return "missing" } sort.Strings(snap.ExpectedCallIDs) return fmt.Sprintf("%s|%s|%s|%d|%s|%v", snap.State, snap.ActiveStageID, record.phase, len(record.pending), record.pendingHash, snap.ExpectedCallIDs) } func TestArtifactPairFailureCleanupKeepsMalformedFailClosed(t *testing.T) { for _, endpoint := range []string{"openai", "anthropic"} { endpoint := endpoint t.Run(endpoint+" exact failure", func(t *testing.T) { fixture := newScriptedLightFixture(t, endpoint, false) prepare := fixture.request() fixture.consumeToolResponse(prepare, []string{`{"written":true}`}) pair := fixture.request() fixture.consumeToolResponse(pair, []string{`{"written":true}`, `{"error":"write-failed"}`}) cleanup := fixture.request() if cleanup.Code != http.StatusOK || !strings.Contains(cleanup.Body.String(), "delete_file") { t.Fatalf("exact failure cleanup: status=%d body=%s", cleanup.Code, cleanup.Body.String()) } }) t.Run(endpoint+" malformed result", func(t *testing.T) { fixture := newScriptedLightFixture(t, endpoint, false) prepare := fixture.request() fixture.consumeToolResponse(prepare, []string{`{"written":true}`}) pair := fixture.request() fixture.consumeToolResponse(pair, []string{`{"written":true}`, `not-json`}) response := fixture.request() if response.Code != http.StatusBadRequest || strings.Contains(response.Body.String(), "delete_file") { t.Fatalf("malformed result response: status=%d body=%s", response.Code, response.Body.String()) } if got := len(fixture.service.snapshots()); got != 2 { t.Fatalf("malformed result dispatched provider calls=%d, want 2", got) } }) t.Run(endpoint+" empty result", func(t *testing.T) { fixture := newScriptedLightFixture(t, endpoint, false) prepare := fixture.request() fixture.consumeToolResponse(prepare, []string{`{"written":true}`}) pair := fixture.request() fixture.consumeToolResponse(pair, []string{`{"written":true}`, ``}) response := fixture.request() if response.Code != http.StatusBadRequest || strings.Contains(response.Body.String(), "delete_file") { t.Fatalf("empty result response: status=%d body=%s", response.Code, response.Body.String()) } if got := len(fixture.service.snapshots()); got != 2 { t.Fatalf("empty result dispatched provider calls=%d, want 2", got) } }) } }