package openai import ( "context" "encoding/json" "errors" "net/http" "strings" "sync" "sync/atomic" "testing" "time" edgeservice "iop/apps/edge/internal/service" iop "iop/proto/gen/iop" ) type qualityGateController struct { mu sync.Mutex envelopes []edgeservice.SingleRequestEnvelope } func (*qualityGateController) RequestID() string { return "quality-request" } func (*qualityGateController) Binding() *edgeservice.SingleRequestBinding { return nil } func (*qualityGateController) Context() context.Context { return context.Background() } func (*qualityGateController) State() edgeservice.SingleRequestState { return edgeservice.SingleRequestStatePlanning } func (*qualityGateController) ReadInternalArtifact(context.Context, edgeservice.SingleRequestArtifactKind) ([]byte, error) { return nil, errors.New("unused") } func (*qualityGateController) WriteInternalArtifact(context.Context, edgeservice.SingleRequestArtifactKind, []byte) error { return errors.New("unused") } func (c *qualityGateController) SubmitEnvelope(envelope edgeservice.SingleRequestEnvelope) error { c.mu.Lock() c.envelopes = append(c.envelopes, envelope) c.mu.Unlock() return nil } func TestSingleRequestQualityGateTerminalMatrix(t *testing.T) { cancelledCtx, cancel := context.WithCancel(context.Background()) cancel() timedOutCtx, timeoutCancel := context.WithDeadline(context.Background(), time.Now().Add(-time.Second)) defer timeoutCancel() tests := []struct { name string err func(*singleRequestQualityGate) error want edgeservice.SingleRequestTerminalDisposition }{ {name: "provider", err: func(g *singleRequestQualityGate) error { return g.providerFailure(context.Background(), errors.New("private provider detail"), errProviderStageGeneric) }, want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorProvider}}, {name: "provider timeout", err: func(g *singleRequestQualityGate) error { return g.providerFailure(timedOutCtx, context.DeadlineExceeded, errProviderStageGeneric) }, want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorTimeout}}, {name: "stage budget", err: func(g *singleRequestQualityGate) error { return g.budget(errSingleRequestWorkStage) }, want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorBudget}}, {name: "malformed call", err: func(g *singleRequestQualityGate) error { return g.malformed(errSingleRequestWorkStage) }, want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorMalformed}}, {name: "context limit", err: func(g *singleRequestQualityGate) error { return g.contextLimit(errSingleRequestPlanStage) }, want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorContext}}, {name: "output limit", err: func(g *singleRequestQualityGate) error { return g.length(errSingleRequestReviewStage) }, want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalLength}}, {name: "caller cancel", err: func(g *singleRequestQualityGate) error { return g.providerFailure(cancelledCtx, context.Canceled, errProviderStageGeneric) }, want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalCancelled}}, {name: "tool failure", err: func(g *singleRequestQualityGate) error { return g.internalTool(errSingleRequestWorkStage) }, want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorInternalTool}}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { gate := newSingleRequestQualityGate() stageErr := test.err(gate) got, ok := singleRequestTerminalDisposition(stageErr) if !ok || got != test.want || got.Validate() != nil { t.Fatalf("disposition=(%+v, %v), want %+v", got, ok, test.want) } controller := &qualityGateController{} sequence := &singleRequestSequenceController{SingleRequestController: controller} if err := submitSingleRequestClosedTerminal(context.Background(), "quality-request", sequence, stageErr); err != nil { t.Fatalf("submit terminal: %v", err) } if len(controller.envelopes) != 1 { t.Fatalf("terminal envelopes=%d, want 1", len(controller.envelopes)) } envelope := controller.envelopes[0] switch test.want.Kind { case edgeservice.SingleRequestTerminalLength: if envelope.Stage != edgeservice.SingleRequestStateFinalizing || envelope.Result == nil || envelope.Result.Output != "" || envelope.Result.Terminal != test.want { t.Fatalf("length envelope=%+v", envelope) } case edgeservice.SingleRequestTerminalCancelled: if envelope.Stage != edgeservice.SingleRequestStateCancelled || envelope.Terminal == nil || *envelope.Terminal != test.want { t.Fatalf("cancel envelope=%+v", envelope) } default: if envelope.Stage != edgeservice.SingleRequestStateFailed || envelope.Terminal == nil || *envelope.Terminal != test.want || !errors.Is(envelope.Err, edgeservice.ErrSingleRequestFailed) { t.Fatalf("error envelope=%+v", envelope) } } }) } } func TestSingleRequestQualityGateCancellationOwnership(t *testing.T) { cancelledCtx, cancel := context.WithCancel(context.Background()) cancel() tests := []struct { name string err func(*singleRequestQualityGate) error want edgeservice.SingleRequestTerminalDisposition }{ { name: "provider raw cancellation with live context", err: func(g *singleRequestQualityGate) error { return g.providerFailure(context.Background(), context.Canceled, errProviderStageGeneric) }, want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorProvider}, }, { name: "provider cancellation with cancelled context", err: func(g *singleRequestQualityGate) error { return g.providerFailure(cancelledCtx, context.Canceled, errProviderStageGeneric) }, want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalCancelled}, }, { name: "service raw cancellation with live context", err: func(g *singleRequestQualityGate) error { return g.serviceFailure(context.Background(), context.Canceled, errSingleRequestWorkStage) }, want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorInternalTool}, }, { name: "service cancellation with cancelled context", err: func(g *singleRequestQualityGate) error { return g.serviceFailure(cancelledCtx, context.Canceled, errSingleRequestWorkStage) }, want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalCancelled}, }, { name: "service owned cancellation sentinel", err: func(g *singleRequestQualityGate) error { return g.serviceFailure(context.Background(), edgeservice.ErrSingleRequestCancelled, errSingleRequestWorkStage) }, want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalCancelled}, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { got, ok := singleRequestTerminalDisposition(test.err(newSingleRequestQualityGate())) if !ok || got != test.want { t.Fatalf("disposition=(%+v, %v), want %+v", got, ok, test.want) } }) } } func TestSingleRequestQualityGateProviderHTTPStatusClassification(t *testing.T) { tests := []struct { name string status int32 want edgeservice.SingleRequestTerminalDisposition }{ {name: "400 remains provider", status: http.StatusBadRequest, want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorProvider}}, {name: "413 is context", status: http.StatusRequestEntityTooLarge, want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorContext}}, {name: "502 remains provider", status: http.StatusBadGateway, want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorProvider}}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { frames := make(chan *iop.ProviderTunnelFrame, 2) frames <- &iop.ProviderTunnelFrame{ Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START, StatusCode: test.status, } frames <- &iop.ProviderTunnelFrame{ Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte("PRIVATE_UPSTREAM_BODY"), } close(frames) body, providerErr := collectProviderStageFrames(context.Background(), frames, 1024) if providerErr == nil || len(body) != 0 { t.Fatalf("collect=(%q, %v), want empty body and status failure", body, providerErr) } stageErr := newSingleRequestQualityGate().providerFailure(context.Background(), providerErr, errProviderStageGeneric) got, ok := singleRequestTerminalDisposition(stageErr) if !ok || got != test.want || strings.Contains(stageErr.Error(), "PRIVATE_UPSTREAM_BODY") { t.Fatalf("disposition=(%+v, %v) error=%q, want %+v without private body", got, ok, stageErr, test.want) } controller := &qualityGateController{} if err := submitSingleRequestClosedTerminal(context.Background(), "quality-request", controller, stageErr); err != nil { t.Fatalf("submit terminal: %v", err) } if len(controller.envelopes) != 1 || controller.envelopes[0].Terminal == nil || *controller.envelopes[0].Terminal != test.want { t.Fatalf("envelopes=%+v, want one closed %+v terminal", controller.envelopes, test.want) } }) } } func TestSingleRequestQualityGateProviderCodecClassification(t *testing.T) { finishBody := func(reason string) []byte { return []byte(`{"id":"id","object":"chat.completion","created":1,"model":"model","choices":[{"index":0,"finish_reason":"` + reason + `","message":{"role":"assistant","content":"private partial"}}]}`) } tests := []struct { name string body []byte want edgeservice.SingleRequestTerminalDisposition }{ {name: "length", body: finishBody("length"), want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalLength}}, {name: "context", body: finishBody("context_length_exceeded"), want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorContext}}, {name: "malformed", body: []byte(`{"private":"value"}`), want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorMalformed}}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { _, codecErr := decodeSingleRequestChatResponse(test.body, edgeservice.RunDispatch{}) if codecErr == nil { t.Fatal("codec unexpectedly accepted terminal fixture") } stageErr := newSingleRequestQualityGate().providerFailure(context.Background(), codecErr, errProviderStageGeneric) got, ok := singleRequestTerminalDisposition(stageErr) if !ok || got != test.want { t.Fatalf("disposition=(%+v, %v), want %+v", got, ok, test.want) } }) } } func TestSingleRequestQualityGateProviderTerminalStopsBeforeLaterDispatch(t *testing.T) { finishBody := func(reason string) []byte { return []byte(`{"id":"id","object":"chat.completion","created":1,"model":"model","choices":[{"index":0,"finish_reason":"` + reason + `","message":{"role":"assistant","content":"PRIVATE_PARTIAL"}}]}`) } tests := []struct { name string body []byte dispatchErr error want edgeservice.SingleRequestTerminalDisposition }{ {name: "provider", dispatchErr: errors.New("PRIVATE_PROVIDER_ERROR"), want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorProvider}}, {name: "timeout", dispatchErr: context.DeadlineExceeded, want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorTimeout}}, {name: "length", body: finishBody("length"), want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalLength}}, {name: "context", body: finishBody("context_length_exceeded"), want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorContext}}, {name: "malformed", body: []byte(`{"private":"provider payload"}`), want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorMalformed}}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { var providerCalls atomic.Int32 mockSvc := &mockService{submit: func(_ context.Context, _ edgeservice.ProviderPoolDispatchRequest) (*edgeservice.ProviderPoolDispatchResult, error) { providerCalls.Add(1) if test.dispatchErr != nil { return nil, test.dispatchErr } return &edgeservice.ProviderPoolDispatchResult{Path: edgeservice.ProviderPoolPathTunnel, Tunnel: &mockTunnel{frames: framesFor(test.body)}, DispatchInfo: matchingDispatch()}, nil }} executor := NewSingleRequestExecutor(mockSvc) service, binding, node := newTestServiceHarness(t, executor) execution, err := service.StartSingleRequest(context.Background(), edgeservice.SingleRequestRequest{RequestID: "quality-provider-" + test.name, Binding: binding, Prompt: "provider terminal"}) if err != nil { t.Fatal(err) } var terminal edgeservice.SingleRequestTerminalDisposition terminalCount := 0 for progress := range execution.Progress() { if progress.Terminal != nil { terminal = *progress.Terminal terminalCount++ } if progress.Stage == edgeservice.SingleRequestStateFinalizing { if err := execution.AcknowledgeTerminal(true); err != nil { t.Fatalf("acknowledge length: %v", err) } } } result, waitErr := execution.Wait() if terminal != test.want || terminalCount != 1 { t.Fatalf("terminal=%+v count=%d, want %+v/1", terminal, terminalCount, test.want) } if test.want.Kind == edgeservice.SingleRequestTerminalLength { if waitErr != nil || result.Output != "" || result.Terminal != test.want { t.Fatalf("length result=%+v err=%v", result, waitErr) } } else if waitErr == nil { t.Fatal("error terminal returned nil Wait error") } if providerCalls.Load() != 1 || node.toolCount.Load() != 0 || node.cleanupCount.Load() != 0 || executor.bridge.pendingCount() != 0 { t.Fatalf("provider=%d tool=%d cleanup=%d pending=%d", providerCalls.Load(), node.toolCount.Load(), node.cleanupCount.Load(), executor.bridge.pendingCount()) } }) } } func TestSingleRequestQualityGateBudgetAndMalformedCallStopComposite(t *testing.T) { tests := []struct { name string responses [][]byte maxIterations int want edgeservice.SingleRequestTerminalDisposition wantProviders int32 wantTools int32 }{ { name: "iteration budget", responses: [][]byte{ executorPlanBody("Read bounded inputs", "Stop at the bound"), workToolBody("budget-1", edgeservice.InternalWorkspaceToolRead, `{"relative_path":"first.txt"}`), workToolBody("budget-2", edgeservice.InternalWorkspaceToolRead, `{"relative_path":"second.txt"}`), }, maxIterations: 1, want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorBudget}, wantProviders: 3, wantTools: 1, }, { name: "malformed workspace call", responses: [][]byte{ executorPlanBody("Reject an invalid path", "No tool effect"), workToolBody("malformed-1", edgeservice.InternalWorkspaceToolRead, `{"relative_path":"../private"}`), }, maxIterations: 4, want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorMalformed}, wantProviders: 2, wantTools: 0, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { var providerCalls atomic.Int32 mockSvc := &mockService{submit: func(_ context.Context, _ edgeservice.ProviderPoolDispatchRequest) (*edgeservice.ProviderPoolDispatchResult, error) { index := int(providerCalls.Add(1) - 1) if index >= len(test.responses) { t.Fatalf("unexpected later provider dispatch %d", index) } return &edgeservice.ProviderPoolDispatchResult{Path: edgeservice.ProviderPoolPathTunnel, Tunnel: &mockTunnel{frames: framesFor(test.responses[index])}, DispatchInfo: matchingDispatch()}, nil }} executor := NewSingleRequestExecutor(mockSvc) service, binding, node := newTestServiceHarness(t, executor) binding.Limits.MaxToolIterations = test.maxIterations execution, err := service.StartSingleRequest(context.Background(), edgeservice.SingleRequestRequest{RequestID: "quality-call-" + test.name, Binding: binding, Prompt: "tool terminal"}) if err != nil { t.Fatal(err) } var terminal edgeservice.SingleRequestTerminalDisposition terminalCount := 0 for progress := range execution.Progress() { if progress.Terminal != nil { terminal = *progress.Terminal terminalCount++ } } _, waitErr := execution.Wait() if waitErr == nil || terminal != test.want || terminalCount != 1 { t.Fatalf("Wait=%v terminal=%+v count=%d, want %+v/1", waitErr, terminal, terminalCount, test.want) } if providerCalls.Load() != test.wantProviders || node.toolCount.Load() != test.wantTools || node.cleanupCount.Load() != 1 || executor.bridge.pendingCount() != 0 { t.Fatalf("provider=%d tool=%d cleanup=%d pending=%d", providerCalls.Load(), node.toolCount.Load(), node.cleanupCount.Load(), executor.bridge.pendingCount()) } }) } } func TestSingleRequestQualityGateRepetitionStopsBeforeLaterDispatch(t *testing.T) { responses := [][]byte{ executorPlanBody("Read once", "Verify once"), workToolBody("repeat-1", edgeservice.InternalWorkspaceToolRead, `{"relative_path":"result.txt"}`), workToolBody("repeat-2", edgeservice.InternalWorkspaceToolRead, `{"relative_path":"result.txt"}`), } var providerCalls atomic.Int32 mockSvc := &mockService{submit: func(_ context.Context, _ edgeservice.ProviderPoolDispatchRequest) (*edgeservice.ProviderPoolDispatchResult, error) { index := int(providerCalls.Add(1) - 1) if index >= len(responses) { t.Fatalf("unexpected later provider dispatch %d", index) } return &edgeservice.ProviderPoolDispatchResult{Path: edgeservice.ProviderPoolPathTunnel, Tunnel: &mockTunnel{frames: framesFor(responses[index])}, DispatchInfo: matchingDispatch()}, nil }} executor := NewSingleRequestExecutor(mockSvc) service, binding, node := newTestServiceHarness(t, executor) execution, err := service.StartSingleRequest(context.Background(), edgeservice.SingleRequestRequest{RequestID: "quality-repeat", Binding: binding, Prompt: "repeat guard"}) if err != nil { t.Fatal(err) } terminalCount := 0 var terminal edgeservice.SingleRequestTerminalDisposition for progress := range execution.Progress() { if progress.Terminal != nil { terminalCount++ terminal = *progress.Terminal } } _, waitErr := execution.Wait() want := edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorRepetition} if !errors.Is(waitErr, edgeservice.ErrSingleRequestFailed) || terminal != want { t.Fatalf("Wait=%v terminal=%+v, want repetition", waitErr, terminal) } if providerCalls.Load() != 3 || node.toolCount.Load() != 2 || node.cleanupCount.Load() != 1 || executor.bridge.pendingCount() != 0 || terminalCount != 1 { t.Fatalf("provider=%d tool=%d cleanup=%d pending=%d terminals=%d", providerCalls.Load(), node.toolCount.Load(), node.cleanupCount.Load(), executor.bridge.pendingCount(), terminalCount) } } func TestSingleRequestQualityGateAdmitsOnlyRepairableToolResults(t *testing.T) { arguments := json.RawMessage(`{"relative_path":"result.txt"}`) cause := errSingleRequestReviewStage gate := newSingleRequestQualityGate() notFound := edgeservice.InternalWorkspaceToolResult{Status: "error", ErrorCode: "not_found"} if err := gate.observeToolCycle(singleRequestReviewStageID, edgeservice.InternalWorkspaceToolRead, arguments, notFound, cause); err != nil { t.Fatalf("first not_found rejected: %v", err) } repeated := gate.observeToolCycle(singleRequestReviewStageID, edgeservice.InternalWorkspaceToolRead, arguments, notFound, cause) wantRepeat := edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorRepetition} if got, ok := singleRequestTerminalDisposition(repeated); !ok || got != wantRepeat { t.Fatalf("repeated not_found disposition=(%+v,%v), want %+v", got, ok, wantRepeat) } if err := newSingleRequestQualityGate().observeToolCycle(singleRequestWorkStageID, edgeservice.InternalWorkspaceToolRead, arguments, edgeservice.InternalWorkspaceToolResult{Status: "success"}, cause); err != nil { t.Fatalf("success rejected: %v", err) } tests := []struct { name string result edgeservice.InternalWorkspaceToolResult want edgeservice.SingleRequestTerminalDisposition }{ {name: "success with error code", result: edgeservice.InternalWorkspaceToolResult{Status: "success", ErrorCode: "not_found"}, want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorInternalTool}}, {name: "error without code", result: edgeservice.InternalWorkspaceToolResult{Status: "error"}, want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorInternalTool}}, {name: "internal error", result: edgeservice.InternalWorkspaceToolResult{Status: "error", ErrorCode: "internal"}, want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorInternalTool}}, {name: "timeout", result: edgeservice.InternalWorkspaceToolResult{Status: "timeout", ErrorCode: "timeout"}, want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorTimeout}}, {name: "invalid request", result: edgeservice.InternalWorkspaceToolResult{Status: "invalid", ErrorCode: "invalid_request"}, want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorMalformed}}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { err := newSingleRequestQualityGate().observeToolCycle(singleRequestReviewStageID, edgeservice.InternalWorkspaceToolRead, arguments, test.result, cause) got, ok := singleRequestTerminalDisposition(err) if !ok || got != test.want { t.Fatalf("disposition=(%+v,%v), want %+v", got, ok, test.want) } }) } } func TestSingleRequestQualityGateToolTimeoutStopsBeforeContinuation(t *testing.T) { responses := [][]byte{ executorPlanBody("Run command", "Verify command"), workToolBody("timeout-1", edgeservice.InternalWorkspaceToolCommand, `{"command_id":"verify"}`), } var providerCalls atomic.Int32 mockSvc := &mockService{submit: func(_ context.Context, _ edgeservice.ProviderPoolDispatchRequest) (*edgeservice.ProviderPoolDispatchResult, error) { index := int(providerCalls.Add(1) - 1) if index >= len(responses) { t.Fatalf("unexpected later provider dispatch %d", index) } return &edgeservice.ProviderPoolDispatchResult{Path: edgeservice.ProviderPoolPathTunnel, Tunnel: &mockTunnel{frames: framesFor(responses[index])}, DispatchInfo: matchingDispatch()}, nil }} executor := NewSingleRequestExecutor(mockSvc) service, binding, node := newTestServiceHarness(t, executor) node.toolResponder = func(request *iop.WorkspaceToolRequest) *iop.WorkspaceToolResponse { return &iop.WorkspaceToolResponse{ RequestId: request.GetRequestId(), StageId: request.GetStageId(), ToolCallId: request.GetToolCallId(), Status: iop.WorkspaceStatus_WORKSPACE_STATUS_TIMEOUT, ErrorCode: iop.WorkspaceErrorCode_WORKSPACE_ERROR_CODE_TIMEOUT, Error: "workspace command timed out", Stderr: []byte("private timeout detail"), ExitCode: -1, } } execution, err := service.StartSingleRequest(context.Background(), edgeservice.SingleRequestRequest{RequestID: "quality-timeout", Binding: binding, Prompt: "timeout guard"}) if err != nil { t.Fatal(err) } var terminal edgeservice.SingleRequestTerminalDisposition terminalCount := 0 for progress := range execution.Progress() { if progress.Terminal != nil { terminal = *progress.Terminal terminalCount++ } } _, waitErr := execution.Wait() want := edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorTimeout} if !errors.Is(waitErr, edgeservice.ErrSingleRequestInternalToolFailed) || terminal != want { t.Fatalf("Wait=%v terminal=%+v, want timeout", waitErr, terminal) } if providerCalls.Load() != 2 || node.toolCount.Load() != 1 || node.cleanupCount.Load() != 1 || executor.bridge.pendingCount() != 0 || terminalCount != 1 { t.Fatalf("provider=%d tool=%d cleanup=%d pending=%d terminals=%d", providerCalls.Load(), node.toolCount.Load(), node.cleanupCount.Load(), executor.bridge.pendingCount(), terminalCount) } }