toki
d7a150c7fe
Claude의 단일 Anthropic 요청 안에서 IOP가 Plan, Work, Review와 workspace 도구 실행을 끝내고 실제 dev smoke로 계약을 검증할 수 있어야 한다.\n\n완료 task evidence와 마일스톤 검토 상태도 같은 변경에 고정한다.
490 lines
26 KiB
Go
490 lines
26 KiB
Go
package openai
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"net/http"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
edgeservice "iop/apps/edge/internal/service"
|
|
iop "iop/proto/gen/iop"
|
|
)
|
|
|
|
type qualityGateController struct {
|
|
mu sync.Mutex
|
|
envelopes []edgeservice.SingleRequestEnvelope
|
|
}
|
|
|
|
func (*qualityGateController) RequestID() string { return "quality-request" }
|
|
func (*qualityGateController) Binding() *edgeservice.SingleRequestBinding {
|
|
return nil
|
|
}
|
|
func (*qualityGateController) Context() context.Context { return context.Background() }
|
|
func (*qualityGateController) State() edgeservice.SingleRequestState {
|
|
return edgeservice.SingleRequestStatePlanning
|
|
}
|
|
func (*qualityGateController) ReadInternalArtifact(context.Context, edgeservice.SingleRequestArtifactKind) ([]byte, error) {
|
|
return nil, errors.New("unused")
|
|
}
|
|
func (*qualityGateController) WriteInternalArtifact(context.Context, edgeservice.SingleRequestArtifactKind, []byte) error {
|
|
return errors.New("unused")
|
|
}
|
|
func (c *qualityGateController) SubmitEnvelope(envelope edgeservice.SingleRequestEnvelope) error {
|
|
c.mu.Lock()
|
|
c.envelopes = append(c.envelopes, envelope)
|
|
c.mu.Unlock()
|
|
return nil
|
|
}
|
|
|
|
func TestSingleRequestQualityGateTerminalMatrix(t *testing.T) {
|
|
cancelledCtx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
timedOutCtx, timeoutCancel := context.WithDeadline(context.Background(), time.Now().Add(-time.Second))
|
|
defer timeoutCancel()
|
|
|
|
tests := []struct {
|
|
name string
|
|
err func(*singleRequestQualityGate) error
|
|
want edgeservice.SingleRequestTerminalDisposition
|
|
}{
|
|
{name: "provider", err: func(g *singleRequestQualityGate) error {
|
|
return g.providerFailure(context.Background(), errors.New("private provider detail"), errProviderStageGeneric)
|
|
}, want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorProvider}},
|
|
{name: "provider timeout", err: func(g *singleRequestQualityGate) error {
|
|
return g.providerFailure(timedOutCtx, context.DeadlineExceeded, errProviderStageGeneric)
|
|
}, want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorTimeout}},
|
|
{name: "stage budget", err: func(g *singleRequestQualityGate) error { return g.budget(errSingleRequestWorkStage) }, want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorBudget}},
|
|
{name: "malformed call", err: func(g *singleRequestQualityGate) error { return g.malformed(errSingleRequestWorkStage) }, want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorMalformed}},
|
|
{name: "context limit", err: func(g *singleRequestQualityGate) error { return g.contextLimit(errSingleRequestPlanStage) }, want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorContext}},
|
|
{name: "output limit", err: func(g *singleRequestQualityGate) error { return g.length(errSingleRequestReviewStage) }, want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalLength}},
|
|
{name: "caller cancel", err: func(g *singleRequestQualityGate) error {
|
|
return g.providerFailure(cancelledCtx, context.Canceled, errProviderStageGeneric)
|
|
}, want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalCancelled}},
|
|
{name: "tool failure", err: func(g *singleRequestQualityGate) error { return g.internalTool(errSingleRequestWorkStage) }, want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorInternalTool}},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
gate := newSingleRequestQualityGate()
|
|
stageErr := test.err(gate)
|
|
got, ok := singleRequestTerminalDisposition(stageErr)
|
|
if !ok || got != test.want || got.Validate() != nil {
|
|
t.Fatalf("disposition=(%+v, %v), want %+v", got, ok, test.want)
|
|
}
|
|
controller := &qualityGateController{}
|
|
sequence := &singleRequestSequenceController{SingleRequestController: controller}
|
|
if err := submitSingleRequestClosedTerminal(context.Background(), "quality-request", sequence, stageErr); err != nil {
|
|
t.Fatalf("submit terminal: %v", err)
|
|
}
|
|
if len(controller.envelopes) != 1 {
|
|
t.Fatalf("terminal envelopes=%d, want 1", len(controller.envelopes))
|
|
}
|
|
envelope := controller.envelopes[0]
|
|
switch test.want.Kind {
|
|
case edgeservice.SingleRequestTerminalLength:
|
|
if envelope.Stage != edgeservice.SingleRequestStateFinalizing || envelope.Result == nil || envelope.Result.Output != "" || envelope.Result.Terminal != test.want {
|
|
t.Fatalf("length envelope=%+v", envelope)
|
|
}
|
|
case edgeservice.SingleRequestTerminalCancelled:
|
|
if envelope.Stage != edgeservice.SingleRequestStateCancelled || envelope.Terminal == nil || *envelope.Terminal != test.want {
|
|
t.Fatalf("cancel envelope=%+v", envelope)
|
|
}
|
|
default:
|
|
if envelope.Stage != edgeservice.SingleRequestStateFailed || envelope.Terminal == nil || *envelope.Terminal != test.want || !errors.Is(envelope.Err, edgeservice.ErrSingleRequestFailed) {
|
|
t.Fatalf("error envelope=%+v", envelope)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSingleRequestQualityGateCancellationOwnership(t *testing.T) {
|
|
cancelledCtx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
|
|
tests := []struct {
|
|
name string
|
|
err func(*singleRequestQualityGate) error
|
|
want edgeservice.SingleRequestTerminalDisposition
|
|
}{
|
|
{
|
|
name: "provider raw cancellation with live context",
|
|
err: func(g *singleRequestQualityGate) error {
|
|
return g.providerFailure(context.Background(), context.Canceled, errProviderStageGeneric)
|
|
},
|
|
want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorProvider},
|
|
},
|
|
{
|
|
name: "provider cancellation with cancelled context",
|
|
err: func(g *singleRequestQualityGate) error {
|
|
return g.providerFailure(cancelledCtx, context.Canceled, errProviderStageGeneric)
|
|
},
|
|
want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalCancelled},
|
|
},
|
|
{
|
|
name: "service raw cancellation with live context",
|
|
err: func(g *singleRequestQualityGate) error {
|
|
return g.serviceFailure(context.Background(), context.Canceled, errSingleRequestWorkStage)
|
|
},
|
|
want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorInternalTool},
|
|
},
|
|
{
|
|
name: "service cancellation with cancelled context",
|
|
err: func(g *singleRequestQualityGate) error {
|
|
return g.serviceFailure(cancelledCtx, context.Canceled, errSingleRequestWorkStage)
|
|
},
|
|
want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalCancelled},
|
|
},
|
|
{
|
|
name: "service owned cancellation sentinel",
|
|
err: func(g *singleRequestQualityGate) error {
|
|
return g.serviceFailure(context.Background(), edgeservice.ErrSingleRequestCancelled, errSingleRequestWorkStage)
|
|
},
|
|
want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalCancelled},
|
|
},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
got, ok := singleRequestTerminalDisposition(test.err(newSingleRequestQualityGate()))
|
|
if !ok || got != test.want {
|
|
t.Fatalf("disposition=(%+v, %v), want %+v", got, ok, test.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSingleRequestQualityGateProviderHTTPStatusClassification(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
status int32
|
|
want edgeservice.SingleRequestTerminalDisposition
|
|
}{
|
|
{name: "400 remains provider", status: http.StatusBadRequest, want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorProvider}},
|
|
{name: "413 is context", status: http.StatusRequestEntityTooLarge, want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorContext}},
|
|
{name: "502 remains provider", status: http.StatusBadGateway, want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorProvider}},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
frames := make(chan *iop.ProviderTunnelFrame, 2)
|
|
frames <- &iop.ProviderTunnelFrame{
|
|
Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START,
|
|
StatusCode: test.status,
|
|
}
|
|
frames <- &iop.ProviderTunnelFrame{
|
|
Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY,
|
|
Body: []byte("PRIVATE_UPSTREAM_BODY"),
|
|
}
|
|
close(frames)
|
|
|
|
body, providerErr := collectProviderStageFrames(context.Background(), frames, 1024)
|
|
if providerErr == nil || len(body) != 0 {
|
|
t.Fatalf("collect=(%q, %v), want empty body and status failure", body, providerErr)
|
|
}
|
|
stageErr := newSingleRequestQualityGate().providerFailure(context.Background(), providerErr, errProviderStageGeneric)
|
|
got, ok := singleRequestTerminalDisposition(stageErr)
|
|
if !ok || got != test.want || strings.Contains(stageErr.Error(), "PRIVATE_UPSTREAM_BODY") {
|
|
t.Fatalf("disposition=(%+v, %v) error=%q, want %+v without private body", got, ok, stageErr, test.want)
|
|
}
|
|
|
|
controller := &qualityGateController{}
|
|
if err := submitSingleRequestClosedTerminal(context.Background(), "quality-request", controller, stageErr); err != nil {
|
|
t.Fatalf("submit terminal: %v", err)
|
|
}
|
|
if len(controller.envelopes) != 1 || controller.envelopes[0].Terminal == nil || *controller.envelopes[0].Terminal != test.want {
|
|
t.Fatalf("envelopes=%+v, want one closed %+v terminal", controller.envelopes, test.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSingleRequestQualityGateProviderCodecClassification(t *testing.T) {
|
|
finishBody := func(reason string) []byte {
|
|
return []byte(`{"id":"id","object":"chat.completion","created":1,"model":"model","choices":[{"index":0,"finish_reason":"` + reason + `","message":{"role":"assistant","content":"private partial"}}]}`)
|
|
}
|
|
tests := []struct {
|
|
name string
|
|
body []byte
|
|
want edgeservice.SingleRequestTerminalDisposition
|
|
}{
|
|
{name: "length", body: finishBody("length"), want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalLength}},
|
|
{name: "context", body: finishBody("context_length_exceeded"), want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorContext}},
|
|
{name: "malformed", body: []byte(`{"private":"value"}`), want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorMalformed}},
|
|
}
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
_, codecErr := decodeSingleRequestChatResponse(test.body, edgeservice.RunDispatch{})
|
|
if codecErr == nil {
|
|
t.Fatal("codec unexpectedly accepted terminal fixture")
|
|
}
|
|
stageErr := newSingleRequestQualityGate().providerFailure(context.Background(), codecErr, errProviderStageGeneric)
|
|
got, ok := singleRequestTerminalDisposition(stageErr)
|
|
if !ok || got != test.want {
|
|
t.Fatalf("disposition=(%+v, %v), want %+v", got, ok, test.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSingleRequestQualityGateProviderTerminalStopsBeforeLaterDispatch(t *testing.T) {
|
|
finishBody := func(reason string) []byte {
|
|
return []byte(`{"id":"id","object":"chat.completion","created":1,"model":"model","choices":[{"index":0,"finish_reason":"` + reason + `","message":{"role":"assistant","content":"PRIVATE_PARTIAL"}}]}`)
|
|
}
|
|
tests := []struct {
|
|
name string
|
|
body []byte
|
|
dispatchErr error
|
|
want edgeservice.SingleRequestTerminalDisposition
|
|
}{
|
|
{name: "provider", dispatchErr: errors.New("PRIVATE_PROVIDER_ERROR"), want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorProvider}},
|
|
{name: "timeout", dispatchErr: context.DeadlineExceeded, want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorTimeout}},
|
|
{name: "length", body: finishBody("length"), want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalLength}},
|
|
{name: "context", body: finishBody("context_length_exceeded"), want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorContext}},
|
|
{name: "malformed", body: []byte(`{"private":"provider payload"}`), want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorMalformed}},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
var providerCalls atomic.Int32
|
|
mockSvc := &mockService{submit: func(_ context.Context, _ edgeservice.ProviderPoolDispatchRequest) (*edgeservice.ProviderPoolDispatchResult, error) {
|
|
providerCalls.Add(1)
|
|
if test.dispatchErr != nil {
|
|
return nil, test.dispatchErr
|
|
}
|
|
return &edgeservice.ProviderPoolDispatchResult{Path: edgeservice.ProviderPoolPathTunnel, Tunnel: &mockTunnel{frames: framesFor(test.body)}, DispatchInfo: matchingDispatch()}, nil
|
|
}}
|
|
executor := NewSingleRequestExecutor(mockSvc)
|
|
service, binding, node := newTestServiceHarness(t, executor)
|
|
execution, err := service.StartSingleRequest(context.Background(), edgeservice.SingleRequestRequest{RequestID: "quality-provider-" + test.name, Binding: binding, Prompt: "provider terminal"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var terminal edgeservice.SingleRequestTerminalDisposition
|
|
terminalCount := 0
|
|
for progress := range execution.Progress() {
|
|
if progress.Terminal != nil {
|
|
terminal = *progress.Terminal
|
|
terminalCount++
|
|
}
|
|
if progress.Stage == edgeservice.SingleRequestStateFinalizing {
|
|
if err := execution.AcknowledgeTerminal(true); err != nil {
|
|
t.Fatalf("acknowledge length: %v", err)
|
|
}
|
|
}
|
|
}
|
|
result, waitErr := execution.Wait()
|
|
if terminal != test.want || terminalCount != 1 {
|
|
t.Fatalf("terminal=%+v count=%d, want %+v/1", terminal, terminalCount, test.want)
|
|
}
|
|
if test.want.Kind == edgeservice.SingleRequestTerminalLength {
|
|
if waitErr != nil || result.Output != "" || result.Terminal != test.want {
|
|
t.Fatalf("length result=%+v err=%v", result, waitErr)
|
|
}
|
|
} else if waitErr == nil {
|
|
t.Fatal("error terminal returned nil Wait error")
|
|
}
|
|
if providerCalls.Load() != 1 || node.toolCount.Load() != 0 || node.cleanupCount.Load() != 0 || executor.bridge.pendingCount() != 0 {
|
|
t.Fatalf("provider=%d tool=%d cleanup=%d pending=%d", providerCalls.Load(), node.toolCount.Load(), node.cleanupCount.Load(), executor.bridge.pendingCount())
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSingleRequestQualityGateBudgetAndMalformedCallStopComposite(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
responses [][]byte
|
|
maxIterations int
|
|
want edgeservice.SingleRequestTerminalDisposition
|
|
wantProviders int32
|
|
wantTools int32
|
|
}{
|
|
{
|
|
name: "iteration budget",
|
|
responses: [][]byte{
|
|
executorPlanBody("Read bounded inputs", "Stop at the bound"),
|
|
workToolBody("budget-1", edgeservice.InternalWorkspaceToolRead, `{"relative_path":"first.txt"}`),
|
|
workToolBody("budget-2", edgeservice.InternalWorkspaceToolRead, `{"relative_path":"second.txt"}`),
|
|
},
|
|
maxIterations: 1,
|
|
want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorBudget},
|
|
wantProviders: 3,
|
|
wantTools: 1,
|
|
},
|
|
{
|
|
name: "malformed workspace call",
|
|
responses: [][]byte{
|
|
executorPlanBody("Reject an invalid path", "No tool effect"),
|
|
workToolBody("malformed-1", edgeservice.InternalWorkspaceToolRead, `{"relative_path":"../private"}`),
|
|
},
|
|
maxIterations: 4,
|
|
want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorMalformed},
|
|
wantProviders: 2,
|
|
wantTools: 0,
|
|
},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
var providerCalls atomic.Int32
|
|
mockSvc := &mockService{submit: func(_ context.Context, _ edgeservice.ProviderPoolDispatchRequest) (*edgeservice.ProviderPoolDispatchResult, error) {
|
|
index := int(providerCalls.Add(1) - 1)
|
|
if index >= len(test.responses) {
|
|
t.Fatalf("unexpected later provider dispatch %d", index)
|
|
}
|
|
return &edgeservice.ProviderPoolDispatchResult{Path: edgeservice.ProviderPoolPathTunnel, Tunnel: &mockTunnel{frames: framesFor(test.responses[index])}, DispatchInfo: matchingDispatch()}, nil
|
|
}}
|
|
executor := NewSingleRequestExecutor(mockSvc)
|
|
service, binding, node := newTestServiceHarness(t, executor)
|
|
binding.Limits.MaxToolIterations = test.maxIterations
|
|
execution, err := service.StartSingleRequest(context.Background(), edgeservice.SingleRequestRequest{RequestID: "quality-call-" + test.name, Binding: binding, Prompt: "tool terminal"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var terminal edgeservice.SingleRequestTerminalDisposition
|
|
terminalCount := 0
|
|
for progress := range execution.Progress() {
|
|
if progress.Terminal != nil {
|
|
terminal = *progress.Terminal
|
|
terminalCount++
|
|
}
|
|
}
|
|
_, waitErr := execution.Wait()
|
|
if waitErr == nil || terminal != test.want || terminalCount != 1 {
|
|
t.Fatalf("Wait=%v terminal=%+v count=%d, want %+v/1", waitErr, terminal, terminalCount, test.want)
|
|
}
|
|
if providerCalls.Load() != test.wantProviders || node.toolCount.Load() != test.wantTools || node.cleanupCount.Load() != 1 || executor.bridge.pendingCount() != 0 {
|
|
t.Fatalf("provider=%d tool=%d cleanup=%d pending=%d", providerCalls.Load(), node.toolCount.Load(), node.cleanupCount.Load(), executor.bridge.pendingCount())
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSingleRequestQualityGateRepetitionStopsBeforeLaterDispatch(t *testing.T) {
|
|
responses := [][]byte{
|
|
executorPlanBody("Read once", "Verify once"),
|
|
workToolBody("repeat-1", edgeservice.InternalWorkspaceToolRead, `{"relative_path":"result.txt"}`),
|
|
workToolBody("repeat-2", edgeservice.InternalWorkspaceToolRead, `{"relative_path":"result.txt"}`),
|
|
}
|
|
var providerCalls atomic.Int32
|
|
mockSvc := &mockService{submit: func(_ context.Context, _ edgeservice.ProviderPoolDispatchRequest) (*edgeservice.ProviderPoolDispatchResult, error) {
|
|
index := int(providerCalls.Add(1) - 1)
|
|
if index >= len(responses) {
|
|
t.Fatalf("unexpected later provider dispatch %d", index)
|
|
}
|
|
return &edgeservice.ProviderPoolDispatchResult{Path: edgeservice.ProviderPoolPathTunnel, Tunnel: &mockTunnel{frames: framesFor(responses[index])}, DispatchInfo: matchingDispatch()}, nil
|
|
}}
|
|
executor := NewSingleRequestExecutor(mockSvc)
|
|
service, binding, node := newTestServiceHarness(t, executor)
|
|
execution, err := service.StartSingleRequest(context.Background(), edgeservice.SingleRequestRequest{RequestID: "quality-repeat", Binding: binding, Prompt: "repeat guard"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
terminalCount := 0
|
|
var terminal edgeservice.SingleRequestTerminalDisposition
|
|
for progress := range execution.Progress() {
|
|
if progress.Terminal != nil {
|
|
terminalCount++
|
|
terminal = *progress.Terminal
|
|
}
|
|
}
|
|
_, waitErr := execution.Wait()
|
|
want := edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorRepetition}
|
|
if !errors.Is(waitErr, edgeservice.ErrSingleRequestFailed) || terminal != want {
|
|
t.Fatalf("Wait=%v terminal=%+v, want repetition", waitErr, terminal)
|
|
}
|
|
if providerCalls.Load() != 3 || node.toolCount.Load() != 2 || node.cleanupCount.Load() != 1 || executor.bridge.pendingCount() != 0 || terminalCount != 1 {
|
|
t.Fatalf("provider=%d tool=%d cleanup=%d pending=%d terminals=%d", providerCalls.Load(), node.toolCount.Load(), node.cleanupCount.Load(), executor.bridge.pendingCount(), terminalCount)
|
|
}
|
|
}
|
|
|
|
func TestSingleRequestQualityGateAdmitsOnlyRepairableToolResults(t *testing.T) {
|
|
arguments := json.RawMessage(`{"relative_path":"result.txt"}`)
|
|
cause := errSingleRequestReviewStage
|
|
|
|
gate := newSingleRequestQualityGate()
|
|
notFound := edgeservice.InternalWorkspaceToolResult{Status: "error", ErrorCode: "not_found"}
|
|
if err := gate.observeToolCycle(singleRequestReviewStageID, edgeservice.InternalWorkspaceToolRead, arguments, notFound, cause); err != nil {
|
|
t.Fatalf("first not_found rejected: %v", err)
|
|
}
|
|
repeated := gate.observeToolCycle(singleRequestReviewStageID, edgeservice.InternalWorkspaceToolRead, arguments, notFound, cause)
|
|
wantRepeat := edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorRepetition}
|
|
if got, ok := singleRequestTerminalDisposition(repeated); !ok || got != wantRepeat {
|
|
t.Fatalf("repeated not_found disposition=(%+v,%v), want %+v", got, ok, wantRepeat)
|
|
}
|
|
|
|
if err := newSingleRequestQualityGate().observeToolCycle(singleRequestWorkStageID, edgeservice.InternalWorkspaceToolRead, arguments, edgeservice.InternalWorkspaceToolResult{Status: "success"}, cause); err != nil {
|
|
t.Fatalf("success rejected: %v", err)
|
|
}
|
|
|
|
tests := []struct {
|
|
name string
|
|
result edgeservice.InternalWorkspaceToolResult
|
|
want edgeservice.SingleRequestTerminalDisposition
|
|
}{
|
|
{name: "success with error code", result: edgeservice.InternalWorkspaceToolResult{Status: "success", ErrorCode: "not_found"}, want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorInternalTool}},
|
|
{name: "error without code", result: edgeservice.InternalWorkspaceToolResult{Status: "error"}, want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorInternalTool}},
|
|
{name: "internal error", result: edgeservice.InternalWorkspaceToolResult{Status: "error", ErrorCode: "internal"}, want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorInternalTool}},
|
|
{name: "timeout", result: edgeservice.InternalWorkspaceToolResult{Status: "timeout", ErrorCode: "timeout"}, want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorTimeout}},
|
|
{name: "invalid request", result: edgeservice.InternalWorkspaceToolResult{Status: "invalid", ErrorCode: "invalid_request"}, want: edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorMalformed}},
|
|
}
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
err := newSingleRequestQualityGate().observeToolCycle(singleRequestReviewStageID, edgeservice.InternalWorkspaceToolRead, arguments, test.result, cause)
|
|
got, ok := singleRequestTerminalDisposition(err)
|
|
if !ok || got != test.want {
|
|
t.Fatalf("disposition=(%+v,%v), want %+v", got, ok, test.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSingleRequestQualityGateToolTimeoutStopsBeforeContinuation(t *testing.T) {
|
|
responses := [][]byte{
|
|
executorPlanBody("Run command", "Verify command"),
|
|
workToolBody("timeout-1", edgeservice.InternalWorkspaceToolCommand, `{"command_id":"verify"}`),
|
|
}
|
|
var providerCalls atomic.Int32
|
|
mockSvc := &mockService{submit: func(_ context.Context, _ edgeservice.ProviderPoolDispatchRequest) (*edgeservice.ProviderPoolDispatchResult, error) {
|
|
index := int(providerCalls.Add(1) - 1)
|
|
if index >= len(responses) {
|
|
t.Fatalf("unexpected later provider dispatch %d", index)
|
|
}
|
|
return &edgeservice.ProviderPoolDispatchResult{Path: edgeservice.ProviderPoolPathTunnel, Tunnel: &mockTunnel{frames: framesFor(responses[index])}, DispatchInfo: matchingDispatch()}, nil
|
|
}}
|
|
executor := NewSingleRequestExecutor(mockSvc)
|
|
service, binding, node := newTestServiceHarness(t, executor)
|
|
node.toolResponder = func(request *iop.WorkspaceToolRequest) *iop.WorkspaceToolResponse {
|
|
return &iop.WorkspaceToolResponse{
|
|
RequestId: request.GetRequestId(), StageId: request.GetStageId(), ToolCallId: request.GetToolCallId(),
|
|
Status: iop.WorkspaceStatus_WORKSPACE_STATUS_TIMEOUT, ErrorCode: iop.WorkspaceErrorCode_WORKSPACE_ERROR_CODE_TIMEOUT,
|
|
Error: "workspace command timed out", Stderr: []byte("private timeout detail"), ExitCode: -1,
|
|
}
|
|
}
|
|
execution, err := service.StartSingleRequest(context.Background(), edgeservice.SingleRequestRequest{RequestID: "quality-timeout", Binding: binding, Prompt: "timeout guard"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var terminal edgeservice.SingleRequestTerminalDisposition
|
|
terminalCount := 0
|
|
for progress := range execution.Progress() {
|
|
if progress.Terminal != nil {
|
|
terminal = *progress.Terminal
|
|
terminalCount++
|
|
}
|
|
}
|
|
_, waitErr := execution.Wait()
|
|
want := edgeservice.SingleRequestTerminalDisposition{Kind: edgeservice.SingleRequestTerminalError, ErrorClass: edgeservice.SingleRequestTerminalErrorTimeout}
|
|
if !errors.Is(waitErr, edgeservice.ErrSingleRequestInternalToolFailed) || terminal != want {
|
|
t.Fatalf("Wait=%v terminal=%+v, want timeout", waitErr, terminal)
|
|
}
|
|
if providerCalls.Load() != 2 || node.toolCount.Load() != 1 || node.cleanupCount.Load() != 1 || executor.bridge.pendingCount() != 0 || terminalCount != 1 {
|
|
t.Fatalf("provider=%d tool=%d cleanup=%d pending=%d terminals=%d", providerCalls.Load(), node.toolCount.Load(), node.cleanupCount.Load(), executor.bridge.pendingCount(), terminalCount)
|
|
}
|
|
}
|