iop/apps/edge/internal/openai/hot_path_cleanup_test.go

491 lines
20 KiB
Go

package openai
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"
"time"
edgeservice "iop/apps/edge/internal/service"
)
func nilRequestWithContext(ctx context.Context) *http.Request {
return httptest.NewRequest(http.MethodPost, "/", nil).WithContext(ctx)
}
func TestHotPathCleanupTerminalMatrix(t *testing.T) {
for _, endpoint := range []string{"openai", "anthropic"} {
endpoint := endpoint
t.Run(endpoint+" success waits for exact delete", func(t *testing.T) {
fixture := newScriptedLightFixture(t, endpoint, false)
cleanup := fixture.runToCleanup()
if cleanup.Code != http.StatusOK || !strings.Contains(cleanup.Body.String(), "delete_file") ||
!strings.Contains(cleanup.Body.String(), ".iop/job/") || strings.Contains(cleanup.Body.String(), "review-resolution-visible") {
t.Fatalf("cleanup frontier response: status=%d body=%s", cleanup.Code, cleanup.Body.String())
}
fixture.server.requestCoordinator.mu.Lock()
if len(fixture.server.requestCoordinator.requests) != 1 {
fixture.server.requestCoordinator.mu.Unlock()
t.Fatalf("cleanup coordinator records=%d, want 1", len(fixture.server.requestCoordinator.requests))
}
for _, record := range fixture.server.requestCoordinator.requests {
if record.state != logicalRequestStateCleanup || !record.cleanup || record.terminalClass != "success" {
fixture.server.requestCoordinator.mu.Unlock()
t.Fatalf("cleanup coordinator state=%q cleanup=%t terminal=%q", record.state, record.cleanup, record.terminalClass)
}
}
fixture.server.requestCoordinator.mu.Unlock()
fixture.consumeToolResponse(cleanup, []string{`{"written":true}`})
final := fixture.request()
if final.Code != http.StatusOK || !strings.Contains(final.Body.String(), "review-resolution-visible") {
t.Fatalf("terminal response: status=%d body=%s", final.Code, final.Body.String())
}
fixture.assertCleanupCommitted(7)
})
t.Run(endpoint+" cleanup mismatch cannot become success", func(t *testing.T) {
fixture := newScriptedLightFixture(t, endpoint, false)
cleanup := fixture.runToCleanup()
fixture.consumeToolResponse(cleanup, []string{`{"written":false}`})
final := fixture.request()
if final.Code != http.StatusBadGateway || !strings.Contains(final.Body.String(), "workspace cleanup failed") ||
strings.Contains(final.Body.String(), "review-resolution-visible") {
t.Fatalf("cleanup failure response: status=%d body=%s", final.Code, final.Body.String())
}
fixture.assertCleanupCommitted(7)
})
}
}
func TestHotPathCleanupPrimaryErrorPrecedence(t *testing.T) {
for _, endpoint := range []string{"openai", "anthropic"} {
endpoint := endpoint
for _, frontier := range []struct {
name string
wantProviderCalls int
wantResponseID string
consumePrimaryFail func(*scriptedLightFixture)
}{
{
name: "prepare", wantProviderCalls: 1,
wantResponseID: map[string]string{"openai": "chatcmpl-scripted", "anthropic": "msg-scripted"}[endpoint],
consumePrimaryFail: func(fixture *scriptedLightFixture) {
prepare := fixture.request()
fixture.consumeToolResponse(prepare, []string{`{"error":"prepare-denied"}`})
},
},
{
name: "pair", wantProviderCalls: 2,
wantResponseID: map[string]string{"openai": "chatcmpl-scripted-pair", "anthropic": "msg-scripted-pair"}[endpoint],
consumePrimaryFail: func(fixture *scriptedLightFixture) {
prepare := fixture.request()
fixture.consumeToolResponse(prepare, []string{`{"written":true}`})
pair := fixture.request()
fixture.consumeToolResponse(pair, []string{`{"written":true}`, `{"error":"pair-denied"}`})
},
},
{
// A partial pair whose Plan write matches but whose Review
// result only fails the configured receipt matcher (no explicit
// error signal) must still author the same canonical delete
// frontier so a possible sibling artifact cannot leak.
name: "pair-matcher-failure", wantProviderCalls: 2,
wantResponseID: map[string]string{"openai": "chatcmpl-scripted-pair", "anthropic": "msg-scripted-pair"}[endpoint],
consumePrimaryFail: func(fixture *scriptedLightFixture) {
prepare := fixture.request()
fixture.consumeToolResponse(prepare, []string{`{"written":true}`})
pair := fixture.request()
fixture.consumeToolResponse(pair, []string{`{"written":true}`, `{"written":false}`})
},
},
} {
frontier := frontier
for _, cleanupReceipt := range []struct {
name string
body string
}{
{name: "acknowledged", body: `{"written":true}`},
{name: "acknowledgement-failed", body: `{"written":false,"error":"delete-denied"}`},
} {
cleanupReceipt := cleanupReceipt
t.Run(endpoint+"/"+frontier.name+"/"+cleanupReceipt.name, func(t *testing.T) {
fixture := newScriptedLightFixture(t, endpoint, false)
frontier.consumePrimaryFail(fixture)
cleanup := fixture.request()
if cleanup.Code != http.StatusOK || !strings.Contains(cleanup.Body.String(), "delete_file") ||
!strings.Contains(cleanup.Body.String(), frontier.wantResponseID) {
t.Fatalf("primary cleanup response: status=%d body=%s", cleanup.Code, cleanup.Body.String())
}
fixture.consumeToolResponse(cleanup, []string{cleanupReceipt.body})
final := fixture.request()
if final.Code != http.StatusBadRequest || !strings.Contains(final.Body.String(), "artifact receipt rejected") ||
strings.Contains(final.Body.String(), "workspace cleanup failed") || strings.Contains(final.Body.String(), "denied") {
t.Fatalf("primary error response: status=%d body=%s", final.Code, final.Body.String())
}
if got := len(fixture.service.snapshots()); got != frontier.wantProviderCalls {
t.Fatalf("provider calls=%d, want selector-only %d", got, frontier.wantProviderCalls)
}
fixture.assertCleanupStoresRemoved()
})
}
}
}
}
type primaryErrorPoolService struct {
*scriptedLightPoolService
failAt int
failure error
}
func (s *primaryErrorPoolService) SubmitProviderPool(ctx context.Context, req edgeservice.ProviderPoolDispatchRequest) (*edgeservice.ProviderPoolDispatchResult, error) {
s.mu.Lock()
index := len(s.requests)
if index == s.failAt {
s.requests = append(s.requests, req)
s.mu.Unlock()
return nil, s.failure
}
s.mu.Unlock()
return s.scriptedLightPoolService.SubmitProviderPool(ctx, req)
}
func TestHotPathCleanupPrimaryErrorStageMatrix(t *testing.T) {
for _, endpoint := range []string{"openai", "anthropic"} {
endpoint := endpoint
for _, stageCase := range []struct {
name string
wantStatus int
wantMessage string
wantProviderCalls int
prepare func(*scriptedLightFixture)
}{
{
name: "local-dispatch", wantStatus: http.StatusBadGateway,
wantMessage: "local dispatch sentinel", wantProviderCalls: 3,
prepare: func(fixture *scriptedLightFixture) {
fixture.server.service = &primaryErrorPoolService{
scriptedLightPoolService: fixture.service, failAt: 2, failure: errors.New("local dispatch sentinel"),
}
},
},
{
name: "local-tool-frontier", wantStatus: http.StatusBadRequest,
wantMessage: "stage tool \"cleanup_unknown_tool\" is not in the immutable caller tool set", wantProviderCalls: 3,
prepare: func(fixture *scriptedLightFixture) {
fixture.service.responses[2] = func(string) string { return primaryErrorUnknownToolOutput(endpoint) }
},
},
{
name: "review-dispatch", wantStatus: http.StatusBadGateway,
wantMessage: "review dispatch sentinel", wantProviderCalls: 5,
prepare: func(fixture *scriptedLightFixture) {
fixture.server.service = &primaryErrorPoolService{
scriptedLightPoolService: fixture.service, failAt: 4, failure: errors.New("review dispatch sentinel"),
}
},
},
{
name: "review-classification", wantStatus: http.StatusBadRequest,
wantMessage: "review stage completed before writing the issued review artifact", wantProviderCalls: 5,
prepare: func(fixture *scriptedLightFixture) {
fixture.service.responses[4] = func(string) string {
return scriptedLightCompletion(endpoint, "review completed without its required write")
}
},
},
{
name: "review-tool-frontier", wantStatus: http.StatusBadRequest,
wantMessage: "stage tool \"cleanup_unknown_tool\" is not in the immutable caller tool set", wantProviderCalls: 5,
prepare: func(fixture *scriptedLightFixture) {
fixture.service.responses[4] = func(string) string { return primaryErrorUnknownToolOutput(endpoint) }
},
},
} {
stageCase := stageCase
t.Run(endpoint+"/"+stageCase.name, func(t *testing.T) {
fixture := newScriptedLightFixture(t, endpoint, false)
stageCase.prepare(fixture)
preparePrimaryErrorStage(t, fixture, strings.HasPrefix(stageCase.name, "review-"))
cleanup := fixture.request()
if cleanup.Code != http.StatusOK || !strings.Contains(cleanup.Body.String(), "delete_file") {
t.Fatalf("primary cleanup response: status=%d body=%s", cleanup.Code, cleanup.Body.String())
}
fixture.consumeToolResponse(cleanup, []string{`{"written":false,"error":"cleanup-denied"}`})
final := fixture.request()
errorType, message := decodePrimaryEndpointError(t, endpoint, final.Body.Bytes())
wantType := hotPathLightEndpointError(endpoint, stageCase.wantStatus, stageCase.wantMessage).Type
if final.Code != stageCase.wantStatus || errorType != wantType || message != stageCase.wantMessage ||
strings.Contains(message, "workspace cleanup failed") {
t.Fatalf("primary terminal response: status=%d body=%s", final.Code, final.Body.String())
}
if got := len(fixture.service.snapshots()); got != stageCase.wantProviderCalls {
t.Fatalf("provider calls=%d, want %d", got, stageCase.wantProviderCalls)
}
fixture.assertCleanupStoresRemoved()
})
}
t.Run(endpoint+"/cancellation", func(t *testing.T) {
fixture := newScriptedLightFixture(t, endpoint, false)
preparePrimaryErrorStage(t, fixture, false)
raw := scriptedArtifactRequestBody(t, endpoint, fixture.tools, fixture.history)
dispatch, err := fixture.server.resolveRouteDispatchForPrincipal(context.Background(), "virtual-model")
if err != nil {
t.Fatal(err)
}
metadata := map[string]string{}
var ingress presetIngressResult
if endpoint == "anthropic" {
ingress, err = fixture.server.joinPresetAnthropicIngress(nilRequestWithContext(context.Background()), dispatch, raw, metadata)
} else {
ingress, err = fixture.server.joinPresetChatIngress(nilRequestWithContext(context.Background()), dispatch, raw, metadata)
}
if err != nil || !ingress.localStageEligible() {
t.Fatalf("local admission: ingress=%+v err=%v", ingress, err)
}
requestID := metadata["iop_logical_request_id"]
if _, err := fixture.server.lightFlows.startLocal(requestID, fixture.server.edgeIDValue(), fixture.server.requestCoordinator); err != nil {
t.Fatal(err)
}
if _, err := fixture.server.lightFlows.beginDispatch(requestID, fixture.server.edgeIDValue(), false); err != nil {
t.Fatal(err)
}
cancelled, cancel := context.WithCancel(context.Background())
cancel()
recorder := httptest.NewRecorder()
err = fixture.server.writeHotPathPrimaryError(
recorder, nilRequestWithContext(cancelled), dispatch, endpoint, false, requestID,
hotPathLightEndpointError(endpoint, http.StatusBadGateway, "cancelled primary sentinel"),
)
if !errors.Is(err, context.Canceled) || strings.Contains(recorder.Body.String(), "delete_file") {
t.Fatalf("cancelled primary cleanup: err=%v body=%s", err, recorder.Body.String())
}
if got := len(fixture.service.snapshots()); got != 2 {
t.Fatalf("provider calls after cancellation=%d, want 2", got)
}
fixture.server.requestCoordinator.mu.Lock()
record := fixture.server.requestCoordinator.requests[requestID]
fixture.server.requestCoordinator.mu.Unlock()
if record == nil || record.state != logicalRequestStateDetached || record.terminalClass != "cancelled" {
t.Fatalf("cancelled coordinator state=%+v", record)
}
fixture.server.lightFlows.mu.Lock()
light := fixture.server.lightFlows.records[requestID]
fixture.server.lightFlows.mu.Unlock()
if light == nil || light.running || light.cleanupTransitions != 0 || light.pendingKind == hotPathPendingCleanup {
t.Fatalf("cancelled light state=%+v", light)
}
})
}
}
func TestHotPathCleanupPrimaryErrorStartFailure(t *testing.T) {
for _, endpoint := range []string{"openai", "anthropic"} {
endpoint := endpoint
t.Run(endpoint, func(t *testing.T) {
fixture := newScriptedLightFixture(t, endpoint, false)
fixture.server.service = &primaryErrorPoolService{
scriptedLightPoolService: fixture.service, failAt: 2, failure: errors.New("cleanup start primary sentinel"),
}
preparePrimaryErrorStage(t, fixture, false)
fixture.server.lightFlows.mu.Lock()
var requestID string
for id, record := range fixture.server.lightFlows.records {
requestID = id
delete(record.binding.operations, opKindDelete)
}
fixture.server.lightFlows.mu.Unlock()
if requestID == "" {
t.Fatal("light request was not retained")
}
terminal := fixture.request()
errorType, message := decodePrimaryEndpointError(t, endpoint, terminal.Body.Bytes())
wantType := hotPathLightEndpointError(endpoint, http.StatusBadGateway, "cleanup start primary sentinel").Type
if terminal.Code != http.StatusBadGateway || errorType != wantType || message != "cleanup start primary sentinel" ||
strings.Contains(terminal.Body.String(), "cleanup delete binding is unavailable") || strings.Contains(terminal.Body.String(), "delete_file") {
t.Fatalf("cleanup-start fallback: status=%d body=%s", terminal.Code, terminal.Body.String())
}
if got := len(fixture.service.snapshots()); got != 3 {
t.Fatalf("provider calls=%d, want 3", got)
}
fixture.server.requestCoordinator.mu.Lock()
record := fixture.server.requestCoordinator.requests[requestID]
if record == nil || record.state != logicalRequestStateDetached || record.terminalClass != "primary_error" {
fixture.server.requestCoordinator.mu.Unlock()
t.Fatalf("retained coordinator state=%+v", record)
}
expireAt := record.updatedAt.Add(fixture.server.requestCoordinator.ttl + time.Second)
fixture.server.requestCoordinator.now = func() time.Time { return expireAt }
fixture.server.requestCoordinator.mu.Unlock()
fixture.server.sweepLogicalRequestTTL()
fixture.assertCleanupStoresRemoved()
})
}
}
func preparePrimaryErrorStage(t *testing.T, fixture *scriptedLightFixture, review bool) {
t.Helper()
prepare := fixture.request()
fixture.consumeToolResponse(prepare, []string{`{"written":true}`})
pair := fixture.request()
fixture.consumeToolResponse(pair, []string{`{"written":true}`, `{"written":true}`})
if review {
localRead := fixture.request()
fixture.consumeToolResponse(localRead, []string{`{"written":true}`})
}
}
func primaryErrorUnknownToolOutput(endpoint string) string {
if endpoint == "anthropic" {
return `{"id":"msg-primary-tool-error","type":"message","role":"assistant","content":[{"type":"tool_use","id":"provider-primary-tool-error","name":"cleanup_unknown_tool","input":{"value":"x"}}],"stop_reason":"tool_use"}`
}
return fmt.Sprintf(`{"id":"chatcmpl-primary-tool-error","created":10,"choices":[{"message":{"role":"assistant","tool_calls":[{"id":"provider-primary-tool-error","type":"function","function":{"name":"cleanup_unknown_tool","arguments":%q}}]},"finish_reason":"tool_calls"}]}`, `{"value":"x"}`)
}
func decodePrimaryEndpointError(t *testing.T, endpoint string, body []byte) (string, string) {
t.Helper()
if endpoint == "anthropic" {
var envelope struct {
Error struct {
Type string `json:"type"`
Message string `json:"message"`
} `json:"error"`
}
if err := json.Unmarshal(body, &envelope); err != nil {
t.Fatalf("decode Anthropic error: %v body=%s", err, body)
}
return envelope.Error.Type, envelope.Error.Message
}
var envelope struct {
Error struct {
Type string `json:"type"`
Message string `json:"message"`
} `json:"error"`
}
if err := json.Unmarshal(body, &envelope); err != nil {
t.Fatalf("decode OpenAI error: %v body=%s", err, body)
}
return envelope.Error.Type, envelope.Error.Message
}
func TestHotPathCleanupConcurrentExactlyOnce(t *testing.T) {
for _, endpoint := range []string{"openai", "anthropic"} {
endpoint := endpoint
t.Run(endpoint, func(t *testing.T) {
fixture := newScriptedLightFixture(t, endpoint, false)
cleanup := fixture.runToCleanup()
fixture.consumeToolResponse(cleanup, []string{`{"written":true}`})
body := scriptedArtifactRequestBody(t, endpoint, fixture.tools, fixture.history)
const contenders = 8
responses := make(chan int, contenders)
var wg sync.WaitGroup
for i := 0; i < contenders; i++ {
wg.Add(1)
go func() {
defer wg.Done()
responses <- serveScriptedArtifactRequest(t, fixture.server, endpoint, body).Code
}()
}
wg.Wait()
close(responses)
successes := 0
for status := range responses {
if status == http.StatusOK {
successes++
}
}
if successes != 1 {
t.Fatalf("terminal winners=%d, want 1", successes)
}
if got := len(fixture.service.snapshots()); got != 7 {
t.Fatalf("duplicate cleanup dispatched provider calls=%d, want 7", got)
}
fixture.assertCleanupCommitted(7)
})
}
}
func TestHotPathCleanupCancellationStopsWork(t *testing.T) {
for _, endpoint := range []string{"openai", "anthropic"} {
endpoint := endpoint
t.Run(endpoint, func(t *testing.T) {
fixture := newScriptedLightFixture(t, endpoint, false)
prepare := fixture.request()
fixture.consumeToolResponse(prepare, []string{`{"written":true}`})
pair := fixture.request()
fixture.consumeToolResponse(pair, []string{`{"written":true}`, `{"written":true}`})
localRead := fixture.request()
fixture.consumeToolResponse(localRead, []string{`{"written":true}`})
reviewWrite := fixture.request()
fixture.consumeToolResponse(reviewWrite, []string{`{"written":true}`})
reviewRead := fixture.request()
fixture.consumeToolResponse(reviewRead, []string{`{"written":true}`})
raw := scriptedArtifactRequestBody(t, endpoint, fixture.tools, fixture.history)
dispatch, err := fixture.server.resolveRouteDispatchForPrincipal(context.Background(), "virtual-model")
if err != nil {
t.Fatal(err)
}
metadata := map[string]string{}
var ingress presetIngressResult
if endpoint == "anthropic" {
ingress, err = fixture.server.joinPresetAnthropicIngress(nilRequestWithContext(context.Background()), dispatch, raw, metadata)
} else {
ingress, err = fixture.server.joinPresetChatIngress(nilRequestWithContext(context.Background()), dispatch, raw, metadata)
}
if err != nil || !ingress.lightStageContinuation() {
t.Fatalf("consume review-read frontier: ingress=%+v err=%v", ingress, err)
}
requestID := ingress.Light.RequestID
if _, err := fixture.server.lightFlows.beginDispatch(requestID, fixture.server.edgeIDValue(), false); err != nil {
t.Fatal(err)
}
cancelled, cancel := context.WithCancel(context.Background())
cancel()
if _, err := fixture.server.lightFlows.beginCleanup(cancelled, requestID, fixture.server.edgeIDValue(), hotPathTerminalIntent{
Output: normalizedStageOutput{ResponseID: "provider-final", Content: "must-not-commit"},
}, fixture.server.requestCoordinator); err == nil {
t.Fatal("cancelled cleanup unexpectedly issued")
}
before := len(fixture.service.snapshots())
fixture.server.requestCoordinator.mu.Lock()
record := fixture.server.requestCoordinator.requests[requestID]
if record == nil || record.state != logicalRequestStateDetached {
fixture.server.requestCoordinator.mu.Unlock()
t.Fatalf("cancelled state=%v", record)
}
fixture.server.requestCoordinator.mu.Unlock()
fixture.server.lightFlows.mu.Lock()
light := fixture.server.lightFlows.records[requestID]
if light == nil || light.pendingKind == hotPathPendingCleanup || light.cleanupTransitions != 0 {
fixture.server.lightFlows.mu.Unlock()
t.Fatalf("cancelled light state=%+v", light)
}
fixture.server.lightFlows.mu.Unlock()
replay := serveScriptedArtifactRequest(t, fixture.server, endpoint, raw)
if replay.Code == http.StatusOK || strings.Contains(replay.Body.String(), "delete_file") {
t.Fatalf("cancelled replay response: status=%d body=%s", replay.Code, replay.Body.String())
}
if after := len(fixture.service.snapshots()); after != before {
t.Fatalf("cancelled replay dispatched provider calls: before=%d after=%d", before, after)
}
})
}
}