491 lines
20 KiB
Go
491 lines
20 KiB
Go
package openai
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
edgeservice "iop/apps/edge/internal/service"
|
|
)
|
|
|
|
func nilRequestWithContext(ctx context.Context) *http.Request {
|
|
return httptest.NewRequest(http.MethodPost, "/", nil).WithContext(ctx)
|
|
}
|
|
|
|
func TestHotPathCleanupTerminalMatrix(t *testing.T) {
|
|
for _, endpoint := range []string{"openai", "anthropic"} {
|
|
endpoint := endpoint
|
|
t.Run(endpoint+" success waits for exact delete", func(t *testing.T) {
|
|
fixture := newScriptedLightFixture(t, endpoint, false)
|
|
cleanup := fixture.runToCleanup()
|
|
if cleanup.Code != http.StatusOK || !strings.Contains(cleanup.Body.String(), "delete_file") ||
|
|
!strings.Contains(cleanup.Body.String(), ".iop/job/") || strings.Contains(cleanup.Body.String(), "review-resolution-visible") {
|
|
t.Fatalf("cleanup frontier response: status=%d body=%s", cleanup.Code, cleanup.Body.String())
|
|
}
|
|
fixture.server.requestCoordinator.mu.Lock()
|
|
if len(fixture.server.requestCoordinator.requests) != 1 {
|
|
fixture.server.requestCoordinator.mu.Unlock()
|
|
t.Fatalf("cleanup coordinator records=%d, want 1", len(fixture.server.requestCoordinator.requests))
|
|
}
|
|
for _, record := range fixture.server.requestCoordinator.requests {
|
|
if record.state != logicalRequestStateCleanup || !record.cleanup || record.terminalClass != "success" {
|
|
fixture.server.requestCoordinator.mu.Unlock()
|
|
t.Fatalf("cleanup coordinator state=%q cleanup=%t terminal=%q", record.state, record.cleanup, record.terminalClass)
|
|
}
|
|
}
|
|
fixture.server.requestCoordinator.mu.Unlock()
|
|
|
|
fixture.consumeToolResponse(cleanup, []string{`{"written":true}`})
|
|
final := fixture.request()
|
|
if final.Code != http.StatusOK || !strings.Contains(final.Body.String(), "review-resolution-visible") {
|
|
t.Fatalf("terminal response: status=%d body=%s", final.Code, final.Body.String())
|
|
}
|
|
fixture.assertCleanupCommitted(7)
|
|
})
|
|
|
|
t.Run(endpoint+" cleanup mismatch cannot become success", func(t *testing.T) {
|
|
fixture := newScriptedLightFixture(t, endpoint, false)
|
|
cleanup := fixture.runToCleanup()
|
|
fixture.consumeToolResponse(cleanup, []string{`{"written":false}`})
|
|
final := fixture.request()
|
|
if final.Code != http.StatusBadGateway || !strings.Contains(final.Body.String(), "workspace cleanup failed") ||
|
|
strings.Contains(final.Body.String(), "review-resolution-visible") {
|
|
t.Fatalf("cleanup failure response: status=%d body=%s", final.Code, final.Body.String())
|
|
}
|
|
fixture.assertCleanupCommitted(7)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestHotPathCleanupPrimaryErrorPrecedence(t *testing.T) {
|
|
for _, endpoint := range []string{"openai", "anthropic"} {
|
|
endpoint := endpoint
|
|
for _, frontier := range []struct {
|
|
name string
|
|
wantProviderCalls int
|
|
wantResponseID string
|
|
consumePrimaryFail func(*scriptedLightFixture)
|
|
}{
|
|
{
|
|
name: "prepare", wantProviderCalls: 1,
|
|
wantResponseID: map[string]string{"openai": "chatcmpl-scripted", "anthropic": "msg-scripted"}[endpoint],
|
|
consumePrimaryFail: func(fixture *scriptedLightFixture) {
|
|
prepare := fixture.request()
|
|
fixture.consumeToolResponse(prepare, []string{`{"error":"prepare-denied"}`})
|
|
},
|
|
},
|
|
{
|
|
name: "pair", wantProviderCalls: 2,
|
|
wantResponseID: map[string]string{"openai": "chatcmpl-scripted-pair", "anthropic": "msg-scripted-pair"}[endpoint],
|
|
consumePrimaryFail: func(fixture *scriptedLightFixture) {
|
|
prepare := fixture.request()
|
|
fixture.consumeToolResponse(prepare, []string{`{"written":true}`})
|
|
pair := fixture.request()
|
|
fixture.consumeToolResponse(pair, []string{`{"written":true}`, `{"error":"pair-denied"}`})
|
|
},
|
|
},
|
|
{
|
|
// A partial pair whose Plan write matches but whose Review
|
|
// result only fails the configured receipt matcher (no explicit
|
|
// error signal) must still author the same canonical delete
|
|
// frontier so a possible sibling artifact cannot leak.
|
|
name: "pair-matcher-failure", wantProviderCalls: 2,
|
|
wantResponseID: map[string]string{"openai": "chatcmpl-scripted-pair", "anthropic": "msg-scripted-pair"}[endpoint],
|
|
consumePrimaryFail: func(fixture *scriptedLightFixture) {
|
|
prepare := fixture.request()
|
|
fixture.consumeToolResponse(prepare, []string{`{"written":true}`})
|
|
pair := fixture.request()
|
|
fixture.consumeToolResponse(pair, []string{`{"written":true}`, `{"written":false}`})
|
|
},
|
|
},
|
|
} {
|
|
frontier := frontier
|
|
for _, cleanupReceipt := range []struct {
|
|
name string
|
|
body string
|
|
}{
|
|
{name: "acknowledged", body: `{"written":true}`},
|
|
{name: "acknowledgement-failed", body: `{"written":false,"error":"delete-denied"}`},
|
|
} {
|
|
cleanupReceipt := cleanupReceipt
|
|
t.Run(endpoint+"/"+frontier.name+"/"+cleanupReceipt.name, func(t *testing.T) {
|
|
fixture := newScriptedLightFixture(t, endpoint, false)
|
|
frontier.consumePrimaryFail(fixture)
|
|
|
|
cleanup := fixture.request()
|
|
if cleanup.Code != http.StatusOK || !strings.Contains(cleanup.Body.String(), "delete_file") ||
|
|
!strings.Contains(cleanup.Body.String(), frontier.wantResponseID) {
|
|
t.Fatalf("primary cleanup response: status=%d body=%s", cleanup.Code, cleanup.Body.String())
|
|
}
|
|
fixture.consumeToolResponse(cleanup, []string{cleanupReceipt.body})
|
|
final := fixture.request()
|
|
if final.Code != http.StatusBadRequest || !strings.Contains(final.Body.String(), "artifact receipt rejected") ||
|
|
strings.Contains(final.Body.String(), "workspace cleanup failed") || strings.Contains(final.Body.String(), "denied") {
|
|
t.Fatalf("primary error response: status=%d body=%s", final.Code, final.Body.String())
|
|
}
|
|
if got := len(fixture.service.snapshots()); got != frontier.wantProviderCalls {
|
|
t.Fatalf("provider calls=%d, want selector-only %d", got, frontier.wantProviderCalls)
|
|
}
|
|
fixture.assertCleanupStoresRemoved()
|
|
})
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
type primaryErrorPoolService struct {
|
|
*scriptedLightPoolService
|
|
failAt int
|
|
failure error
|
|
}
|
|
|
|
func (s *primaryErrorPoolService) SubmitProviderPool(ctx context.Context, req edgeservice.ProviderPoolDispatchRequest) (*edgeservice.ProviderPoolDispatchResult, error) {
|
|
s.mu.Lock()
|
|
index := len(s.requests)
|
|
if index == s.failAt {
|
|
s.requests = append(s.requests, req)
|
|
s.mu.Unlock()
|
|
return nil, s.failure
|
|
}
|
|
s.mu.Unlock()
|
|
return s.scriptedLightPoolService.SubmitProviderPool(ctx, req)
|
|
}
|
|
|
|
func TestHotPathCleanupPrimaryErrorStageMatrix(t *testing.T) {
|
|
for _, endpoint := range []string{"openai", "anthropic"} {
|
|
endpoint := endpoint
|
|
for _, stageCase := range []struct {
|
|
name string
|
|
wantStatus int
|
|
wantMessage string
|
|
wantProviderCalls int
|
|
prepare func(*scriptedLightFixture)
|
|
}{
|
|
{
|
|
name: "local-dispatch", wantStatus: http.StatusBadGateway,
|
|
wantMessage: "local dispatch sentinel", wantProviderCalls: 3,
|
|
prepare: func(fixture *scriptedLightFixture) {
|
|
fixture.server.service = &primaryErrorPoolService{
|
|
scriptedLightPoolService: fixture.service, failAt: 2, failure: errors.New("local dispatch sentinel"),
|
|
}
|
|
},
|
|
},
|
|
{
|
|
name: "local-tool-frontier", wantStatus: http.StatusBadRequest,
|
|
wantMessage: "stage tool \"cleanup_unknown_tool\" is not in the immutable caller tool set", wantProviderCalls: 3,
|
|
prepare: func(fixture *scriptedLightFixture) {
|
|
fixture.service.responses[2] = func(string) string { return primaryErrorUnknownToolOutput(endpoint) }
|
|
},
|
|
},
|
|
{
|
|
name: "review-dispatch", wantStatus: http.StatusBadGateway,
|
|
wantMessage: "review dispatch sentinel", wantProviderCalls: 5,
|
|
prepare: func(fixture *scriptedLightFixture) {
|
|
fixture.server.service = &primaryErrorPoolService{
|
|
scriptedLightPoolService: fixture.service, failAt: 4, failure: errors.New("review dispatch sentinel"),
|
|
}
|
|
},
|
|
},
|
|
{
|
|
name: "review-classification", wantStatus: http.StatusBadRequest,
|
|
wantMessage: "review stage completed before writing the issued review artifact", wantProviderCalls: 5,
|
|
prepare: func(fixture *scriptedLightFixture) {
|
|
fixture.service.responses[4] = func(string) string {
|
|
return scriptedLightCompletion(endpoint, "review completed without its required write")
|
|
}
|
|
},
|
|
},
|
|
{
|
|
name: "review-tool-frontier", wantStatus: http.StatusBadRequest,
|
|
wantMessage: "stage tool \"cleanup_unknown_tool\" is not in the immutable caller tool set", wantProviderCalls: 5,
|
|
prepare: func(fixture *scriptedLightFixture) {
|
|
fixture.service.responses[4] = func(string) string { return primaryErrorUnknownToolOutput(endpoint) }
|
|
},
|
|
},
|
|
} {
|
|
stageCase := stageCase
|
|
t.Run(endpoint+"/"+stageCase.name, func(t *testing.T) {
|
|
fixture := newScriptedLightFixture(t, endpoint, false)
|
|
stageCase.prepare(fixture)
|
|
preparePrimaryErrorStage(t, fixture, strings.HasPrefix(stageCase.name, "review-"))
|
|
|
|
cleanup := fixture.request()
|
|
if cleanup.Code != http.StatusOK || !strings.Contains(cleanup.Body.String(), "delete_file") {
|
|
t.Fatalf("primary cleanup response: status=%d body=%s", cleanup.Code, cleanup.Body.String())
|
|
}
|
|
fixture.consumeToolResponse(cleanup, []string{`{"written":false,"error":"cleanup-denied"}`})
|
|
final := fixture.request()
|
|
errorType, message := decodePrimaryEndpointError(t, endpoint, final.Body.Bytes())
|
|
wantType := hotPathLightEndpointError(endpoint, stageCase.wantStatus, stageCase.wantMessage).Type
|
|
if final.Code != stageCase.wantStatus || errorType != wantType || message != stageCase.wantMessage ||
|
|
strings.Contains(message, "workspace cleanup failed") {
|
|
t.Fatalf("primary terminal response: status=%d body=%s", final.Code, final.Body.String())
|
|
}
|
|
if got := len(fixture.service.snapshots()); got != stageCase.wantProviderCalls {
|
|
t.Fatalf("provider calls=%d, want %d", got, stageCase.wantProviderCalls)
|
|
}
|
|
fixture.assertCleanupStoresRemoved()
|
|
})
|
|
}
|
|
|
|
t.Run(endpoint+"/cancellation", func(t *testing.T) {
|
|
fixture := newScriptedLightFixture(t, endpoint, false)
|
|
preparePrimaryErrorStage(t, fixture, false)
|
|
|
|
raw := scriptedArtifactRequestBody(t, endpoint, fixture.tools, fixture.history)
|
|
dispatch, err := fixture.server.resolveRouteDispatchForPrincipal(context.Background(), "virtual-model")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
metadata := map[string]string{}
|
|
var ingress presetIngressResult
|
|
if endpoint == "anthropic" {
|
|
ingress, err = fixture.server.joinPresetAnthropicIngress(nilRequestWithContext(context.Background()), dispatch, raw, metadata)
|
|
} else {
|
|
ingress, err = fixture.server.joinPresetChatIngress(nilRequestWithContext(context.Background()), dispatch, raw, metadata)
|
|
}
|
|
if err != nil || !ingress.localStageEligible() {
|
|
t.Fatalf("local admission: ingress=%+v err=%v", ingress, err)
|
|
}
|
|
requestID := metadata["iop_logical_request_id"]
|
|
if _, err := fixture.server.lightFlows.startLocal(requestID, fixture.server.edgeIDValue(), fixture.server.requestCoordinator); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := fixture.server.lightFlows.beginDispatch(requestID, fixture.server.edgeIDValue(), false); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
cancelled, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
recorder := httptest.NewRecorder()
|
|
err = fixture.server.writeHotPathPrimaryError(
|
|
recorder, nilRequestWithContext(cancelled), dispatch, endpoint, false, requestID,
|
|
hotPathLightEndpointError(endpoint, http.StatusBadGateway, "cancelled primary sentinel"),
|
|
)
|
|
if !errors.Is(err, context.Canceled) || strings.Contains(recorder.Body.String(), "delete_file") {
|
|
t.Fatalf("cancelled primary cleanup: err=%v body=%s", err, recorder.Body.String())
|
|
}
|
|
if got := len(fixture.service.snapshots()); got != 2 {
|
|
t.Fatalf("provider calls after cancellation=%d, want 2", got)
|
|
}
|
|
fixture.server.requestCoordinator.mu.Lock()
|
|
record := fixture.server.requestCoordinator.requests[requestID]
|
|
fixture.server.requestCoordinator.mu.Unlock()
|
|
if record == nil || record.state != logicalRequestStateDetached || record.terminalClass != "cancelled" {
|
|
t.Fatalf("cancelled coordinator state=%+v", record)
|
|
}
|
|
fixture.server.lightFlows.mu.Lock()
|
|
light := fixture.server.lightFlows.records[requestID]
|
|
fixture.server.lightFlows.mu.Unlock()
|
|
if light == nil || light.running || light.cleanupTransitions != 0 || light.pendingKind == hotPathPendingCleanup {
|
|
t.Fatalf("cancelled light state=%+v", light)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestHotPathCleanupPrimaryErrorStartFailure(t *testing.T) {
|
|
for _, endpoint := range []string{"openai", "anthropic"} {
|
|
endpoint := endpoint
|
|
t.Run(endpoint, func(t *testing.T) {
|
|
fixture := newScriptedLightFixture(t, endpoint, false)
|
|
fixture.server.service = &primaryErrorPoolService{
|
|
scriptedLightPoolService: fixture.service, failAt: 2, failure: errors.New("cleanup start primary sentinel"),
|
|
}
|
|
preparePrimaryErrorStage(t, fixture, false)
|
|
|
|
fixture.server.lightFlows.mu.Lock()
|
|
var requestID string
|
|
for id, record := range fixture.server.lightFlows.records {
|
|
requestID = id
|
|
delete(record.binding.operations, opKindDelete)
|
|
}
|
|
fixture.server.lightFlows.mu.Unlock()
|
|
if requestID == "" {
|
|
t.Fatal("light request was not retained")
|
|
}
|
|
|
|
terminal := fixture.request()
|
|
errorType, message := decodePrimaryEndpointError(t, endpoint, terminal.Body.Bytes())
|
|
wantType := hotPathLightEndpointError(endpoint, http.StatusBadGateway, "cleanup start primary sentinel").Type
|
|
if terminal.Code != http.StatusBadGateway || errorType != wantType || message != "cleanup start primary sentinel" ||
|
|
strings.Contains(terminal.Body.String(), "cleanup delete binding is unavailable") || strings.Contains(terminal.Body.String(), "delete_file") {
|
|
t.Fatalf("cleanup-start fallback: status=%d body=%s", terminal.Code, terminal.Body.String())
|
|
}
|
|
if got := len(fixture.service.snapshots()); got != 3 {
|
|
t.Fatalf("provider calls=%d, want 3", got)
|
|
}
|
|
|
|
fixture.server.requestCoordinator.mu.Lock()
|
|
record := fixture.server.requestCoordinator.requests[requestID]
|
|
if record == nil || record.state != logicalRequestStateDetached || record.terminalClass != "primary_error" {
|
|
fixture.server.requestCoordinator.mu.Unlock()
|
|
t.Fatalf("retained coordinator state=%+v", record)
|
|
}
|
|
expireAt := record.updatedAt.Add(fixture.server.requestCoordinator.ttl + time.Second)
|
|
fixture.server.requestCoordinator.now = func() time.Time { return expireAt }
|
|
fixture.server.requestCoordinator.mu.Unlock()
|
|
|
|
fixture.server.sweepLogicalRequestTTL()
|
|
fixture.assertCleanupStoresRemoved()
|
|
})
|
|
}
|
|
}
|
|
|
|
func preparePrimaryErrorStage(t *testing.T, fixture *scriptedLightFixture, review bool) {
|
|
t.Helper()
|
|
prepare := fixture.request()
|
|
fixture.consumeToolResponse(prepare, []string{`{"written":true}`})
|
|
pair := fixture.request()
|
|
fixture.consumeToolResponse(pair, []string{`{"written":true}`, `{"written":true}`})
|
|
if review {
|
|
localRead := fixture.request()
|
|
fixture.consumeToolResponse(localRead, []string{`{"written":true}`})
|
|
}
|
|
}
|
|
|
|
func primaryErrorUnknownToolOutput(endpoint string) string {
|
|
if endpoint == "anthropic" {
|
|
return `{"id":"msg-primary-tool-error","type":"message","role":"assistant","content":[{"type":"tool_use","id":"provider-primary-tool-error","name":"cleanup_unknown_tool","input":{"value":"x"}}],"stop_reason":"tool_use"}`
|
|
}
|
|
return fmt.Sprintf(`{"id":"chatcmpl-primary-tool-error","created":10,"choices":[{"message":{"role":"assistant","tool_calls":[{"id":"provider-primary-tool-error","type":"function","function":{"name":"cleanup_unknown_tool","arguments":%q}}]},"finish_reason":"tool_calls"}]}`, `{"value":"x"}`)
|
|
}
|
|
|
|
func decodePrimaryEndpointError(t *testing.T, endpoint string, body []byte) (string, string) {
|
|
t.Helper()
|
|
if endpoint == "anthropic" {
|
|
var envelope struct {
|
|
Error struct {
|
|
Type string `json:"type"`
|
|
Message string `json:"message"`
|
|
} `json:"error"`
|
|
}
|
|
if err := json.Unmarshal(body, &envelope); err != nil {
|
|
t.Fatalf("decode Anthropic error: %v body=%s", err, body)
|
|
}
|
|
return envelope.Error.Type, envelope.Error.Message
|
|
}
|
|
var envelope struct {
|
|
Error struct {
|
|
Type string `json:"type"`
|
|
Message string `json:"message"`
|
|
} `json:"error"`
|
|
}
|
|
if err := json.Unmarshal(body, &envelope); err != nil {
|
|
t.Fatalf("decode OpenAI error: %v body=%s", err, body)
|
|
}
|
|
return envelope.Error.Type, envelope.Error.Message
|
|
}
|
|
|
|
func TestHotPathCleanupConcurrentExactlyOnce(t *testing.T) {
|
|
for _, endpoint := range []string{"openai", "anthropic"} {
|
|
endpoint := endpoint
|
|
t.Run(endpoint, func(t *testing.T) {
|
|
fixture := newScriptedLightFixture(t, endpoint, false)
|
|
cleanup := fixture.runToCleanup()
|
|
fixture.consumeToolResponse(cleanup, []string{`{"written":true}`})
|
|
body := scriptedArtifactRequestBody(t, endpoint, fixture.tools, fixture.history)
|
|
|
|
const contenders = 8
|
|
responses := make(chan int, contenders)
|
|
var wg sync.WaitGroup
|
|
for i := 0; i < contenders; i++ {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
responses <- serveScriptedArtifactRequest(t, fixture.server, endpoint, body).Code
|
|
}()
|
|
}
|
|
wg.Wait()
|
|
close(responses)
|
|
successes := 0
|
|
for status := range responses {
|
|
if status == http.StatusOK {
|
|
successes++
|
|
}
|
|
}
|
|
if successes != 1 {
|
|
t.Fatalf("terminal winners=%d, want 1", successes)
|
|
}
|
|
if got := len(fixture.service.snapshots()); got != 7 {
|
|
t.Fatalf("duplicate cleanup dispatched provider calls=%d, want 7", got)
|
|
}
|
|
fixture.assertCleanupCommitted(7)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestHotPathCleanupCancellationStopsWork(t *testing.T) {
|
|
for _, endpoint := range []string{"openai", "anthropic"} {
|
|
endpoint := endpoint
|
|
t.Run(endpoint, func(t *testing.T) {
|
|
fixture := newScriptedLightFixture(t, endpoint, false)
|
|
prepare := fixture.request()
|
|
fixture.consumeToolResponse(prepare, []string{`{"written":true}`})
|
|
pair := fixture.request()
|
|
fixture.consumeToolResponse(pair, []string{`{"written":true}`, `{"written":true}`})
|
|
localRead := fixture.request()
|
|
fixture.consumeToolResponse(localRead, []string{`{"written":true}`})
|
|
reviewWrite := fixture.request()
|
|
fixture.consumeToolResponse(reviewWrite, []string{`{"written":true}`})
|
|
reviewRead := fixture.request()
|
|
fixture.consumeToolResponse(reviewRead, []string{`{"written":true}`})
|
|
|
|
raw := scriptedArtifactRequestBody(t, endpoint, fixture.tools, fixture.history)
|
|
dispatch, err := fixture.server.resolveRouteDispatchForPrincipal(context.Background(), "virtual-model")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
metadata := map[string]string{}
|
|
var ingress presetIngressResult
|
|
if endpoint == "anthropic" {
|
|
ingress, err = fixture.server.joinPresetAnthropicIngress(nilRequestWithContext(context.Background()), dispatch, raw, metadata)
|
|
} else {
|
|
ingress, err = fixture.server.joinPresetChatIngress(nilRequestWithContext(context.Background()), dispatch, raw, metadata)
|
|
}
|
|
if err != nil || !ingress.lightStageContinuation() {
|
|
t.Fatalf("consume review-read frontier: ingress=%+v err=%v", ingress, err)
|
|
}
|
|
requestID := ingress.Light.RequestID
|
|
if _, err := fixture.server.lightFlows.beginDispatch(requestID, fixture.server.edgeIDValue(), false); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
cancelled, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
if _, err := fixture.server.lightFlows.beginCleanup(cancelled, requestID, fixture.server.edgeIDValue(), hotPathTerminalIntent{
|
|
Output: normalizedStageOutput{ResponseID: "provider-final", Content: "must-not-commit"},
|
|
}, fixture.server.requestCoordinator); err == nil {
|
|
t.Fatal("cancelled cleanup unexpectedly issued")
|
|
}
|
|
before := len(fixture.service.snapshots())
|
|
fixture.server.requestCoordinator.mu.Lock()
|
|
record := fixture.server.requestCoordinator.requests[requestID]
|
|
if record == nil || record.state != logicalRequestStateDetached {
|
|
fixture.server.requestCoordinator.mu.Unlock()
|
|
t.Fatalf("cancelled state=%v", record)
|
|
}
|
|
fixture.server.requestCoordinator.mu.Unlock()
|
|
fixture.server.lightFlows.mu.Lock()
|
|
light := fixture.server.lightFlows.records[requestID]
|
|
if light == nil || light.pendingKind == hotPathPendingCleanup || light.cleanupTransitions != 0 {
|
|
fixture.server.lightFlows.mu.Unlock()
|
|
t.Fatalf("cancelled light state=%+v", light)
|
|
}
|
|
fixture.server.lightFlows.mu.Unlock()
|
|
|
|
replay := serveScriptedArtifactRequest(t, fixture.server, endpoint, raw)
|
|
if replay.Code == http.StatusOK || strings.Contains(replay.Body.String(), "delete_file") {
|
|
t.Fatalf("cancelled replay response: status=%d body=%s", replay.Code, replay.Body.String())
|
|
}
|
|
if after := len(fixture.service.snapshots()); after != before {
|
|
t.Fatalf("cancelled replay dispatched provider calls: before=%d after=%d", before, after)
|
|
}
|
|
})
|
|
}
|
|
}
|