package openai import ( "bytes" "context" "encoding/json" "errors" "fmt" "io" "net/http" "net/http/httptest" "os" "path/filepath" "reflect" "sort" "strconv" "strings" "sync" "sync/atomic" "testing" "time" "go.uber.org/zap" "go.uber.org/zap/zapcore" edgeservice "iop/apps/edge/internal/service" "iop/packages/go/config" "iop/packages/go/streamgate" iop "iop/proto/gen/iop" ) // --- test-only InjectedViolationFilter -------------------------------------- // // injectedViolationFilter is the test-only filter named by the vertical-slice // plan (SDD S16). It is deliberately kept local to this test file: production // wiring (openAIStreamGateRegistrations) never registers it. It triggers an // exact_replay RecoveryIntent on the first `trigger` text_delta evaluations it // sees, then passes for the rest of the request. // // It uses terminal_gate hold mode rather than none: none mode is reserved // for provider-error-only observers (its trigger-ready rule only fires on // provider_error/terminal events regardless of content subscription — see // packages/go/streamgate/evidence_tail.go deriveTriggerReady), so a filter // that wants to judge accumulated content must hold it until the terminal // trigger like a real output-validation filter would. // injectedViolationPriority is shared by every test registration of // injectedViolationFilter and its emitted RecoveryIntent: the Arbiter // requires an exact match between a violation's intent priority and the // filter's effective registration priority. const injectedViolationPriority = 10 type injectedViolationFilter struct { streamgate.FilterBase requestRef string continuationCursor int passBeforeTrigger int mu sync.Mutex trigger int } func newInjectedViolationFilter(t *testing.T, id string, trigger int, requestRef string) *injectedViolationFilter { t.Helper() base, err := streamgate.NewFilterBase(id) if err != nil { t.Fatalf("NewFilterBase: %v", err) } return &injectedViolationFilter{FilterBase: base, requestRef: requestRef, trigger: trigger} } // newInjectedContinuationViolationFilter is the test-only stand-in for the // follow-up repeat guard. It exercises S20's Rebuilder seam without making the // semantic detector itself part of this task: it requests continuation from the // request-local recovery source through the stable ingress snapshot reference. func newInjectedContinuationViolationFilter(t *testing.T, id string, trigger int, requestRef string, cursor int) *injectedViolationFilter { t.Helper() filter := newInjectedViolationFilter(t, id, trigger, requestRef) filter.continuationCursor = cursor // A rolling filter needs one safe evaluation to open the downstream stream // before a continuation plan can use the Core's continue-stream mode. filter.passBeforeTrigger = 1 return filter } func (f *injectedViolationFilter) Applies(streamgate.FilterContext) bool { return true } func (f *injectedViolationFilter) HoldRequirement(streamgate.FilterContext) streamgate.FilterHoldRequirement { if f.continuationCursor > 0 { req, err := streamgate.NewFilterHoldRequirementRolling(streamGateChannelDefault, []streamgate.EventKind{streamgate.EventKindTextDelta}, 1) if err != nil { panic(err) } return req } req, err := streamgate.NewFilterHoldRequirementTerminalGate(streamGateChannelDefault, []streamgate.EventKind{streamgate.EventKindTextDelta}, streamgate.EventKindTerminal) if err != nil { panic(err) } return req } func (f *injectedViolationFilter) Evaluate(ctx context.Context, fctx streamgate.FilterContext, batch streamgate.EvidenceBatch) (streamgate.FilterDecision, error) { f.mu.Lock() fire := f.passBeforeTrigger == 0 && f.trigger > 0 if f.passBeforeTrigger > 0 { f.passBeforeTrigger-- } else if fire { f.trigger-- } f.mu.Unlock() events := batch.Events() kind := streamgate.EventKindTextDelta if len(events) > 0 { kind = events[len(events)-1].Kind() } evidence, err := streamgate.NewSanitizedEvidence(kind, streamGateChannelDefault, "test.rule", "test_injected_violation", streamgate.FixedFingerprint{0x02}, 1, 0, streamgate.FilterOutcomeKindEvaluated, batch.CapturedAt()) if err != nil { return streamgate.FilterDecision{}, err } if !fire { return streamgate.NewFilterDecision(streamgate.FilterDecisionKindPass, "test.consumer", f.ID(), "test.rule", evidence, nil) } var directive streamgate.RecoveryDirective if f.continuationCursor > 0 { directive, err = streamgate.NewRecoveryDirectiveContinuation(f.continuationCursor, f.requestRef) } else { directive, err = streamgate.NewRecoveryDirectiveExact(f.requestRef) } if err != nil { return streamgate.FilterDecision{}, err } strategy := streamgate.RecoveryStrategyExactReplay if f.continuationCursor > 0 { strategy = streamgate.RecoveryStrategyContinuationRepair } intent, err := streamgate.NewRecoveryIntent(strategy, directive, "test_injected", injectedViolationPriority) if err != nil { return streamgate.FilterDecision{}, err } return streamgate.NewFilterDecision(streamgate.FilterDecisionKindViolation, "test.consumer", f.ID(), "test.rule", evidence, &intent) } var _ streamgate.Filter = (*injectedViolationFilter)(nil) func streamGateTestRegistry(t *testing.T, extra ...streamgate.FilterRegistration) streamgate.FilterRegistrySnapshot { t.Helper() regs, err := openAIStreamGateNoopRegistrations() if err != nil { t.Fatalf("openAIStreamGateNoopRegistrations: %v", err) } regs = append(regs, extra...) snap, err := streamgate.NewFilterRegistrySnapshot(streamGateConfigGeneration, regs, nil) if err != nil { t.Fatalf("NewFilterRegistrySnapshot: %v", err) } return snap } // --- recording ResponseWriter ------------------------------------------------- // recordingResponseWriter records the ordered sequence of Header/WriteHeader/ // Write calls so tests can assert that no status/header/role write happens // before the Core's first safe release. type recordingResponseWriter struct { header http.Header body bytes.Buffer code int calls []string } func newRecordingResponseWriter() *recordingResponseWriter { return &recordingResponseWriter{header: make(http.Header)} } func (w *recordingResponseWriter) Header() http.Header { return w.header } func (w *recordingResponseWriter) WriteHeader(code int) { w.code = code w.calls = append(w.calls, fmt.Sprintf("header:%d", code)) } func (w *recordingResponseWriter) Write(p []byte) (int, error) { w.calls = append(w.calls, fmt.Sprintf("write:%d", len(p))) return w.body.Write(p) } func (w *recordingResponseWriter) Flush() {} func (w *recordingResponseWriter) headerCallCount() int { n := 0 for _, c := range w.calls { if strings.HasPrefix(c, "header:") { n++ } } return n } type notifyingResponseWriter struct { *recordingResponseWriter writes chan struct{} } func newNotifyingResponseWriter() *notifyingResponseWriter { return ¬ifyingResponseWriter{recordingResponseWriter: newRecordingResponseWriter(), writes: make(chan struct{}, 8)} } func (w *notifyingResponseWriter) Write(p []byte) (int, error) { w.calls = append(w.calls, fmt.Sprintf("write:%d", len(p))) n, err := w.body.Write(p) select { case w.writes <- struct{}{}: default: } return n, err } var ( _ http.ResponseWriter = (*recordingResponseWriter)(nil) _ http.Flusher = (*recordingResponseWriter)(nil) ) // --- SSE parsing helper ------------------------------------------------------- func parseSSEChatChunks(t *testing.T, body string) []chatCompletionChunk { t.Helper() var chunks []chatCompletionChunk for _, frame := range strings.Split(body, "\n\n") { frame = strings.TrimSpace(frame) if frame == "" { continue } payload := strings.TrimPrefix(frame, "data: ") if payload == "[DONE]" { continue } var chunk chatCompletionChunk if err := json.Unmarshal([]byte(payload), &chunk); err != nil { continue // error chunks decode into errorResponse, not chatCompletionChunk; skip } if len(chunk.Choices) == 0 { continue } chunks = append(chunks, chunk) } return chunks } // parseCompleteResponsesSSE rejects any byte that is not part of a complete // Responses SSE frame. Unlike the chat helper above, it does not forgive a // malformed payload: recovery tests use it to guard the caller wire boundary. func parseCompleteResponsesSSE(t *testing.T, body string) []map[string]any { t.Helper() remaining := []byte(body) var payloads []map[string]any for len(remaining) > 0 { frame, rest, ok := takeOpenAISSEFrame(remaining) if !ok { t.Fatalf("Responses SSE has a non-frame remainder: %q", remaining) } if !bytes.HasPrefix(frame, []byte("data: ")) { t.Fatalf("Responses SSE frame does not begin with data: %q", frame) } data := openAISSEData(frame) if data != "[DONE]" { var payload map[string]any if err := json.Unmarshal([]byte(data), &payload); err != nil { t.Fatalf("Responses SSE payload is not JSON: %q: %v", data, err) } payloads = append(payloads, payload) } remaining = rest } return payloads } // responsesOracleItem records the event-specific state of one output item so // the lifecycle oracle can enforce expected type/status, stable identity and // indexes, legal transition order, exact done counts, and value agreement // between every accumulated delta, its done payload, the completed item, and // the terminal output. type responsesOracleItem struct { id string itemType string outputIndex int contentIndex int callID string name string role string contentOpened bool value string textDone int contentDone int argumentsDone int itemDone int } // assertResponsesSSELifecycle validates the full public Responses SSE contract // for a caller stream and returns the parsed payloads. wantTerminal is the // required last event type ("response.completed" or "error"). func assertResponsesSSELifecycle(t *testing.T, body, wantTerminal string) []map[string]any { t.Helper() payloads := parseCompleteResponsesSSE(t, body) if got := strings.Count(body, "data: [DONE]\n\n"); got != 1 { t.Fatalf("Responses [DONE] count = %d, want 1; body=%q", got, body) } var previousSequence float64 terminalCount := 0 sawCompleted := false responseID := "" items := make(map[string]*responsesOracleItem) order := make([]string, 0) indexOwner := make(map[int]string) for _, payload := range payloads { kind, _ := payload["type"].(string) sequence, ok := payload["sequence_number"].(float64) if !ok || sequence <= previousSequence { t.Fatalf("Responses event %q sequence_number = %#v after %v; payload=%#v", kind, payload["sequence_number"], previousSequence, payload) } previousSequence = sequence requireString := func(key string) { t.Helper() if value, ok := payload[key].(string); !ok || value == "" { t.Fatalf("Responses event %q missing %s: %#v", kind, key, payload) } } requireNumber := func(key string) { t.Helper() if _, ok := payload[key].(float64); !ok { t.Fatalf("Responses event %q missing %s: %#v", kind, key, payload) } } forbid := func(key string) { t.Helper() if _, ok := payload[key]; ok { t.Fatalf("Responses event %q must not carry %s: %#v", kind, key, payload) } } known := func(itemID string) *responsesOracleItem { t.Helper() it, ok := items[itemID] if !ok { t.Fatalf("Responses event %q references unknown item %q: %#v", kind, itemID, payload) } return it } assertOutputIndex := func(it *responsesOracleItem) { t.Helper() if got := int(payload["output_index"].(float64)); got != it.outputIndex { t.Fatalf("Responses event %q output_index = %d, want stable %d for item %q: %#v", kind, got, it.outputIndex, it.id, payload) } } assertContentIndex := func(it *responsesOracleItem) { t.Helper() if err := validateResponsesContentIndex(payload, it.contentIndex); err != nil { t.Fatalf("Responses event %q %v", kind, err) } } switch kind { case "response.created": response, ok := payload["response"].(map[string]any) if !ok || response["id"] == "" { t.Fatalf("response.created has no response identity: %#v", payload) } id, _ := response["id"].(string) if responseID != "" && responseID != id { t.Fatalf("response.created identity = %q, want stable %q", id, responseID) } responseID = id case "response.output_item.added": requireNumber("output_index") item, ok := payload["item"].(map[string]any) id, _ := item["id"].(string) itemType, _ := item["type"].(string) status, _ := item["status"].(string) if !ok || id == "" || itemType == "" { t.Fatalf("response.output_item.added has no item identity: %#v", payload) } if status != "in_progress" { t.Fatalf("response.output_item.added status = %q, want in_progress: %#v", status, payload) } if _, exists := items[id]; exists { t.Fatalf("response.output_item.added duplicates item %q", id) } index := int(payload["output_index"].(float64)) if prior, exists := indexOwner[index]; exists { t.Fatalf("response.output_item.added reuses output index %d for %q and %q", index, prior, id) } rec := &responsesOracleItem{id: id, itemType: itemType, outputIndex: index} switch itemType { case "function_call": callID, _ := item["call_id"].(string) name, _ := item["name"].(string) if callID == "" || name == "" { t.Fatalf("function output_item.added missing call_id/name: %#v", item) } rec.callID = callID rec.name = name case "message": role, _ := item["role"].(string) if role == "" { t.Fatalf("message output_item.added missing role: %#v", item) } rec.role = role if content, ok := item["content"].([]any); !ok || len(content) != 0 { t.Fatalf("message output_item.added opening content must be empty: %#v", item) } case "reasoning": if content, ok := item["content"].([]any); !ok || len(content) != 0 { t.Fatalf("reasoning output_item.added opening content must be empty: %#v", item) } default: t.Fatalf("response.output_item.added has an unsupported item type %q: %#v", itemType, item) } items[id] = rec order = append(order, id) indexOwner[index] = id case "response.content_part.added": requireString("item_id") requireNumber("output_index") requireNumber("content_index") it := known(payload["item_id"].(string)) assertOutputIndex(it) if it.itemType == "function_call" { t.Fatalf("response.content_part.added opened a function item: %#v", payload) } assertResponsesPartShape(t, it.itemType, payload["part"], "") it.contentOpened = true idx, err := parseResponsesContentIndex(payload) if err != nil { t.Fatalf("response.content_part.added invalid content_index: %v", err) } it.contentIndex = idx case "response.output_text.delta", "response.reasoning_text.delta": requireString("item_id") requireNumber("output_index") requireString("delta") it := known(payload["item_id"].(string)) assertOutputIndex(it) assertContentIndex(it) assertResponsesTextEventKind(t, kind, it, "delta") if !it.contentOpened || it.textDone > 0 { t.Fatalf("Responses text delta has no opened content or follows a done: %#v", payload) } it.value += payload["delta"].(string) case "response.function_call_arguments.delta": requireString("item_id") requireNumber("output_index") requireString("delta") forbid("call_id") forbid("name") it := known(payload["item_id"].(string)) assertOutputIndex(it) if it.itemType != "function_call" || it.argumentsDone > 0 { t.Fatalf("function delta references a non-function or completed item: %#v", payload) } it.value += payload["delta"].(string) case "response.output_text.done", "response.reasoning_text.done": requireString("item_id") requireNumber("output_index") requireString("text") it := known(payload["item_id"].(string)) assertOutputIndex(it) assertContentIndex(it) assertResponsesTextEventKind(t, kind, it, "done") if !it.contentOpened { t.Fatalf("Responses text done has no opened content: %#v", payload) } if got := payload["text"]; got != it.value { t.Fatalf("Responses text done = %#v, want accumulated delta %q: %#v", got, it.value, payload) } it.textDone++ case "response.content_part.done": requireString("item_id") requireNumber("output_index") it := known(payload["item_id"].(string)) assertOutputIndex(it) assertContentIndex(it) if !it.contentOpened || it.textDone == 0 || it.contentDone > 0 { t.Fatalf("Responses content done is out of order: %#v", payload) } assertResponsesPartShape(t, it.itemType, payload["part"], it.value) it.contentDone++ case "response.function_call_arguments.done": requireString("item_id") requireNumber("output_index") requireString("name") requireString("arguments") it := known(payload["item_id"].(string)) assertOutputIndex(it) if it.itemType != "function_call" { t.Fatalf("function arguments done references a non-function item: %#v", payload) } if got := payload["arguments"]; got != it.value { t.Fatalf("function arguments done = %#v, want accumulated delta %q: %#v", got, it.value, payload) } if got := payload["name"]; got != it.name { t.Fatalf("function arguments done name = %#v, want %q: %#v", got, it.name, payload) } it.argumentsDone++ case "response.output_item.done": requireNumber("output_index") item, ok := payload["item"].(map[string]any) id, _ := item["id"].(string) if !ok || id == "" { t.Fatalf("response.output_item.done has no item identity: %#v", payload) } it := known(id) assertOutputIndex(it) if status, _ := item["status"].(string); status != "completed" { t.Fatalf("response.output_item.done status = %#v, want completed: %#v", item["status"], payload) } if it.itemDone > 0 { t.Fatalf("response.output_item.done repeats item %q", id) } if it.itemType == "function_call" { if it.argumentsDone == 0 { t.Fatalf("function output_item.done precedes arguments done: %#v", payload) } } else if it.textDone == 0 || it.contentDone == 0 { t.Fatalf("output_item.done precedes text/content done: %#v", payload) } assertResponsesCompletedItem(t, it, item) it.itemDone++ case "response.completed": terminalCount++ sawCompleted = true response, ok := payload["response"].(map[string]any) if !ok || response["id"] == "" || response["object"] != "response" || response["status"] != "completed" || response["output"] == nil || response["usage"] == nil { t.Fatalf("response.completed has an incomplete response object: %#v", payload) } if responseID != "" && response["id"] != responseID { t.Fatalf("response.completed identity = %q, want %q", response["id"], responseID) } for _, id := range order { if items[id].itemDone != 1 { t.Fatalf("response.completed preceded item completion for %q: %#v", id, payloads) } } output, ok := response["output"].([]any) if !ok { t.Fatalf("response.completed output is not an array: %#v", response) } expectedOrder := append([]string(nil), order...) sort.Slice(expectedOrder, func(i, j int) bool { return items[expectedOrder[i]].outputIndex < items[expectedOrder[j]].outputIndex }) if len(output) != len(expectedOrder) { t.Fatalf("response.completed output length = %d, want %d: %#v", len(output), len(expectedOrder), output) } for position, rawItem := range output { item, ok := rawItem.(map[string]any) if !ok { t.Fatalf("response.completed output item is invalid: %#v", rawItem) } id, _ := item["id"].(string) if id != expectedOrder[position] { t.Fatalf("response.completed output item %d identity = %q, want %q: %#v", position, id, expectedOrder[position], item) } it := known(id) assertResponsesCompletedItem(t, it, item) } case "error": terminalCount++ requireString("code") requireString("message") default: t.Fatalf("unexpected Responses SSE event type %q: %#v", kind, payload) } } if terminalCount != 1 { t.Fatalf("Responses terminal event count = %d, want 1; payloads=%#v", terminalCount, payloads) } if payloads[len(payloads)-1]["type"] != wantTerminal { t.Fatalf("Responses terminal type = %q, want %q; payloads=%#v", payloads[len(payloads)-1]["type"], wantTerminal, payloads) } if sawCompleted { for _, id := range order { it := items[id] if it.itemType == "function_call" { if it.argumentsDone != 1 || it.itemDone != 1 { t.Fatalf("function item %q done counts = (args %d, item %d), want exactly one each", id, it.argumentsDone, it.itemDone) } continue } if it.textDone != 1 || it.contentDone != 1 || it.itemDone != 1 { t.Fatalf("item %q done counts = (text %d, content %d, item %d), want exactly one each", id, it.textDone, it.contentDone, it.itemDone) } } } return payloads } // assertResponsesTextEventKind rejects a text lifecycle event whose type does // not match the item's type (output_text for message, reasoning_text for // reasoning). func assertResponsesTextEventKind(t *testing.T, kind string, it *responsesOracleItem, phase string) { t.Helper() want := "response.output_text." + phase if it.itemType == "reasoning" { want = "response.reasoning_text." + phase } if kind != want { t.Fatalf("Responses %s event %q does not match item %q of type %q", phase, kind, it.id, it.itemType) } } // assertResponsesPartShape validates a content part against the item type. An // output-text part must carry annotations/logprobs arrays; a reasoning part // must not. wantText is compared when non-empty is expected (done parts). func assertResponsesPartShape(t *testing.T, itemType string, raw any, wantText string) { t.Helper() part, ok := raw.(map[string]any) if !ok { t.Fatalf("Responses content part is not an object: %#v", raw) } wantType := "output_text" if itemType == "reasoning" { wantType = "reasoning_text" } if part["type"] != wantType { t.Fatalf("Responses content part type = %#v, want %q: %#v", part["type"], wantType, part) } if got, ok := part["text"].(string); !ok || got != wantText { t.Fatalf("Responses content part text = %#v, want %q: %#v", part["text"], wantText, part) } if itemType == "reasoning" { return } if _, ok := part["annotations"].([]any); !ok { t.Fatalf("output_text content part missing annotations array: %#v", part) } if _, ok := part["logprobs"].([]any); !ok { t.Fatalf("output_text content part missing logprobs array: %#v", part) } } // assertResponsesCompletedItem validates a completed item object (in // response.output_item.done and terminal output) against the accumulated // oracle state. func assertResponsesCompletedItem(t *testing.T, it *responsesOracleItem, item map[string]any) { t.Helper() if item["id"] != it.id || item["type"] != it.itemType || item["status"] != "completed" { t.Fatalf("completed item identity/type/status = (%#v,%#v,%#v), want (%q,%q,%q): %#v", item["id"], item["type"], item["status"], it.id, it.itemType, "completed", item) } if it.itemType == "function_call" { if item["arguments"] != it.value { t.Fatalf("completed function arguments = %#v, want %q: %#v", item["arguments"], it.value, item) } if item["call_id"] != it.callID || item["name"] != it.name { t.Fatalf("completed function metadata = (%#v,%#v), want (%q,%q): %#v", item["call_id"], item["name"], it.callID, it.name, item) } return } content, ok := item["content"].([]any) if !ok || len(content) != 1 { t.Fatalf("completed content item is invalid: %#v", item) } assertResponsesPartShape(t, it.itemType, content[0], it.value) } // parseResponsesContentIndex checks that payload["content_index"] is present, // numeric (float64 in parsed JSON payload), and an exact integer. It returns // the integer value or a descriptive error. func parseResponsesContentIndex(payload map[string]any) (int, error) { raw, ok := payload["content_index"] if !ok { return 0, fmt.Errorf("missing content_index: %#v", payload) } floatVal, ok := raw.(float64) if !ok { return 0, fmt.Errorf("non-numeric content_index type %T (%#v): %#v", raw, raw, payload) } if floatVal != float64(int(floatVal)) { return 0, fmt.Errorf("non-integer content_index value %v: %#v", floatVal, payload) } return int(floatVal), nil } // validateResponsesContentIndex checks that payload["content_index"] is present, // numeric (float64 in parsed JSON payload), an exact integer, and equals expectedIndex. // It returns a descriptive error for missing, non-numeric, non-integer, or changed values. func validateResponsesContentIndex(payload map[string]any, expectedIndex int) error { got, err := parseResponsesContentIndex(payload) if err != nil { return err } if got != expectedIndex { return fmt.Errorf("content_index = %d, want stable %d: %#v", got, expectedIndex, payload) } return nil } func TestResponsesLifecycleContentIndexInvariant(t *testing.T) { tests := []struct { name string payload map[string]any expectedIndex int wantErr bool }{ { name: "matching content_index", payload: map[string]any{ "content_index": float64(0), }, expectedIndex: 0, wantErr: false, }, { name: "matching content_index non-zero", payload: map[string]any{ "content_index": float64(2), }, expectedIndex: 2, wantErr: false, }, { name: "changed content_index", payload: map[string]any{ "content_index": float64(9), }, expectedIndex: 0, wantErr: true, }, { name: "missing content_index", payload: map[string]any{ "item_id": "msg_1", }, expectedIndex: 0, wantErr: true, }, { name: "non-numeric string content_index", payload: map[string]any{ "content_index": "0", }, expectedIndex: 0, wantErr: true, }, { name: "non-numeric nil content_index", payload: map[string]any{ "content_index": nil, }, expectedIndex: 0, wantErr: true, }, { name: "fractional content_index matching truncated integer", payload: map[string]any{ "content_index": float64(0.5), }, expectedIndex: 0, wantErr: true, }, { name: "fractional content_index non-zero", payload: map[string]any{ "content_index": float64(2.5), }, expectedIndex: 2, wantErr: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { err := validateResponsesContentIndex(tt.payload, tt.expectedIndex) if (err != nil) != tt.wantErr { t.Errorf("validateResponsesContentIndex() error = %v, wantErr %v", err, tt.wantErr) } }) } } func responsesSSEOutputText(payloads []map[string]any) string { var output strings.Builder for _, payload := range payloads { if payload["type"] != "response.output_text.delta" { continue } text, _ := payload["delta"].(string) output.WriteString(text) } return output.String() } func joinedContent(chunks []chatCompletionChunk) string { var b strings.Builder for _, c := range chunks { if len(c.Choices) > 0 { b.WriteString(c.Choices[0].Delta.Content) } } return b.String() } // --- chat handler-level: disabled vs enabled Noop parity --------------------- func streamGateChatHandlerFixture(t *testing.T, enabled bool) (*Server, *fakeRunService) { t.Helper() fake := &fakeRunService{events: make(chan *iop.RunEvent, 3)} fake.events <- &iop.RunEvent{Type: "delta", Delta: "hello"} fake.events <- &iop.RunEvent{Type: "delta", Delta: " world"} fake.events <- &iop.RunEvent{Type: "complete", Usage: &iop.Usage{InputTokens: 1, OutputTokens: 2}} srv := NewServer(config.EdgeOpenAIConf{ Adapter: "ollama", Target: "llama-fixed", SessionID: "cli", TimeoutSec: 15, StreamEvidenceGate: config.StreamEvidenceGateConf{ Enabled: enabled, }, }, fake, nil) return srv, fake } func TestStreamGateChatHandlerDisabledVsEnabledNoopPass(t *testing.T) { body := `{"model":"client-model","messages":[{"role":"user","content":"hi"}],"stream":true}` for _, enabled := range []bool{false, true} { enabled := enabled name := "disabled" if enabled { name = "enabled" } t.Run(name, func(t *testing.T) { srv, fake := streamGateChatHandlerFixture(t, enabled) req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(body)) w := httptest.NewRecorder() srv.handleChatCompletions(w, req) if w.Code != http.StatusOK { t.Fatalf("status: got %d body=%s", w.Code, w.Body.String()) } chunks := parseSSEChatChunks(t, w.Body.String()) if len(chunks) == 0 { t.Fatalf("no SSE chunks decoded from body=%s", w.Body.String()) } if chunks[0].Choices[0].Delta.Role != "assistant" { t.Fatalf("first chunk role: got %+v", chunks[0].Choices[0].Delta) } if got := joinedContent(chunks); got != "hello world" { t.Fatalf("joined content: got %q", got) } last := chunks[len(chunks)-1] if last.Choices[0].FinishReason != "stop" { t.Fatalf("finish reason: got %q", last.Choices[0].FinishReason) } if !strings.Contains(w.Body.String(), "data: [DONE]") { t.Fatalf("missing [DONE] sentinel: body=%s", w.Body.String()) } if len(fake.reqsSnapshot()) != 1 { t.Fatalf("dispatch attempts: got %d, want 1", len(fake.reqsSnapshot())) } }) } } // --- adapter-level: chat runtime + injected violation recovery --------------- // buildStreamGateChatFixture builds a ready-to-run chatDispatchContext plus the // initial fakeRunResult a handler would already have dispatched, mirroring // production ingress without going through the HTTP decode path. func buildStreamGateChatFixture(t *testing.T, enabled bool, maxRequestFaultRecovery int) (*Server, *chatDispatchContext, *fakeRunResult, *fakeRunService) { t.Helper() rawBody := []byte(`{"model":"client-model","messages":[{"role":"user","content":"hi"}],"stream":true}`) var req chatCompletionRequest if err := json.Unmarshal(rawBody, &req); err != nil { t.Fatalf("decode fixture body: %v", err) } fault := maxRequestFaultRecovery fake := &fakeRunService{} srv := NewServer(config.EdgeOpenAIConf{ Adapter: "ollama", Target: "llama-fixed", SessionID: "cli", TimeoutSec: 15, StreamEvidenceGate: config.StreamEvidenceGateConf{ Enabled: enabled, MaxRequestFaultRecovery: &fault, }, }, fake, nil) route := routeDispatch{Adapter: "ollama", Target: "llama-fixed", SessionID: "cli", TimeoutSec: 15} requestCtx := newTestRequestContext(t, route, rawBody) basePrompt := promptFromMessages(req.Messages) outputPolicy := srv.resolveOutputPolicy(basePrompt) dc := srv.newChatDispatchContext(requestCtx, req, basePrompt, outputPolicy) initial := &fakeRunResult{ dispatch: edgeservice.RunDispatch{RunID: "run-attempt-1", Adapter: "ollama", Target: "llama-fixed", SessionID: "cli", TimeoutSec: 15}, events: bufferedRunEvents( &iop.RunEvent{Type: "delta", Delta: "should-be-discarded"}, &iop.RunEvent{Type: "complete"}, ), } return srv, dc, initial, fake } func TestStreamGateChatBuildRuntimeInjectedViolationSingleRetrySuccess(t *testing.T) { srv, dc, initial, fake := buildStreamGateChatFixture(t, true, 3) fake.eventRuns = []chan *iop.RunEvent{bufferedRunEvents( &iop.RunEvent{Type: "delta", Delta: "final answer"}, &iop.RunEvent{Type: "complete"}, )} fake.runIDs = []string{"run-attempt-2"} snapRef, err := dc.ingress.recoveryRef() if err != nil { t.Fatalf("recoveryRef: %v", err) } violation := newInjectedViolationFilter(t, "test.injected_violation", 1, snapRef.SnapshotRef()) reg, err := streamgate.NewFilterRegistration(violation, streamGateNoopCapability, true, streamgate.FilterEnforcementBlocking, streamGateFilterTimeout, injectedViolationPriority) if err != nil { t.Fatalf("NewFilterRegistration: %v", err) } registry := streamGateTestRegistry(t, reg) w := newRecordingResponseWriter() rt, usage, err := srv.buildOpenAIChatStreamGateRuntime(dc, initial, newOpenAIChatSSEReleaseSink(w, w, "chatcmpl-run-attempt-1", time.Now().Unix(), "client-model"), registry) if err != nil { t.Fatalf("buildOpenAIChatStreamGateRuntime: %v", err) } if err := rt.Run(context.Background()); err != nil { t.Fatalf("rt.Run: %v", err) } _ = usage if got := len(fake.reqsSnapshot()); got != 1 { t.Fatalf("recovery dispatch count: got %d, want 1 (exact_replay resubmits once)", got) } if got := w.headerCallCount(); got != 1 { t.Fatalf("WriteHeader call count: got %d, want 1 (no eager header from the discarded first attempt)", got) } chunks := parseSSEChatChunks(t, w.body.String()) roleCount := 0 for _, c := range chunks { if c.Choices[0].Delta.Role == "assistant" { roleCount++ } } if roleCount != 1 { t.Fatalf("assistant role chunk count: got %d, want 1", roleCount) } if got := joinedContent(chunks); got != "final answer" { t.Fatalf("joined content: got %q, want only the retry attempt's content (no leak from the discarded first attempt)", got) } } func TestOpenAIRepeatResumeBuildRunsAfterAbort(t *testing.T) { srv, dc, initial, fake := buildStreamGateChatFixture(t, true, 3) recorder := &recordingOpenAIObservationSink{} srv.SetObservationSink(recorder) // The catalog is the request-start source of the context window used by the // resume builder. The direct fixture still uses the normalized fake service. srv.SetModelCatalog([]config.ModelCatalogEntry{{ID: "client-model", ContextWindowTokens: 4096}}) initial.events = bufferedRunEvents( &iop.RunEvent{Type: "delta", Delta: "safe prefix "}, &iop.RunEvent{Type: "delta", Delta: "repeated-tail"}, &iop.RunEvent{Type: "complete"}, ) fake.eventRuns = []chan *iop.RunEvent{bufferedRunEvents( &iop.RunEvent{Type: "delta", Delta: "recovered answer"}, &iop.RunEvent{Type: "complete"}, )} fake.runIDs = []string{"run-attempt-2"} snapRef, err := dc.ingress.recoveryRef() if err != nil { t.Fatalf("recoveryRef: %v", err) } violation := newInjectedContinuationViolationFilter(t, "test.resume", 1, snapRef.SnapshotRef(), len("safe prefix ")) reg, err := streamgate.NewFilterRegistration(violation, streamGateNoopCapability, true, streamgate.FilterEnforcementBlocking, streamGateFilterTimeout, injectedViolationPriority) if err != nil { t.Fatalf("NewFilterRegistration: %v", err) } w := newRecordingResponseWriter() rt, _, err := srv.buildOpenAIChatStreamGateRuntime(dc, initial, newOpenAIChatSSEReleaseSink(w, w, "chatcmpl-run-attempt-1", time.Now().Unix(), "client-model"), streamGateTestRegistry(t, reg)) if err != nil { t.Fatalf("buildOpenAIChatStreamGateRuntime: %v", err) } if err := rt.Run(context.Background()); err != nil { t.Fatalf("rt.Run: %v", err) } if got := len(fake.reqsSnapshot()); got != 1 { t.Fatalf("recovery dispatch count = %d, want 1 after the aborted attempt; response=%s", got, w.body.String()) } if got := joinedContent(parseSSEChatChunks(t, w.body.String())); !strings.Contains(got, "recovered answer") { t.Fatalf("released content = %q, want the continuation attempt after abort", got) } var recovery []streamgate.ObservationKind for _, observation := range recorder.Snapshot() { switch observation.Kind() { case streamgate.ObservationKindRecoveryAttemptAborted, streamgate.ObservationKindRecoveryRebuilt, streamgate.ObservationKindRecoveryDispatched: recovery = append(recovery, observation.Kind()) } } wantRecovery := []streamgate.ObservationKind{ streamgate.ObservationKindRecoveryAttemptAborted, streamgate.ObservationKindRecoveryRebuilt, streamgate.ObservationKindRecoveryDispatched, } if len(recovery) != len(wantRecovery) { t.Fatalf("recovery lifecycle rows = %v, want %v", recovery, wantRecovery) } for i := range wantRecovery { if recovery[i] != wantRecovery[i] { t.Fatalf("recovery lifecycle order = %v, want %v", recovery, wantRecovery) } } } func TestRepeatGuardStreamOpenNoDuplicatePrefix(t *testing.T) { srv, dc, initial, fake := buildStreamGateChatFixture(t, true, 3) srv.SetModelCatalog([]config.ModelCatalogEntry{{ID: "client-model", ContextWindowTokens: 8192}}) safePrefix := uniqueKoreanRunes(520) initial.events = bufferedRunEvents( &iop.RunEvent{Type: "delta", Delta: safePrefix}, &iop.RunEvent{Type: "delta", Delta: safePrefix}, &iop.RunEvent{Type: "complete"}, ) fake.eventRuns = []chan *iop.RunEvent{bufferedRunEvents( // The replacement provider echoes the assistant prefill. The // request-local source must suppress that known prefix once. &iop.RunEvent{Type: "delta", Delta: safePrefix + "recovered suffix"}, &iop.RunEvent{Type: "complete"}, )} fake.runIDs = []string{"run-repeat-recovery"} snapRef, err := dc.ingress.recoveryRef() if err != nil { t.Fatalf("recoveryRef: %v", err) } gateCfg := config.StreamEvidenceGateConf{ Enabled: true, Filters: []config.StreamGateFilterPolicyConf{{ Filter: config.StreamGateFilterRepeatGuard, Enforcement: config.StreamGateFilterEnforcementBlocking, Priority: 10, HoldEvidenceRunes: 500, }}, } registry, err := openAIStreamGateRegistrySnapshotFor(gateCfg, openAIOutputFilterContext{ endpoint: openAIRebuildEndpointChat, requestRef: snapRef.SnapshotRef(), }) if err != nil { t.Fatalf("openAIStreamGateRegistrySnapshotFor: %v", err) } w := newRecordingResponseWriter() rt, _, err := srv.buildOpenAIChatStreamGateRuntime( dc, initial, newOpenAIChatSSEReleaseSink(w, w, "chatcmpl-repeat", time.Now().Unix(), "client-model"), registry, ) if err != nil { t.Fatalf("buildOpenAIChatStreamGateRuntime: %v", err) } if err := rt.Run(context.Background()); err != nil { t.Fatalf("rt.Run: %v", err) } if err := rt.CloseRequestResources(context.Background(), true); err != nil { t.Fatalf("CloseRequestResources: %v", err) } requests := fake.reqsSnapshot() if len(requests) != 1 { t.Fatalf("recovery dispatches = %d, want 1", len(requests)) } resumeMessages, _ := requests[0].Input["messages"].([]any) if len(resumeMessages) < 1 { t.Fatalf("recovery messages = %#v", requests[0].Input["messages"]) } resumeAssistant, _ := resumeMessages[0].(map[string]any) if got := resumeAssistant["content"]; got != safePrefix { t.Fatalf("recovery safe prefix length/value = (%T, %d), want %d bytes", got, len(fmt.Sprint(got)), len(safePrefix)) } options, _ := requests[0].Input["options"].(map[string]any) if got := options["temperature"]; got != 0.2 { t.Fatalf("recovery temperature = %#v, want 0.2", got) } body := w.body.String() chunks := parseSSEChatChunks(t, body) if got := joinedContent(chunks); got != safePrefix+"recovered suffix" { t.Fatalf("downstream content length=%d, want safe prefix once plus recovered suffix", len(got)) } roleCount := 0 for _, chunk := range chunks { if len(chunk.Choices) > 0 && chunk.Choices[0].Delta.Role == "assistant" { roleCount++ } } if roleCount != 1 { t.Fatalf("assistant opening count = %d, want 1", roleCount) } if got := strings.Count(body, "data: [DONE]"); got != 1 { t.Fatalf("[DONE] count = %d, want 1", got) } if got := len(fake.cancelCallsSnapshot()); got != 1 { t.Fatalf("initial abort count = %d, want 1", got) } } func TestStreamGateChatTunnelRepeatRecoveryExhaustionTerminatesSSE(t *testing.T) { safePrefix := uniqueKoreanRunes(520) service := newScriptedPoolRunService( scriptedPoolAttempt{ path: string(edgeservice.ProviderPoolPathTunnel), runID: "repeat-tunnel-1", provider: "provider-a", target: "served-a", frames: poolChatRepeatSSEFrames("chatcmpl-repeat-1", safePrefix), }, scriptedPoolAttempt{ path: string(edgeservice.ProviderPoolPathTunnel), runID: "repeat-tunnel-2", provider: "provider-b", target: "served-b", frames: poolChatRepeatSSEFrames("chatcmpl-repeat-2", safePrefix), }, scriptedPoolAttempt{ path: string(edgeservice.ProviderPoolPathTunnel), runID: "repeat-tunnel-3", provider: "provider-c", target: "served-c", frames: poolChatRepeatSSEFrames("chatcmpl-repeat-3", safePrefix), }, scriptedPoolAttempt{ path: string(edgeservice.ProviderPoolPathTunnel), runID: "repeat-tunnel-4", provider: "provider-d", target: "served-d", frames: poolChatRepeatSSEFrames("chatcmpl-repeat-4", safePrefix), }, ) rawBody := []byte(`{"model":"client-model","messages":[{"role":"user","content":"hi"}],"stream":true}`) srv, dc := buildStreamGatePoolChatFixture(t, service, rawBody, 3) srv.SetModelCatalog([]config.ModelCatalogEntry{{ID: "client-model", ContextWindowTokens: 32768}}) srv.mu.Lock() srv.cfg.StreamEvidenceGate.Filters = []config.StreamGateFilterPolicyConf{{ Filter: config.StreamGateFilterRepeatGuard, Enforcement: config.StreamGateFilterEnforcementBlocking, Priority: 10, HoldEvidenceRunes: 500, }} srv.mu.Unlock() w, sink, runErr := runPoolChatStreamGate(t, srv, dc) if runErr != nil { t.Fatalf("runtime.Run: %v", runErr) } if got := service.poolSubmits() - 1; got != 3 { t.Fatalf("recovery dispatch count=%d, want 3", got) } if got := w.headerCallCount(); got != 1 { t.Fatalf("WriteHeader count=%d, want 1", got) } if w.code != http.StatusOK { t.Fatalf("status=%d, want %d", w.code, http.StatusOK) } composite, ok := sink.(*openAICompositeReleaseSink) if !ok { t.Fatalf("release sink=%T, want *openAICompositeReleaseSink", sink) } if got := composite.resolvedCodec(); got != openAIStreamGateCodecTunnel { t.Fatalf("resolved codec=%q, want tunnel", got) } if committed, success := composite.terminalStatus(); !committed || success { t.Fatalf("terminal status=(%v,%v), want (true,false)", committed, success) } body := w.body.String() if got := strings.Count(body, safePrefix); got != 1 { t.Fatalf("accepted prefix occurrences=%d, want 1", got) } if got := strings.Count(body, `"error"`); got != 1 { t.Fatalf("error envelope count=%d, want 1; body=%q", got, body) } if got := strings.Count(body, "data: [DONE]"); got != 1 { t.Fatalf("[DONE] count=%d, want 1; body=%q", got, body) } for _, rejectedID := range []string{ "chatcmpl-repeat-2", "chatcmpl-repeat-3", "chatcmpl-repeat-4", } { if strings.Contains(body, rejectedID) { t.Fatalf("rejected attempt fragment %q reached the caller", rejectedID) } } evidence, err := validateDevRepeatGuardSSE([]byte(body)) if err != nil { t.Fatalf("caller SSE validation: %v; body=%q", err, body) } if evidence.terminal != "error" || len(evidence.providerResponseIDs) != 1 || evidence.providerResponseIDs[0] != "chatcmpl-repeat-1" { t.Fatalf("caller SSE evidence=%+v, want initial provider identity and error terminal", evidence) } if !strings.HasSuffix(strings.TrimSpace(body), "data: [DONE]") { t.Fatalf("[DONE] is not the final SSE data event: body=%q", body) } } func TestRepeatGuardStreamOpenNoDuplicatePrefixResponses(t *testing.T) { safePrefix := uniqueKoreanRunes(520) initialPayload := responsesStreamPrefix("resp-repeat", "msg-repeat", safePrefix) + responsesTextDelta("msg-repeat", safePrefix, 5) service := newScriptedPoolRunService( scriptedPoolAttempt{ path: string(edgeservice.ProviderPoolPathTunnel), runID: "responses-repeat-initial", provider: "provider-a", target: "served-a", frames: responsesTunnelFrames(initialPayload), }, scriptedPoolAttempt{ path: string(edgeservice.ProviderPoolPathNormalized), runID: "responses-repeat-recovery", provider: "provider-b", target: "served-b", runEvents: bufferedRunEvents( &iop.RunEvent{Type: "delta", Delta: safePrefix + "recovered suffix"}, &iop.RunEvent{Type: "complete"}, ), }, ) srv, requestCtx, poolReq := buildStreamGateResponsesPoolFixture(t, service, 3) srv.SetModelCatalog([]config.ModelCatalogEntry{{ID: "client-model", ContextWindowTokens: 8192}}) result, err := service.SubmitProviderPool(context.Background(), poolReq) if err != nil { t.Fatalf("initial SubmitProviderPool: %v", err) } dc := newOpenAIResponsesPoolTunnelDispatchContext(requestCtx, poolReq) snapRef, err := requestCtx.ingress.recoveryRef() if err != nil { t.Fatalf("recoveryRef: %v", err) } gateCfg := config.StreamEvidenceGateConf{ Enabled: true, Filters: []config.StreamGateFilterPolicyConf{{ Filter: config.StreamGateFilterRepeatGuard, Enforcement: config.StreamGateFilterEnforcementBlocking, Priority: 10, HoldEvidenceRunes: 500, }}, } registry, err := openAIStreamGateRegistrySnapshotFor(gateCfg, openAIOutputFilterContext{ endpoint: openAIRebuildEndpointResponses, requestRef: snapRef.SnapshotRef(), }) if err != nil { t.Fatalf("openAIStreamGateRegistrySnapshotFor: %v", err) } w := newRecordingResponseWriter() holder := &openAIResponsesResultHolder{} selector := newOpenAIStreamGateCodecSelector(openAIStreamGateCodecTunnel) sink := newOpenAIResponsesPoolReleaseSink(w, holder, selector) runtime, _, err := srv.buildOpenAIResponsesStreamGateRuntimeFromAttempt( dc, openAIAttemptTransport{path: openAIAdmissionTunnel, tunnel: result.Tunnel}, result.Tunnel.Dispatch(), result.Tunnel.Close, sink, registry, ) if err != nil { t.Fatalf("buildOpenAIResponsesStreamGateRuntimeFromAttempt: %v", err) } if err := runtime.Run(context.Background()); err != nil { t.Fatalf("runtime.Run: %v", err) } if err := runtime.CloseRequestResources(context.Background(), true); err != nil { t.Fatalf("CloseRequestResources: %v", err) } if got := service.poolSubmits(); got != 2 { t.Fatalf("SubmitProviderPool calls = %d, want initial plus one recovery", got) } payloads := assertResponsesSSELifecycle(t, w.body.String(), "response.completed") if got := responsesSSEOutputText(payloads); got != safePrefix+"recovered suffix" { t.Fatalf("Responses downstream content length=%d, want safe prefix once plus recovered suffix", len(got)) } responseStarts := 0 terminals := 0 for _, payload := range payloads { switch payload["type"] { case "response.created": responseStarts++ case "response.completed", "error": terminals++ } } if responseStarts != 1 || terminals != 1 { t.Fatalf("Responses lifecycle starts=%d terminals=%d, want 1/1", responseStarts, terminals) } } func TestOpenAIRepeatResumeContextOverflowDoesNotDispatch(t *testing.T) { srv, dc, initial, fake := buildStreamGateChatFixture(t, true, 3) // No catalog entry means the context window is unknown. S20 must refuse the // continuation before the dispatcher can submit a replacement attempt. initial.events = bufferedRunEvents( &iop.RunEvent{Type: "delta", Delta: "output awaiting a repeat resume"}, &iop.RunEvent{Type: "complete"}, ) snapRef, err := dc.ingress.recoveryRef() if err != nil { t.Fatalf("recoveryRef: %v", err) } violation := newInjectedContinuationViolationFilter(t, "test.resume.unknown-context", 1, snapRef.SnapshotRef(), len("output ")) reg, err := streamgate.NewFilterRegistration(violation, streamGateNoopCapability, true, streamgate.FilterEnforcementBlocking, streamGateFilterTimeout, injectedViolationPriority) if err != nil { t.Fatalf("NewFilterRegistration: %v", err) } w := newRecordingResponseWriter() rt, _, err := srv.buildOpenAIChatStreamGateRuntime(dc, initial, newOpenAIChatSSEReleaseSink(w, w, "chatcmpl-run-attempt-1", time.Now().Unix(), "client-model"), streamGateTestRegistry(t, reg)) if err != nil { t.Fatalf("buildOpenAIChatStreamGateRuntime: %v", err) } if err := rt.Run(context.Background()); err != nil { t.Fatalf("rt.Run: %v", err) } if got := len(fake.reqsSnapshot()); got != 0 { t.Fatalf("context refusal dispatched %d recovery attempts, want 0", got) } } func buildStreamGateResponsesFixture(t *testing.T, service runService, providerPool bool) (*Server, *responsesDispatchContext) { t.Helper() body := []byte(`{"model":"client-model","input":"CALLER_RESPONSES_INPUT_MUST_NOT_BE_COPIED"}`) fault := 3 srv := NewServer(config.EdgeOpenAIConf{ Adapter: "ollama", Target: "llama-fixed", SessionID: "cli", TimeoutSec: 15, StreamEvidenceGate: config.StreamEvidenceGateConf{Enabled: true, MaxRequestFaultRecovery: &fault}, }, service, nil) srv.SetModelCatalog([]config.ModelCatalogEntry{{ID: "client-model", ContextWindowTokens: 4096}}) route := routeDispatch{Adapter: "ollama", Target: "llama-fixed", SessionID: "cli", TimeoutSec: 15, ProviderPool: providerPool} base := newTestRequestContext(t, route, body) base.r = httptest.NewRequest(http.MethodPost, "/v1/responses", nil) base.endpoint = usageEndpointResponses requestCtx := &responsesRequestContext{openAIRequestContext: base, envelope: responsesEnvelope{Model: "client-model"}} dc, err := srv.newResponsesDispatchContext(requestCtx, responsesRequest{Model: "client-model", Input: json.RawMessage(`"CALLER_RESPONSES_INPUT_MUST_NOT_BE_COPIED"`)}) if err != nil { t.Fatalf("newResponsesDispatchContext: %v", err) } if !providerPool { return srv, dc } pool := edgeservice.ProviderPoolDispatchRequest{ Run: dc.submitReq, Tunnel: edgeservice.SubmitProviderTunnelRequest{ ModelGroupKey: "client-model", SessionID: route.SessionID, Method: http.MethodPost, Path: "/v1/responses", ProviderPool: true, }, } return srv, dc.withPoolDispatch(pool) } func TestOpenAIResponsesRepeatResumeDispatchesNormalized(t *testing.T) { fake := &fakeRunService{eventRuns: []chan *iop.RunEvent{bufferedRunEvents(&iop.RunEvent{Type: "complete"})}, runIDs: []string{"responses-attempt-2"}} srv, dc := buildStreamGateResponsesFixture(t, fake, false) source := newOpenAIRecoverySourceStore(dc.ingress) source.recordText("safe prefix repeated tail") ref, err := dc.ingress.recoveryRef() if err != nil { t.Fatalf("recoveryRef: %v", err) } rebuilder, err := newOpenAIRequestRebuilder(dc.ingress, openAIRebuildEndpointResponses, source, 4096) if err != nil { t.Fatalf("newOpenAIRequestRebuilder: %v", err) } defer rebuilder.Close() directive, err := streamgate.NewRecoveryDirectiveContinuation(len("safe prefix "), ref.SnapshotRef()) if err != nil { t.Fatalf("NewRecoveryDirectiveContinuation: %v", err) } plan := mustOpenAIRecoveryPlan(t, "plan.responses.dispatch.normalized", streamgate.RecoveryStrategyContinuationRepair, directive) draft, err := rebuilder.RebuildRequest(context.Background(), ref, plan) if err != nil { t.Fatalf("RebuildRequest: %v", err) } _, request, err := plan.FinalizeRebuiltRequest(draft) if err != nil { t.Fatalf("FinalizeRebuiltRequest: %v", err) } state := &openAIResponsesAttemptContext{} dispatcher, err := newOpenAIAttemptDispatcher(srv.service, rebuilder.RebuiltStore(), newOpenAIResponsesRecoveryAdmissionBuilder(srv, dc, state), func(transport openAIAttemptTransport) (streamgate.NormalizedEventSource, error) { if transport.run == nil || state.get() == nil { return nil, fmt.Errorf("normalized Responses replacement was not admitted") } return newOpenAIResponsesEventSource(state.get(), transport.run, &openAIResponsesResultHolder{}, nil), nil }) if err != nil { t.Fatalf("newOpenAIAttemptDispatcher: %v", err) } binding, err := dispatcher.DispatchAttempt(context.Background(), request) if err != nil { t.Fatalf("DispatchAttempt: %v", err) } for { event, eventErr := binding.EventSource().NextEvent(context.Background()) if eventErr != nil { t.Fatalf("replacement event source: %v", eventErr) } if event.Kind() == streamgate.EventKindTerminal { break } } requests := fake.reqsSnapshot() if len(requests) != 1 { t.Fatalf("recovery SubmitRun calls = %d, want 1", len(requests)) } if strings.Contains(requests[0].Prompt, "CALLER_RESPONSES_INPUT_MUST_NOT_BE_COPIED") { t.Fatalf("recovery request copied caller input: %q", requests[0].Prompt) } if got := requests[0].Input["responses_resume_content"]; got != "safe prefix " { t.Fatalf("recovery content provenance = %#v", got) } } func TestOpenAIResponsesRepeatResumeDispatchesProviderPool(t *testing.T) { service := newScriptedPoolRunService( scriptedPoolAttempt{path: string(edgeservice.ProviderPoolPathNormalized), runID: "responses-pool-attempt-2", provider: "provider-b", target: "served-b", runEvents: bufferedRunEvents( &iop.RunEvent{Type: "complete"}, )}, ) srv, dc := buildStreamGateResponsesFixture(t, service, true) source := newOpenAIRecoverySourceStore(dc.ingress) source.recordText("safe prefix repeated tail") ref, err := dc.ingress.recoveryRef() if err != nil { t.Fatalf("recoveryRef: %v", err) } rebuilder, err := newOpenAIRequestRebuilder(dc.ingress, openAIRebuildEndpointResponses, source, 4096) if err != nil { t.Fatalf("newOpenAIRequestRebuilder: %v", err) } defer rebuilder.Close() directive, err := streamgate.NewRecoveryDirectiveContinuation(len("safe prefix "), ref.SnapshotRef()) if err != nil { t.Fatalf("NewRecoveryDirectiveContinuation: %v", err) } plan := mustOpenAIRecoveryPlan(t, "plan.responses.dispatch.pool", streamgate.RecoveryStrategyContinuationRepair, directive) draft, err := rebuilder.RebuildRequest(context.Background(), ref, plan) if err != nil { t.Fatalf("RebuildRequest: %v", err) } _, request, err := plan.FinalizeRebuiltRequest(draft) if err != nil { t.Fatalf("FinalizeRebuiltRequest: %v", err) } state := &openAIResponsesAttemptContext{} dispatcher, err := newOpenAIAttemptDispatcher(srv.service, rebuilder.RebuiltStore(), newOpenAIResponsesRecoveryAdmissionBuilder(srv, dc, state), func(transport openAIAttemptTransport) (streamgate.NormalizedEventSource, error) { if transport.run == nil || state.get() == nil { return nil, fmt.Errorf("provider-pool Responses replacement was not admitted as normalized") } return newOpenAIResponsesEventSource(state.get(), transport.run, &openAIResponsesResultHolder{}, nil), nil }) if err != nil { t.Fatalf("newOpenAIAttemptDispatcher: %v", err) } binding, err := dispatcher.DispatchAttempt(context.Background(), request) if err != nil { t.Fatalf("DispatchAttempt: %v", err) } for { event, eventErr := binding.EventSource().NextEvent(context.Background()) if eventErr != nil { t.Fatalf("provider-pool replacement event source: %v", eventErr) } if event.Kind() == streamgate.EventKindTerminal { break } } pools, _, _, _, runRequests, _ := service.snapshot() if pools != 1 || len(runRequests) != 1 { t.Fatalf("provider-pool replacement = pools:%d runs:%d, want 1 admission", pools, len(runRequests)) } resume := runRequests[0] if !resume.ProviderPool || strings.Contains(resume.Prompt, "CALLER_RESPONSES_INPUT_MUST_NOT_BE_COPIED") { t.Fatalf("pool resume request = %#v", resume) } if got := resume.Input["responses_resume_content"]; got != "safe prefix " { t.Fatalf("pool recovery content provenance = %#v", got) } } func TestStreamGateChatBuildRuntimeRecoveryBudgetExhaustionTerminatesWithError(t *testing.T) { srv, dc, initial, fake := buildStreamGateChatFixture(t, true, 1) fake.eventRuns = []chan *iop.RunEvent{bufferedRunEvents( &iop.RunEvent{Type: "delta", Delta: "still-bad"}, &iop.RunEvent{Type: "complete"}, )} fake.runIDs = []string{"run-attempt-2"} snapRef, err := dc.ingress.recoveryRef() if err != nil { t.Fatalf("recoveryRef: %v", err) } // trigger=2: violates on both the initial attempt and the one allowed // recovery attempt, so the request-total cap (1) is exhausted after the // first recovery and the second violation must terminate, not dispatch a // third attempt. violation := newInjectedViolationFilter(t, "test.injected_violation", 2, snapRef.SnapshotRef()) reg, err := streamgate.NewFilterRegistration(violation, streamGateNoopCapability, true, streamgate.FilterEnforcementBlocking, streamGateFilterTimeout, injectedViolationPriority) if err != nil { t.Fatalf("NewFilterRegistration: %v", err) } registry := streamGateTestRegistry(t, reg) w := newRecordingResponseWriter() rt, _, err := srv.buildOpenAIChatStreamGateRuntime(dc, initial, newOpenAIChatSSEReleaseSink(w, w, "chatcmpl-run-attempt-1", time.Now().Unix(), "client-model"), registry) if err != nil { t.Fatalf("buildOpenAIChatStreamGateRuntime: %v", err) } if err := rt.Run(context.Background()); err != nil { t.Fatalf("rt.Run: %v", err) } if got := len(fake.reqsSnapshot()); got != 1 { t.Fatalf("recovery dispatch count: got %d, want 1 (cap=1 allows exactly one recovery dispatch)", got) } if !strings.Contains(w.body.String(), `"error"`) { t.Fatalf("expected an SSE error terminal, got body=%s", w.body.String()) } if strings.Contains(w.body.String(), "still-bad") { t.Fatalf("exhausted recovery must not leak the still-violating attempt's content: body=%s", w.body.String()) } } func TestStreamGateChatBuildRuntimeObserveOnlyDoesNotBlock(t *testing.T) { srv, dc, initial, fake := buildStreamGateChatFixture(t, true, 3) initial.events = bufferedRunEvents( &iop.RunEvent{Type: "delta", Delta: "observed content"}, &iop.RunEvent{Type: "complete"}, ) snapRef, err := dc.ingress.recoveryRef() if err != nil { t.Fatalf("recoveryRef: %v", err) } violation := newInjectedViolationFilter(t, "test.injected_violation_observe", 1, snapRef.SnapshotRef()) reg, err := streamgate.NewFilterRegistration(violation, streamGateNoopCapability, true, streamgate.FilterEnforcementObserveOnly, streamGateFilterTimeout, injectedViolationPriority) if err != nil { t.Fatalf("NewFilterRegistration: %v", err) } registry := streamGateTestRegistry(t, reg) w := newRecordingResponseWriter() rt, _, err := srv.buildOpenAIChatStreamGateRuntime(dc, initial, newOpenAIChatSSEReleaseSink(w, w, "chatcmpl-run-attempt-1", time.Now().Unix(), "client-model"), registry) if err != nil { t.Fatalf("buildOpenAIChatStreamGateRuntime: %v", err) } if err := rt.Run(context.Background()); err != nil { t.Fatalf("rt.Run: %v", err) } if got := len(fake.reqsSnapshot()); got != 0 { t.Fatalf("observe-only violation must not trigger a recovery dispatch: got %d dispatches", got) } chunks := parseSSEChatChunks(t, w.body.String()) if got := joinedContent(chunks); got != "observed content" { t.Fatalf("joined content: got %q, want the pass-through content unblocked by the observe-only violation", got) } } // --- adapter-level: tunnel runtime Noop passthrough --------------------------- func TestStreamGateTunnelBuildRuntimeNoopPassRelaysRawBytes(t *testing.T) { rawBody := []byte(`{"model":"client-model","messages":[{"role":"user","content":"hi"}],"stream":true}`) fault := 3 fake := &providerFakeRunService{} srv := NewServer(config.EdgeOpenAIConf{ Adapter: "openai-compat", Target: "served-model", TimeoutSec: 15, StreamEvidenceGate: config.StreamEvidenceGateConf{ Enabled: true, MaxRequestFaultRecovery: &fault, }, }, fake, nil) route := routeDispatch{Adapter: "openai-compat", Target: "served-model", TimeoutSec: 15} requestCtx := newTestRequestContext(t, route, rawBody) frames := make(chan *iop.ProviderTunnelFrame, 4) frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START, StatusCode: 200, Headers: map[string]string{"Content-Type": "text/event-stream"}} frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte("data: {\"choices\":[{\"delta\":{\"content\":\"raw\"}}]}\n\n")} frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END} close(frames) handle := &fakeTunnelHandle{ dispatch: edgeservice.RunDispatch{RunID: "tunnel-attempt-1", Adapter: "openai-compat", Target: "served-model"}, frames: frames, } req := openAITunnelStreamGateRequest{ route: route, ingress: requestCtx.ingress, endpoint: openAIRebuildEndpointChat, method: http.MethodPost, path: "/v1/chat/completions", modelGroupKey: "client-model", requestModel: "", // no model echo present in the fixture body; rewrite is a no-op authorize: func(context.Context) (map[string]string, error) { return nil, nil }, rewriteBody: func(body []byte, target string) ([]byte, error) { return body, nil }, } w := newRecordingResponseWriter() registry := streamGateTestRegistry(t) rt, _, err := srv.buildOpenAITunnelStreamGateRuntime(req, handle, newOpenAITunnelReleaseSink(w, w), registry) if err != nil { t.Fatalf("buildOpenAITunnelStreamGateRuntime: %v", err) } if err := rt.Run(context.Background()); err != nil { t.Fatalf("rt.Run: %v", err) } if w.code != http.StatusOK { t.Fatalf("status: got %d", w.code) } if got := w.headerCallCount(); got != 1 { t.Fatalf("WriteHeader call count: got %d, want 1", got) } if !strings.Contains(w.body.String(), `"content":"raw"`) { t.Fatalf("raw provider bytes were not relayed byte-for-byte: body=%s", w.body.String()) } } // --- S16: strategy recovery cap enforcement (REVIEW_EDGE_VERTICAL_SLICE-4) --- // newStrategyCapChatFixture mirrors buildStreamGateChatFixture but pins an // explicit per-strategy recovery cap so the request-start RuntimeOptions // snapshot carries a non-empty strategy-limit map into the Core policy. func newStrategyCapChatFixture(t *testing.T, maxRequestFaultRecovery, maxStrategyFaultRecovery int) (*Server, *chatDispatchContext, *fakeRunResult, *fakeRunService) { t.Helper() rawBody := []byte(`{"model":"client-model","messages":[{"role":"user","content":"hi"}],"stream":true}`) var req chatCompletionRequest if err := json.Unmarshal(rawBody, &req); err != nil { t.Fatalf("decode fixture body: %v", err) } total := maxRequestFaultRecovery strategy := maxStrategyFaultRecovery fake := &fakeRunService{} srv := NewServer(config.EdgeOpenAIConf{ Adapter: "ollama", Target: "llama-fixed", SessionID: "cli", TimeoutSec: 15, StreamEvidenceGate: config.StreamEvidenceGateConf{ Enabled: true, MaxRequestFaultRecovery: &total, MaxStrategyFaultRecovery: &strategy, }, }, fake, nil) route := routeDispatch{Adapter: "ollama", Target: "llama-fixed", SessionID: "cli", TimeoutSec: 15} requestCtx := newTestRequestContext(t, route, rawBody) basePrompt := promptFromMessages(req.Messages) outputPolicy := srv.resolveOutputPolicy(basePrompt) dc := srv.newChatDispatchContext(requestCtx, req, basePrompt, outputPolicy) initial := &fakeRunResult{ dispatch: edgeservice.RunDispatch{RunID: "run-attempt-1", Adapter: "ollama", Target: "llama-fixed", SessionID: "cli", TimeoutSec: 15}, events: bufferedRunEvents( &iop.RunEvent{Type: "delta", Delta: "should-be-discarded"}, &iop.RunEvent{Type: "complete"}, ), } return srv, dc, initial, fake } func runStrategyCapChatFixture(t *testing.T, srv *Server, dc *chatDispatchContext, initial *fakeRunResult, trigger int) (*recordingResponseWriter, error) { t.Helper() snapRef, err := dc.ingress.recoveryRef() if err != nil { t.Fatalf("recoveryRef: %v", err) } violation := newInjectedViolationFilter(t, "test.injected_violation", trigger, snapRef.SnapshotRef()) reg, err := streamgate.NewFilterRegistration(violation, streamGateNoopCapability, true, streamgate.FilterEnforcementBlocking, streamGateFilterTimeout, injectedViolationPriority) if err != nil { t.Fatalf("NewFilterRegistration: %v", err) } registry := streamGateTestRegistry(t, reg) w := newRecordingResponseWriter() rt, _, err := srv.buildOpenAIChatStreamGateRuntime(dc, initial, newOpenAIChatSSEReleaseSink(w, w, "chatcmpl-run-attempt-1", time.Now().Unix(), "client-model"), registry) if err != nil { t.Fatalf("buildOpenAIChatStreamGateRuntime: %v", err) } return w, rt.Run(context.Background()) } // TestStreamGateChatStrategyCapZeroBlocksExactReplayRecovery verifies that an // explicit strategy cap of 0 forbids the exact_replay fault strategy even when // the request-total cap allows recovery: the single violation must terminate // with an error and dispatch zero recovery attempts. func TestStreamGateChatStrategyCapZeroBlocksExactReplayRecovery(t *testing.T) { srv, dc, initial, fake := newStrategyCapChatFixture(t, 3, 0) fake.eventRuns = []chan *iop.RunEvent{bufferedRunEvents( &iop.RunEvent{Type: "delta", Delta: "must-not-be-reached"}, &iop.RunEvent{Type: "complete"}, )} fake.runIDs = []string{"run-attempt-2"} w, runErr := runStrategyCapChatFixture(t, srv, dc, initial, 1) if runErr != nil { t.Fatalf("rt.Run: %v", runErr) } if got := len(fake.reqsSnapshot()); got != 0 { t.Fatalf("strategy cap 0 must block all exact_replay dispatch: got %d dispatches", got) } if !strings.Contains(w.body.String(), `"error"`) { t.Fatalf("expected an SSE error terminal for a strategy-capped violation, body=%s", w.body.String()) } } // TestStreamGateChatStrategyCapOneExhaustsOnSecondViolation verifies that a // strategy cap of 1 (below the request-total cap of 3) allows exactly one // exact_replay recovery dispatch and then exhausts, terminating on the second // violation instead of dispatching a second recovery. func TestStreamGateChatStrategyCapOneExhaustsOnSecondViolation(t *testing.T) { srv, dc, initial, fake := newStrategyCapChatFixture(t, 3, 1) fake.eventRuns = []chan *iop.RunEvent{bufferedRunEvents( &iop.RunEvent{Type: "delta", Delta: "still-bad"}, &iop.RunEvent{Type: "complete"}, )} fake.runIDs = []string{"run-attempt-2"} w, runErr := runStrategyCapChatFixture(t, srv, dc, initial, 2) if runErr != nil { t.Fatalf("rt.Run: %v", runErr) } if got := len(fake.reqsSnapshot()); got != 1 { t.Fatalf("strategy cap 1 allows exactly one exact_replay dispatch: got %d", got) } if !strings.Contains(w.body.String(), `"error"`) { t.Fatalf("expected an SSE error terminal once the strategy cap is exhausted, body=%s", w.body.String()) } } // --- S16: current-binding lifecycle + terminal outcome (REVIEW_EDGE_VERTICAL_SLICE-3) --- // TestStreamGateChatCloseRequestResourcesAbortsLatestRun verifies that after a // recovery cycle the request runtime cleans up the *current* (latest) attempt // binding — not just the initial handle — and that a non-graceful close cancels // the latest provider run while a graceful close does not, and that repeated // closes are idempotent. func TestStreamGateChatCloseRequestResourcesAbortsLatestRun(t *testing.T) { for _, graceful := range []bool{false, true} { graceful := graceful name := "abort" if graceful { name = "graceful" } t.Run(name, func(t *testing.T) { srv, dc, initial, fake := buildStreamGateChatFixture(t, true, 3) fake.eventRuns = []chan *iop.RunEvent{bufferedRunEvents( &iop.RunEvent{Type: "delta", Delta: "final answer"}, &iop.RunEvent{Type: "complete"}, )} fake.runIDs = []string{"run-attempt-2"} snapRef, err := dc.ingress.recoveryRef() if err != nil { t.Fatalf("recoveryRef: %v", err) } violation := newInjectedViolationFilter(t, "test.injected_violation", 1, snapRef.SnapshotRef()) reg, err := streamgate.NewFilterRegistration(violation, streamGateNoopCapability, true, streamgate.FilterEnforcementBlocking, streamGateFilterTimeout, injectedViolationPriority) if err != nil { t.Fatalf("NewFilterRegistration: %v", err) } registry := streamGateTestRegistry(t, reg) w := newRecordingResponseWriter() rt, _, err := srv.buildOpenAIChatStreamGateRuntime(dc, initial, newOpenAIChatSSEReleaseSink(w, w, "chatcmpl-run-attempt-1", time.Now().Unix(), "client-model"), registry) if err != nil { t.Fatalf("buildOpenAIChatStreamGateRuntime: %v", err) } if err := rt.Run(context.Background()); err != nil { t.Fatalf("rt.Run: %v", err) } // The recovery cycle already aborted the initial attempt (run-attempt-1). cancelsAfterRun := fake.cancelCallsSnapshot() if !cancelRunIDsContain(cancelsAfterRun, "run-attempt-1") { t.Fatalf("recovery must abort the initial attempt run-attempt-1, cancels=%v", cancelRunIDs(cancelsAfterRun)) } if cancelRunIDsContain(cancelsAfterRun, "run-attempt-2") { t.Fatalf("the latest attempt must not be canceled by a successful terminal, cancels=%v", cancelRunIDs(cancelsAfterRun)) } if err := rt.CloseRequestResources(context.Background(), graceful); err != nil { t.Fatalf("CloseRequestResources: %v", err) } // Idempotent: a second close must be a no-op regardless of the flag. if err := rt.CloseRequestResources(context.Background(), !graceful); err != nil { t.Fatalf("second CloseRequestResources: %v", err) } cancelsAfterClose := fake.cancelCallsSnapshot() latestCanceled := cancelRunIDsContain(cancelsAfterClose, "run-attempt-2") if graceful && latestCanceled { t.Fatalf("graceful close must not cancel the latest run, cancels=%v", cancelRunIDs(cancelsAfterClose)) } if !graceful && !latestCanceled { t.Fatalf("non-graceful close must cancel the latest run once, cancels=%v", cancelRunIDs(cancelsAfterClose)) } // Exactly-once: at most one cancel per attempt id. if n := cancelRunIDCount(cancelsAfterClose, "run-attempt-1"); n != 1 { t.Fatalf("initial attempt canceled %d times, want exactly 1", n) } if n := cancelRunIDCount(cancelsAfterClose, "run-attempt-2"); n > 1 { t.Fatalf("latest attempt canceled %d times, want at most 1", n) } }) } } func cancelRunIDs(cancels []edgeservice.CancelRunRequest) []string { ids := make([]string, 0, len(cancels)) for _, c := range cancels { ids = append(ids, c.RunID) } return ids } func cancelRunIDsContain(cancels []edgeservice.CancelRunRequest, runID string) bool { return cancelRunIDCount(cancels, runID) > 0 } func cancelRunIDCount(cancels []edgeservice.CancelRunRequest, runID string) int { n := 0 for _, c := range cancels { if c.RunID == runID { n++ } } return n } // TestStreamGateUsageStatusFromTerminalOutcome verifies the host maps a // Core-committed terminal into the correct usage-metric status: a committed // error terminal is reported as error (never success), a committed success // terminal as success, and a caller cancel/timeout as cancel. func TestStreamGateUsageStatusFromTerminalOutcome(t *testing.T) { cases := []struct { name string runErr error committed bool success bool want string }{ {"success terminal", nil, true, true, usageStatusSuccess}, {"error terminal not success metric", nil, true, false, usageStatusError}, {"caller cancel", context.Canceled, false, false, usageStatusCancel}, {"deadline exceeded", context.DeadlineExceeded, false, false, usageStatusCancel}, {"internal run error", fmt.Errorf("boom"), false, false, usageStatusError}, {"no terminal committed", nil, false, false, usageStatusError}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { if got := streamGateUsageStatus(tc.runErr, tc.committed, tc.success); got != tc.want { t.Fatalf("streamGateUsageStatus(%v,%v,%v) = %q, want %q", tc.runErr, tc.committed, tc.success, got, tc.want) } }) } } // TestOpenAIChatSSESinkCapturesTerminalOutcome verifies the release sink records // the terminal disposition the Core commits so the host outcome is authoritative. func TestOpenAIChatSSESinkCapturesTerminalOutcome(t *testing.T) { successTerm, err := streamgate.NewSuccessTerminalResult(streamGateChannelDefault, time.Now()) if err != nil { t.Fatalf("NewSuccessTerminalResult: %v", err) } w := newRecordingResponseWriter() sink := newOpenAIChatSSEReleaseSink(w, w, "chatcmpl-x", time.Now().Unix(), "m") if committed, _ := sink.terminalStatus(); committed { t.Fatalf("sink reported a committed terminal before any commit") } if _, err := sink.CommitTerminal(context.Background(), successTerm); err != nil { t.Fatalf("CommitTerminal: %v", err) } committed, success := sink.terminalStatus() if !committed || !success { t.Fatalf("sink terminalStatus after success = (%v,%v), want (true,true)", committed, success) } } // Note: the separate fixed-run recovery and fixed-tunnel pass fixtures above // do not prove the S16 provider/path-switch requirement. That evidence needs a // provider-pool host fixture in which one RequestRuntime recovery cycle changes // the actual AttemptBinding execution path before release. // --- S16: provider-pool one cycle, path switch, and single re-admission ------ // buildStreamGatePoolChatFixture builds a provider-pool chat dispatch context // bound to a scripted pool admission double, mirroring what // handleChatCompletionsProviderPool freezes before its first SubmitProviderPool // call. The returned pool request is the same template the recovery admission // builder re-enters for every recovery attempt. func buildStreamGatePoolChatFixture(t *testing.T, service runService, rawBody []byte, maxRequestFaultRecovery int) (*Server, *chatDispatchContext) { t.Helper() var req chatCompletionRequest if err := json.Unmarshal(rawBody, &req); err != nil { t.Fatalf("decode fixture body: %v", err) } fault := maxRequestFaultRecovery srv := NewServer(config.EdgeOpenAIConf{ Adapter: "ollama", Target: "llama-fixed", TimeoutSec: 15, StreamEvidenceGate: config.StreamEvidenceGateConf{ Enabled: true, MaxRequestFaultRecovery: &fault, }, }, service, nil) route := routeDispatch{ProviderPool: true, SessionID: "cli", TimeoutSec: 15} requestCtx := newTestRequestContext(t, route, rawBody) basePrompt := promptFromMessages(req.Messages) outputPolicy := srv.resolveOutputPolicy(basePrompt) dc := srv.newChatDispatchContext(requestCtx, req, basePrompt, outputPolicy) poolReq := edgeservice.ProviderPoolDispatchRequest{ Run: dc.submitReq, Tunnel: edgeservice.SubmitProviderTunnelRequest{ ModelGroupKey: strings.TrimSpace(req.Model), SessionID: route.SessionID, Method: http.MethodPost, Path: "/v1/chat/completions", Stream: req.Stream, TimeoutSec: route.TimeoutSec, Metadata: dc.runMetadata, EstimatedInputTokens: dc.estimate, ContextClass: dc.contextClass, ProviderPool: true, }, PrepareTunnel: func(tunnelReq edgeservice.SubmitProviderTunnelRequest) (edgeservice.SubmitProviderTunnelRequest, error) { return tunnelReq, nil }, PrepareRun: func(runReq edgeservice.SubmitRunRequest) (edgeservice.SubmitRunRequest, error) { return runReq, nil }, } poolReq.Tunnel.BuildBody = func(target string) ([]byte, error) { body, err := dc.ingress.canonicalBody() if err != nil { return nil, err } return rewriteChatCompletionModel(body, target, req) } return srv, dc.withPoolDispatch(poolReq) } // runPoolChatStreamGate dispatches the initial pool admission exactly the way // handleChatCompletionsProviderPool does, then drives the production // provider-pool wiring (config, composite sink, dual event-source factory, // recovery admission builder) through one request runtime. func runPoolChatStreamGate(t *testing.T, srv *Server, dc *chatDispatchContext, extra ...streamgate.FilterRegistration) (*recordingResponseWriter, openAIStreamGateSink, error) { t.Helper() result, err := srv.service.SubmitProviderPool(context.Background(), *dc.poolDispatch) if err != nil { t.Fatalf("initial SubmitProviderPool: %v", err) } w := newRecordingResponseWriter() cfg, sink, err := srv.newOpenAIChatPoolStreamGateConfig(w, w, dc, result, extra) if err != nil { t.Fatalf("newOpenAIChatPoolStreamGateConfig: %v", err) } rt, _, err := srv.buildOpenAIChatStreamGateRuntimeFor(dc, cfg) if err != nil { t.Fatalf("buildOpenAIChatStreamGateRuntimeFor: %v", err) } runErr := rt.Run(context.Background()) _ = rt.CloseRequestResources(context.Background(), false) return w, sink, runErr } func newInjectedViolationRegistration(t *testing.T, dc *chatDispatchContext, id string, trigger int) streamgate.FilterRegistration { t.Helper() snapRef, err := dc.ingress.recoveryRef() if err != nil { t.Fatalf("recoveryRef: %v", err) } filter := newInjectedViolationFilter(t, id, trigger, snapRef.SnapshotRef()) reg, err := streamgate.NewFilterRegistration(filter, streamGateNoopCapability, true, streamgate.FilterEnforcementBlocking, streamGateFilterTimeout, injectedViolationPriority) if err != nil { t.Fatalf("NewFilterRegistration: %v", err) } return reg } func poolChatSSEFrames(content string) chan *iop.ProviderTunnelFrame { return bufferedTunnelFrames( &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START, StatusCode: 200, Headers: map[string]string{"Content-Type": "text/event-stream"}}, &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte("data: {\"choices\":[{\"delta\":{\"content\":\"" + content + "\"}}]}\n\n")}, &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true}, ) } func poolChatRepeatSSEFrames(responseID, content string) chan *iop.ProviderTunnelFrame { event := func(value string) []byte { return []byte(fmt.Sprintf( "data: {\"id\":%q,\"object\":\"chat.completion.chunk\",\"choices\":[{\"index\":0,\"delta\":{\"content\":%q},\"finish_reason\":null}]}\n\n", responseID, value, )) } return bufferedTunnelFrames( &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START, StatusCode: 200, Headers: map[string]string{"Content-Type": "text/event-stream"}}, &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: event(content)}, &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: event(content)}, &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true}, ) } type recordingOpenAIObservationSink struct { mu sync.Mutex got []streamgate.FilterObservation delegate streamgate.ObservationSink fail error } func (s *recordingOpenAIObservationSink) Emit(ctx context.Context, obs streamgate.FilterObservation) error { s.mu.Lock() s.got = append(s.got, obs) s.mu.Unlock() if s.delegate != nil { if err := s.delegate.Emit(ctx, obs); err != nil { return err } } return s.fail } func (s *recordingOpenAIObservationSink) Snapshot() []streamgate.FilterObservation { s.mu.Lock() defer s.mu.Unlock() return append([]streamgate.FilterObservation(nil), s.got...) } func observationKinds(observations []streamgate.FilterObservation) []streamgate.ObservationKind { kinds := make([]streamgate.ObservationKind, 0, len(observations)) for _, obs := range observations { kinds = append(kinds, obs.Kind()) } return kinds } func assertProviderPoolObservationTimeline(t *testing.T, observations []streamgate.FilterObservation) { t.Helper() if len(observations) == 0 { t.Fatal("observation timeline is empty") } initialTarget, err := streamgate.NewObservationAttemptTarget("client-model", "served-normalized", "prov-normalized", "normalized") if err != nil { t.Fatalf("initial target: %v", err) } reboundTarget, err := streamgate.NewObservationAttemptTarget("client-model", "served-tunnel", "prov-tunnel", "provider_tunnel") if err != nil { t.Fatalf("rebound target: %v", err) } // Per-observation invariants over the whole timeline: every row is raw-free // valid, sequence is strictly monotonic from 1, correlation is stable, causes // and evidence stay bounded, and the actual attempt id/target matches the // segment the row belongs to. The segment switches to the rebound attempt at // recovery_dispatched (inclusive), so any extra interleaved filter row is // still attributed to the correct actual attempt. seenDispatch := false for i, obs := range observations { if err := obs.Validate(); err != nil { t.Fatalf("observation[%d] invalid: %v", i, err) } if obs.Sequence() != uint64(i+1) { t.Fatalf("observation[%d] sequence=%d, want %d", i, obs.Sequence(), i+1) } if obs.StableCorrelation() != "req.pool-attempt-1" || obs.ConfigGeneration() != streamGateConfigGeneration { t.Fatalf("observation[%d] correlation=(%q,%q)", i, obs.StableCorrelation(), obs.ConfigGeneration()) } if obs.Kind() == streamgate.ObservationKindRecoveryDispatched { seenDispatch = true } wantID, wantTarget := "attempt.pool-attempt-1", initialTarget if seenDispatch { wantID, wantTarget = "attempt.pool-attempt-2", reboundTarget } if obs.AttemptID() != wantID || obs.AttemptTarget() != wantTarget { t.Fatalf("observation[%d] kind=%s attempt/target=(%q,%+v), want (%q,%+v)", i, obs.Kind(), obs.AttemptID(), obs.AttemptTarget(), wantID, wantTarget) } if obs.Causes().Len() > streamgate.MaxFailureCauses { t.Fatalf("observation[%d] cause count=%d", i, obs.Causes().Len()) } if evidence := obs.Evidence(); evidence != nil { if err := evidence.Validate(); err != nil { t.Fatalf("observation[%d] evidence invalid: %v", i, err) } } } // Exact chronological order: the actual normalized→tunnel recovery must // appear as this ordered subsequence, consumed front-to-back. Extra filter // rows (one evaluation_started/evaluated pair per registered filter) may // interleave, but the relative order of these required rows must hold — a // swapped recovery/terminal or a rebound segment that starts before dispatch // can no longer pass. Each row pins the actual attempt segment, the commit // state, and per-segment epoch consistency. const ( segInitial = "initial" segRebound = "rebound" ) type wantRow struct { kind streamgate.ObservationKind segment string commit streamgate.CommitState // epochGroup buckets rows that must share the same non-zero epoch. The // recovery rows carry the triggering (initial evaluation) epoch, so they // share the initial group. An empty group means the row is epoch-unbound. epochGroup string } uncommitted := streamgate.CommitStateTransportUncommitted rows := []wantRow{ {streamgate.ObservationKindResponseStaged, segInitial, uncommitted, ""}, {streamgate.ObservationKindFilterEvaluationStarted, segInitial, uncommitted, "eval-initial"}, {streamgate.ObservationKindFilterEvaluated, segInitial, uncommitted, "eval-initial"}, {streamgate.ObservationKindArbitrationDecided, segInitial, uncommitted, "eval-initial"}, {streamgate.ObservationKindRecoveryPlanSelected, segInitial, uncommitted, "eval-initial"}, {streamgate.ObservationKindRecoveryAttemptAborted, segInitial, uncommitted, "eval-initial"}, {streamgate.ObservationKindRecoveryRebuilt, segInitial, uncommitted, "eval-initial"}, {streamgate.ObservationKindRecoveryDispatched, segRebound, uncommitted, "eval-initial"}, {streamgate.ObservationKindResponseStaged, segRebound, uncommitted, ""}, {streamgate.ObservationKindFilterEvaluationStarted, segRebound, uncommitted, "eval-rebound"}, {streamgate.ObservationKindFilterEvaluated, segRebound, uncommitted, "eval-rebound"}, {streamgate.ObservationKindArbitrationDecided, segRebound, uncommitted, "eval-rebound"}, {streamgate.ObservationKindReleaseCommitted, segRebound, streamgate.CommitStateStreamOpen, "eval-rebound"}, {streamgate.ObservationKindTerminalCommitted, segRebound, streamgate.CommitStateTerminalCommitted, "eval-rebound"}, } epochByGroup := make(map[string]uint64) idx := 0 for r, want := range rows { match := -1 for j := idx; j < len(observations); j++ { if observations[j].Kind() == want.kind { match = j break } } if match < 0 { t.Fatalf("required row %d kind=%s not found in order after index %d; timeline=%v", r, want.kind, idx, observationKinds(observations)) } obs := observations[match] wantID, wantTarget := "attempt.pool-attempt-1", initialTarget if want.segment == segRebound { wantID, wantTarget = "attempt.pool-attempt-2", reboundTarget } if obs.AttemptID() != wantID || obs.AttemptTarget() != wantTarget { t.Fatalf("required row %d kind=%s attempt/target=(%q,%+v), want (%q,%+v)", r, want.kind, obs.AttemptID(), obs.AttemptTarget(), wantID, wantTarget) } if obs.CommitState() != want.commit { t.Fatalf("required row %d kind=%s commit=%s, want %s", r, want.kind, obs.CommitState(), want.commit) } if want.epochGroup == "" { if obs.EpochID() != 0 { t.Fatalf("required row %d kind=%s epoch=%d, want 0 (unbound)", r, want.kind, obs.EpochID()) } } else { if obs.EpochID() == 0 { t.Fatalf("required row %d kind=%s epoch=0, want non-zero epoch for group %s", r, want.kind, want.epochGroup) } if prev, ok := epochByGroup[want.epochGroup]; ok { if prev != obs.EpochID() { t.Fatalf("required row %d kind=%s epoch=%d, want %d consistent with group %s", r, want.kind, obs.EpochID(), prev, want.epochGroup) } } else { epochByGroup[want.epochGroup] = obs.EpochID() } } idx = match + 1 } } func TestStreamGateProviderPoolObservationCorrelationTimeline(t *testing.T) { rawSentinels := []string{"SECRET_PROMPT_CONTENT", "SECRET_OUTPUT_CONTENT", "SECRET_TOOL_ARGS", "SECRET_AUTH_TOKEN", "SECRET_PREPARER_INPUT"} rawBody := []byte(`{"model":"client-model","messages":[{"role":"user","content":"SECRET_PROMPT_CONTENT SECRET_TOOL_ARGS SECRET_AUTH_TOKEN SECRET_PREPARER_INPUT"}],"stream":true}`) service := newScriptedPoolRunService( scriptedPoolAttempt{ path: string(edgeservice.ProviderPoolPathNormalized), runID: "pool-attempt-1", provider: "prov-normalized", target: "served-normalized", runEvents: bufferedRunEvents( &iop.RunEvent{Type: "delta", Delta: "SECRET_OUTPUT_CONTENT"}, &iop.RunEvent{Type: "complete"}, ), }, scriptedPoolAttempt{ path: string(edgeservice.ProviderPoolPathTunnel), runID: "pool-attempt-2", provider: "prov-tunnel", target: "served-tunnel", frames: poolChatSSEFrames("switched"), }, ) var logOutput bytes.Buffer encoderConfig := zap.NewProductionEncoderConfig() encoderConfig.TimeKey = "" logger := zap.New(zapcore.NewCore(zapcore.NewJSONEncoder(encoderConfig), zapcore.AddSync(&logOutput), zapcore.InfoLevel)) recorder := &recordingOpenAIObservationSink{delegate: newZapFilterObservationSink(logger)} srv, dc := buildStreamGatePoolChatFixture(t, service, rawBody, 3) srv.SetObservationSink(recorder) w, releaseSink, runErr := runPoolChatStreamGate(t, srv, dc, newInjectedViolationRegistration(t, dc, "test.injected_violation", 1)) if runErr != nil { t.Fatalf("rt.Run: %v", runErr) } if !strings.Contains(w.body.String(), `"content":"switched"`) || strings.Contains(w.body.String(), "SECRET_OUTPUT_CONTENT") { t.Fatalf("unexpected response body: %q", w.body.String()) } composite := releaseSink.(*openAICompositeReleaseSink) if committed, success := composite.terminalStatus(); !committed || !success { t.Fatalf("terminal status=(%v,%v), want successful commit", committed, success) } observations := recorder.Snapshot() assertProviderPoolObservationTimeline(t, observations) eventDump, encoded := fmt.Sprintf("%+v", observations), logOutput.String() for _, sentinel := range rawSentinels { if strings.Contains(eventDump, sentinel) { t.Fatalf("observation event contains raw sentinel %q", sentinel) } if strings.Contains(encoded, sentinel) { t.Fatalf("observation log contains raw sentinel %q", sentinel) } } for _, field := range []string{`"model_group":"client-model"`, `"actual_model":"served-normalized"`, `"actual_model":"served-tunnel"`, `"actual_provider":"prov-tunnel"`, `"execution_path":"provider_tunnel"`} { if !strings.Contains(encoded, field) { t.Errorf("observation log missing %s", field) } } } func TestStreamGateProviderPoolObservationSinkFailureIsolation(t *testing.T) { rawBody := []byte(`{"model":"client-model","messages":[{"role":"user","content":"hi"}],"stream":true}`) service := newScriptedPoolRunService( scriptedPoolAttempt{ path: string(edgeservice.ProviderPoolPathNormalized), runID: "pool-attempt-1", provider: "prov-normalized", target: "served-normalized", runEvents: bufferedRunEvents(&iop.RunEvent{Type: "delta", Delta: "discarded"}, &iop.RunEvent{Type: "complete"}), }, scriptedPoolAttempt{ path: string(edgeservice.ProviderPoolPathTunnel), runID: "pool-attempt-2", provider: "prov-tunnel", target: "served-tunnel", frames: poolChatSSEFrames("switched"), }, ) recorder := &recordingOpenAIObservationSink{fail: errors.New("injected observation sink failure")} srv, dc := buildStreamGatePoolChatFixture(t, service, rawBody, 3) srv.SetObservationSink(recorder) w, releaseSink, runErr := runPoolChatStreamGate(t, srv, dc, newInjectedViolationRegistration(t, dc, "test.injected_violation", 1)) if runErr != nil { t.Fatalf("sink failure escaped runtime: %v", runErr) } if !strings.Contains(w.body.String(), `"content":"switched"`) { t.Fatalf("successful response missing after sink failures: %q", w.body.String()) } composite := releaseSink.(*openAICompositeReleaseSink) if committed, success := composite.terminalStatus(); !committed || !success { t.Fatalf("terminal status=(%v,%v), want successful commit", committed, success) } if service.poolSubmits() != 2 { t.Fatalf("pool submits=%d, want 2", service.poolSubmits()) } _, cancels, _, _, _, _ := service.snapshot() if n := countString(cancels, "pool-attempt-1"); n != 1 { t.Fatalf("replaced attempt canceled %d times, want exactly once", n) } if n := countString(cancels, "pool-attempt-2"); n > 1 { t.Fatalf("terminal attempt canceled %d times, want at most once", n) } assertProviderPoolObservationTimeline(t, recorder.Snapshot()) } func countString(values []string, want string) int { n := 0 for _, value := range values { if value == want { n++ } } return n } // TestStreamGateProviderPoolRecoverySwitchesActualPathOnce is the S16 // vertical-slice evidence: one RequestRuntime admits the initial provider-pool // attempt, a blocking violation triggers a single re-admission through // SubmitProviderPool, the pool selects a candidate on the *other* execution // path, and only that attempt's framing and bytes reach the caller. func TestStreamGateProviderPoolRecoverySwitchesActualPathOnce(t *testing.T) { rawBody := []byte(`{"model":"client-model","messages":[{"role":"user","content":"hi"}],"stream":true}`) t.Run("normalized to tunnel", func(t *testing.T) { service := newScriptedPoolRunService( scriptedPoolAttempt{ path: string(edgeservice.ProviderPoolPathNormalized), runID: "pool-attempt-1", provider: "prov-normalized", target: "served-normalized", runEvents: bufferedRunEvents( &iop.RunEvent{Type: "delta", Delta: "discarded-normalized"}, &iop.RunEvent{Type: "complete"}, ), }, scriptedPoolAttempt{ path: string(edgeservice.ProviderPoolPathTunnel), runID: "pool-attempt-2", provider: "prov-tunnel", target: "served-tunnel", frames: poolChatSSEFrames("switched"), }, ) srv, dc := buildStreamGatePoolChatFixture(t, service, rawBody, 3) w, sink, runErr := runPoolChatStreamGate(t, srv, dc, newInjectedViolationRegistration(t, dc, "test.injected_violation", 1)) if runErr != nil { t.Fatalf("rt.Run: %v", runErr) } if got := service.poolSubmits(); got != 2 { t.Fatalf("SubmitProviderPool calls: got %d, want 2 (initial admission + exactly one re-admission)", got) } composite, ok := sink.(*openAICompositeReleaseSink) if !ok { t.Fatalf("provider-pool request must use the composite sink, got %T", sink) } if got := composite.resolvedCodec(); got != openAIStreamGateCodecTunnel { t.Fatalf("resolved codec: got %q, want tunnel framing from the recovered attempt", got) } if got := w.headerCallCount(); got != 1 { t.Fatalf("WriteHeader call count: got %d, want 1", got) } body := w.body.String() if !strings.Contains(body, `"content":"switched"`) { t.Fatalf("recovered tunnel attempt bytes missing: body=%q", body) } if strings.Contains(body, "discarded-normalized") { t.Fatalf("discarded normalized attempt leaked into the response: body=%q", body) } if strings.Contains(body, "chat.completion.chunk") { t.Fatalf("normalized SSE framing leaked into a tunnel-framed response: body=%q", body) } _, cancels, _, _, _, _ := service.snapshot() if !containsString(cancels, "pool-attempt-1") { t.Fatalf("the replaced attempt must be aborted, cancels=%v", cancels) } }) t.Run("tunnel to normalized", func(t *testing.T) { service := newScriptedPoolRunService( scriptedPoolAttempt{ path: string(edgeservice.ProviderPoolPathTunnel), runID: "pool-attempt-1", provider: "prov-tunnel", target: "served-tunnel", frames: poolChatSSEFrames("discarded-tunnel"), }, scriptedPoolAttempt{ path: string(edgeservice.ProviderPoolPathNormalized), runID: "pool-attempt-2", provider: "prov-normalized", target: "served-normalized", runEvents: bufferedRunEvents( &iop.RunEvent{Type: "delta", Delta: "switched back"}, &iop.RunEvent{Type: "complete"}, ), }, ) srv, dc := buildStreamGatePoolChatFixture(t, service, rawBody, 3) w, sink, runErr := runPoolChatStreamGate(t, srv, dc, newInjectedViolationRegistration(t, dc, "test.injected_violation", 1)) if runErr != nil { t.Fatalf("rt.Run: %v", runErr) } if got := service.poolSubmits(); got != 2 { t.Fatalf("SubmitProviderPool calls: got %d, want 2", got) } composite := sink.(*openAICompositeReleaseSink) if got := composite.resolvedCodec(); got != openAIStreamGateCodecNormalized { t.Fatalf("resolved codec: got %q, want normalized framing from the recovered attempt", got) } if got := w.headerCallCount(); got != 1 { t.Fatalf("WriteHeader call count: got %d, want 1", got) } chunks := parseSSEChatChunks(t, w.body.String()) if got := joinedContent(chunks); got != "switched back" { t.Fatalf("joined content: got %q, want only the recovered normalized attempt's content", got) } if strings.Contains(w.body.String(), "discarded-tunnel") { t.Fatalf("discarded tunnel attempt leaked into the response: body=%q", w.body.String()) } }) } // TestStreamGateProviderPoolExhaustionTerminatesWithoutPathLeak verifies the // bounded budget still holds across a path switch: with a request-total cap of // 1 the second violation terminates instead of re-admitting a third candidate, // and no attempt's content is released. func TestStreamGateProviderPoolExhaustionTerminatesWithoutPathLeak(t *testing.T) { rawBody := []byte(`{"model":"client-model","messages":[{"role":"user","content":"hi"}],"stream":true}`) service := newScriptedPoolRunService( scriptedPoolAttempt{ path: string(edgeservice.ProviderPoolPathNormalized), runID: "pool-attempt-1", provider: "prov-normalized", target: "served-normalized", runEvents: bufferedRunEvents(&iop.RunEvent{Type: "delta", Delta: "still-bad-1"}, &iop.RunEvent{Type: "complete"}), }, scriptedPoolAttempt{ path: string(edgeservice.ProviderPoolPathTunnel), runID: "pool-attempt-2", provider: "prov-tunnel", target: "served-tunnel", frames: poolChatSSEFrames("still-bad-2"), }, ) srv, dc := buildStreamGatePoolChatFixture(t, service, rawBody, 1) w, _, runErr := runPoolChatStreamGate(t, srv, dc, newInjectedViolationRegistration(t, dc, "test.injected_violation", 2)) if runErr != nil { t.Fatalf("rt.Run: %v", runErr) } if got := service.poolSubmits(); got != 2 { t.Fatalf("SubmitProviderPool calls: got %d, want 2 (cap=1 allows exactly one re-admission)", got) } body := w.body.String() if !strings.Contains(body, `"error"`) { t.Fatalf("expected an error terminal once the recovery budget is exhausted, body=%q", body) } for _, leak := range []string{"still-bad-1", "still-bad-2"} { if strings.Contains(body, leak) { t.Fatalf("exhausted recovery leaked attempt content %q: body=%q", leak, body) } } } // TestStreamGateProviderPoolSimultaneousViolationsDispatchOnce verifies that two // independent blocking filters violating on the same terminal produce exactly // one recovery plan and one re-admission, not one per filter. func TestStreamGateProviderPoolSimultaneousViolationsDispatchOnce(t *testing.T) { rawBody := []byte(`{"model":"client-model","messages":[{"role":"user","content":"hi"}],"stream":true}`) service := newScriptedPoolRunService( scriptedPoolAttempt{ path: string(edgeservice.ProviderPoolPathNormalized), runID: "pool-attempt-1", provider: "prov-normalized", target: "served-normalized", runEvents: bufferedRunEvents(&iop.RunEvent{Type: "delta", Delta: "discarded"}, &iop.RunEvent{Type: "complete"}), }, scriptedPoolAttempt{ path: string(edgeservice.ProviderPoolPathNormalized), runID: "pool-attempt-2", provider: "prov-normalized", target: "served-normalized", runEvents: bufferedRunEvents(&iop.RunEvent{Type: "delta", Delta: "final answer"}, &iop.RunEvent{Type: "complete"}), }, ) srv, dc := buildStreamGatePoolChatFixture(t, service, rawBody, 3) w, _, runErr := runPoolChatStreamGate(t, srv, dc, newInjectedViolationRegistration(t, dc, "test.injected_violation_a", 1), newInjectedViolationRegistration(t, dc, "test.injected_violation_b", 1), ) if runErr != nil { t.Fatalf("rt.Run: %v", runErr) } if got := service.poolSubmits(); got != 2 { t.Fatalf("SubmitProviderPool calls: got %d, want 2 (two simultaneous violations must share one plan/dispatch)", got) } chunks := parseSSEChatChunks(t, w.body.String()) if got := joinedContent(chunks); got != "final answer" { t.Fatalf("joined content: got %q, want the single recovered attempt's content", got) } if got := w.headerCallCount(); got != 1 { t.Fatalf("WriteHeader call count: got %d, want 1", got) } } type devRepeatGuardSmokeEvidence struct { Run int `json:"run"` Attempt int `json:"attempt"` Model string `json:"model"` StatusCode int `json:"status_code"` Terminal string `json:"terminal"` ProviderResponseIDs []string `json:"provider_response_ids"` downstreamRepeated bool } type devRepeatGuardLifecycleEvidence struct { Run int `json:"run"` Model string `json:"model"` ActualModel string `json:"actual_model"` Provider string `json:"provider"` CorrelationID string `json:"correlation_id"` Result string `json:"result"` Decision string `json:"decision"` Lifecycle string `json:"lifecycle"` TerminalReason string `json:"terminal_reason"` Fingerprint string `json:"fingerprint,omitempty"` Offset int `json:"offset"` } type devRepeatGuardSmokeSummary struct { Result string `json:"result"` Model string `json:"model"` Concurrency int `json:"concurrency"` Runs int `json:"runs"` ProviderResponses []devRepeatGuardSmokeEvidence `json:"provider_responses"` Lifecycles []devRepeatGuardLifecycleEvidence `json:"lifecycles"` CapacityRuns []devRepeatGuardCapacityEvidence `json:"capacity_runs"` } type devRepeatGuardCapacityEvidence struct { Run int `json:"run"` ConfiguredCapacity int `json:"configured_capacity"` PeakInFlight int `json:"peak_in_flight"` PeakQueued int `json:"peak_queued"` FinalInFlight int `json:"final_in_flight"` FinalQueued int `json:"final_queued"` } type devRepeatGuardCapacitySnapshot struct { configuredCapacity int inFlight int queued int } type devRepeatGuardObservation struct { Message string `json:"msg"` Sequence uint64 `json:"sequence"` ObservationKind string `json:"observation_kind"` CorrelationID string `json:"correlation_id"` AttemptID string `json:"attempt_id"` ModelGroup string `json:"model_group"` ActualModel string `json:"actual_model"` ActualProvider string `json:"actual_provider"` FilterID string `json:"filter_id"` RuleID string `json:"rule_id"` FilterOutcome string `json:"filter_outcome"` DecisionKind string `json:"decision_kind"` ArbitrationAction string `json:"arbitration_action"` TerminalReason string `json:"terminal_reason"` RecoveryStrategy string `json:"recovery_strategy"` EvidenceDescriptorCode string `json:"evidence_descriptor_code"` EvidenceFingerprint string `json:"evidence_fingerprint"` EvidenceOffset int `json:"evidence_offset"` line int } type devRepeatGuardSSEEvidence struct { terminal string providerResponseIDs []string } const ( devRepeatGuardMaxProviderResponseIDs = 4 devRepeatGuardDefaultRequestTimeout = 25 * time.Minute devRepeatGuardMaxRequestTimeout = 25 * time.Minute devRepeatGuardPerRunSettlementAllowance = 2 * time.Minute devRepeatGuardFinalCleanupReserve = 5 * time.Minute devRepeatGuardRequestTimeoutEnvironmentName = "IOP_OPENAI_SMOKE_REQUEST_TIMEOUT" ) // TestDevRepeatGuardOrnithSmoke is an env-gated live entry. Raw prompt and SSE // bytes are read/written only in ignored agent-test/runs paths. Stdout and the // sanitized summary contain no prompt, output, token, tool, auth, or endpoint // value. Reproduction is classified only from correlated raw-free guard // observations; released text is inspected only to fail a run if a lifecycle // observation says a blocked duplicate reached the downstream stream. func TestDevRepeatGuardOrnithSmoke(t *testing.T) { baseURL := strings.TrimRight(strings.TrimSpace(os.Getenv("IOP_OPENAI_BASE_URL")), "/") model := strings.TrimSpace(os.Getenv("IOP_OPENAI_MODEL")) promptFile := strings.TrimSpace(os.Getenv("IOP_OPENAI_SMOKE_PROMPT_FILE")) artifactDir := strings.TrimSpace(os.Getenv("IOP_OPENAI_SMOKE_ARTIFACT_DIR")) observationFile := strings.TrimSpace(os.Getenv("IOP_OPENAI_SMOKE_OBSERVATION_FILE")) statusURL := strings.TrimSpace(os.Getenv("IOP_OPENAI_SMOKE_STATUS_URL")) rawProviderIDs := strings.TrimSpace(os.Getenv("IOP_OPENAI_SMOKE_PROVIDER_IDS")) required := map[string]string{ "IOP_OPENAI_BASE_URL": baseURL, "IOP_OPENAI_MODEL": model, "IOP_OPENAI_SMOKE_PROMPT_FILE": promptFile, "IOP_OPENAI_SMOKE_ARTIFACT_DIR": artifactDir, "IOP_OPENAI_SMOKE_OBSERVATION_FILE": observationFile, "IOP_OPENAI_SMOKE_STATUS_URL": statusURL, "IOP_OPENAI_SMOKE_PROVIDER_IDS": rawProviderIDs, } configured := 0 for _, value := range required { if value != "" { configured++ } } if configured == 0 { t.Skip("live repeat-guard smoke environment is not configured") } for key, value := range required { if value == "" { t.Fatalf("%s is required when the live repeat-guard smoke is configured", key) } } providerIDs, err := parseDevRepeatGuardProviderIDs(rawProviderIDs) if err != nil { t.Fatalf("parse selected provider IDs: %v", err) } concurrency := devRepeatGuardEnvInt(t, "IOP_OPENAI_SMOKE_CONCURRENCY", 5) runs := devRepeatGuardEnvInt(t, "IOP_OPENAI_SMOKE_RUNS", 3) if concurrency != 5 || runs != 3 { t.Fatalf("live repeat-guard evidence requires concurrency=5 and runs=3") } requestTimeout := devRepeatGuardRequestTimeout(t, runs) prompt, err := os.ReadFile(promptFile) if err != nil { t.Fatalf("read ignored smoke prompt: %v", err) } if len(prompt) == 0 { t.Fatal("ignored smoke prompt is empty") } if err := os.MkdirAll(artifactDir, 0o700); err != nil { t.Fatalf("create ignored smoke artifact directory: %v", err) } client := &http.Client{Timeout: requestTimeout} statusClient := &http.Client{Timeout: 5 * time.Second} providerResponses := make([]devRepeatGuardSmokeEvidence, 0, concurrency*runs) lifecycles := make([]devRepeatGuardLifecycleEvidence, 0, concurrency*runs) capacityRuns := make([]devRepeatGuardCapacityEvidence, 0, runs) for run := 1; run <= runs; run++ { if _, err := waitDevRepeatGuardCapacityIdle( statusClient, statusURL, providerIDs, 30*time.Second, ); err != nil { t.Fatalf("wait for idle selected capacity before run=%d: %v", run, err) } observationOffset, err := devRepeatGuardObservationQuietOffset( observationFile, 2*time.Second, ) if err != nil { t.Fatalf("observation sink is not quiescent before run=%d: %v", run, err) } var ( wg sync.WaitGroup resultC = make(chan devRepeatGuardSmokeEvidence, concurrency) errorC = make(chan error, concurrency) startC = make(chan struct{}) doneC = make(chan struct{}) ) for attempt := 1; attempt <= concurrency; attempt++ { wg.Add(1) go func(run, attempt int) { defer wg.Done() <-startC evidence, requestErr := runDevRepeatGuardSmokeAttempt( client, baseURL, model, string(prompt), artifactDir, run, attempt, ) if requestErr != nil { errorC <- requestErr return } resultC <- evidence }(run, attempt) } go func() { wg.Wait() close(doneC) }() capacityC := make(chan devRepeatGuardCapacityEvidence, 1) capacityErrC := make(chan error, 1) observerReadyC := make(chan struct{}) go func(run int) { evidence, observeErr := observeDevRepeatGuardCapacityRun( statusClient, statusURL, providerIDs, run, doneC, observerReadyC, requestTimeout+devRepeatGuardPerRunSettlementAllowance, ) if observeErr != nil { capacityErrC <- observeErr return } capacityC <- evidence }(run) <-observerReadyC close(startC) wg.Wait() close(resultC) close(errorC) for requestErr := range errorC { if requestErr != nil { t.Errorf("live repeat-guard request failed: %v", requestErr) } } batch := make([]devRepeatGuardSmokeEvidence, 0, concurrency) for evidence := range resultC { batch = append(batch, evidence) } if t.Failed() { return } select { case capacity := <-capacityC: capacityRuns = append(capacityRuns, capacity) case capacityErr := <-capacityErrC: t.Fatalf("observe selected provider capacity for run=%d: %v", run, capacityErr) } sort.Slice(batch, func(i, j int) bool { return batch[i].Attempt < batch[j].Attempt }) observations, err := waitDevRepeatGuardObservationSegment( observationFile, observationOffset, model, concurrency, prompt, []byte(strings.TrimSpace(os.Getenv("IOP_OPENAI_API_KEY"))), ) if err != nil { t.Fatalf("read observation segment for run=%d: %v", run, err) } correlated, err := correlateDevRepeatGuardBatch(batch, observations) if err != nil { t.Fatalf("correlate observation segment for run=%d: %v", run, err) } providerResponses = append(providerResponses, batch...) lifecycles = append(lifecycles, correlated...) } if t.Failed() { return } if len(providerResponses) != 15 || len(lifecycles) != 15 || len(capacityRuns) != 3 { t.Fatalf( "sanitized evidence rows: provider_responses=%d lifecycles=%d capacity_runs=%d, want 15, 15, and 3", len(providerResponses), len(lifecycles), len(capacityRuns), ) } sort.Slice(providerResponses, func(i, j int) bool { if providerResponses[i].Run == providerResponses[j].Run { return providerResponses[i].Attempt < providerResponses[j].Attempt } return providerResponses[i].Run < providerResponses[j].Run }) sort.Slice(lifecycles, func(i, j int) bool { if lifecycles[i].Run == lifecycles[j].Run { return lifecycles[i].CorrelationID < lifecycles[j].CorrelationID } return lifecycles[i].Run < lifecycles[j].Run }) if err := validateDevRepeatGuardUniqueEvidence(providerResponses, lifecycles); err != nil { t.Fatalf("validate final evidence identity: %v", err) } overall := "not_reproduced" for _, result := range lifecycles { if result.Result == "reproduced" { overall = "reproduced" break } } summary := devRepeatGuardSmokeSummary{ Result: overall, Model: model, Concurrency: concurrency, Runs: runs, ProviderResponses: providerResponses, Lifecycles: lifecycles, CapacityRuns: capacityRuns, } encoded, err := json.MarshalIndent(summary, "", " ") if err != nil { t.Fatalf("encode sanitized smoke summary: %v", err) } encoded = append(encoded, '\n') if err := os.WriteFile(filepath.Join(artifactDir, "summary.json"), encoded, 0o600); err != nil { t.Fatalf("write sanitized smoke summary: %v", err) } t.Logf( "repeat-guard live result=%s provider_responses=%d edge_correlations=%d capacity_runs=%d", overall, len(providerResponses), len(lifecycles), len(capacityRuns), ) } func validateDevRepeatGuardUniqueEvidence( providerResponses []devRepeatGuardSmokeEvidence, lifecycles []devRepeatGuardLifecycleEvidence, ) error { providerIDs := make(map[string]struct{}, len(providerResponses)) for _, response := range providerResponses { if err := collectDevRepeatGuardProviderResponseIDs( response.ProviderResponseIDs, providerIDs, fmt.Sprintf("run=%d attempt=%d", response.Run, response.Attempt), ); err != nil { return err } } correlations := make(map[string]struct{}, len(lifecycles)) for _, lifecycle := range lifecycles { correlation := strings.TrimSpace(lifecycle.CorrelationID) if correlation == "" { return errors.New("Edge lifecycle correlation is empty") } if _, found := correlations[correlation]; found { return fmt.Errorf("Edge lifecycle correlation %q is not globally unique", correlation) } correlations[correlation] = struct{}{} } return nil } func devRepeatGuardRequestTimeout(t *testing.T, runs int) time.Duration { t.Helper() deadline, hasDeadline := t.Deadline() requestTimeout, err := resolveDevRepeatGuardRequestTimeout( os.Getenv(devRepeatGuardRequestTimeoutEnvironmentName), runs, time.Now(), deadline, hasDeadline, ) if err != nil { t.Fatalf("%s: %v", devRepeatGuardRequestTimeoutEnvironmentName, err) } return requestTimeout } func resolveDevRepeatGuardRequestTimeout( raw string, runs int, now time.Time, deadline time.Time, hasDeadline bool, ) (time.Duration, error) { if runs <= 0 || runs > 32 { return 0, errors.New("run count must be between 1 and 32") } requestTimeout := devRepeatGuardDefaultRequestTimeout if value := strings.TrimSpace(raw); value != "" { parsed, err := time.ParseDuration(value) if err != nil { return 0, errors.New("request timeout must be a valid Go duration") } requestTimeout = parsed } if requestTimeout <= 0 { return 0, errors.New("request timeout must be positive") } if requestTimeout > devRepeatGuardMaxRequestTimeout { return 0, fmt.Errorf( "request timeout must not exceed %s", devRepeatGuardMaxRequestTimeout, ) } requiredBudget := time.Duration(runs) * (requestTimeout + devRepeatGuardPerRunSettlementAllowance) requiredBudget += devRepeatGuardFinalCleanupReserve if hasDeadline && deadline.Sub(now) < requiredBudget { return 0, fmt.Errorf( "test deadline leaves %s, need at least %s for %d runs, settlement, and cleanup", deadline.Sub(now), requiredBudget, runs, ) } return requestTimeout, nil } func collectDevRepeatGuardProviderResponseIDs( ids []string, seen map[string]struct{}, context string, ) error { if len(ids) == 0 || len(ids) > devRepeatGuardMaxProviderResponseIDs { return fmt.Errorf( "%s provider response identity count=%d, want 1..%d", context, len(ids), devRepeatGuardMaxProviderResponseIDs, ) } const chatCompletionPrefix = "chatcmpl-" local := make(map[string]struct{}, len(ids)) for _, rawID := range ids { id := strings.TrimSpace(rawID) if !strings.HasPrefix(id, chatCompletionPrefix) || strings.TrimPrefix(id, chatCompletionPrefix) == "" { return fmt.Errorf("%s has invalid provider response identity", context) } if _, found := local[id]; found { return fmt.Errorf("%s repeats provider response identity %q", context, id) } local[id] = struct{}{} if _, found := seen[id]; found { return fmt.Errorf("provider response identity %q is not globally unique", id) } seen[id] = struct{}{} } return nil } func devRepeatGuardEnvInt(t *testing.T, key string, fallback int) int { t.Helper() raw := strings.TrimSpace(os.Getenv(key)) if raw == "" { return fallback } value, err := strconv.Atoi(raw) if err != nil || value <= 0 || value > 32 { t.Fatalf("%s must be between 1 and 32", key) } return value } func parseDevRepeatGuardProviderIDs(raw string) ([]string, error) { parts := strings.Split(raw, ",") if len(parts) == 0 || len(parts) > 16 { return nil, errors.New("provider ID count must be between 1 and 16") } seen := make(map[string]struct{}, len(parts)) providerIDs := make([]string, 0, len(parts)) for _, part := range parts { providerID := strings.TrimSpace(part) if providerID == "" { return nil, errors.New("provider IDs must be non-empty") } if _, found := seen[providerID]; found { return nil, fmt.Errorf("duplicate provider ID %q", providerID) } seen[providerID] = struct{}{} providerIDs = append(providerIDs, providerID) } return providerIDs, nil } func parseDevRepeatGuardCapacityStatus( raw []byte, providerIDs []string, ) (devRepeatGuardCapacitySnapshot, error) { var status struct { Nodes []struct { ProviderSnapshots []struct { ID string `json:"id"` Capacity int `json:"capacity"` InFlight int `json:"in_flight"` Queued int `json:"queued"` } `json:"provider_snapshots"` } `json:"nodes"` } if err := json.Unmarshal(raw, &status); err != nil { return devRepeatGuardCapacitySnapshot{}, errors.New("status response is not valid JSON") } selected := make(map[string]struct{}, len(providerIDs)) for _, providerID := range providerIDs { if strings.TrimSpace(providerID) == "" { return devRepeatGuardCapacitySnapshot{}, errors.New("selected provider ID is empty") } selected[providerID] = struct{}{} } found := make(map[string]struct{}, len(providerIDs)) var snapshot devRepeatGuardCapacitySnapshot for _, node := range status.Nodes { for _, provider := range node.ProviderSnapshots { if _, wanted := selected[provider.ID]; !wanted { continue } if _, duplicate := found[provider.ID]; duplicate { return devRepeatGuardCapacitySnapshot{}, fmt.Errorf( "selected provider %q appears more than once", provider.ID, ) } if provider.Capacity < 0 || provider.InFlight < 0 || provider.Queued < 0 { return devRepeatGuardCapacitySnapshot{}, fmt.Errorf( "selected provider %q has a negative capacity counter", provider.ID, ) } found[provider.ID] = struct{}{} snapshot.configuredCapacity += provider.Capacity snapshot.inFlight += provider.InFlight snapshot.queued += provider.Queued } } for _, providerID := range providerIDs { if _, ok := found[providerID]; !ok { return devRepeatGuardCapacitySnapshot{}, fmt.Errorf( "selected provider %q is missing from status", providerID, ) } } return snapshot, nil } func fetchDevRepeatGuardCapacityStatus( client *http.Client, statusURL string, providerIDs []string, ) (devRepeatGuardCapacitySnapshot, error) { req, err := http.NewRequest(http.MethodGet, statusURL, nil) if err != nil { return devRepeatGuardCapacitySnapshot{}, errors.New("build status request") } resp, err := client.Do(req) if err != nil { return devRepeatGuardCapacitySnapshot{}, errors.New("status request failed") } defer resp.Body.Close() if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { return devRepeatGuardCapacitySnapshot{}, fmt.Errorf( "status request returned HTTP %d", resp.StatusCode, ) } const statusBodyLimit = 4 * 1024 * 1024 raw, err := io.ReadAll(io.LimitReader(resp.Body, statusBodyLimit+1)) if err != nil { return devRepeatGuardCapacitySnapshot{}, errors.New("read status response") } if len(raw) > statusBodyLimit { return devRepeatGuardCapacitySnapshot{}, errors.New("status response exceeds 4 MiB") } return parseDevRepeatGuardCapacityStatus(raw, providerIDs) } func waitDevRepeatGuardCapacityIdle( client *http.Client, statusURL string, providerIDs []string, timeout time.Duration, ) (devRepeatGuardCapacitySnapshot, error) { deadline := time.Now().Add(timeout) var lastErr error for { snapshot, err := fetchDevRepeatGuardCapacityStatus(client, statusURL, providerIDs) if err == nil { if snapshot.configuredCapacity != 4 { return devRepeatGuardCapacitySnapshot{}, fmt.Errorf( "selected provider capacity=%d, want 4", snapshot.configuredCapacity, ) } if snapshot.inFlight == 0 && snapshot.queued == 0 { return snapshot, nil } lastErr = fmt.Errorf( "selected providers are not idle: in_flight=%d queued=%d", snapshot.inFlight, snapshot.queued, ) } else { lastErr = err } if time.Now().After(deadline) { return devRepeatGuardCapacitySnapshot{}, lastErr } time.Sleep(100 * time.Millisecond) } } func observeDevRepeatGuardCapacityRun( client *http.Client, statusURL string, providerIDs []string, run int, requestsDone <-chan struct{}, observerReady chan<- struct{}, timeout time.Duration, ) (devRepeatGuardCapacityEvidence, error) { close(observerReady) deadline := time.Now().Add(timeout) requestsComplete := false evidence := devRepeatGuardCapacityEvidence{Run: run} for { snapshot, err := fetchDevRepeatGuardCapacityStatus(client, statusURL, providerIDs) if err != nil { return devRepeatGuardCapacityEvidence{}, err } if snapshot.configuredCapacity != 4 { return devRepeatGuardCapacityEvidence{}, fmt.Errorf( "selected provider capacity=%d, want 4", snapshot.configuredCapacity, ) } evidence.ConfiguredCapacity = snapshot.configuredCapacity if snapshot.inFlight > evidence.PeakInFlight { evidence.PeakInFlight = snapshot.inFlight } if snapshot.queued > evidence.PeakQueued { evidence.PeakQueued = snapshot.queued } if !requestsComplete { select { case <-requestsDone: requestsComplete = true default: } } if requestsComplete && snapshot.inFlight == 0 && snapshot.queued == 0 { evidence.FinalInFlight = snapshot.inFlight evidence.FinalQueued = snapshot.queued if err := validateDevRepeatGuardCapacityEvidence(evidence); err != nil { return devRepeatGuardCapacityEvidence{}, err } return evidence, nil } if time.Now().After(deadline) { return devRepeatGuardCapacityEvidence{}, errors.New( "selected provider capacity did not recover before the deadline", ) } time.Sleep(100 * time.Millisecond) } } func validateDevRepeatGuardCapacityEvidence(evidence devRepeatGuardCapacityEvidence) error { if evidence.Run <= 0 { return errors.New("capacity evidence has no run index") } if evidence.ConfiguredCapacity != 4 { return fmt.Errorf( "configured capacity=%d, want 4", evidence.ConfiguredCapacity, ) } if evidence.PeakInFlight != 4 { return fmt.Errorf("peak in_flight=%d, want 4", evidence.PeakInFlight) } if evidence.PeakQueued < 1 { return fmt.Errorf("peak queued=%d, want at least 1", evidence.PeakQueued) } if evidence.FinalInFlight != 0 || evidence.FinalQueued != 0 { return fmt.Errorf( "final in_flight/queued=%d/%d, want 0/0", evidence.FinalInFlight, evidence.FinalQueued, ) } return nil } func runDevRepeatGuardSmokeAttempt( client *http.Client, baseURL, model, prompt, artifactDir string, run, attempt int, ) (devRepeatGuardSmokeEvidence, error) { body, err := json.Marshal(map[string]any{ "model": model, "messages": []any{ map[string]any{"role": "user", "content": prompt}, }, "stream": true, }) if err != nil { return devRepeatGuardSmokeEvidence{}, fmt.Errorf("encode request: %w", err) } req, err := http.NewRequest(http.MethodPost, baseURL+"/v1/chat/completions", bytes.NewReader(body)) if err != nil { return devRepeatGuardSmokeEvidence{}, fmt.Errorf("build request: %w", err) } req.Header.Set("Content-Type", "application/json") if apiKey := strings.TrimSpace(os.Getenv("IOP_OPENAI_API_KEY")); apiKey != "" { req.Header.Set("Authorization", "Bearer "+apiKey) } resp, err := client.Do(req) if err != nil { return devRepeatGuardSmokeEvidence{}, fmt.Errorf("dispatch run=%d attempt=%d: %w", run, attempt, err) } defer resp.Body.Close() raw, err := io.ReadAll(io.LimitReader(resp.Body, 64*1024*1024)) if err != nil { return devRepeatGuardSmokeEvidence{}, fmt.Errorf("read run=%d attempt=%d: %w", run, attempt, err) } rawPath := filepath.Join(artifactDir, fmt.Sprintf("run-%02d-attempt-%02d.sse", run, attempt)) if err := os.WriteFile(rawPath, raw, 0o600); err != nil { return devRepeatGuardSmokeEvidence{}, fmt.Errorf("write ignored raw run artifact: %w", err) } if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { return devRepeatGuardSmokeEvidence{}, fmt.Errorf( "run=%d attempt=%d returned non-2xx status %d", run, attempt, resp.StatusCode, ) } sse, err := validateDevRepeatGuardSSE(raw) if err != nil { return devRepeatGuardSmokeEvidence{}, fmt.Errorf( "run=%d attempt=%d invalid SSE: %w", run, attempt, err, ) } channels := devRepeatGuardSmokeChannels(raw) downstreamRepeated := channels.contentRepeated(64) || channels.reasoningRepeated(64) evidence := devRepeatGuardSmokeEvidence{ Run: run, Attempt: attempt, Model: model, StatusCode: resp.StatusCode, Terminal: sse.terminal, ProviderResponseIDs: sse.providerResponseIDs, downstreamRepeated: downstreamRepeated, } return evidence, nil } func validateDevRepeatGuardSSE(raw []byte) (devRepeatGuardSSEEvidence, error) { normalized := bytes.ReplaceAll(raw, []byte("\r\n"), []byte("\n")) lines := bytes.Split(normalized, []byte{'\n'}) var ( dataLines [][]byte eventCount int errorCount int successFinishCount int sawDone bool currentResponseID string providerResponseIDs []string closedResponseIDs = make(map[string]struct{}) ) processCurrentData := func() error { if len(dataLines) == 0 { return nil } eventCount++ data := bytes.Join(dataLines, []byte{'\n'}) dataLines = dataLines[:0] if bytes.Equal(bytes.TrimSpace(data), []byte("[DONE]")) { if sawDone { return errors.New("duplicate [DONE] terminal marker") } if (errorCount == 1) == (successFinishCount == 1) { return fmt.Errorf( "[DONE] has invalid logical terminal state: errors=%d success_finishes=%d", errorCount, successFinishCount, ) } sawDone = true return nil } if sawDone { return fmt.Errorf("data event %d appears after [DONE]", eventCount) } var envelope map[string]json.RawMessage if err := json.Unmarshal(data, &envelope); err != nil { return fmt.Errorf("data event %d is not JSON", eventCount) } hasError := false if rawError := envelope["error"]; len(rawError) > 0 && !bytes.Equal(bytes.TrimSpace(rawError), []byte("null")) { hasError = true } rawID := envelope["id"] if len(rawID) > 0 { var eventResponseID string if err := json.Unmarshal(rawID, &eventResponseID); err != nil { return fmt.Errorf("data event %d has a non-string response id", eventCount) } eventResponseID = strings.TrimSpace(eventResponseID) if eventResponseID == "" { return fmt.Errorf("data event %d has an empty response id", eventCount) } const chatCompletionPrefix = "chatcmpl-" if !strings.HasPrefix(eventResponseID, chatCompletionPrefix) || strings.TrimPrefix(eventResponseID, chatCompletionPrefix) == "" { return fmt.Errorf( "data event %d has an invalid provider-native response identity", eventCount, ) } if eventResponseID != currentResponseID { if _, reused := closedResponseIDs[eventResponseID]; reused { return fmt.Errorf( "data event %d reused a non-contiguous provider response identity", eventCount, ) } if len(providerResponseIDs) == devRepeatGuardMaxProviderResponseIDs { return fmt.Errorf( "data event %d exceeds the provider response identity bound of %d", eventCount, devRepeatGuardMaxProviderResponseIDs, ) } if currentResponseID != "" { closedResponseIDs[currentResponseID] = struct{}{} } currentResponseID = eventResponseID providerResponseIDs = append(providerResponseIDs, eventResponseID) } } else if !hasError { return fmt.Errorf( "data event %d has no provider-native response identity", eventCount, ) } finishes := 0 if rawChoices := envelope["choices"]; len(rawChoices) > 0 && !bytes.Equal(bytes.TrimSpace(rawChoices), []byte("null")) { var choices []struct { FinishReason json.RawMessage `json:"finish_reason"` } if err := json.Unmarshal(rawChoices, &choices); err != nil { return fmt.Errorf("data event %d has invalid choices", eventCount) } for _, choice := range choices { rawFinish := bytes.TrimSpace(choice.FinishReason) if len(rawFinish) == 0 || bytes.Equal(rawFinish, []byte("null")) { continue } var finish string if err := json.Unmarshal(rawFinish, &finish); err != nil || strings.TrimSpace(finish) == "" { return fmt.Errorf("data event %d has invalid finish_reason", eventCount) } finishes++ } } switch { case hasError && (successFinishCount > 0 || finishes > 0): return errors.New("SSE mixes error and success finish terminals") case finishes > 0 && errorCount > 0: return errors.New("SSE mixes success finish and error terminals") case hasError && errorCount > 0: return errors.New("SSE has duplicate error terminals") case finishes > 0 && successFinishCount > 0: return errors.New("SSE has duplicate success finish terminals") case (errorCount > 0 || successFinishCount > 0) && !hasError && finishes == 0: return errors.New("SSE has data after its logical terminal") } if hasError { errorCount++ } successFinishCount += finishes if successFinishCount > 1 { return fmt.Errorf("success finish count=%d, want 1", successFinishCount) } return nil } for _, rawLine := range lines { line := bytes.TrimSuffix(rawLine, []byte{'\r'}) if len(line) == 0 { if err := processCurrentData(); err != nil { return devRepeatGuardSSEEvidence{}, err } continue } if line[0] == ':' { continue } field, value, found := bytes.Cut(line, []byte{':'}) if !found { field, value = line, nil } else if len(value) > 0 && value[0] == ' ' { value = value[1:] } switch string(field) { case "data": dataLines = append(dataLines, append([]byte(nil), value...)) case "event", "id", "retry": // Valid SSE fields that do not carry the OpenAI-compatible payload. default: return devRepeatGuardSSEEvidence{}, fmt.Errorf("unsupported SSE field %q", field) } } if err := processCurrentData(); err != nil { return devRepeatGuardSSEEvidence{}, err } if eventCount == 0 { return devRepeatGuardSSEEvidence{}, errors.New("no SSE data events") } if !sawDone { return devRepeatGuardSSEEvidence{}, errors.New("SSE is missing final [DONE]") } if (errorCount == 1) == (successFinishCount == 1) { return devRepeatGuardSSEEvidence{}, fmt.Errorf( "invalid logical terminal state: errors=%d success_finishes=%d", errorCount, successFinishCount, ) } terminal := "done" if errorCount == 1 { terminal = "error" } evidence := devRepeatGuardSSEEvidence{terminal: terminal} if len(providerResponseIDs) == 0 { return devRepeatGuardSSEEvidence{}, errors.New( "SSE response has no provider-native chat completion identity", ) } evidence.providerResponseIDs = append([]string(nil), providerResponseIDs...) return evidence, nil } // devRepeatGuardSmokeChannelEvidence carries the downstream content and // reasoning channels as independent rune streams. Repetition must be evaluated // per channel, never across the channel boundary, otherwise identical tails in // two distinct channels produce a false downstream repeat. type devRepeatGuardSmokeChannelEvidence struct { content []rune reasoning []rune } // contentRepeated reports whether the content channel alone contains a // duplicated suffix of at least minimum runes. func (e devRepeatGuardSmokeChannelEvidence) contentRepeated(minimum int) bool { _, _, repeated := openAIRepeatedSuffix(e.content, minimum) return repeated } // reasoningRepeated reports whether the canonical reasoning channel alone // contains a duplicated suffix of at least minimum runes. func (e devRepeatGuardSmokeChannelEvidence) reasoningRepeated(minimum int) bool { _, _, repeated := openAIRepeatedSuffix(e.reasoning, minimum) return repeated } // devRepeatGuardSmokeChannels reconstructs the content and reasoning channels // from a raw SSE stream. It mirrors decodeChatTunnelFrame reasoning alias // precedence: reasoning_content wins over reasoning, and the two aliases are // never concatenated for a single choice. func devRepeatGuardSmokeChannels(raw []byte) devRepeatGuardSmokeChannelEvidence { var content, reasoning strings.Builder for _, line := range bytes.Split(raw, []byte{'\n'}) { line = bytes.TrimSpace(line) if !bytes.HasPrefix(line, []byte("data:")) { continue } data := bytes.TrimSpace(bytes.TrimPrefix(line, []byte("data:"))) if bytes.Equal(data, []byte("[DONE]")) { continue } var chunk struct { Choices []struct { Delta struct { Content string `json:"content"` ReasoningContent string `json:"reasoning_content"` Reasoning string `json:"reasoning"` } `json:"delta"` } `json:"choices"` } if json.Unmarshal(data, &chunk) != nil { continue } for _, choice := range chunk.Choices { content.WriteString(choice.Delta.Content) canonicalReasoning := choice.Delta.ReasoningContent if canonicalReasoning == "" { canonicalReasoning = choice.Delta.Reasoning } reasoning.WriteString(canonicalReasoning) } } return devRepeatGuardSmokeChannelEvidence{ content: []rune(content.String()), reasoning: []rune(reasoning.String()), } } func devRepeatGuardObservationQuietOffset(path string, quietWindow time.Duration) (int64, error) { if quietWindow < 0 { return 0, errors.New("observation quiet window must not be negative") } return devRepeatGuardObservationQuietOffsetWithWait(path, func() error { timer := time.NewTimer(quietWindow) defer timer.Stop() <-timer.C return nil }) } func devRepeatGuardObservationQuietOffsetWithWait( path string, wait func() error, ) (int64, error) { before, err := os.Stat(path) if err != nil { return 0, err } if !before.Mode().IsRegular() { return 0, errors.New("observation path is not a regular file") } if wait == nil { return 0, errors.New("observation quiet-window wait is nil") } if err := wait(); err != nil { return 0, err } after, err := os.Stat(path) if err != nil { return 0, err } if !after.Mode().IsRegular() { return 0, errors.New("observation path stopped being a regular file") } if !os.SameFile(before, after) { return 0, errors.New("observation file was rotated during the quiet window") } if after.Size() != before.Size() { return 0, fmt.Errorf( "observation file size changed during the quiet window: before=%d after=%d", before.Size(), after.Size(), ) } return after.Size(), nil } func waitDevRepeatGuardObservationSegment( path string, offset int64, model string, expectedCorrelations int, forbiddenValues ...[]byte, ) ([]devRepeatGuardObservation, error) { deadline := time.Now().Add(15 * time.Second) var lastErr error for { observations, err := readDevRepeatGuardObservationSegment(path, offset, forbiddenValues...) if err == nil { groups := groupDevRepeatGuardObservations(observations, model) if len(groups) > expectedCorrelations { return nil, fmt.Errorf( "observation segment is contaminated: groups=%d, want exactly %d", len(groups), expectedCorrelations, ) } complete := 0 for _, group := range groups { for _, observation := range group { if observation.ObservationKind == string(streamgate.ObservationKindTerminalCommitted) { complete++ break } } } if len(groups) == expectedCorrelations && complete == expectedCorrelations { return observations, nil } lastErr = fmt.Errorf( "observation groups=%d complete=%d, want %d", len(groups), complete, expectedCorrelations, ) } else { lastErr = err } if time.Now().After(deadline) { return nil, lastErr } time.Sleep(250 * time.Millisecond) } } func readDevRepeatGuardObservationSegment( path string, offset int64, forbiddenValues ...[]byte, ) ([]devRepeatGuardObservation, error) { file, err := os.Open(path) if err != nil { return nil, err } defer file.Close() info, err := file.Stat() if err != nil { return nil, err } if info.Size() < offset { return nil, errors.New("observation file was truncated or rotated") } if _, err := file.Seek(offset, io.SeekStart); err != nil { return nil, err } const observationSegmentLimit = 64 * 1024 * 1024 raw, err := io.ReadAll(io.LimitReader(file, observationSegmentLimit+1)) if err != nil { return nil, err } if len(raw) > observationSegmentLimit { return nil, errors.New("observation segment exceeds 64 MiB") } return parseDevRepeatGuardObservations(raw, forbiddenValues...) } func parseDevRepeatGuardObservations( raw []byte, forbiddenValues ...[]byte, ) ([]devRepeatGuardObservation, error) { for _, forbidden := range forbiddenValues { if len(forbidden) > 0 && bytes.Contains(raw, forbidden) { return nil, errors.New("observation segment contains a forbidden raw value") } } observations := make([]devRepeatGuardObservation, 0) for lineIndex, rawLine := range bytes.Split(raw, []byte{'\n'}) { line := bytes.TrimSpace(rawLine) if len(line) == 0 { continue } var fields map[string]json.RawMessage if err := json.Unmarshal(line, &fields); err != nil { return nil, fmt.Errorf("observation line %d is not JSON", lineIndex+1) } if err := validateDevRepeatGuardObservationFields(fields); err != nil { return nil, fmt.Errorf("observation line %d: %w", lineIndex+1, err) } var observation devRepeatGuardObservation if err := json.Unmarshal(line, &observation); err != nil { return nil, fmt.Errorf("decode observation line %d: %w", lineIndex+1, err) } if observation.Message != filterObservationLogMessage { continue } observation.line = lineIndex if strings.TrimSpace(observation.CorrelationID) == "" { return nil, fmt.Errorf("observation line %d has no correlation_id", lineIndex+1) } observations = append(observations, observation) } if len(observations) == 0 { return nil, errors.New("observation segment has no streamgate filter observations") } return observations, nil } func validateDevRepeatGuardObservationFields(fields map[string]json.RawMessage) error { forbidden := map[string]struct{}{ "authorization": {}, "api_key": {}, "token": {}, "prompt": {}, "raw_prompt": {}, "output": {}, "raw_output": {}, "arguments": {}, "tool_arguments": {}, "tool_result": {}, "request_body": {}, "response_body": {}, } for key := range fields { if _, found := forbidden[strings.ToLower(strings.TrimSpace(key))]; found { return fmt.Errorf("forbidden raw field %q", key) } } return nil } func groupDevRepeatGuardObservations( observations []devRepeatGuardObservation, model string, ) map[string][]devRepeatGuardObservation { groups := make(map[string][]devRepeatGuardObservation) for _, observation := range observations { if observation.ModelGroup != model { continue } groups[observation.CorrelationID] = append(groups[observation.CorrelationID], observation) } return groups } func correlateDevRepeatGuardBatch( batch []devRepeatGuardSmokeEvidence, observations []devRepeatGuardObservation, ) ([]devRepeatGuardLifecycleEvidence, error) { if len(batch) == 0 { return nil, errors.New("no request results to correlate") } run := batch[0].Run model := batch[0].Model providerResponseIDs := make(map[string]struct{}, len(batch)) attempts := make(map[int]struct{}, len(batch)) providerTerminalCounts := map[string]int{"done": 0, "error": 0} for _, response := range batch { if response.Run != run || response.Model != model { return nil, errors.New("provider response batch mixes run or model identity") } if response.Attempt <= 0 { return nil, errors.New("provider response has no attempt index") } if _, found := attempts[response.Attempt]; found { return nil, fmt.Errorf("duplicate provider response attempt=%d", response.Attempt) } attempts[response.Attempt] = struct{}{} if response.StatusCode < http.StatusOK || response.StatusCode >= http.StatusMultipleChoices { return nil, fmt.Errorf( "run=%d attempt=%d returned non-2xx status %d", response.Run, response.Attempt, response.StatusCode, ) } if _, found := providerTerminalCounts[response.Terminal]; !found { return nil, fmt.Errorf( "run=%d attempt=%d has invalid SSE terminal %q", response.Run, response.Attempt, response.Terminal, ) } providerTerminalCounts[response.Terminal]++ if err := collectDevRepeatGuardProviderResponseIDs( response.ProviderResponseIDs, providerResponseIDs, fmt.Sprintf("run=%d attempt=%d", response.Run, response.Attempt), ); err != nil { return nil, err } if response.downstreamRepeated { return nil, fmt.Errorf( "run=%d attempt=%d released repeated downstream content", response.Run, response.Attempt, ) } } groups := groupDevRepeatGuardObservations(observations, model) if len(groups) != len(batch) { return nil, fmt.Errorf("correlation groups=%d, requests=%d", len(groups), len(batch)) } correlations := make([]string, 0, len(groups)) for correlation := range groups { correlations = append(correlations, correlation) } sort.Strings(correlations) edgeTerminalCounts := map[string]int{"done": 0, "error": 0} correlated := make([]devRepeatGuardLifecycleEvidence, 0, len(groups)) for _, correlation := range correlations { evidence, err := correlateRepeatGuardObservations(run, model, groups[correlation]) if err != nil { return nil, fmt.Errorf( "run=%d correlation=%s: %w", run, correlation, err, ) } terminal := "done" if evidence.TerminalReason == "error" { terminal = "error" } edgeTerminalCounts[terminal]++ correlated = append(correlated, evidence) } if providerTerminalCounts["done"] != edgeTerminalCounts["done"] || providerTerminalCounts["error"] != edgeTerminalCounts["error"] { return nil, fmt.Errorf( "provider/Edge terminal cardinality mismatch: provider done/error=%d/%d Edge done/error=%d/%d", providerTerminalCounts["done"], providerTerminalCounts["error"], edgeTerminalCounts["done"], edgeTerminalCounts["error"], ) } return correlated, nil } func correlateRepeatGuardObservations( run int, model string, observations []devRepeatGuardObservation, ) (devRepeatGuardLifecycleEvidence, error) { if len(observations) == 0 { return devRepeatGuardLifecycleEvidence{}, errors.New("no correlated observations") } correlation := observations[0].CorrelationID if strings.TrimSpace(correlation) == "" { return devRepeatGuardLifecycleEvidence{}, errors.New("missing correlation_id") } var ( actualModel, actualProvider string repeatPass, repeatViolation bool repeatFatal, arbitrationRecover bool repeatEvidence bool arbitrationSeen int arbitrationTerminal, terminalSeen int terminalReason string recoveryKinds = make(map[string]int) recoveryStage int recoveryLifecycleComplete bool fingerprint string offset int ) for _, observation := range observations { if observation.CorrelationID != correlation { return devRepeatGuardLifecycleEvidence{}, errors.New("mixed correlation IDs") } if observation.ActualModel != "" { actualModel = observation.ActualModel } if observation.ActualProvider != "" { actualProvider = observation.ActualProvider } switch observation.ObservationKind { case string(streamgate.ObservationKindFilterEvaluated): if observation.FilterID != openAIRepeatGuardFilterID && observation.FilterID != openAIRepeatActionGuardFilterID { continue } if observation.FilterOutcome != string(streamgate.FilterOutcomeKindEvaluated) { continue } switch observation.DecisionKind { case string(streamgate.FilterDecisionKindPass): repeatPass = true case string(streamgate.FilterDecisionKindViolation): repeatViolation = true case string(streamgate.FilterDecisionKindFatal): repeatFatal = true } if observation.EvidenceFingerprint != "" && observation.DecisionKind != string(streamgate.FilterDecisionKindPass) { fingerprint = observation.EvidenceFingerprint offset = observation.EvidenceOffset } switch observation.EvidenceDescriptorCode { case "repeat_content_detected", "assistant_history_anchor_repeat", "repeated_action_no_progress", "repeat_after_tool_boundary", "repeat_recovery_source_unavailable": repeatEvidence = true } case string(streamgate.ObservationKindArbitrationDecided): arbitrationSeen++ switch observation.ArbitrationAction { case string(streamgate.ArbitrationActionRecover): arbitrationRecover = true case string(streamgate.ArbitrationActionTerminal): arbitrationTerminal++ } case string(streamgate.ObservationKindTerminalCommitted): terminalSeen++ switch observation.TerminalReason { case "": terminalReason = "error" case string(streamgate.TerminalReasonCompleted): terminalReason = observation.TerminalReason default: return devRepeatGuardLifecycleEvidence{}, fmt.Errorf( "unexpected terminal reason %q", observation.TerminalReason, ) } case string(streamgate.ObservationKindRecoveryPlanSelected), string(streamgate.ObservationKindRecoveryAttemptAborted), string(streamgate.ObservationKindRecoveryRebuilt), string(streamgate.ObservationKindRecoveryDispatched): recoveryKinds[observation.ObservationKind]++ switch observation.ObservationKind { case string(streamgate.ObservationKindRecoveryPlanSelected): recoveryStage = 1 case string(streamgate.ObservationKindRecoveryAttemptAborted): if recoveryStage == 1 { recoveryStage = 2 } case string(streamgate.ObservationKindRecoveryRebuilt): if recoveryStage == 2 { recoveryStage = 3 } case string(streamgate.ObservationKindRecoveryDispatched): if recoveryStage == 3 { recoveryLifecycleComplete = true } } if observation.RecoveryStrategy != "" && observation.RecoveryStrategy != string(streamgate.RecoveryStrategyContinuationRepair) { return devRepeatGuardLifecycleEvidence{}, fmt.Errorf( "unexpected recovery strategy %q", observation.RecoveryStrategy, ) } } } if actualModel == "" { return devRepeatGuardLifecycleEvidence{}, errors.New("missing actual_model") } if actualProvider == "" || actualProvider == "unknown" || actualProvider == "unavailable" { return devRepeatGuardLifecycleEvidence{}, errors.New("missing actual_provider") } if arbitrationSeen == 0 { return devRepeatGuardLifecycleEvidence{}, errors.New("missing arbitration_decided") } if terminalSeen != 1 { return devRepeatGuardLifecycleEvidence{}, fmt.Errorf( "terminal_committed count=%d, want 1", terminalSeen, ) } evidence := devRepeatGuardLifecycleEvidence{ Run: run, Model: model, ActualModel: actualModel, Provider: actualProvider, CorrelationID: correlation, TerminalReason: terminalReason, Fingerprint: fingerprint, Offset: offset, } switch { case repeatFatal: if !repeatEvidence { return devRepeatGuardLifecycleEvidence{}, errors.New("fatal guard decision has no repeat evidence") } if arbitrationTerminal == 0 { return devRepeatGuardLifecycleEvidence{}, errors.New("fatal repeat has no terminal arbitration") } if terminalReason != "error" { return devRepeatGuardLifecycleEvidence{}, errors.New("fatal repeat did not end with an Edge error terminal") } evidence.Result = "reproduced" evidence.Decision = "safe_stop" evidence.Lifecycle = "evaluated_fatal_terminal_committed" case repeatViolation: if !repeatEvidence { return devRepeatGuardLifecycleEvidence{}, errors.New("guard violation has no repeat evidence") } if !arbitrationRecover { return devRepeatGuardLifecycleEvidence{}, errors.New("repeat violation has no recover arbitration") } if terminalReason == "error" { for _, kind := range []streamgate.ObservationKind{ streamgate.ObservationKindRecoveryPlanSelected, streamgate.ObservationKindRecoveryAttemptAborted, } { if recoveryKinds[string(kind)] == 0 { return devRepeatGuardLifecycleEvidence{}, fmt.Errorf("repeat safe stop missing %s", kind) } } evidence.Result = "reproduced" evidence.Decision = "safe_stop" evidence.Lifecycle = "evaluated_violation_abort_terminal_committed" break } if terminalReason != string(streamgate.TerminalReasonCompleted) { return devRepeatGuardLifecycleEvidence{}, errors.New("repeat recovery has no completed Edge terminal") } required := []streamgate.ObservationKind{ streamgate.ObservationKindRecoveryPlanSelected, streamgate.ObservationKindRecoveryAttemptAborted, streamgate.ObservationKindRecoveryRebuilt, streamgate.ObservationKindRecoveryDispatched, } for _, kind := range required { if recoveryKinds[string(kind)] == 0 { return devRepeatGuardLifecycleEvidence{}, fmt.Errorf("repeat violation missing %s", kind) } } if !recoveryLifecycleComplete { return devRepeatGuardLifecycleEvidence{}, errors.New("repeat recovery lifecycle is out of order") } evidence.Result = "reproduced" evidence.Decision = "continuation_repair" evidence.Lifecycle = "evaluated_violation_abort_rebuild_dispatch_terminal_committed" case repeatPass: if terminalReason != string(streamgate.TerminalReasonCompleted) { return devRepeatGuardLifecycleEvidence{}, errors.New("passing repeat guard did not end successfully") } evidence.Result = "not_reproduced" evidence.Decision = "guard_evaluated_pass" evidence.Lifecycle = "evaluated_pass_terminal_committed" default: return devRepeatGuardLifecycleEvidence{}, errors.New("repeat guard was not evaluated") } return evidence, nil } func TestDevRepeatGuardSmokeEvidence(t *testing.T) { const ( errorEvent = "data: {\"error\":{\"type\":\"run_error\"}}\n\n" doneEvent = "data: [DONE]\n\n" ) providerEvent := func(id, content string) string { return fmt.Sprintf( "data: {\"id\":%q,\"choices\":[{\"delta\":{\"content\":%q},\"finish_reason\":null}]}\n\n", id, content, ) } successFinish := func(id string) string { return fmt.Sprintf( "data: {\"id\":%q,\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n\n", id, ) } validSuccess := providerEvent("chatcmpl-run-1", "ok") + successFinish("chatcmpl-run-1") + doneEvent validContinuationSuccess := providerEvent("chatcmpl-run-1", "first") + providerEvent("chatcmpl-run-2", "continued") + successFinish("chatcmpl-run-2") + doneEvent validContinuationError := providerEvent("chatcmpl-run-1", "first") + providerEvent("chatcmpl-run-2", "continued") + errorEvent + doneEvent tests := []struct { name string status int body string wantTerminal string wantIDs []string wantErr bool }{ { name: "valid one identity success", status: http.StatusOK, body: validSuccess, wantTerminal: "done", wantIDs: []string{"chatcmpl-run-1"}, }, { name: "valid two identity continuation success", status: http.StatusOK, body: validContinuationSuccess, wantTerminal: "done", wantIDs: []string{"chatcmpl-run-1", "chatcmpl-run-2"}, }, { name: "valid two identity continuation error", status: http.StatusOK, body: validContinuationError, wantTerminal: "error", wantIDs: []string{"chatcmpl-run-1", "chatcmpl-run-2"}, }, {name: "non-2xx", status: http.StatusBadGateway, body: validSuccess, wantErr: true}, {name: "malformed JSON", status: http.StatusOK, body: "data: {\n\ndata: [DONE]\n\n", wantErr: true}, { name: "missing provider response identity", status: http.StatusOK, body: "data: {\"choices\":[{\"finish_reason\":\"stop\"}]}\n\n" + doneEvent, wantErr: true, }, { name: "empty provider response identity", status: http.StatusOK, body: "data: {\"id\":\"\",\"choices\":[{\"finish_reason\":\"stop\"}]}\n\n" + doneEvent, wantErr: true, }, { name: "invalid provider response identity prefix", status: http.StatusOK, body: "data: {\"id\":\"resp-run-1\",\"choices\":[{\"finish_reason\":\"stop\"}]}\n\n" + doneEvent, wantErr: true, }, { name: "non contiguous provider response identity reuse", status: http.StatusOK, body: providerEvent("chatcmpl-run-1", "first") + providerEvent("chatcmpl-run-2", "continued") + successFinish("chatcmpl-run-1") + doneEvent, wantErr: true, }, { name: "more than four provider response identities", status: http.StatusOK, body: providerEvent("chatcmpl-run-1", "one") + providerEvent("chatcmpl-run-2", "two") + providerEvent("chatcmpl-run-3", "three") + providerEvent("chatcmpl-run-4", "four") + successFinish("chatcmpl-run-5") + doneEvent, wantErr: true, }, { name: "missing success finish", status: http.StatusOK, body: providerEvent("chatcmpl-run-1", "ok") + doneEvent, wantErr: true, }, { name: "missing success done", status: http.StatusOK, body: providerEvent("chatcmpl-run-1", "ok") + successFinish("chatcmpl-run-1"), wantErr: true, }, { name: "missing error done", status: http.StatusOK, body: providerEvent("chatcmpl-run-1", "ok") + errorEvent, wantErr: true, }, { name: "duplicate error", status: http.StatusOK, body: providerEvent("chatcmpl-run-1", "ok") + errorEvent + errorEvent + doneEvent, wantErr: true, }, { name: "duplicate success finish", status: http.StatusOK, body: providerEvent("chatcmpl-run-1", "ok") + successFinish("chatcmpl-run-1") + successFinish("chatcmpl-run-1") + doneEvent, wantErr: true, }, {name: "duplicate done", status: http.StatusOK, body: validSuccess + doneEvent, wantErr: true}, { name: "event after done", status: http.StatusOK, body: validSuccess + providerEvent("chatcmpl-run-1", "late"), wantErr: true, }, { name: "data after logical terminal", status: http.StatusOK, body: providerEvent("chatcmpl-run-1", "ok") + successFinish("chatcmpl-run-1") + providerEvent("chatcmpl-run-1", "late") + doneEvent, wantErr: true, }, { name: "finish then error", status: http.StatusOK, body: providerEvent("chatcmpl-run-1", "ok") + successFinish("chatcmpl-run-1") + errorEvent + doneEvent, wantErr: true, }, { name: "error then finish", status: http.StatusOK, body: providerEvent("chatcmpl-run-1", "ok") + errorEvent + successFinish("chatcmpl-run-1") + doneEvent, wantErr: true, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Content-Type", "text/event-stream") w.WriteHeader(tc.status) _, _ = io.WriteString(w, tc.body) })) defer server.Close() artifactDir := t.TempDir() evidence, err := runDevRepeatGuardSmokeAttempt( server.Client(), server.URL, "ornith:35b", "test prompt", artifactDir, 1, 1, ) if (err != nil) != tc.wantErr { t.Fatalf("runDevRepeatGuardSmokeAttempt error=%v, wantErr=%v", err, tc.wantErr) } if !tc.wantErr { if evidence.StatusCode != http.StatusOK || evidence.Terminal != tc.wantTerminal { t.Fatalf("evidence=%+v", evidence) } if !reflect.DeepEqual(evidence.ProviderResponseIDs, tc.wantIDs) { t.Fatalf("provider response identities=%v, want %v", evidence.ProviderResponseIDs, tc.wantIDs) } } }) } } func TestDevRepeatGuardDownstreamRepeatEvidence(t *testing.T) { const minimum = 64 contentToken := strings.Repeat("C", minimum) reasoningToken := strings.Repeat("R", minimum) aliasToken := strings.Repeat("A", minimum) if len([]rune(contentToken)) != minimum || len([]rune(reasoningToken)) != minimum || len([]rune(aliasToken)) != minimum { t.Fatalf("fixture tokens must each be %d runes", minimum) } deltaEvent := func(fields string) string { return "data: {\"choices\":[{\"delta\":{" + fields + "}}]}\n\n" } contentField := func(value string) string { return fmt.Sprintf("%q:%q", "content", value) } reasoningContentField := func(value string) string { return fmt.Sprintf("%q:%q", "reasoning_content", value) } reasoningField := func(value string) string { return fmt.Sprintf("%q:%q", "reasoning", value) } const done = "data: [DONE]\n\n" t.Run("cross-channel identical tails are not a repeat", func(t *testing.T) { raw := deltaEvent(contentField(contentToken)) + deltaEvent(reasoningContentField(contentToken)) + done channels := devRepeatGuardSmokeChannels([]byte(raw)) if channels.contentRepeated(minimum) || channels.reasoningRepeated(minimum) { t.Fatalf("identical tails in distinct channels must not be a downstream repeat") } }) t.Run("same-channel content repeat is detected", func(t *testing.T) { raw := deltaEvent(contentField(contentToken)) + deltaEvent(contentField(contentToken)) + done channels := devRepeatGuardSmokeChannels([]byte(raw)) if !channels.contentRepeated(minimum) { t.Fatalf("duplicated content channel must be a downstream repeat") } if channels.reasoningRepeated(minimum) { t.Fatalf("empty reasoning channel must not repeat") } }) t.Run("same-channel reasoning repeat is detected", func(t *testing.T) { raw := deltaEvent(reasoningContentField(reasoningToken)) + deltaEvent(reasoningContentField(reasoningToken)) + done channels := devRepeatGuardSmokeChannels([]byte(raw)) if !channels.reasoningRepeated(minimum) { t.Fatalf("duplicated reasoning channel must be a downstream repeat") } if channels.contentRepeated(minimum) { t.Fatalf("empty content channel must not repeat") } }) t.Run("reasoning alias precedence uses reasoning_content only", func(t *testing.T) { raw := deltaEvent(reasoningContentField(reasoningToken)+","+reasoningField(aliasToken)) + deltaEvent(reasoningContentField(reasoningToken)+","+reasoningField(aliasToken)) + done channels := devRepeatGuardSmokeChannels([]byte(raw)) if got, want := string(channels.reasoning), reasoningToken+reasoningToken; got != want { t.Fatalf("reasoning channel must use reasoning_content precedence only") } if strings.Contains(string(channels.reasoning), aliasToken) { t.Fatalf("reasoning alias must not be appended alongside reasoning_content") } if !channels.reasoningRepeated(minimum) { t.Fatalf("duplicated reasoning_content must be a downstream repeat") } }) t.Run("single event does not concatenate reasoning aliases", func(t *testing.T) { raw := deltaEvent(reasoningContentField(reasoningToken)+","+reasoningField(reasoningToken)) + done channels := devRepeatGuardSmokeChannels([]byte(raw)) if channels.reasoningRepeated(minimum) { t.Fatalf("a single reasoning value must not be doubled by alias concatenation") } }) } func TestDevRepeatGuardRequestDeadline(t *testing.T) { now := time.Unix(1_800_000_000, 0) exactThreeRunBudget := 3* (devRepeatGuardDefaultRequestTimeout+devRepeatGuardPerRunSettlementAllowance) + devRepeatGuardFinalCleanupReserve tests := []struct { name string raw string runs int deadline time.Time hasDeadline bool want time.Duration wantErr bool }{ { name: "default duration without test deadline", runs: 3, want: devRepeatGuardDefaultRequestTimeout, }, { name: "explicit valid duration", raw: "24m", runs: 3, want: 24 * time.Minute, }, {name: "malformed duration", raw: "twenty-five", runs: 3, wantErr: true}, {name: "zero duration", raw: "0s", runs: 3, wantErr: true}, {name: "negative duration", raw: "-1s", runs: 3, wantErr: true}, {name: "above policy duration", raw: "25m1s", runs: 3, wantErr: true}, { name: "exact fit includes settlement and cleanup", raw: "25m", runs: 3, deadline: now.Add(exactThreeRunBudget), hasDeadline: true, want: devRepeatGuardDefaultRequestTimeout, }, { name: "insufficient final cleanup reserve", raw: "25m", runs: 3, deadline: now.Add(exactThreeRunBudget - time.Nanosecond), hasDeadline: true, wantErr: true, }, { name: "missing deadline falls back to policy validation", raw: "25m", runs: 3, deadline: now.Add(time.Second), hasDeadline: false, want: devRepeatGuardDefaultRequestTimeout, }, { name: "three runs fit a ninety minute envelope", raw: "25m", runs: 3, deadline: now.Add(90 * time.Minute), hasDeadline: true, want: devRepeatGuardDefaultRequestTimeout, }, {name: "invalid run count", raw: "25m", runs: 0, wantErr: true}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { got, err := resolveDevRepeatGuardRequestTimeout( tc.raw, tc.runs, now, tc.deadline, tc.hasDeadline, ) if (err != nil) != tc.wantErr { t.Fatalf("resolve error=%v, wantErr=%v", err, tc.wantErr) } if !tc.wantErr && got != tc.want { t.Fatalf("request timeout=%s, want %s", got, tc.want) } }) } } func TestDevRepeatGuardObservationCorrelation(t *testing.T) { base := devRepeatGuardSmokeEvidence{ Run: 1, Attempt: 1, Model: "ornith:35b", StatusCode: http.StatusOK, Terminal: "done", ProviderResponseIDs: []string{"chatcmpl-provider-1"}, } observation := func(kind string) devRepeatGuardObservation { return devRepeatGuardObservation{ Message: filterObservationLogMessage, ObservationKind: kind, CorrelationID: "req.run-1", ModelGroup: "ornith:35b", ActualModel: "served-model", ActualProvider: "provider-a", } } filterDecision := func(decision string) devRepeatGuardObservation { value := observation(string(streamgate.ObservationKindFilterEvaluated)) value.FilterID = openAIRepeatGuardFilterID value.FilterOutcome = string(streamgate.FilterOutcomeKindEvaluated) value.DecisionKind = decision value.EvidenceFingerprint = "0123456789abcdef" value.EvidenceOffset = 23 if decision != string(streamgate.FilterDecisionKindPass) { value.EvidenceDescriptorCode = "repeat_content_detected" } return value } terminal := observation(string(streamgate.ObservationKindTerminalCommitted)) terminal.TerminalReason = string(streamgate.TerminalReasonCompleted) errorTerminal := observation(string(streamgate.ObservationKindTerminalCommitted)) t.Run("pass is not reproduced", func(t *testing.T) { evidence, err := correlateRepeatGuardObservations(1, base.Model, []devRepeatGuardObservation{ filterDecision(string(streamgate.FilterDecisionKindPass)), observation(string(streamgate.ObservationKindArbitrationDecided)), terminal, }) if err != nil { t.Fatal(err) } if evidence.Result != "not_reproduced" || evidence.Decision != "guard_evaluated_pass" { t.Fatalf("evidence=%+v", evidence) } }) t.Run("violation requires continuation lifecycle", func(t *testing.T) { violation := filterDecision(string(streamgate.FilterDecisionKindViolation)) arbitration := observation(string(streamgate.ObservationKindArbitrationDecided)) arbitration.ArbitrationAction = string(streamgate.ArbitrationActionRecover) lifecycle := []devRepeatGuardObservation{violation, arbitration} for _, kind := range []streamgate.ObservationKind{ streamgate.ObservationKindRecoveryPlanSelected, streamgate.ObservationKindRecoveryAttemptAborted, streamgate.ObservationKindRecoveryRebuilt, streamgate.ObservationKindRecoveryDispatched, } { value := observation(string(kind)) value.RecoveryStrategy = string(streamgate.RecoveryStrategyContinuationRepair) lifecycle = append(lifecycle, value) } lifecycle = append(lifecycle, terminal) evidence, err := correlateRepeatGuardObservations(1, base.Model, lifecycle) if err != nil { t.Fatal(err) } if evidence.Result != "reproduced" || evidence.Decision != "continuation_repair" { t.Fatalf("evidence=%+v", evidence) } }) t.Run("violation can end in safe stop", func(t *testing.T) { violation := filterDecision(string(streamgate.FilterDecisionKindViolation)) arbitration := observation(string(streamgate.ObservationKindArbitrationDecided)) arbitration.ArbitrationAction = string(streamgate.ArbitrationActionRecover) selected := observation(string(streamgate.ObservationKindRecoveryPlanSelected)) selected.RecoveryStrategy = string(streamgate.RecoveryStrategyContinuationRepair) aborted := observation(string(streamgate.ObservationKindRecoveryAttemptAborted)) aborted.RecoveryStrategy = string(streamgate.RecoveryStrategyContinuationRepair) evidence, err := correlateRepeatGuardObservations(1, base.Model, []devRepeatGuardObservation{ violation, arbitration, selected, aborted, errorTerminal, }) if err != nil { t.Fatal(err) } if evidence.Result != "reproduced" || evidence.Decision != "safe_stop" { t.Fatalf("evidence=%+v", evidence) } }) t.Run("fatal is safe stop", func(t *testing.T) { arbitration := observation(string(streamgate.ObservationKindArbitrationDecided)) arbitration.ArbitrationAction = string(streamgate.ArbitrationActionTerminal) evidence, err := correlateRepeatGuardObservations(1, base.Model, []devRepeatGuardObservation{ filterDecision(string(streamgate.FilterDecisionKindFatal)), arbitration, errorTerminal, }) if err != nil { t.Fatal(err) } if evidence.Result != "reproduced" || evidence.Decision != "safe_stop" { t.Fatalf("evidence=%+v", evidence) } }) for _, tc := range []struct { name string mutate func([]devRepeatGuardObservation) []devRepeatGuardObservation }{ { name: "missing provider", mutate: func(values []devRepeatGuardObservation) []devRepeatGuardObservation { for i := range values { values[i].ActualProvider = "" } return values }, }, { name: "missing correlation", mutate: func(values []devRepeatGuardObservation) []devRepeatGuardObservation { for i := range values { values[i].CorrelationID = "" } return values }, }, } { t.Run(tc.name, func(t *testing.T) { values := tc.mutate([]devRepeatGuardObservation{ filterDecision(string(streamgate.FilterDecisionKindPass)), terminal, }) if _, err := correlateRepeatGuardObservations(1, base.Model, values); err == nil { t.Fatal("expected correlation failure") } }) } t.Run("raw sentinel is rejected", func(t *testing.T) { raw := []byte( "{\"msg\":\"streamgate_filter_observation\",\"correlation_id\":\"req.run-1\"," + "\"model_group\":\"ornith:35b\",\"prompt\":\"raw-sentinel\"}\n", ) if _, err := parseDevRepeatGuardObservations(raw, []byte("raw-sentinel")); err == nil { t.Fatal("expected raw sentinel rejection") } }) passGroup := func(correlation string) []devRepeatGuardObservation { filter := filterDecision(string(streamgate.FilterDecisionKindPass)) arbitration := observation(string(streamgate.ObservationKindArbitrationDecided)) committed := terminal for _, value := range []*devRepeatGuardObservation{&filter, &arbitration, &committed} { value.CorrelationID = correlation } return []devRepeatGuardObservation{filter, arbitration, committed} } request := func(attempt int, providerResponseIDs ...string) devRepeatGuardSmokeEvidence { value := base value.Attempt = attempt value.ProviderResponseIDs = append([]string(nil), providerResponseIDs...) return value } t.Run("batch accepts disjoint provider and Edge identities", func(t *testing.T) { batch := []devRepeatGuardSmokeEvidence{ request(1, "chatcmpl-provider-a", "chatcmpl-provider-a-continuation"), request(2, "chatcmpl-provider-b"), } observations := append(passGroup("req.edge-z"), passGroup("req.edge-a")...) correlated, err := correlateDevRepeatGuardBatch(batch, observations) if err != nil { t.Fatal(err) } if correlated[0].CorrelationID != "req.edge-a" || correlated[1].CorrelationID != "req.edge-z" { t.Fatalf("correlated=%+v", correlated) } }) t.Run("final evidence rejects overlap across runs", func(t *testing.T) { firstRun := request(1, "chatcmpl-provider-a", "chatcmpl-provider-shared") secondRun := request(1, "chatcmpl-provider-shared", "chatcmpl-provider-b") secondRun.Run = 2 err := validateDevRepeatGuardUniqueEvidence( []devRepeatGuardSmokeEvidence{firstRun, secondRun}, []devRepeatGuardLifecycleEvidence{ {CorrelationID: "req.edge-a"}, {CorrelationID: "req.edge-b"}, }, ) if err == nil { t.Fatal("expected final flattened provider identity overlap to fail") } }) for _, tc := range []struct { name string batch []devRepeatGuardSmokeEvidence observations []devRepeatGuardObservation }{ { name: "batch rejects missing provider identity", batch: []devRepeatGuardSmokeEvidence{request(1)}, observations: passGroup("req.run-1"), }, { name: "batch rejects duplicate provider identity", batch: []devRepeatGuardSmokeEvidence{ request(1, "chatcmpl-provider-a"), request(2, "chatcmpl-provider-a"), }, observations: append(passGroup("req.run-1"), passGroup("req.run-2")...), }, { name: "batch rejects cross-request provider identity overlap", batch: []devRepeatGuardSmokeEvidence{ request(1, "chatcmpl-provider-a", "chatcmpl-provider-shared"), request(2, "chatcmpl-provider-shared", "chatcmpl-provider-b"), }, observations: append(passGroup("req.run-1"), passGroup("req.run-2")...), }, { name: "batch rejects extra Edge group", batch: []devRepeatGuardSmokeEvidence{ request(1, "chatcmpl-provider-a"), request(2, "chatcmpl-provider-b"), }, observations: append( append(passGroup("req.run-1"), passGroup("req.run-2")...), passGroup("req.manual")..., ), }, { name: "batch rejects provider and Edge terminal count mismatch", batch: []devRepeatGuardSmokeEvidence{ request(1, "chatcmpl-provider-a"), func() devRepeatGuardSmokeEvidence { value := request(2, "chatcmpl-provider-b") value.Terminal = "error" return value }(), }, observations: append(passGroup("req.run-1"), passGroup("req.run-2")...), }, { name: "batch rejects released repeat without identity pairing", batch: []devRepeatGuardSmokeEvidence{ func() devRepeatGuardSmokeEvidence { value := request(1, "chatcmpl-provider-a") value.downstreamRepeated = true return value }(), }, observations: passGroup("req.run-1"), }, } { t.Run(tc.name, func(t *testing.T) { if _, err := correlateDevRepeatGuardBatch(tc.batch, tc.observations); err == nil { t.Fatal("expected strict batch correlation failure") } }) } } func TestDevRepeatGuardObservationQuiescence(t *testing.T) { path := filepath.Join(t.TempDir(), "observations.jsonl") const initial = "{\"msg\":\"sanitized\"}\n" if err := os.WriteFile(path, []byte(initial), 0o600); err != nil { t.Fatal(err) } t.Run("unchanged regular file passes", func(t *testing.T) { offset, err := devRepeatGuardObservationQuietOffsetWithWait(path, func() error { return nil }) if err != nil { t.Fatal(err) } if offset != int64(len(initial)) { t.Fatalf("offset=%d, want %d", offset, len(initial)) } }) t.Run("append during quiet window fails", func(t *testing.T) { if _, err := devRepeatGuardObservationQuietOffsetWithWait(path, func() error { file, openErr := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0) if openErr != nil { return openErr } if _, writeErr := file.WriteString("{\"msg\":\"concurrent\"}\n"); writeErr != nil { _ = file.Close() return writeErr } return file.Close() }); err == nil { t.Fatal("expected append during quiet window to fail") } }) t.Run("rotation during quiet window fails", func(t *testing.T) { if _, err := devRepeatGuardObservationQuietOffsetWithWait(path, func() error { rotated := path + ".rotated" if renameErr := os.Rename(path, rotated); renameErr != nil { return renameErr } return os.WriteFile(path, []byte(initial), 0o600) }); err == nil { t.Fatal("expected rotation during quiet window to fail") } }) } func TestDevRepeatGuardCapacityEvidence(t *testing.T) { const validStatus = `{ "nodes": [ {"provider_snapshots": [ {"id":"provider-a","capacity":2,"in_flight":2,"queued":1}, {"id":"ignored-provider","capacity":99,"in_flight":99,"queued":99} ]}, {"provider_snapshots": [ {"id":"provider-b","capacity":2,"in_flight":2,"queued":2} ]} ] }` providerIDs, err := parseDevRepeatGuardProviderIDs(" provider-a,provider-b ") if err != nil { t.Fatal(err) } snapshot, err := parseDevRepeatGuardCapacityStatus([]byte(validStatus), providerIDs) if err != nil { t.Fatal(err) } if snapshot.configuredCapacity != 4 || snapshot.inFlight != 4 || snapshot.queued != 3 { t.Fatalf("snapshot=%+v", snapshot) } t.Run("fetches selected aggregate", func(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Content-Type", "application/json") _, _ = io.WriteString(w, validStatus) })) defer server.Close() got, fetchErr := fetchDevRepeatGuardCapacityStatus( server.Client(), server.URL, providerIDs, ) if fetchErr != nil { t.Fatal(fetchErr) } if got != snapshot { t.Fatalf("fetched snapshot=%+v, want %+v", got, snapshot) } }) t.Run("observes peak queue and final recovery", func(t *testing.T) { var requestCount atomic.Int32 snapshots := []struct { inFlight int queued int }{ {inFlight: 2}, {inFlight: 4, queued: 1}, {inFlight: 1}, {}, } server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { index := int(requestCount.Add(1)) - 1 if index >= len(snapshots) { index = len(snapshots) - 1 } current := snapshots[index] w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(map[string]any{ "nodes": []any{map[string]any{ "provider_snapshots": []any{map[string]any{ "id": "provider-a", "capacity": 4, "in_flight": current.inFlight, "queued": current.queued, }}, }}, }) })) defer server.Close() doneC := make(chan struct{}) close(doneC) evidence, observeErr := observeDevRepeatGuardCapacityRun( server.Client(), server.URL, []string{"provider-a"}, 1, doneC, make(chan struct{}), 2*time.Second, ) if observeErr != nil { t.Fatal(observeErr) } if evidence.ConfiguredCapacity != 4 || evidence.PeakInFlight != 4 || evidence.PeakQueued != 1 || evidence.FinalInFlight != 0 || evidence.FinalQueued != 0 { t.Fatalf("capacity evidence=%+v", evidence) } }) for _, tc := range []struct { name string raw string }{ {name: "malformed JSON", raw: `{"nodes":[`}, { name: "duplicate selected provider", raw: `{"nodes":[{"provider_snapshots":[ {"id":"provider-a","capacity":2,"in_flight":0,"queued":0}, {"id":"provider-a","capacity":2,"in_flight":0,"queued":0}, {"id":"provider-b","capacity":2,"in_flight":0,"queued":0} ]}]}`, }, { name: "missing selected provider", raw: `{"nodes":[{"provider_snapshots":[ {"id":"provider-a","capacity":2,"in_flight":0,"queued":0} ]}]}`, }, { name: "negative selected counter", raw: `{"nodes":[{"provider_snapshots":[ {"id":"provider-a","capacity":2,"in_flight":-1,"queued":0}, {"id":"provider-b","capacity":2,"in_flight":0,"queued":0} ]}]}`, }, } { t.Run(tc.name, func(t *testing.T) { if _, parseErr := parseDevRepeatGuardCapacityStatus( []byte(tc.raw), providerIDs, ); parseErr == nil { t.Fatal("expected capacity status validation failure") } }) } for _, rawIDs := range []string{"", "provider-a,", "provider-a,provider-a"} { t.Run("rejects provider IDs "+strconv.Quote(rawIDs), func(t *testing.T) { if _, parseErr := parseDevRepeatGuardProviderIDs(rawIDs); parseErr == nil { t.Fatal("expected provider ID validation failure") } }) } validEvidence := devRepeatGuardCapacityEvidence{ Run: 1, ConfiguredCapacity: 4, PeakInFlight: 4, PeakQueued: 1, FinalInFlight: 0, FinalQueued: 0, } if err := validateDevRepeatGuardCapacityEvidence(validEvidence); err != nil { t.Fatal(err) } for _, tc := range []struct { name string mutate func(*devRepeatGuardCapacityEvidence) }{ {name: "wrong capacity", mutate: func(value *devRepeatGuardCapacityEvidence) { value.ConfiguredCapacity = 3 }}, {name: "missing in flight peak", mutate: func(value *devRepeatGuardCapacityEvidence) { value.PeakInFlight = 3 }}, {name: "missing queue pressure", mutate: func(value *devRepeatGuardCapacityEvidence) { value.PeakQueued = 0 }}, {name: "not recovered", mutate: func(value *devRepeatGuardCapacityEvidence) { value.FinalQueued = 1 }}, } { t.Run(tc.name, func(t *testing.T) { value := validEvidence tc.mutate(&value) if validateErr := validateDevRepeatGuardCapacityEvidence(value); validateErr == nil { t.Fatal("expected capacity evidence validation failure") } }) } } // --- S16: host recovery preparer lifecycle ---------------------------------- // recordingPreparer is a test-only host RecoveryPlanPreparer. Production OpenAI // surfaces register no preparer; this fixture exercises the host wiring of the // optional one-shot preparation seam (S23): it must run at most once per plan, // after attempt ownership is closed and before the rebuild/dispatch. type recordingPreparer struct { mu sync.Mutex calls int failWith error directive streamgate.RecoveryDirective } func (p *recordingPreparer) PrepareRecoveryPlan(ctx context.Context, plan streamgate.RecoveryPlan, snapshot streamgate.RecoveryPreparationSnapshot) (streamgate.RecoveryDirective, error) { p.mu.Lock() p.calls++ failWith := p.failWith directive := p.directive p.mu.Unlock() if failWith != nil { return streamgate.RecoveryDirective{}, failWith } return directive, nil } func (p *recordingPreparer) callCount() int { p.mu.Lock() defer p.mu.Unlock() return p.calls } // recordingPreparationSnapshot is the minimal bounded snapshot seam; it only // records that Core released it exactly once. type recordingPreparationSnapshot struct { ref string releases *int32 } func (s *recordingPreparationSnapshot) SnapshotRef() string { return s.ref } func (s *recordingPreparationSnapshot) Release() error { atomic.AddInt32(s.releases, 1) return nil } type recordingPreparationSnapshotFactory struct { ref string creates *int32 releases *int32 } func (f *recordingPreparationSnapshotFactory) CreatePreparationSnapshot(ctx context.Context, snapshot streamgate.RequestRuntimeSnapshot, binding streamgate.AttemptBinding) (streamgate.RecoveryPreparationSnapshot, error) { atomic.AddInt32(f.creates, 1) return &recordingPreparationSnapshot{ref: f.ref, releases: f.releases}, nil } // TestStreamGateChatRecoveryPreparerLifecycle pins the host preparer contract // for one recovery cycle: on success the prepared directive is what gets // rebuilt and dispatched exactly once, and on failure the cycle stops before // any dispatch and converges on a single error terminal with no leaked content. func TestStreamGateChatRecoveryPreparerLifecycle(t *testing.T) { for _, tc := range []struct { name string preparerFail bool wantDispatch int }{ {"preparer success dispatches once", false, 1}, {"preparer failure dispatches nothing", true, 0}, } { tc := tc t.Run(tc.name, func(t *testing.T) { srv, dc, initial, fake := buildStreamGateChatFixture(t, true, 3) fake.eventRuns = []chan *iop.RunEvent{bufferedRunEvents( &iop.RunEvent{Type: "delta", Delta: "final answer"}, &iop.RunEvent{Type: "complete"}, )} fake.runIDs = []string{"run-attempt-2"} snapRef, err := dc.ingress.recoveryRef() if err != nil { t.Fatalf("recoveryRef: %v", err) } directive, err := streamgate.NewRecoveryDirectiveExact(snapRef.SnapshotRef()) if err != nil { t.Fatalf("NewRecoveryDirectiveExact: %v", err) } preparer := &recordingPreparer{directive: directive} if tc.preparerFail { preparer.failWith = fmt.Errorf("injected preparer failure") } var creates, releases int32 factory := &recordingPreparationSnapshotFactory{ref: "openai.prep.1", creates: &creates, releases: &releases} registry := streamGateTestRegistry(t, newInjectedViolationRegistration(t, dc, "test.injected_violation", 1)) w := newRecordingResponseWriter() sink := newOpenAIChatSSEReleaseSink(w, w, "chatcmpl-run-attempt-1", time.Now().Unix(), "client-model") rt, _, err := srv.buildOpenAIChatStreamGateRuntimeFor(dc, openAIChatStreamGateConfig{ mode: openAIChatGateModeLive, initial: openAIAttemptTransport{path: openAIAdmissionRun, run: initial}, dispatch: initial.Dispatch(), closeAll: initial.Close, sink: sink, selector: newOpenAIStreamGateCodecSelector(openAIStreamGateCodecNormalized), registry: registry, preparer: preparer, prepFactory: factory, }) if err != nil { t.Fatalf("buildOpenAIChatStreamGateRuntimeFor: %v", err) } if err := rt.Run(context.Background()); err != nil { t.Fatalf("rt.Run: %v", err) } if got := preparer.callCount(); got != 1 { t.Fatalf("preparer calls: got %d, want exactly 1 per recovery plan", got) } if got := atomic.LoadInt32(&creates); got != 1 { t.Fatalf("preparation snapshot creations: got %d, want 1", got) } if got := atomic.LoadInt32(&releases); got != 1 { t.Fatalf("preparation snapshot releases: got %d, want exactly 1", got) } if got := len(fake.reqsSnapshot()); got != tc.wantDispatch { t.Fatalf("recovery dispatches: got %d, want %d", got, tc.wantDispatch) } body := w.body.String() if strings.Contains(body, "should-be-discarded") { t.Fatalf("the aborted attempt leaked into the response: %s", body) } if tc.preparerFail { if !strings.Contains(body, `"error"`) { t.Fatalf("a failed preparation must terminate with an error, body=%s", body) } return } if got := joinedContent(parseSSEChatChunks(t, body)); got != "final answer" { t.Fatalf("joined content: got %q, want the prepared recovery attempt's content", got) } }) } } // --- S16: producer backpressure while an evaluator holds -------------------- // blockingEvaluationFilter blocks inside Evaluate until it is released, so a // fixture can observe that the unbuffered provider producer cannot advance // while the Core is evaluating an epoch. type blockingEvaluationFilter struct { streamgate.FilterBase entered chan struct{} release chan struct{} once sync.Once } func newBlockingEvaluationFilter(t *testing.T, id string) *blockingEvaluationFilter { t.Helper() base, err := streamgate.NewFilterBase(id) if err != nil { t.Fatalf("NewFilterBase: %v", err) } return &blockingEvaluationFilter{ FilterBase: base, entered: make(chan struct{}, 1), release: make(chan struct{}), } } func (f *blockingEvaluationFilter) Applies(streamgate.FilterContext) bool { return true } func (f *blockingEvaluationFilter) HoldRequirement(streamgate.FilterContext) streamgate.FilterHoldRequirement { req, err := streamgate.NewFilterHoldRequirementRolling(streamGateChannelDefault, []streamgate.EventKind{streamgate.EventKindTextDelta}, 1) if err != nil { panic(err) } return req } func (f *blockingEvaluationFilter) Evaluate(ctx context.Context, fctx streamgate.FilterContext, batch streamgate.EvidenceBatch) (streamgate.FilterDecision, error) { f.once.Do(func() { f.entered <- struct{}{} select { case <-f.release: case <-ctx.Done(): } }) kind := streamgate.EventKindTextDelta if events := batch.Events(); len(events) > 0 { kind = events[len(events)-1].Kind() } evidence, err := streamgate.NewSanitizedEvidence(kind, streamGateChannelDefault, "test.rule", "test_blocking_eval", streamgate.FixedFingerprint{0x03}, 1, 0, streamgate.FilterOutcomeKindEvaluated, batch.CapturedAt()) if err != nil { return streamgate.FilterDecision{}, err } return streamgate.NewFilterDecision(streamgate.FilterDecisionKindPass, "test.consumer", f.ID(), "test.rule", evidence, nil) } var _ streamgate.Filter = (*blockingEvaluationFilter)(nil) // TestStreamGateChatProducerBackpressureDuringEvaluation verifies the Core // applies real backpressure: while a blocking filter is still evaluating the // first epoch, the unbuffered provider producer cannot complete its next send, // and it resumes only once the evaluation is released. func TestStreamGateChatProducerBackpressureDuringEvaluation(t *testing.T) { srv, dc, initial, _ := buildStreamGateChatFixture(t, true, 3) events := make(chan *iop.RunEvent) // unbuffered: the producer blocks unless the source is reading initial.events = events sent := make(chan int, 3) go func() { for i, ev := range []*iop.RunEvent{ {Type: "delta", Delta: "one "}, {Type: "delta", Delta: "two"}, {Type: "complete"}, } { events <- ev sent <- i + 1 } }() blocking := newBlockingEvaluationFilter(t, "test.blocking_eval") reg, err := streamgate.NewFilterRegistration(blocking, streamGateNoopCapability, true, streamgate.FilterEnforcementBlocking, streamGateFilterTimeout, 5) if err != nil { t.Fatalf("NewFilterRegistration: %v", err) } registry := streamGateTestRegistry(t, reg) w := newRecordingResponseWriter() rt, _, err := srv.buildOpenAIChatStreamGateRuntime(dc, initial, newOpenAIChatSSEReleaseSink(w, w, "chatcmpl-run-attempt-1", time.Now().Unix(), "client-model"), registry) if err != nil { t.Fatalf("buildOpenAIChatStreamGateRuntime: %v", err) } done := make(chan error, 1) go func() { done <- rt.Run(context.Background()) }() select { case <-blocking.entered: case <-time.After(5 * time.Second): t.Fatal("the blocking evaluator was never entered") } // The first event was consumed by the event source; the second send must be // parked because the runtime is inside the evaluator and not reading. select { case got := <-sent: if got != 1 { t.Fatalf("first completed send: got %d, want 1", got) } case <-time.After(5 * time.Second): t.Fatal("the first send never completed") } select { case got := <-sent: t.Fatalf("send %d completed while the evaluator still held the epoch: producer backpressure is missing", got) case <-time.After(150 * time.Millisecond): } close(blocking.release) select { case runErr := <-done: if runErr != nil { t.Fatalf("rt.Run: %v", runErr) } case <-time.After(10 * time.Second): t.Fatal("rt.Run did not finish after the evaluation was released") } for want := 2; want <= 3; want++ { select { case got := <-sent: if got != want { t.Fatalf("resumed send: got %d, want %d", got, want) } case <-time.After(5 * time.Second): t.Fatalf("send %d never completed after the evaluation was released", want) } } if got := joinedContent(parseSSEChatChunks(t, w.body.String())); got != "one two" { t.Fatalf("joined content: got %q, want %q", got, "one two") } } // TestOpenAIStreamGateRecoveryCandidateRejectionIsBadRequest proves that a // provider-pool capability rejection during Core recovery keeps the same public // admission contract as initial and queued rejection. The rejected recovery // performs no provider transport dispatch and commits a single pre-stream 400. func TestOpenAIStreamGateRecoveryCandidateRejectionIsBadRequest(t *testing.T) { service := newScriptedPoolRunService( scriptedPoolAttempt{ path: string(edgeservice.ProviderPoolPathNormalized), runID: "recovery-reject-1", provider: "prov-a", target: "served-a", runEvents: bufferedRunEvents( &iop.RunEvent{Type: "delta", Delta: "discarded"}, &iop.RunEvent{Type: "complete"}, ), }, scriptedPoolAttempt{err: edgeservice.ErrProviderPoolCandidateRejected}, ) rawBody := []byte(`{"model":"client-model","messages":[{"role":"user","content":"hi"}],"stream":true}`) srv, dc := buildStreamGatePoolChatFixture(t, service, rawBody, 3) w, sink, runErr := runPoolChatStreamGate(t, srv, dc, newInjectedViolationRegistration(t, dc, "test.injected_violation", 1)) if runErr != nil { t.Fatalf("runtime.Run: %v", runErr) } if w.code != http.StatusBadRequest { t.Fatalf("status: got %d, want %d; body=%q", w.code, http.StatusBadRequest, w.body.String()) } if !strings.Contains(w.body.String(), openAIStreamGateCandidateRejectedMessage) { t.Fatalf("candidate rejection message missing: %q", w.body.String()) } if strings.Contains(w.body.String(), "discarded") { t.Fatalf("rejected attempt content leaked: %q", w.body.String()) } committed, success := sink.terminalStatus() if !committed || success { t.Fatalf("terminal status: got (%v,%v), want (true,false)", committed, success) } pools, _, _, _, runReqs, tunnelReqs := service.snapshot() if pools != 2 { t.Fatalf("pool admissions: got %d, want initial + one rejected recovery", pools) } if len(runReqs) != 1 || len(tunnelReqs) != 0 { t.Fatalf("provider transports: normalized=%d tunnel=%d, want only the initial normalized dispatch", len(runReqs), len(tunnelReqs)) } } // --- S16: Responses provider-pool tunnel recovery --------------------------- // buildStreamGateResponsesPoolFixture builds a streaming /v1/responses // provider-pool passthrough request bound to a scripted pool double, mirroring // what handleResponsesProviderPool freezes before its first SubmitProviderPool. func buildStreamGateResponsesPoolFixture(t *testing.T, service runService, maxRequestFaultRecovery int) (*Server, *responsesRequestContext, edgeservice.ProviderPoolDispatchRequest) { t.Helper() rawBody := []byte(`{"model":"client-model","input":"CALLER_RESPONSES_INPUT_MUST_NOT_BE_COPIED","stream":true}`) fault := maxRequestFaultRecovery srv := NewServer(config.EdgeOpenAIConf{ Adapter: "openai-compat", Target: "served-model", TimeoutSec: 15, StreamEvidenceGate: config.StreamEvidenceGateConf{ Enabled: true, MaxRequestFaultRecovery: &fault, }, }, service, nil) srv.SetModelCatalog([]config.ModelCatalogEntry{{ID: "client-model", ContextWindowTokens: 4096}}) route := routeDispatch{ProviderPool: true, TimeoutSec: 15} base := newTestRequestContext(t, route, rawBody) base.endpoint = usageEndpointResponses base.callerMetadata = map[string]string{"principal_ref": "principal-test"} base.estimate = 73 base.contextClass = "long" requestCtx := &responsesRequestContext{ openAIRequestContext: base, envelope: responsesEnvelope{Model: "client-model", Stream: true}, } metadata := map[string]string{ "principal_ref": "principal-test", "openai_model": "client-model", "openai_stream": "true", "estimated_input_tokens": "73", "context_class": "long", } poolReq := edgeservice.ProviderPoolDispatchRequest{ Run: edgeservice.SubmitRunRequest{ ModelGroupKey: "client-model", Metadata: metadata, ProviderPool: true, }, Tunnel: edgeservice.SubmitProviderTunnelRequest{ ModelGroupKey: "client-model", Method: http.MethodPost, Path: "/v1/responses", Stream: true, TimeoutSec: 15, Metadata: metadata, ProviderPool: true, }, PrepareTunnel: func(tunnelReq edgeservice.SubmitProviderTunnelRequest) (edgeservice.SubmitProviderTunnelRequest, error) { return tunnelReq, nil }, PrepareRun: func(runReq edgeservice.SubmitRunRequest) (edgeservice.SubmitRunRequest, error) { return runReq, nil }, } poolReq.Tunnel.BuildBody = func(target string) ([]byte, error) { body, err := requestCtx.ingress.canonicalBody() if err != nil { return nil, err } return rewriteResponsesModel(body, target) } return srv, requestCtx, poolReq } func responsesTunnelFrames(payload string) chan *iop.ProviderTunnelFrame { return bufferedTunnelFrames( &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START, StatusCode: 200, Headers: map[string]string{"Content-Type": "text/event-stream"}}, &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(payload)}, &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true}, ) } func responsesStreamPrefix(responseID, itemID, text string) string { return fmt.Sprintf("data: {\"type\":\"response.created\",\"response\":{\"id\":\"%s\",\"object\":\"response\",\"status\":\"in_progress\"},\"sequence_number\":1}\n\n"+ "data: {\"type\":\"response.output_item.added\",\"output_index\":0,\"item\":{\"id\":\"%s\",\"type\":\"message\",\"status\":\"in_progress\",\"role\":\"assistant\",\"content\":[]},\"sequence_number\":2}\n\n"+ "data: {\"type\":\"response.content_part.added\",\"item_id\":\"%s\",\"output_index\":0,\"content_index\":0,\"part\":{\"type\":\"output_text\",\"text\":\"\",\"annotations\":[],\"logprobs\":[]},\"sequence_number\":3}\n\n"+ "data: {\"type\":\"response.output_text.delta\",\"item_id\":\"%s\",\"output_index\":0,\"content_index\":0,\"delta\":\"%s\",\"logprobs\":[],\"sequence_number\":4}\n\n", responseID, itemID, itemID, itemID, text) } func responsesStreamTerminal(responseID, itemID, text string, sequence int) string { return fmt.Sprintf("data: {\"type\":\"response.output_text.done\",\"item_id\":\"%s\",\"output_index\":0,\"content_index\":0,\"text\":\"%s\",\"logprobs\":[],\"sequence_number\":%d}\n\n"+ "data: {\"type\":\"response.content_part.done\",\"item_id\":\"%s\",\"output_index\":0,\"content_index\":0,\"part\":{\"type\":\"output_text\",\"text\":\"%s\",\"annotations\":[],\"logprobs\":[]},\"sequence_number\":%d}\n\n"+ "data: {\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{\"id\":\"%s\",\"type\":\"message\",\"status\":\"completed\",\"role\":\"assistant\",\"content\":[{\"type\":\"output_text\",\"text\":\"%s\",\"annotations\":[],\"logprobs\":[]}]},\"sequence_number\":%d}\n\n"+ "data: {\"type\":\"response.completed\",\"response\":{\"id\":\"%s\",\"object\":\"response\",\"status\":\"completed\",\"output\":[{\"id\":\"%s\",\"type\":\"message\",\"status\":\"completed\",\"role\":\"assistant\",\"content\":[{\"type\":\"output_text\",\"text\":\"%s\",\"annotations\":[],\"logprobs\":[]}]}],\"usage\":{\"input_tokens\":1,\"output_tokens\":1,\"total_tokens\":2}},\"sequence_number\":%d}\n\n"+ "data: [DONE]\n\n", itemID, text, sequence, itemID, text, sequence+1, itemID, text, sequence+2, responseID, itemID, text, sequence+3) } func responsesTextDelta(itemID, text string, sequence int) string { return fmt.Sprintf("data: {\"type\":\"response.output_text.delta\",\"item_id\":\"%s\",\"output_index\":0,\"content_index\":0,\"delta\":\"%s\",\"sequence_number\":%d}\n\n", itemID, text, sequence) } func responsesRecoveryTunnelFrames(text string, inputTokens, outputTokens int) chan *iop.ProviderTunnelFrame { return bufferedTunnelFrames( &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START, StatusCode: 200, Headers: map[string]string{"Content-Type": "application/json"}}, &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(fmt.Sprintf("{\"type\":\"response.output_text.delta\",\"delta\":\"%s\"}", text))}, &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_USAGE, Usage: &iop.Usage{InputTokens: int32(inputTokens), OutputTokens: int32(outputTokens)}}, &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true}, ) } // TestStreamGateResponsesPoolRecoveryTerminal proves the real Responses Core // transaction from an initial provider tunnel through one continuation // readmission. The replacement may stay on the tunnel path or switch to a // normalized RunEvent path; both must commit only the replacement attempt. func TestStreamGateResponsesPoolRecoveryTerminal(t *testing.T) { for _, tc := range []struct { name string path string wantCodec openAIStreamGateCodec wantOutput string wantUsage usageObservation }{ { name: "tunnel replacement", path: string(edgeservice.ProviderPoolPathTunnel), wantCodec: openAIStreamGateCodecTunnel, wantOutput: "recovered tunnel", wantUsage: usageObservation{inputTokens: 31, outputTokens: 17}, }, { name: "normalized replacement", path: string(edgeservice.ProviderPoolPathNormalized), wantCodec: openAIStreamGateCodecNormalized, wantOutput: "recovered normalized", wantUsage: usageObservation{inputTokens: 29, outputTokens: 13}, }, } { tc := tc t.Run(tc.name, func(t *testing.T) { initialPayload := responsesStreamPrefix("resp-initial", "msg-initial", "safe prefix ") + "data: {\"type\":\"response.output_text.delta\",\"item_id\":\"msg-initial\",\"output_index\":0,\"content_index\":0,\"delta\":\"discarded repeat tail\",\"sequence_number\":5}\n\n" replacement := scriptedPoolAttempt{ path: tc.path, runID: "resp-attempt-2", provider: "prov-b", target: "served-b", } if tc.path == string(edgeservice.ProviderPoolPathTunnel) { replacement.frames = responsesRecoveryTunnelFrames(tc.wantOutput, tc.wantUsage.inputTokens, tc.wantUsage.outputTokens) } else { replacement.runEvents = bufferedRunEvents( &iop.RunEvent{Type: "delta", Delta: tc.wantOutput}, &iop.RunEvent{Type: "complete", Usage: &iop.Usage{InputTokens: int32(tc.wantUsage.inputTokens), OutputTokens: int32(tc.wantUsage.outputTokens)}}, ) } service := newScriptedPoolRunService( scriptedPoolAttempt{ path: string(edgeservice.ProviderPoolPathTunnel), runID: "resp-attempt-1", provider: "prov-a", target: "served-a", frames: responsesTunnelFrames(initialPayload), }, replacement, ) srv, requestCtx, poolReq := buildStreamGateResponsesPoolFixture(t, service, 3) result, err := service.SubmitProviderPool(context.Background(), poolReq) if err != nil { t.Fatalf("initial SubmitProviderPool: %v", err) } dc := newOpenAIResponsesPoolTunnelDispatchContext(requestCtx, poolReq) snapRef, err := requestCtx.ingress.recoveryRef() if err != nil { t.Fatalf("recoveryRef: %v", err) } violation := newInjectedContinuationViolationFilter(t, "test.responses.resume", 1, snapRef.SnapshotRef(), len("safe prefix ")) reg, err := streamgate.NewFilterRegistration(violation, streamGateNoopCapability, true, streamgate.FilterEnforcementBlocking, streamGateFilterTimeout, injectedViolationPriority) if err != nil { t.Fatalf("NewFilterRegistration: %v", err) } w := newRecordingResponseWriter() holder := &openAIResponsesResultHolder{} selector := newOpenAIStreamGateCodecSelector(openAIStreamGateCodecTunnel) sink := newOpenAIResponsesPoolReleaseSink(w, holder, selector) runtime, usage, err := srv.buildOpenAIResponsesStreamGateRuntimeFromAttempt( dc, openAIAttemptTransport{path: openAIAdmissionTunnel, tunnel: result.Tunnel}, result.Tunnel.Dispatch(), result.Tunnel.Close, sink, streamGateTestRegistry(t, reg), ) if err != nil { t.Fatalf("buildOpenAIResponsesStreamGateRuntimeFromAttempt: %v", err) } if err := runtime.Run(context.Background()); err != nil { t.Fatalf("runtime.Run: %v", err) } if err := runtime.CloseRequestResources(context.Background(), true); err != nil { t.Fatalf("CloseRequestResources: %v", err) } if got := service.poolSubmits(); got != 2 { t.Fatalf("SubmitProviderPool calls: got %d, want initial + one recovery", got) } if got := selector.get(); got != tc.wantCodec { t.Fatalf("resolved codec = %q, want %q", got, tc.wantCodec) } committed, success := sink.terminalStatus() if !committed || !success { t.Fatalf("terminal status = (%v,%v), want (true,true)", committed, success) } body := w.body.String() payloads := assertResponsesSSELifecycle(t, body, "response.completed") var terminal map[string]any for _, payload := range payloads { if payload["type"] == "response.completed" { terminal, _ = payload["response"].(map[string]any) break } } if got, _ := terminal["model"].(string); got != "served-b" { t.Fatalf("response.completed model = %q, want selected replacement model served-b: %#v", got, terminal) } terminalUsage, _ := terminal["usage"].(map[string]any) if got := terminalUsage["input_tokens"]; got != float64(tc.wantUsage.inputTokens) { t.Fatalf("response.completed input_tokens = %#v, want %d: %#v", got, tc.wantUsage.inputTokens, terminalUsage) } if got := terminalUsage["output_tokens"]; got != float64(tc.wantUsage.outputTokens) { t.Fatalf("response.completed output_tokens = %#v, want %d: %#v", got, tc.wantUsage.outputTokens, terminalUsage) } if got := terminalUsage["total_tokens"]; got != float64(tc.wantUsage.inputTokens+tc.wantUsage.outputTokens) { t.Fatalf("response.completed total_tokens = %#v, want %d: %#v", got, tc.wantUsage.inputTokens+tc.wantUsage.outputTokens, terminalUsage) } if got := responsesSSEOutputText(payloads); got != "safe prefix "+tc.wantOutput { t.Fatalf("Responses SSE output = %q, want safe prefix plus replacement; body=%q", got, body) } if strings.Contains(body, "discarded repeat tail") || strings.Contains(body, "CALLER_RESPONSES_INPUT_MUST_NOT_BE_COPIED") { t.Fatalf("response body did not preserve only the safe prefix and replacement: %q", body) } if got := w.Header().Get("Content-Type"); got != "text/event-stream" { t.Fatalf("Content-Type = %q, want text/event-stream", got) } if got := w.headerCallCount(); got != 1 { t.Fatalf("WriteHeader call count: got %d, want one terminal commit", got) } if got := usage.get(); got.inputTokens != tc.wantUsage.inputTokens || got.outputTokens != tc.wantUsage.outputTokens { t.Fatalf("terminal usage = %#v, want %#v", got, tc.wantUsage) } pools, cancels, _, _, runRequests, tunnelRequests := service.snapshot() if pools != 2 || countString(cancels, "resp-attempt-1") != 1 { t.Fatalf("recovery lifecycle pools=%d cancels=%v, want two admissions and one initial abort", pools, cancels) } if tc.path == string(edgeservice.ProviderPoolPathNormalized) { if len(runRequests) != 1 || runRequests[0].Metadata["openai_stream"] != "false" || runRequests[0].EstimatedInputTokens != 68 || runRequests[0].ContextClass != "normal" { t.Fatalf("normalized replacement request = %#v", runRequests) } if strings.Contains(runRequests[0].Prompt, "CALLER_RESPONSES_INPUT_MUST_NOT_BE_COPIED") || runRequests[0].Input["responses_resume_content"] != "safe prefix " { t.Fatalf("normalized replacement did not preserve private resume provenance: %#v", runRequests[0]) } return } if len(tunnelRequests) != 2 { t.Fatalf("tunnel requests = %d, want initial plus replacement", len(tunnelRequests)) } resumeTunnel := tunnelRequests[1] if resumeTunnel.Stream || resumeTunnel.Metadata["openai_stream"] != "false" || resumeTunnel.EstimatedInputTokens != 68 || resumeTunnel.ContextClass != "normal" { t.Fatalf("tunnel replacement admission did not use the rebuilt attempt: %#v", resumeTunnel) } resumeBody, err := resumeTunnel.BuildBody("served-b") if err != nil { t.Fatalf("replacement BuildBody: %v", err) } if strings.Contains(string(resumeBody), "CALLER_RESPONSES_INPUT_MUST_NOT_BE_COPIED") || !strings.Contains(string(resumeBody), "safe prefix ") || !strings.Contains(string(resumeBody), "Continue from the provided content and reasoning output") { t.Fatalf("tunnel replacement body does not contain only the private resume request: %s", resumeBody) } }) } } func TestOpenAIResponsesPoolSyntheticLifecycle(t *testing.T) { w := newRecordingResponseWriter() sink := newOpenAIResponsesPoolReleaseSink(w, &openAIResponsesResultHolder{}, newOpenAIStreamGateCodecSelector(openAIStreamGateCodecNormalized)) sink.bindAttempt(false, edgeservice.RunDispatch{Target: "served-synthetic"}) usage := &openAIStreamGateUsageHolder{} usage.set(usageObservation{inputTokens: 7, outputTokens: 11}) sink.setUsageHolder(usage) start, err := streamgate.NewResponseStart(streamGateChannelDefault, http.StatusOK, map[string]string{"Content-Type": "application/json"}, time.Now()) if err != nil { t.Fatalf("NewResponseStart: %v", err) } if _, err := sink.CommitResponseStart(context.Background(), start); err != nil { t.Fatalf("CommitResponseStart: %v", err) } for _, event := range []streamgate.ReleaseEvent{ mustReleaseText(t, "answer"), mustReleaseReasoning(t, "because"), mustReleaseToolCall(t, "call-1", "lookup", `{"q":"IOP"}`), } { if _, err := sink.Release(context.Background(), event); err != nil { t.Fatalf("Release(%q): %v", event.Kind(), err) } } terminal, err := streamgate.NewSuccessTerminalResult(streamGateChannelDefault, time.Now()) if err != nil { t.Fatalf("NewSuccessTerminalResult: %v", err) } if _, err := sink.CommitTerminal(context.Background(), terminal); err != nil { t.Fatalf("CommitTerminal: %v", err) } payloads := assertResponsesSSELifecycle(t, w.body.String(), "response.completed") var kinds []string for _, payload := range payloads { kind, _ := payload["type"].(string) kinds = append(kinds, kind) } wantKinds := []string{ "response.created", "response.output_item.added", "response.content_part.added", "response.output_text.delta", "response.output_item.added", "response.content_part.added", "response.reasoning_text.delta", "response.output_item.added", "response.function_call_arguments.delta", "response.output_text.done", "response.content_part.done", "response.output_item.done", "response.reasoning_text.done", "response.content_part.done", "response.output_item.done", "response.function_call_arguments.done", "response.output_item.done", "response.completed", } if !reflect.DeepEqual(kinds, wantKinds) { t.Fatalf("synthetic lifecycle kinds = %#v, want %#v", kinds, wantKinds) } completed := payloads[len(payloads)-1]["response"].(map[string]any) if got := completed["model"]; got != "served-synthetic" { t.Fatalf("synthetic terminal model = %#v", got) } if got := completed["output_text"]; got != "answer" { t.Fatalf("synthetic terminal output_text = %#v", got) } terminalUsage := completed["usage"].(map[string]any) if terminalUsage["input_tokens"] != float64(7) || terminalUsage["output_tokens"] != float64(11) || terminalUsage["total_tokens"] != float64(18) { t.Fatalf("synthetic terminal usage = %#v", terminalUsage) } } func mustReleaseText(t *testing.T, value string) streamgate.ReleaseEvent { t.Helper() event, err := streamgate.NewReleaseTextDeltaEvent(streamGateChannelDefault, value, time.Now()) if err != nil { t.Fatalf("NewReleaseTextDeltaEvent: %v", err) } return event } func mustReleaseReasoning(t *testing.T, value string) streamgate.ReleaseEvent { t.Helper() event, err := streamgate.NewReleaseReasoningDeltaEvent(streamGateChannelDefault, value, time.Now()) if err != nil { t.Fatalf("NewReleaseReasoningDeltaEvent: %v", err) } return event } func mustReleaseToolCall(t *testing.T, id, name, arguments string) streamgate.ReleaseEvent { t.Helper() event, err := streamgate.NewReleaseToolCallFragmentEvent(streamGateChannelDefault, id, name, arguments, time.Now()) if err != nil { t.Fatalf("NewReleaseToolCallFragmentEvent: %v", err) } return event } // rawResponsesItemSpec describes a raw provider item the prefix builder opens. type rawResponsesItemSpec struct { kind string // "message", "reasoning", or "function" responseID string itemID string outputIndex int callID string name string value string // content text for message/reasoning, arguments for function } func responsesRawFrame(seq *int, payload map[string]any) string { *seq++ payload["sequence_number"] = *seq encoded, err := json.Marshal(payload) if err != nil { panic(err) } return fmt.Sprintf("data: %s\n\n", encoded) } // buildRawResponsesItemPrefix emits a schema-valid raw provider SSE prefix that // opens one item and streams it up to boundary ("delta", "value_done", // "content_done", or "item_done"). "content_done" is only valid for content // items; a function item has no content part. func buildRawResponsesItemPrefix(spec rawResponsesItemSpec, boundary string) string { seq := 0 var b strings.Builder b.WriteString(responsesRawFrame(&seq, map[string]any{"type": "response.created", "response": map[string]any{"id": spec.responseID, "object": "response", "status": "in_progress"}})) if spec.kind == "function" { b.WriteString(responsesRawFrame(&seq, map[string]any{"type": "response.output_item.added", "output_index": spec.outputIndex, "item": map[string]any{"id": spec.itemID, "type": "function_call", "status": "in_progress", "call_id": spec.callID, "name": spec.name, "arguments": ""}})) b.WriteString(responsesRawFrame(&seq, map[string]any{"type": "response.function_call_arguments.delta", "item_id": spec.itemID, "output_index": spec.outputIndex, "delta": spec.value})) if boundary == "delta" { return b.String() } b.WriteString(responsesRawFrame(&seq, map[string]any{"type": "response.function_call_arguments.done", "item_id": spec.itemID, "output_index": spec.outputIndex, "name": spec.name, "arguments": spec.value})) if boundary == "value_done" { return b.String() } b.WriteString(responsesRawFrame(&seq, map[string]any{"type": "response.output_item.done", "output_index": spec.outputIndex, "item": map[string]any{"id": spec.itemID, "type": "function_call", "status": "completed", "call_id": spec.callID, "name": spec.name, "arguments": spec.value}})) return b.String() } partType := "output_text" deltaType := "response.output_text.delta" doneType := "response.output_text.done" if spec.kind == "reasoning" { partType, deltaType, doneType = "reasoning_text", "response.reasoning_text.delta", "response.reasoning_text.done" } openingItem := map[string]any{"id": spec.itemID, "type": spec.kind, "status": "in_progress", "content": []any{}} openPart := map[string]any{"type": partType, "text": ""} donePart := map[string]any{"type": partType, "text": spec.value} if spec.kind == "message" { openingItem["role"] = "assistant" openPart["annotations"], openPart["logprobs"] = []any{}, []any{} donePart["annotations"], donePart["logprobs"] = []any{}, []any{} } b.WriteString(responsesRawFrame(&seq, map[string]any{"type": "response.output_item.added", "output_index": spec.outputIndex, "item": openingItem})) b.WriteString(responsesRawFrame(&seq, map[string]any{"type": "response.content_part.added", "item_id": spec.itemID, "output_index": spec.outputIndex, "content_index": 0, "part": openPart})) deltaPayload := map[string]any{"type": deltaType, "item_id": spec.itemID, "output_index": spec.outputIndex, "content_index": 0, "delta": spec.value} if spec.kind == "message" { deltaPayload["logprobs"] = []any{} } b.WriteString(responsesRawFrame(&seq, deltaPayload)) if boundary == "delta" { return b.String() } textDonePayload := map[string]any{"type": doneType, "item_id": spec.itemID, "output_index": spec.outputIndex, "content_index": 0, "text": spec.value} if spec.kind == "message" { textDonePayload["logprobs"] = []any{} } b.WriteString(responsesRawFrame(&seq, textDonePayload)) if boundary == "value_done" { return b.String() } b.WriteString(responsesRawFrame(&seq, map[string]any{"type": "response.content_part.done", "item_id": spec.itemID, "output_index": spec.outputIndex, "content_index": 0, "part": donePart})) if boundary == "content_done" { return b.String() } completedItem := map[string]any{"id": spec.itemID, "type": spec.kind, "status": "completed", "content": []any{donePart}} if spec.kind == "message" { completedItem["role"] = "assistant" } b.WriteString(responsesRawFrame(&seq, map[string]any{"type": "response.output_item.done", "output_index": spec.outputIndex, "item": completedItem})) return b.String() } func responsesEventItemID(payload map[string]any) string { if id, ok := payload["item_id"].(string); ok { return id } if item, ok := payload["item"].(map[string]any); ok { if id, ok := item["id"].(string); ok { return id } } return "" } func countResponsesItemEvents(payloads []map[string]any, itemID string) map[string]int { counts := make(map[string]int) for _, payload := range payloads { if responsesEventItemID(payload) != itemID { continue } kind, _ := payload["type"].(string) counts[kind]++ } return counts } func responsesCompletedOutputItem(t *testing.T, payloads []map[string]any, itemID string) map[string]any { t.Helper() completed := payloads[len(payloads)-1] response, _ := completed["response"].(map[string]any) output, _ := response["output"].([]any) for _, raw := range output { item, _ := raw.(map[string]any) if id, _ := item["id"].(string); id == itemID { return item } } t.Fatalf("terminal output is missing item %q: %#v", itemID, output) return nil } // TestOpenAIResponsesPoolRawLifecycleContinuation feeds a raw provider prefix // that opens one message, reasoning, or function item and stops at each distinct // done boundary, then switches to a normalized continuation of a second item and // commits the terminal. It proves the raw item keeps its provider identity, // metadata, and output index, that the continuation item never replaces it, and // that every remaining lifecycle transition is synthesized exactly once while an // already-released transition is never repeated. func TestOpenAIResponsesPoolRawLifecycleContinuation(t *testing.T) { for _, tc := range []struct { name string kind string boundary string valueDone string // the item-value done event type for the raw item }{ {name: "message after delta", kind: "message", boundary: "delta", valueDone: "response.output_text.done"}, {name: "message after text done", kind: "message", boundary: "value_done", valueDone: "response.output_text.done"}, {name: "message after content done", kind: "message", boundary: "content_done", valueDone: "response.output_text.done"}, {name: "message after item done", kind: "message", boundary: "item_done", valueDone: "response.output_text.done"}, {name: "reasoning after delta", kind: "reasoning", boundary: "delta", valueDone: "response.reasoning_text.done"}, {name: "reasoning after text done", kind: "reasoning", boundary: "value_done", valueDone: "response.reasoning_text.done"}, {name: "reasoning after content done", kind: "reasoning", boundary: "content_done", valueDone: "response.reasoning_text.done"}, {name: "reasoning after item done", kind: "reasoning", boundary: "item_done", valueDone: "response.reasoning_text.done"}, {name: "function after delta", kind: "function", boundary: "delta", valueDone: "response.function_call_arguments.done"}, {name: "function after arguments done", kind: "function", boundary: "value_done", valueDone: "response.function_call_arguments.done"}, {name: "function after item done", kind: "function", boundary: "item_done", valueDone: "response.function_call_arguments.done"}, } { tc := tc for _, outputIndex := range []int{0, 3} { outputIndex := outputIndex t.Run(fmt.Sprintf("%s provider index %d", tc.name, outputIndex), func(t *testing.T) { w := newRecordingResponseWriter() selector := newOpenAIStreamGateCodecSelector(openAIStreamGateCodecTunnel) sink := newOpenAIResponsesPoolReleaseSink(w, &openAIResponsesResultHolder{}, selector) sink.bindAttempt(true, edgeservice.RunDispatch{Target: "served-a"}) usage := &openAIStreamGateUsageHolder{} usage.set(usageObservation{inputTokens: 5, outputTokens: 9}) sink.setUsageHolder(usage) start, err := streamgate.NewResponseStart(streamGateChannelDefault, http.StatusOK, map[string]string{"Content-Type": "text/event-stream"}, time.Now()) if err != nil { t.Fatalf("NewResponseStart: %v", err) } if _, err := sink.CommitResponseStart(context.Background(), start); err != nil { t.Fatalf("CommitResponseStart: %v", err) } spec := rawResponsesItemSpec{kind: tc.kind, responseID: "resp-raw", itemID: "prov-" + tc.kind, outputIndex: outputIndex, value: "raw " + tc.kind} if tc.kind == "function" { spec.callID = "call-provider" spec.name = "provider_fn" spec.value = `{"raw":true}` } sink.codec.pushRelease([]byte(buildRawResponsesItemPrefix(spec, tc.boundary))) if _, err := sink.Release(context.Background(), mustReleaseText(t, "raw wire ignored")); err != nil { t.Fatalf("raw prefix release: %v", err) } // Normalized text continues an open provider message, but must open a // fresh message after raw reasoning or function evidence. A provider // message already closed by its raw lifecycle instead continues with a // different synthetic item. sink.bindAttempt(false, edgeservice.RunDispatch{Target: "served-b"}) selector.set(openAIStreamGateCodecNormalized) continuation := mustReleaseText(t, "continued text") continuationID := "msg-streamgate-1" continuesRawMessage := tc.kind == "message" && tc.boundary == "delta" if continuesRawMessage { continuationID = spec.itemID } else if tc.kind == "message" { continuation = mustReleaseReasoning(t, "continued thought") continuationID = "reasoning-streamgate-1" } if _, err := sink.Release(context.Background(), continuation); err != nil { t.Fatalf("continuation release: %v", err) } terminal, err := streamgate.NewSuccessTerminalResult(streamGateChannelDefault, time.Now()) if err != nil { t.Fatalf("NewSuccessTerminalResult: %v", err) } if _, err := sink.CommitTerminal(context.Background(), terminal); err != nil { t.Fatalf("CommitTerminal: %v", err) } payloads := assertResponsesSSELifecycle(t, w.body.String(), "response.completed") // The raw provider item keeps its identity and appears in terminal // output. A provider message is continued in place; other raw kinds // coexist with a newly opened synthetic message. rawItem := responsesCompletedOutputItem(t, payloads, spec.itemID) if rawItem["type"] != mappedItemType(tc.kind) { t.Fatalf("raw item type = %#v, want %q", rawItem["type"], mappedItemType(tc.kind)) } if !continuesRawMessage { if spec.itemID == continuationID { t.Fatalf("raw item %q collided with continuation item %q", spec.itemID, continuationID) } responsesCompletedOutputItem(t, payloads, continuationID) // continuation coexists continuationAdded := false for _, payload := range payloads { if payload["type"] != "response.output_item.added" || responsesEventItemID(payload) != continuationID { continue } if got := int(payload["output_index"].(float64)); got != outputIndex+1 { t.Fatalf("continuation output_index = %d, want %d", got, outputIndex+1) } continuationAdded = true } if !continuationAdded { t.Fatalf("continuation item %q was not opened", continuationID) } } if tc.kind == "function" { if rawItem["call_id"] != spec.callID || rawItem["name"] != spec.name { t.Fatalf("raw function metadata = (%#v,%#v), want (%q,%q)", rawItem["call_id"], rawItem["name"], spec.callID, spec.name) } if rawItem["arguments"] != spec.value { t.Fatalf("raw function arguments = %#v, want %q", rawItem["arguments"], spec.value) } } else { content, _ := rawItem["content"].([]any) if len(content) != 1 { t.Fatalf("raw content item invalid: %#v", rawItem) } part, _ := content[0].(map[string]any) want := spec.value if continuesRawMessage { want += "continued text" } if part["text"] != want { t.Fatalf("raw content text = %#v, want %q", part["text"], want) } } // Every lifecycle-closing transition for the raw item is present exactly // once, whether it came from the raw prefix or was synthesized. counts := countResponsesItemEvents(payloads, spec.itemID) if counts[tc.valueDone] != 1 { t.Fatalf("%s count for %q = %d, want exactly 1; payloads=%#v", tc.valueDone, spec.itemID, counts[tc.valueDone], payloads) } if counts["response.output_item.done"] != 1 { t.Fatalf("output_item.done count for %q = %d, want exactly 1", spec.itemID, counts["response.output_item.done"]) } if tc.kind != "function" && counts["response.content_part.done"] != 1 { t.Fatalf("content_part.done count for %q = %d, want exactly 1", spec.itemID, counts["response.content_part.done"]) } }) } } } func mappedItemType(kind string) string { if kind == "function" { return "function_call" } return kind } // TestStreamGateResponsesPoolStreamingTunnelReleasesIncrementally isolates the // no-recovery path. It proves the caller-visible initial tunnel is not routed // through a buffered delegate while the terminal frame is still withheld. func TestStreamGateResponsesPoolStreamingTunnelReleasesIncrementally(t *testing.T) { frames := make(chan *iop.ProviderTunnelFrame) service := newScriptedPoolRunService(scriptedPoolAttempt{ path: string(edgeservice.ProviderPoolPathTunnel), runID: "resp-incremental", provider: "prov-a", target: "served-a", frames: frames, }) srv, requestCtx, poolReq := buildStreamGateResponsesPoolFixture(t, service, 3) result, err := service.SubmitProviderPool(context.Background(), poolReq) if err != nil { t.Fatalf("initial SubmitProviderPool: %v", err) } w := newNotifyingResponseWriter() done := make(chan struct{}) go func() { srv.runOpenAIResponsesPoolStreamGate(w, requestCtx, poolReq, result.Tunnel) close(done) }() frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START, StatusCode: http.StatusOK, Headers: map[string]string{"Content-Type": "text/event-stream"}} frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(responsesStreamPrefix("resp-incremental", "msg-incremental", "first safe frame"))} select { case <-w.writes: case <-time.After(5 * time.Second): t.Fatal("initial streaming Responses frame was not written before terminal") } select { case <-done: t.Fatal("Responses runtime completed before the terminal frame was provided") default: } body := w.body.String() if got := responsesSSEOutputText(parseCompleteResponsesSSE(t, body)); got != "first safe frame" { t.Fatalf("pre-terminal Responses SSE output = %q, body=%q", got, body) } frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(responsesStreamTerminal("resp-incremental", "msg-incremental", "first safe frame", 5))} close(frames) select { case <-done: case <-time.After(5 * time.Second): t.Fatal("Responses runtime did not finish after terminal frame") } if got := responsesSSEOutputText(assertResponsesSSELifecycle(t, w.body.String(), "response.completed")); got != "first safe frame" { t.Fatalf("completed Responses SSE output = %q", got) } } func TestStreamGateResponsesPoolInitialErrorPreservesProviderResponse(t *testing.T) { providerBody := `{"error":{"message":"provider capacity exhausted","type":"rate_limit_error"}}` service := newScriptedPoolRunService(scriptedPoolAttempt{ path: string(edgeservice.ProviderPoolPathTunnel), runID: "resp-provider-error", provider: "prov-a", target: "served-a", frames: bufferedTunnelFrames( &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START, StatusCode: http.StatusTooManyRequests, Headers: map[string]string{"Content-Type": "application/json", "X-Provider-Trace": "trace-123"}}, &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(providerBody)}, &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true}, ), }) srv, requestCtx, poolReq := buildStreamGateResponsesPoolFixture(t, service, 3) result, err := service.SubmitProviderPool(context.Background(), poolReq) if err != nil { t.Fatalf("initial SubmitProviderPool: %v", err) } w := newRecordingResponseWriter() srv.runOpenAIResponsesPoolStreamGate(w, requestCtx, poolReq, result.Tunnel) if got := w.code; got != http.StatusTooManyRequests { t.Fatalf("provider status = %d, want %d", got, http.StatusTooManyRequests) } if got := w.Header().Get("Content-Type"); got != "application/json" { t.Fatalf("provider Content-Type = %q, want application/json", got) } if got := w.Header().Get("X-Provider-Trace"); got != "trace-123" { t.Fatalf("provider trace header = %q", got) } if got := w.body.String(); got != providerBody { t.Fatalf("provider body = %q, want byte-for-byte %q", got, providerBody) } if got := w.headerCallCount(); got != 1 { t.Fatalf("WriteHeader call count = %d, want 1", got) } if strings.Contains(w.body.String(), "data:") { t.Fatalf("pre-commit provider error was rewritten as SSE: %q", w.body.String()) } } func TestStreamGateResponsesPoolRecoveryRejectionAfterStreamOpenTerminatesSSE(t *testing.T) { service := newScriptedPoolRunService( scriptedPoolAttempt{ path: string(edgeservice.ProviderPoolPathTunnel), runID: "resp-open-prefix", provider: "prov-a", target: "served-a", frames: responsesTunnelFrames(responsesStreamPrefix("resp-open", "msg-open", "safe prefix ") + "data: {\"type\":\"response.output_text.delta\",\"item_id\":\"msg-open\",\"output_index\":0,\"content_index\":0,\"delta\":\"discarded repeat tail\",\"sequence_number\":5}\n\n"), }, scriptedPoolAttempt{err: edgeservice.ErrProviderPoolCandidateRejected}, ) srv, requestCtx, poolReq := buildStreamGateResponsesPoolFixture(t, service, 3) result, err := service.SubmitProviderPool(context.Background(), poolReq) if err != nil { t.Fatalf("initial SubmitProviderPool: %v", err) } dc := newOpenAIResponsesPoolTunnelDispatchContext(requestCtx, poolReq) snapRef, err := requestCtx.ingress.recoveryRef() if err != nil { t.Fatalf("recoveryRef: %v", err) } violation := newInjectedContinuationViolationFilter(t, "test.responses.rejection", 1, snapRef.SnapshotRef(), len("safe prefix ")) reg, err := streamgate.NewFilterRegistration(violation, streamGateNoopCapability, true, streamgate.FilterEnforcementBlocking, streamGateFilterTimeout, injectedViolationPriority) if err != nil { t.Fatalf("NewFilterRegistration: %v", err) } w := newRecordingResponseWriter() holder := &openAIResponsesResultHolder{} sink := newOpenAIResponsesPoolReleaseSink(w, holder, newOpenAIStreamGateCodecSelector(openAIStreamGateCodecTunnel)) runtime, _, err := srv.buildOpenAIResponsesStreamGateRuntimeFromAttempt( dc, openAIAttemptTransport{path: openAIAdmissionTunnel, tunnel: result.Tunnel}, result.Tunnel.Dispatch(), result.Tunnel.Close, sink, streamGateTestRegistry(t, reg), ) if err != nil { t.Fatalf("buildOpenAIResponsesStreamGateRuntimeFromAttempt: %v", err) } if err := runtime.Run(context.Background()); err != nil { t.Fatalf("runtime.Run: %v", err) } body := w.body.String() payloads := assertResponsesSSELifecycle(t, body, "error") if got := responsesSSEOutputText(payloads); got != "safe prefix " { t.Fatalf("released prefix = %q, want safe prefix only; body=%q", got, body) } if !strings.Contains(body, openAIStreamGateCandidateRejectedMessage) { t.Fatalf("candidate rejection error missing from SSE: %q", body) } if w.code != http.StatusOK || w.Header().Get("Content-Type") != "text/event-stream" { t.Fatalf("post-open rejection status/header = %d/%q, want 200/text-event-stream", w.code, w.Header().Get("Content-Type")) } if got := w.headerCallCount(); got != 1 { t.Fatalf("WriteHeader call count = %d, want 1", got) } if strings.Contains(body, `{"error":{`) { t.Fatalf("post-open rejection appended HTTP JSON: %q", body) } } // --- S16: provider-pool + tool validation (one owner across a path switch) --- const streamGatePoolToolBody = `{"model":"client-model","messages":[{"role":"user","content":"status"}],` + `"tools":[{"type":"function","function":{"name":"run_commands","parameters":{"type":"object","properties":{"commands":{"type":"array","items":{"type":"string"}}}}}}]}` // TestStreamGatePoolToolValidationReAdmitsThroughPool proves the tool-validation // terminal gate and the provider-pool re-admission are the same single cycle: // an invalid tool call on a normalized pool candidate re-enters // SubmitProviderPool once, and the passing candidate's structured output is the // only thing rendered. func TestStreamGatePoolToolValidationReAdmitsThroughPool(t *testing.T) { service := newScriptedPoolRunService( scriptedPoolAttempt{ path: string(edgeservice.ProviderPoolPathNormalized), runID: "pool-tool-1", provider: "prov-a", target: "served-a", runEvents: bufferedRunEvents( &iop.RunEvent{Type: "delta", Delta: "true"}, &iop.RunEvent{Type: "complete"}, ), }, scriptedPoolAttempt{ path: string(edgeservice.ProviderPoolPathNormalized), runID: "pool-tool-2", provider: "prov-b", target: "served-b", runEvents: bufferedRunEvents( &iop.RunEvent{Type: "delta", Delta: "[\"ls\"]\n"}, &iop.RunEvent{Type: "complete"}, ), }, ) srv, dc := buildStreamGatePoolChatFixture(t, service, []byte(streamGatePoolToolBody), 1) if !dc.validation.enabled { t.Fatal("fixture must carry a tool-validation contract") } w, sink, runErr := runPoolChatStreamGate(t, srv, dc) if runErr != nil { t.Fatalf("rt.Run: %v", runErr) } if got := service.poolSubmits(); got != 2 { t.Fatalf("SubmitProviderPool calls: got %d, want 2 (tool-validation recovery must re-enter the pool)", got) } if got := sink.(*openAICompositeReleaseSink).resolvedCodec(); got != openAIStreamGateCodecNormalized { t.Fatalf("resolved codec: got %q, want normalized", got) } var resp chatCompletionResponse if err := json.Unmarshal([]byte(w.body.String()), &resp); err != nil { t.Fatalf("decode response: %v body=%s", err, w.body.String()) } if len(resp.Choices) != 1 || resp.Choices[0].FinishReason != "tool_calls" { t.Fatalf("expected a structured tool_calls completion, got %s", w.body.String()) } if len(resp.Choices[0].Message.ToolCalls) != 1 { t.Fatalf("tool calls: got %d, want 1: %s", len(resp.Choices[0].Message.ToolCalls), w.body.String()) } if strings.Contains(w.body.String(), "delete_everything") { t.Fatalf("the rejected attempt leaked into the response: %s", w.body.String()) } } // TestStreamGatePoolToolValidationDropsOnTunnelCandidate verifies the // tool-validation contract is re-resolved per attempt: after a recovery selects // a raw passthrough candidate the normalized contract no longer applies, and // the response is relayed with the tunnel framing instead of being validated // again. func TestStreamGatePoolToolValidationDropsOnTunnelCandidate(t *testing.T) { service := newScriptedPoolRunService( scriptedPoolAttempt{ path: string(edgeservice.ProviderPoolPathNormalized), runID: "pool-tool-1", provider: "prov-a", target: "served-a", runEvents: bufferedRunEvents( &iop.RunEvent{Type: "delta", Delta: "true"}, &iop.RunEvent{Type: "complete"}, ), }, scriptedPoolAttempt{ path: string(edgeservice.ProviderPoolPathTunnel), runID: "pool-tool-2", provider: "prov-b", target: "served-b", frames: bufferedTunnelFrames( &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START, StatusCode: 200, Headers: map[string]string{"Content-Type": "application/json"}}, &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(`{"id":"provider-native","object":"chat.completion"}`)}, &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true}, ), }, ) srv, dc := buildStreamGatePoolChatFixture(t, service, []byte(streamGatePoolToolBody), 1) w, sink, runErr := runPoolChatStreamGate(t, srv, dc) if runErr != nil { t.Fatalf("rt.Run: %v", runErr) } if got := service.poolSubmits(); got != 2 { t.Fatalf("SubmitProviderPool calls: got %d, want 2", got) } if got := sink.(*openAICompositeReleaseSink).resolvedCodec(); got != openAIStreamGateCodecTunnel { t.Fatalf("resolved codec: got %q, want tunnel", got) } body := w.body.String() if !strings.Contains(body, `"id":"provider-native"`) { t.Fatalf("raw passthrough bytes missing: %q", body) } if strings.Contains(body, "delete_everything") { t.Fatalf("the rejected normalized attempt leaked into the response: %q", body) } if got := w.headerCallCount(); got != 1 { t.Fatalf("WriteHeader call count: got %d, want 1", got) } }