package streamgate import ( "context" "errors" "fmt" "reflect" "sync" "testing" "time" ) type runtimeTrace struct { mu sync.Mutex steps []string } func (r *runtimeTrace) add(step string) { if r == nil { return } r.mu.Lock() r.steps = append(r.steps, step) r.mu.Unlock() } func (r *runtimeTrace) snapshot() []string { r.mu.Lock() defer r.mu.Unlock() return append([]string(nil), r.steps...) } type sliceEventSource struct { mu sync.Mutex events []NormalizedEvent err error index int } func newSliceEventSource(events []NormalizedEvent) *sliceEventSource { return &sliceEventSource{events: events} } func newErrorEventSource(err error) *sliceEventSource { return &sliceEventSource{err: err} } func (s *sliceEventSource) NextEvent(ctx context.Context) (NormalizedEvent, error) { s.mu.Lock() defer s.mu.Unlock() if s.err != nil { return NormalizedEvent{}, s.err } if s.index >= len(s.events) { return NormalizedEvent{}, errors.New("EOF") } ev := s.events[s.index] s.index++ return ev, nil } type fixtureController struct { mu sync.Mutex abortCount int abortErr error name string trace *runtimeTrace } func (c *fixtureController) AbortAttempt(ctx context.Context) error { c.mu.Lock() defer c.mu.Unlock() c.abortCount++ c.trace.add("abort:" + c.name) return c.abortErr } type fixtureDispatcher struct { mu sync.Mutex handler func(ctx context.Context, request RebuiltRequest) (AttemptBinding, error) dispatched []RebuiltRequest trace *runtimeTrace } func (d *fixtureDispatcher) DispatchAttempt(ctx context.Context, request RebuiltRequest) (AttemptBinding, error) { d.mu.Lock() d.dispatched = append(d.dispatched, request) h := d.handler d.mu.Unlock() d.trace.add("dispatch") if h != nil { return h(ctx, request) } return AttemptBinding{}, errors.New("dispatcher error") } type fixtureRebuilder struct { mu sync.Mutex planCount int plans []RecoveryPlan err error trace *runtimeTrace } func (r *fixtureRebuilder) RebuildRequest(ctx context.Context, snapshot RecoveryRequestSnapshotRef, plan RecoveryPlan) (RebuiltRequestDraft, error) { r.mu.Lock() defer r.mu.Unlock() r.planCount++ r.plans = append(r.plans, plan) r.trace.add("rebuild") if r.err != nil { return RebuiltRequestDraft{}, r.err } return NewRebuiltRequestDraftWithIdempotency(plan.PlanID(), plan.IdempotencyKey(), "req-ref-1", "ep", "fam", 10, 20, 100, 5, nil) } // lastPlan returns the most recent recovery plan handed to the rebuilder. func (r *fixtureRebuilder) lastPlan() (RecoveryPlan, bool) { r.mu.Lock() defer r.mu.Unlock() if len(r.plans) == 0 { return RecoveryPlan{}, false } return r.plans[len(r.plans)-1], true } // countingEventSource records how many times the runtime owner loop pulled the // next ingress event so backpressure can be observed without sleeping. type countingEventSource struct { mu sync.Mutex events []NormalizedEvent index int calls int } func newCountingEventSource(events []NormalizedEvent) *countingEventSource { return &countingEventSource{events: events} } func (s *countingEventSource) NextEvent(ctx context.Context) (NormalizedEvent, error) { s.mu.Lock() defer s.mu.Unlock() s.calls++ if s.index >= len(s.events) { return NormalizedEvent{}, errors.New("EOF") } ev := s.events[s.index] s.index++ return ev, nil } func (s *countingEventSource) callCount() int { s.mu.Lock() defer s.mu.Unlock() return s.calls } type fixtureSink struct { mu sync.Mutex starts []ResponseStart events []ReleaseEvent terminals []TerminalResult state CommitState startErr error releaseErr error terminalErr error trace *runtimeTrace } func newFixtureSink() *fixtureSink { return &fixtureSink{state: CommitStateTransportUncommitted} } func (s *fixtureSink) CommitResponseStart(ctx context.Context, rs ResponseStart) (CommitState, error) { s.mu.Lock() defer s.mu.Unlock() s.starts = append(s.starts, rs) s.trace.add("start") if s.startErr != nil { return s.state, s.startErr } s.state = CommitStateStreamOpen return CommitStateStreamOpen, nil } func (s *fixtureSink) Release(ctx context.Context, ev ReleaseEvent) (CommitState, error) { s.mu.Lock() defer s.mu.Unlock() s.events = append(s.events, ev) s.trace.add("release") if s.releaseErr != nil { return s.state, s.releaseErr } s.state = CommitStateStreamOpen return CommitStateStreamOpen, nil } func (s *fixtureSink) CommitTerminal(ctx context.Context, tr TerminalResult) (CommitState, error) { s.mu.Lock() defer s.mu.Unlock() s.terminals = append(s.terminals, tr) if tr.Success() { s.trace.add("terminal:success") } else { s.trace.add("terminal:error") } if s.terminalErr != nil { return s.state, s.terminalErr } s.state = CommitStateTerminalCommitted return CommitStateTerminalCommitted, nil } type customMockFilter struct { id string holdReq *FilterHoldRequirement holdReqFn func(FilterContext) FilterHoldRequirement appliesFn func(FilterContext) bool evaluateFn func(context.Context, FilterContext, EvidenceBatch) (FilterDecision, error) evalCount int mu sync.Mutex } func (m *customMockFilter) ID() string { return m.id } func (m *customMockFilter) Applies(c FilterContext) bool { if m.appliesFn != nil { return m.appliesFn(c) } return true } func (m *customMockFilter) HoldRequirement(c FilterContext) FilterHoldRequirement { if m.holdReqFn != nil { return m.holdReqFn(c) } if m.holdReq != nil { return *m.holdReq } return mustFilterHoldRequirementRolling("default", []EventKind{EventKindTextDelta, EventKindTerminal}, 1) } func (m *customMockFilter) Evaluate(ctx context.Context, fc FilterContext, batch EvidenceBatch) (FilterDecision, error) { m.mu.Lock() m.evalCount++ m.mu.Unlock() if m.evaluateFn != nil { return m.evaluateFn(ctx, fc, batch) } kind := EventKindTextDelta if len(batch.Events()) > 0 { kind = batch.Events()[0].Kind() } fp := FixedFingerprint{1} ev := mustSanitizedEvidence(kind, "default", "rule1", "desc", fp, 1, 0, FilterOutcomeKindEvaluated, time.Now()) return NewFilterDecision(FilterDecisionKindPass, "consumer1", m.id, "rule1", ev, nil) } func createTestRuntimeSnapshot(t *testing.T, regs []FilterRegistration, disp AttemptDispatcher, rebuilder RequestRebuilder, sink ReleaseSink) RequestRuntimeSnapshot { return createTestRuntimeSnapshotWithPreparer(t, regs, disp, rebuilder, nil, nil, sink) } func createTestRuntimeSnapshotWithOptions(t *testing.T, regs []FilterRegistration, disp AttemptDispatcher, rebuilder RequestRebuilder, opts RuntimeOptions, sink ReleaseSink) RequestRuntimeSnapshot { t.Helper() regSnap, err := NewFilterRegistrySnapshot("gen-1", regs, nil) if err != nil { t.Fatalf("NewFilterRegistrySnapshot: %v", err) } snapRef, err := NewRecoveryRequestSnapshotRef("snap.ref.1", 100, 200, 1024) if err != nil { t.Fatalf("NewRecoveryRequestSnapshotRef: %v", err) } snap, err := NewRequestRuntimeSnapshot( "req-123", "gen-1", "test", "ep", "fam", opts, regSnap, nil, snapRef, disp, rebuilder, nil, nil, sink, ) if err != nil { t.Fatalf("NewRequestRuntimeSnapshot: %v", err) } return snap } func createTestRuntimeSnapshotWithPreparer( t *testing.T, regs []FilterRegistration, disp AttemptDispatcher, rebuilder RequestRebuilder, preparer RecoveryPlanPreparer, factory RecoveryPreparationSnapshotFactory, sink ReleaseSink, ) RequestRuntimeSnapshot { regSnap, err := NewFilterRegistrySnapshot("gen-1", regs, nil) if err != nil { t.Fatalf("NewFilterRegistrySnapshot: %v", err) } snapRef, err := NewRecoveryRequestSnapshotRef("snap.ref.1", 100, 200, 1024) if err != nil { t.Fatalf("NewRecoveryRequestSnapshotRef: %v", err) } opts := DefaultRuntimeOptions() snap, err := NewRequestRuntimeSnapshot( "req-123", "gen-1", "test", "ep", "fam", opts, regSnap, nil, snapRef, disp, rebuilder, preparer, factory, sink, ) if err != nil { t.Fatalf("NewRequestRuntimeSnapshot: %v", err) } return snap } type testPrepSnapshot struct { mu sync.Mutex ref string releaseCount int releaseErr error trace *runtimeTrace } func (s *testPrepSnapshot) SnapshotRef() string { return s.ref } func (s *testPrepSnapshot) Release() error { s.mu.Lock() defer s.mu.Unlock() s.releaseCount++ s.trace.add("snapshot:release") return s.releaseErr } type testPrepFactory struct { mu sync.Mutex fail bool callCount int prepared *testPrepSnapshot trace *runtimeTrace } func (f *testPrepFactory) CreatePreparationSnapshot(ctx context.Context, snapshot RequestRuntimeSnapshot, binding AttemptBinding) (RecoveryPreparationSnapshot, error) { f.mu.Lock() defer f.mu.Unlock() f.callCount++ f.trace.add("snapshot:create") if f.fail { return nil, errors.New("prep factory failure") } if f.prepared != nil { return f.prepared, nil } return &testPrepSnapshot{ref: "prep-ref-1", trace: f.trace}, nil } type testPreparer struct { mu sync.Mutex callCount int err error trace *runtimeTrace } func (p *testPreparer) PrepareRecoveryPlan(ctx context.Context, plan RecoveryPlan, snapshot RecoveryPreparationSnapshot) (RecoveryDirective, error) { p.mu.Lock() defer p.mu.Unlock() p.callCount++ p.trace.add("prepare") if p.err != nil { return RecoveryDirective{}, p.err } return NewRecoveryDirectiveExact("req-ref-1") } func TestRequestRuntimeDisabledAndPass(t *testing.T) { t.Run("DisabledAndPass", func(t *testing.T) { sink := newFixtureSink() disp := &fixtureDispatcher{} rebuilder := &fixtureRebuilder{} passF := &customMockFilter{id: "pass-filter"} regPass, err := NewFilterRegistration(passF, "cap1", true, FilterEnforcementBlocking, 5*time.Second, 10) if err != nil { t.Fatalf("NewFilterRegistration pass: %v", err) } disabledF := &customMockFilter{id: "disabled-filter"} regDisabled, err := NewFilterRegistration(disabledF, "cap2", false, FilterEnforcementBlocking, 5*time.Second, 10) if err != nil { t.Fatalf("NewFilterRegistration disabled: %v", err) } snap := createTestRuntimeSnapshot(t, []FilterRegistration{regPass, regDisabled}, disp, rebuilder, sink) rsEv, err := NewResponseStartEvent("default", 200, map[string]string{"content-type": "text/event-stream"}, time.Now()) if err != nil { t.Fatalf("NewResponseStartEvent: %v", err) } txtEv, err := NewTextDeltaEvent("default", "hello world", time.Now()) if err != nil { t.Fatalf("NewTextDeltaEvent: %v", err) } termEv, err := NewTerminalEvent("default", time.Now()) if err != nil { t.Fatalf("NewTerminalEvent: %v", err) } src := newSliceEventSource([]NormalizedEvent{rsEv, txtEv, termEv}) ctrl := &fixtureController{} initialBinding, err := NewAttemptBinding("att-1", "gpt-4", "openai", "primary", src, ctrl) if err != nil { t.Fatalf("NewAttemptBinding: %v", err) } rt, err := NewRequestRuntime(snap, "group-a", initialBinding) if err != nil { t.Fatalf("NewRequestRuntime: %v", err) } ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() if err := rt.Run(ctx); err != nil { t.Fatalf("Run: %v", err) } sink.mu.Lock() defer sink.mu.Unlock() if len(sink.starts) != 1 { t.Errorf("expected 1 response start committed, got %d", len(sink.starts)) } if len(sink.events) != 1 { t.Errorf("expected 1 release event, got %d", len(sink.events)) } if len(sink.terminals) != 1 { t.Fatalf("expected 1 terminal result, got %d", len(sink.terminals)) } if !sink.terminals[0].Success() { t.Logf("terminal: %+v", sink.terminals[0]) t.Errorf("expected success terminal result") } disabledF.mu.Lock() if disabledF.evalCount != 0 { t.Errorf("disabled filter should not be evaluated, count = %d", disabledF.evalCount) } disabledF.mu.Unlock() }) t.Run("ProviderErrorTerminal", func(t *testing.T) { sink := newFixtureSink() disp := &fixtureDispatcher{} rebuilder := &fixtureRebuilder{} passF := &customMockFilter{id: "pass-filter"} regPass, err := NewFilterRegistration(passF, "cap1", true, FilterEnforcementBlocking, 5*time.Second, 10) if err != nil { t.Fatalf("NewFilterRegistration: %v", err) } snap := createTestRuntimeSnapshot(t, []FilterRegistration{regPass}, disp, rebuilder, sink) rsEv, err := NewResponseStartEvent("default", 200, nil, time.Now()) if err != nil { t.Fatalf("NewResponseStartEvent: %v", err) } desc, errDesc := NewExternalDescriptor("error", "provider_error", "internal_error", "") if errDesc != nil { t.Fatalf("NewExternalDescriptor: %v", errDesc) } cause, errCause := NewFailureCause("provider", "internal_error", "", "", "") if errCause != nil { t.Fatalf("NewFailureCause: %v", errCause) } causes, errCauses := NewFailureCauseChain([]FailureCause{cause}) if errCauses != nil { t.Fatalf("NewFailureCauseChain: %v", errCauses) } provErr, err := NewProviderErrorEvent("default", desc, causes, time.Now()) if err != nil { t.Fatalf("NewProviderErrorEvent: %v", err) } src := newSliceEventSource([]NormalizedEvent{rsEv, provErr}) ctrl := &fixtureController{} binding, err := NewAttemptBinding("att-1", "gpt-4", "openai", "primary", src, ctrl) if err != nil { t.Fatalf("NewAttemptBinding: %v", err) } rt, err := NewRequestRuntime(snap, "group-a", binding) if err != nil { t.Fatalf("NewRequestRuntime: %v", err) } ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() if err := rt.Run(ctx); err != nil { t.Fatalf("Run: %v", err) } sink.mu.Lock() defer sink.mu.Unlock() if len(sink.terminals) != 1 { t.Fatalf("expected 1 terminal result for provider error, got %d", len(sink.terminals)) } if sink.terminals[0].Success() { t.Errorf("provider error should produce error terminal result") } }) t.Run("SourceReadFailure", func(t *testing.T) { sink := newFixtureSink() disp := &fixtureDispatcher{} rebuilder := &fixtureRebuilder{} passF := &customMockFilter{id: "pass-filter"} regPass, err := NewFilterRegistration(passF, "cap1", true, FilterEnforcementBlocking, 5*time.Second, 10) if err != nil { t.Fatalf("NewFilterRegistration: %v", err) } snap := createTestRuntimeSnapshot(t, []FilterRegistration{regPass}, disp, rebuilder, sink) src := newErrorEventSource(errors.New("stream abrupt disconnect")) ctrl := &fixtureController{} binding, err := NewAttemptBinding("att-1", "gpt-4", "openai", "primary", src, ctrl) if err != nil { t.Fatalf("NewAttemptBinding: %v", err) } rt, err := NewRequestRuntime(snap, "group-a", binding) if err != nil { t.Fatalf("NewRequestRuntime: %v", err) } ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() if err := rt.Run(ctx); err != nil { t.Fatalf("Run: %v", err) } sink.mu.Lock() defer sink.mu.Unlock() if len(sink.terminals) != 1 { t.Fatalf("expected 1 terminal result on source failure, got %d", len(sink.terminals)) } if sink.terminals[0].Success() { t.Errorf("source failure should produce error terminal result") } }) } func TestRequestRuntimeRecoveryLifecycle(t *testing.T) { t.Run("ReplaceAttemptRecovery", func(t *testing.T) { sink := newFixtureSink() rebuilder := &fixtureRebuilder{} disp := &fixtureDispatcher{} filterID := "violating-filter" var evalCount int var mu sync.Mutex vF := &customMockFilter{ id: filterID, evaluateFn: func(ctx context.Context, fc FilterContext, batch EvidenceBatch) (FilterDecision, error) { mu.Lock() count := evalCount evalCount++ mu.Unlock() fp := FixedFingerprint{1} ev := mustSanitizedEvidence(EventKindTextDelta, "default", "rule1", "desc", fp, 1, 0, FilterOutcomeKindEvaluated, time.Now()) if count == 0 { dir := mustRecoveryDirectiveExact("req-ref-1") intent := mustRecoveryIntent(RecoveryStrategyExactReplay, dir, "rule_violation", 10) return NewFilterDecision(FilterDecisionKindViolation, "consumer1", filterID, "rule1", ev, &intent) } return NewFilterDecision(FilterDecisionKindPass, "consumer1", filterID, "rule1", ev, nil) }, } reg, err := NewFilterRegistration(vF, "cap1", true, FilterEnforcementBlocking, 5*time.Second, 10) if err != nil { t.Fatalf("NewFilterRegistration: %v", err) } txtEv2, err := NewTextDeltaEvent("default", "recovered text", time.Now()) if err != nil { t.Fatalf("NewTextDeltaEvent: %v", err) } termEv2, err := NewTerminalEvent("default", time.Now()) if err != nil { t.Fatalf("NewTerminalEvent: %v", err) } src2 := newSliceEventSource([]NormalizedEvent{txtEv2, termEv2}) ctrl2 := &fixtureController{} disp.handler = func(ctx context.Context, request RebuiltRequest) (AttemptBinding, error) { return NewAttemptBinding("att-2", "gpt-4", "openai", "primary", src2, ctrl2) } snap := createTestRuntimeSnapshot(t, []FilterRegistration{reg}, disp, rebuilder, sink) txtEv1, err := NewTextDeltaEvent("default", "violating text", time.Now()) if err != nil { t.Fatalf("NewTextDeltaEvent: %v", err) } termEv1, err := NewTerminalEvent("default", time.Now()) if err != nil { t.Fatalf("NewTerminalEvent: %v", err) } src1 := newSliceEventSource([]NormalizedEvent{txtEv1, termEv1}) ctrl1 := &fixtureController{} initialBinding, err := NewAttemptBinding("att-1", "gpt-4", "openai", "primary", src1, ctrl1) if err != nil { t.Fatalf("NewAttemptBinding: %v", err) } rt, err := NewRequestRuntime(snap, "group-a", initialBinding) if err != nil { t.Fatalf("NewRequestRuntime: %v", err) } ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() if err := rt.Run(ctx); err != nil { t.Fatalf("Run: %v", err) } sink.mu.Lock() defer sink.mu.Unlock() if len(sink.terminals) != 1 { t.Fatalf("expected exactly 1 terminal result, got %d", len(sink.terminals)) } if !sink.terminals[0].Success() { t.Errorf("expected successful terminal result after recovery") } ctrl1.mu.Lock() if ctrl1.abortCount != 1 { t.Errorf("expected attempt 1 aborted once, got %d", ctrl1.abortCount) } ctrl1.mu.Unlock() }) t.Run("ProviderAndPathSwitching", func(t *testing.T) { sink := newFixtureSink() rebuilder := &fixtureRebuilder{} disp := &fixtureDispatcher{} filterID := "switch-filter" var evaluatedProviders []string var mu sync.Mutex vF := &customMockFilter{ id: filterID, evaluateFn: func(ctx context.Context, fc FilterContext, batch EvidenceBatch) (FilterDecision, error) { mu.Lock() evaluatedProviders = append(evaluatedProviders, fc.ActualProvider()) mu.Unlock() kind := EventKindTextDelta if len(batch.Events()) > 0 { kind = batch.Events()[0].Kind() } fp := FixedFingerprint{1} ev := mustSanitizedEvidence(kind, "default", "rule1", "desc", fp, 1, 0, FilterOutcomeKindEvaluated, time.Now()) if fc.AttemptID() == "att-2" { return NewFilterDecision(FilterDecisionKindPass, "consumer1", filterID, "rule1", ev, nil) } dir := mustRecoveryDirectiveExact("req-ref-1") intent := mustRecoveryIntent(RecoveryStrategyExactReplay, dir, "rule_violation", 10) return NewFilterDecision(FilterDecisionKindViolation, "consumer1", filterID, "rule1", ev, &intent) }, } reg, err := NewFilterRegistration(vF, "cap1", true, FilterEnforcementBlocking, 5*time.Second, 10) if err != nil { t.Fatalf("NewFilterRegistration: %v", err) } txtEv2 := mustTextDelta("default", "fallback text", time.Now()) termEv2 := mustTerminal("default", time.Now()) src2 := newSliceEventSource([]NormalizedEvent{txtEv2, termEv2}) ctrl2 := &fixtureController{} disp.handler = func(ctx context.Context, request RebuiltRequest) (AttemptBinding, error) { // Switch to anthropic / fallback return NewAttemptBinding("att-2", "claude-3-5-sonnet", "anthropic", "fallback", src2, ctrl2) } snap := createTestRuntimeSnapshot(t, []FilterRegistration{reg}, disp, rebuilder, sink) txtEv1 := mustTextDelta("default", "primary text", time.Now()) termEv1 := mustTerminal("default", time.Now()) src1 := newSliceEventSource([]NormalizedEvent{txtEv1, termEv1}) ctrl1 := &fixtureController{} initialBinding, err := NewAttemptBinding("att-1", "gpt-4", "openai", "primary", src1, ctrl1) if err != nil { t.Fatalf("NewAttemptBinding: %v", err) } rt, err := NewRequestRuntime(snap, "group-a", initialBinding) if err != nil { t.Fatalf("NewRequestRuntime: %v", err) } ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() if err := rt.Run(ctx); err != nil { t.Fatalf("Run: %v", err) } mu.Lock() defer mu.Unlock() if len(evaluatedProviders) != 3 { t.Fatalf("expected 3 evaluations across provider switch, got %d", len(evaluatedProviders)) } if evaluatedProviders[0] != "openai" || evaluatedProviders[1] != "anthropic" || evaluatedProviders[2] != "anthropic" { t.Errorf("evaluated providers = %v, want [openai, anthropic, anthropic]", evaluatedProviders) } }) t.Run("PreparerSuccessAndFailure", func(t *testing.T) { t.Run("PreparerSuccess", func(t *testing.T) { sink := newFixtureSink() rebuilder := &fixtureRebuilder{} disp := &fixtureDispatcher{} vF := &customMockFilter{ id: "prep-filter", evaluateFn: func(ctx context.Context, fc FilterContext, batch EvidenceBatch) (FilterDecision, error) { kind := EventKindTextDelta if len(batch.Events()) > 0 { kind = batch.Events()[0].Kind() } fp := FixedFingerprint{1} ev := mustSanitizedEvidence(kind, "default", "rule1", "desc", fp, 1, 0, FilterOutcomeKindEvaluated, time.Now()) if fc.AttemptID() == "att-2" { return NewFilterDecision(FilterDecisionKindPass, "consumer1", "prep-filter", "rule1", ev, nil) } dir := mustRecoveryDirectiveExact("req-ref-1") intent := mustRecoveryIntent(RecoveryStrategyExactReplay, dir, "rule_violation", 10) return NewFilterDecision(FilterDecisionKindViolation, "consumer1", "prep-filter", "rule1", ev, &intent) }, } reg, err := NewFilterRegistration(vF, "cap1", true, FilterEnforcementBlocking, 5*time.Second, 10) if err != nil { t.Fatalf("NewFilterRegistration: %v", err) } txtEv2 := mustTextDelta("default", "prep text", time.Now()) termEv2 := mustTerminal("default", time.Now()) src2 := newSliceEventSource([]NormalizedEvent{txtEv2, termEv2}) ctrl2 := &fixtureController{} disp.handler = func(ctx context.Context, request RebuiltRequest) (AttemptBinding, error) { return NewAttemptBinding("att-2", "gpt-4", "openai", "primary", src2, ctrl2) } prep := &testPreparer{} factory := &testPrepFactory{} snap := createTestRuntimeSnapshotWithPreparer(t, []FilterRegistration{reg}, disp, rebuilder, prep, factory, sink) txtEv1 := mustTextDelta("default", "prep text 1", time.Now()) src1 := newSliceEventSource([]NormalizedEvent{txtEv1}) ctrl1 := &fixtureController{} initialBinding, err := NewAttemptBinding("att-1", "gpt-4", "openai", "primary", src1, ctrl1) if err != nil { t.Fatalf("NewAttemptBinding: %v", err) } rt, err := NewRequestRuntime(snap, "group-a", initialBinding) if err != nil { t.Fatalf("NewRequestRuntime: %v", err) } ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() if err := rt.Run(ctx); err != nil { t.Fatalf("Run: %v", err) } factory.mu.Lock() if factory.callCount != 1 { t.Errorf("expected factory called 1 time, got %d", factory.callCount) } factory.mu.Unlock() prep.mu.Lock() if prep.callCount != 1 { t.Errorf("expected preparer called 1 time, got %d", prep.callCount) } prep.mu.Unlock() }) t.Run("FactoryFailure", func(t *testing.T) { sink := newFixtureSink() rebuilder := &fixtureRebuilder{} disp := &fixtureDispatcher{} vF := &customMockFilter{ id: "prep-filter", evaluateFn: func(ctx context.Context, fc FilterContext, batch EvidenceBatch) (FilterDecision, error) { fp := FixedFingerprint{1} ev := mustSanitizedEvidence(EventKindTextDelta, "default", "rule1", "desc", fp, 1, 0, FilterOutcomeKindEvaluated, time.Now()) dir := mustRecoveryDirectiveExact("req-ref-1") intent := mustRecoveryIntent(RecoveryStrategyExactReplay, dir, "rule_violation", 10) return NewFilterDecision(FilterDecisionKindViolation, "consumer1", "prep-filter", "rule1", ev, &intent) }, } reg, err := NewFilterRegistration(vF, "cap1", true, FilterEnforcementBlocking, 5*time.Second, 10) if err != nil { t.Fatalf("NewFilterRegistration: %v", err) } prep := &testPreparer{} factory := &testPrepFactory{fail: true} snap := createTestRuntimeSnapshotWithPreparer(t, []FilterRegistration{reg}, disp, rebuilder, prep, factory, sink) txtEv1 := mustTextDelta("default", "prep text 1", time.Now()) src1 := newSliceEventSource([]NormalizedEvent{txtEv1}) ctrl1 := &fixtureController{} initialBinding, err := NewAttemptBinding("att-1", "gpt-4", "openai", "primary", src1, ctrl1) if err != nil { t.Fatalf("NewAttemptBinding: %v", err) } rt, err := NewRequestRuntime(snap, "group-a", initialBinding) if err != nil { t.Fatalf("NewRequestRuntime: %v", err) } ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() if err := rt.Run(ctx); err != nil { t.Fatalf("Run: %v", err) } sink.mu.Lock() defer sink.mu.Unlock() if len(sink.terminals) != 1 { t.Fatalf("expected 1 terminal result on factory failure, got %d", len(sink.terminals)) } if sink.terminals[0].Success() { t.Errorf("factory failure should produce error terminal result") } }) }) } func TestRequestRuntimeFailureMatrix(t *testing.T) { t.Run("FatalViolation", func(t *testing.T) { sink := newFixtureSink() disp := &fixtureDispatcher{} rebuilder := &fixtureRebuilder{} fatalF := &customMockFilter{ id: "fatal-filter", evaluateFn: func(ctx context.Context, fc FilterContext, batch EvidenceBatch) (FilterDecision, error) { fp := FixedFingerprint{1} ev := mustSanitizedEvidence(EventKindTextDelta, "default", "rule1", "fatal_desc", fp, 1, 0, FilterOutcomeKindEvaluated, time.Now()) return NewFilterDecision(FilterDecisionKindFatal, "consumer1", "fatal-filter", "rule1", ev, nil) }, } reg, err := NewFilterRegistration(fatalF, "cap1", true, FilterEnforcementBlocking, 5*time.Second, 10) if err != nil { t.Fatalf("NewFilterRegistration: %v", err) } snap := createTestRuntimeSnapshot(t, []FilterRegistration{reg}, disp, rebuilder, sink) txtEv := mustTextDelta("default", "bad text", time.Now()) src := newSliceEventSource([]NormalizedEvent{txtEv}) ctrl := &fixtureController{} initialBinding, err := NewAttemptBinding("att-1", "gpt-4", "openai", "primary", src, ctrl) if err != nil { t.Fatalf("NewAttemptBinding: %v", err) } rt, err := NewRequestRuntime(snap, "group-a", initialBinding) if err != nil { t.Fatalf("NewRequestRuntime: %v", err) } ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() if err := rt.Run(ctx); err != nil { t.Fatalf("Run: %v", err) } sink.mu.Lock() defer sink.mu.Unlock() if len(sink.terminals) != 1 { t.Fatalf("expected 1 terminal result for fatal violation, got %d", len(sink.terminals)) } if sink.terminals[0].Success() { t.Errorf("expected error terminal result for fatal violation") } }) t.Run("ExhaustedRecoveryBudget", func(t *testing.T) { sink := newFixtureSink() disp := &fixtureDispatcher{} rebuilder := &fixtureRebuilder{} vF := &customMockFilter{ id: "violating-filter", evaluateFn: func(ctx context.Context, fc FilterContext, batch EvidenceBatch) (FilterDecision, error) { fp := FixedFingerprint{1} ev := mustSanitizedEvidence(EventKindTextDelta, "default", "rule1", "desc", fp, 1, 0, FilterOutcomeKindEvaluated, time.Now()) dir := mustRecoveryDirectiveExact("req-ref-1") intent := mustRecoveryIntent(RecoveryStrategyExactReplay, dir, "rule_violation", 10) return NewFilterDecision(FilterDecisionKindViolation, "consumer1", "violating-filter", "rule1", ev, &intent) }, } reg, err := NewFilterRegistration(vF, "cap1", true, FilterEnforcementBlocking, 5*time.Second, 10) if err != nil { t.Fatalf("NewFilterRegistration: %v", err) } disp.handler = func(ctx context.Context, request RebuiltRequest) (AttemptBinding, error) { txtEv := mustTextDelta("default", "violating text", time.Now()) src := newSliceEventSource([]NormalizedEvent{txtEv}) ctrl := &fixtureController{} return NewAttemptBinding("att-next", "gpt-4", "openai", "primary", src, ctrl) } snap := createTestRuntimeSnapshot(t, []FilterRegistration{reg}, disp, rebuilder, sink) txtEv := mustTextDelta("default", "violating text 1", time.Now()) src := newSliceEventSource([]NormalizedEvent{txtEv}) ctrl := &fixtureController{} initialBinding, err := NewAttemptBinding("att-1", "gpt-4", "openai", "primary", src, ctrl) if err != nil { t.Fatalf("NewAttemptBinding: %v", err) } rt, err := NewRequestRuntime(snap, "group-a", initialBinding) if err != nil { t.Fatalf("NewRequestRuntime: %v", err) } ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() if err := rt.Run(ctx); err != nil { t.Fatalf("Run: %v", err) } sink.mu.Lock() defer sink.mu.Unlock() if len(sink.terminals) != 1 { t.Fatalf("expected single terminal result on budget exhaustion, got %d", len(sink.terminals)) } if sink.terminals[0].Success() { t.Errorf("expected error terminal result on budget exhaustion") } }) t.Run("CallerCancellation", func(t *testing.T) { sink := newFixtureSink() disp := &fixtureDispatcher{} rebuilder := &fixtureRebuilder{} passF := &customMockFilter{id: "pass-filter"} reg, err := NewFilterRegistration(passF, "cap1", true, FilterEnforcementBlocking, 5*time.Second, 10) if err != nil { t.Fatalf("NewFilterRegistration: %v", err) } snap := createTestRuntimeSnapshot(t, []FilterRegistration{reg}, disp, rebuilder, sink) ctx, cancel := context.WithCancel(context.Background()) cancel() txtEv, err := NewTextDeltaEvent("default", "text", time.Now()) if err != nil { t.Fatalf("NewTextDeltaEvent: %v", err) } src := newSliceEventSource([]NormalizedEvent{txtEv}) ctrl := &fixtureController{} initialBinding, err := NewAttemptBinding("att-1", "gpt-4", "openai", "primary", src, ctrl) if err != nil { t.Fatalf("NewAttemptBinding: %v", err) } rt, err := NewRequestRuntime(snap, "group-a", initialBinding) if err != nil { t.Fatalf("NewRequestRuntime: %v", err) } errRun := rt.Run(ctx) if !errors.Is(errRun, context.Canceled) { t.Errorf("expected context.Canceled, got %v", errRun) } }) } func TestRequestRuntimePassThroughMatrix(t *testing.T) { ctx := context.Background() t.Run("NoFilters", func(t *testing.T) { sink := newFixtureSink() disp := &fixtureDispatcher{} rebuilder := &fixtureRebuilder{} snap := createTestRuntimeSnapshot(t, nil, disp, rebuilder, sink) rsEv, err := NewResponseStartEvent("default", 200, nil, time.Now()) if err != nil { t.Fatalf("NewResponseStartEvent: %v", err) } txtEv, err := NewTextDeltaEvent("default", "hello", time.Now()) if err != nil { t.Fatalf("NewTextDeltaEvent: %v", err) } termEv, err := NewTerminalEvent("default", time.Now()) if err != nil { t.Fatalf("NewTerminalEvent: %v", err) } src := newSliceEventSource([]NormalizedEvent{rsEv, txtEv, termEv}) ctrl := &fixtureController{} initialBinding, err := NewAttemptBinding("att-1", "gpt-4", "openai", "primary", src, ctrl) if err != nil { t.Fatalf("NewAttemptBinding: %v", err) } rt, err := NewRequestRuntime(snap, "group-a", initialBinding) if err != nil { t.Fatalf("NewRequestRuntime: %v", err) } if err := rt.Run(ctx); err != nil { t.Fatalf("Run: %v", err) } sink.mu.Lock() defer sink.mu.Unlock() if len(sink.starts) != 1 { t.Errorf("starts = %d, want 1", len(sink.starts)) } if len(sink.events) != 1 { t.Errorf("events = %d, want 1", len(sink.events)) } if len(sink.terminals) != 1 || !sink.terminals[0].Success() { t.Errorf("terminals = %+v, want 1 success terminal", sink.terminals) } }) t.Run("AllDisabled", func(t *testing.T) { sink := newFixtureSink() disp := &fixtureDispatcher{} rebuilder := &fixtureRebuilder{} disF := &customMockFilter{id: "disabled-1"} regDisabled, err := NewFilterRegistration(disF, "cap1", false, FilterEnforcementBlocking, 5*time.Second, 10) if err != nil { t.Fatalf("NewFilterRegistration: %v", err) } snap := createTestRuntimeSnapshot(t, []FilterRegistration{regDisabled}, disp, rebuilder, sink) rsEv := mustResponseStart("default", 200, nil, time.Now()) txtEv := mustTextDelta("default", "hello", time.Now()) termEv := mustTerminal("default", time.Now()) src := newSliceEventSource([]NormalizedEvent{rsEv, txtEv, termEv}) ctrl := &fixtureController{} initialBinding, err := NewAttemptBinding("att-1", "gpt-4", "openai", "primary", src, ctrl) if err != nil { t.Fatalf("NewAttemptBinding: %v", err) } rt, err := NewRequestRuntime(snap, "group-a", initialBinding) if err != nil { t.Fatalf("NewRequestRuntime: %v", err) } if err := rt.Run(ctx); err != nil { t.Fatalf("Run: %v", err) } disF.mu.Lock() if disF.evalCount != 0 { t.Errorf("disabled filter evalCount = %d, want 0", disF.evalCount) } disF.mu.Unlock() sink.mu.Lock() defer sink.mu.Unlock() if len(sink.starts) != 1 || len(sink.events) != 1 || len(sink.terminals) != 1 { t.Errorf("sink counts mismatch: starts=%d, events=%d, term=%d", len(sink.starts), len(sink.events), len(sink.terminals)) } }) t.Run("ObserveOnly", func(t *testing.T) { sink := newFixtureSink() disp := &fixtureDispatcher{} rebuilder := &fixtureRebuilder{} obsF := &customMockFilter{id: "observe-1"} regObserve, err := NewFilterRegistration(obsF, "cap1", true, FilterEnforcementObserveOnly, 5*time.Second, 10) if err != nil { t.Fatalf("NewFilterRegistration: %v", err) } snap := createTestRuntimeSnapshot(t, []FilterRegistration{regObserve}, disp, rebuilder, sink) rsEv := mustResponseStart("default", 200, nil, time.Now()) txtEv := mustTextDelta("default", "hello", time.Now()) termEv := mustTerminal("default", time.Now()) src := newSliceEventSource([]NormalizedEvent{rsEv, txtEv, termEv}) ctrl := &fixtureController{} initialBinding, err := NewAttemptBinding("att-1", "gpt-4", "openai", "primary", src, ctrl) if err != nil { t.Fatalf("NewAttemptBinding: %v", err) } rt, err := NewRequestRuntime(snap, "group-a", initialBinding) if err != nil { t.Fatalf("NewRequestRuntime: %v", err) } if err := rt.Run(ctx); err != nil { t.Fatalf("Run: %v", err) } sink.mu.Lock() defer sink.mu.Unlock() if len(sink.starts) != 1 || len(sink.events) != 1 || len(sink.terminals) != 1 { t.Errorf("observe-only passthrough sink counts mismatch") } }) t.Run("BlockingFilterNonSubscribedEvent", func(t *testing.T) { sink := newFixtureSink() requirement := mustFilterHoldRequirementRolling("default", []EventKind{EventKindReasoningDelta}, 1) filter := &customMockFilter{id: "reasoning-only", holdReq: &requirement} reg := mustFilterRegistration(filter, "cap1", true, FilterEnforcementBlocking, 5*time.Second, 10) snapshot := createTestRuntimeSnapshot(t, []FilterRegistration{reg}, &fixtureDispatcher{}, &fixtureRebuilder{}, sink) initial := mustAttemptBinding(t, "att-1", newSliceEventSource([]NormalizedEvent{ mustResponseStart("default", 200, nil, testNow), mustTextDelta("default", "unsubscribed", testNow), mustTerminal("default", testNow), }), &fixtureController{}) runtime, err := NewRequestRuntime(snapshot, "group-a", initial) if err != nil { t.Fatalf("NewRequestRuntime: %v", err) } if err := runtime.Run(ctx); err != nil { t.Fatalf("Run: %v", err) } filter.mu.Lock() evalCount := filter.evalCount filter.mu.Unlock() if evalCount != 0 { t.Fatalf("non-subscribed filter eval count = %d, want 0", evalCount) } if len(sink.starts) != 1 || len(sink.events) != 1 || len(sink.terminals) != 1 || !sink.terminals[0].Success() { t.Fatalf("sink counts starts/events/terminals = %d/%d/%d", len(sink.starts), len(sink.events), len(sink.terminals)) } }) } func TestRequestRuntimeContinuationLifecycle(t *testing.T) { ctx := context.Background() trace := &runtimeTrace{} sink := newFixtureSink() sink.trace = trace disp := &fixtureDispatcher{trace: trace} rebuilder := &fixtureRebuilder{trace: trace} oldController := &fixtureController{name: "att-1", trace: trace} newController := &fixtureController{name: "att-2", trace: trace} var capturedLookBehind []string requirement := mustFilterHoldRequirementRolling("default", []EventKind{EventKindTextDelta, EventKindTerminal}, 6) filter := &customMockFilter{ id: "continuation-filter", holdReq: &requirement, evaluateFn: func(ctx context.Context, fc FilterContext, batch EvidenceBatch) (FilterDecision, error) { kind := batch.Events()[0].Kind() evidence := mustSanitizedEvidence(kind, "default", "rule1", "continuation_check", FixedFingerprint{1}, 1, 0, FilterOutcomeKindEvaluated, testNow) text := "" if kind == EventKindTextDelta { var textErr error text, textErr = batch.Events()[0].AsTextDelta() if textErr != nil { return FilterDecision{}, textErr } } if fc.AttemptID() == "att-1" && text == "violation" { directive := mustRecoveryDirectiveContinuation(1, "snapshot.one") intent := mustRecoveryIntent(RecoveryStrategyContinuationRepair, directive, "continuation_required", 10) return NewFilterDecision(FilterDecisionKindViolation, "consumer1", "continuation-filter", "rule1", evidence, &intent) } if fc.AttemptID() == "att-2" && text == "suffix" { for _, ev := range batch.CommittedLookBehind()["default"] { if ev.Kind() == EventKindTextDelta { value, textErr := ev.AsTextDelta() if textErr != nil { return FilterDecision{}, textErr } capturedLookBehind = append(capturedLookBehind, value) } } } return NewFilterDecision(FilterDecisionKindPass, "consumer1", "continuation-filter", "rule1", evidence, nil) }, } reg := mustFilterRegistration(filter, "cap1", true, FilterEnforcementBlocking, 5*time.Second, 10) disp.handler = func(ctx context.Context, request RebuiltRequest) (AttemptBinding, error) { source := newSliceEventSource([]NormalizedEvent{ mustResponseStart("default", 200, nil, testNow.Add(time.Second)), mustTextDelta("default", "suffix", testNow.Add(2*time.Second)), mustTerminal("default", testNow.Add(3*time.Second)), }) return NewAttemptBinding("att-2", "gpt-4", "openai", "primary", source, newController) } snapshot := createTestRuntimeSnapshot(t, []FilterRegistration{reg}, disp, rebuilder, sink) initialSource := newSliceEventSource([]NormalizedEvent{ mustResponseStart("default", 200, nil, testNow), mustTextDelta("default", "prefix", testNow), mustTextDelta("default", "violation", testNow), }) initial := mustAttemptBinding(t, "att-1", initialSource, oldController) runtime, err := NewRequestRuntime(snapshot, "group-a", initial) if err != nil { t.Fatalf("NewRequestRuntime: %v", err) } if err := runtime.Run(ctx); err != nil { t.Fatalf("Run: %v", err) } if runtime.boundary.CurrentAttempt() != "att-2" || runtime.boundary.State() != CommitStateTerminalCommitted { t.Fatalf("boundary attempt/state = %q/%q, want att-2/terminal_committed", runtime.boundary.CurrentAttempt(), runtime.boundary.State()) } assertControllerAbortCount(t, oldController, 1) assertControllerAbortCount(t, newController, 0) if len(sink.starts) != 1 || len(sink.events) != 2 || len(sink.terminals) != 1 || !sink.terminals[0].Success() { t.Fatalf("sink counts starts/events/terminals = %d/%d/%d", len(sink.starts), len(sink.events), len(sink.terminals)) } if got := releaseTexts(t, sink.events); !reflect.DeepEqual(got, []string{"prefix", "suffix"}) { t.Fatalf("released texts = %v, want [prefix suffix]", got) } if !reflect.DeepEqual(capturedLookBehind, []string{"prefix"}) { t.Fatalf("continuation look-behind = %v, want [prefix]", capturedLookBehind) } if got := runtime.tail.CommittedCursor("default"); got != 2 { t.Fatalf("committed cursor = %d, want 2", got) } wantTrace := []string{"start", "release", "abort:att-1", "rebuild", "dispatch", "release", "terminal:success"} if got := trace.snapshot(); !reflect.DeepEqual(got, wantTrace) { t.Fatalf("trace = %v, want %v", got, wantTrace) } } func mustTextDelta(ch, text string, ts time.Time) NormalizedEvent { ev, err := NewTextDeltaEvent(ch, text, ts) if err != nil { panic(err) } return ev } func mustTerminal(ch string, ts time.Time) NormalizedEvent { ev, err := NewTerminalEvent(ch, ts) if err != nil { panic(err) } return ev } func mustSanitizedEvidence(kind EventKind, channel, ruleID, desc string, fp FixedFingerprint, seq, offset int, outcome FilterOutcomeKind, ts time.Time) SanitizedEvidence { ev, err := NewSanitizedEvidence(kind, channel, ruleID, desc, fp, seq, offset, outcome, ts) if err != nil { panic(err) } return ev } func mustRecoveryDirectiveExact(requestRef string) RecoveryDirective { directive, err := NewRecoveryDirectiveExact(requestRef) if err != nil { panic(err) } return directive } func mustRecoveryDirectiveContinuation(cursor int, snapshotRef string) RecoveryDirective { directive, err := NewRecoveryDirectiveContinuation(cursor, snapshotRef) if err != nil { panic(err) } return directive } func mustRecoveryIntent(strategy RecoveryStrategy, directive RecoveryDirective, reason string, cost int) RecoveryIntent { intent, err := NewRecoveryIntent(strategy, directive, reason, cost) if err != nil { panic(err) } return intent } func mustResponseStart(ch string, code int, headers map[string]string, ts time.Time) NormalizedEvent { ev, err := NewResponseStartEvent(ch, code, headers, ts) if err != nil { panic(err) } return ev } func mustReplacementProposal(t *testing.T, events []NormalizedEvent) *ReplacementProposal { t.Helper() proposal, err := NewReplacementProposal(events) if err != nil { t.Fatalf("NewReplacementProposal: %v", err) } return &proposal } func mustFilterRegistration(filter Filter, cap string, enabled bool, enforcement FilterEnforcement, timeout time.Duration, priority int) FilterRegistration { registration, err := NewFilterRegistration(filter, cap, enabled, enforcement, timeout, priority) if err != nil { panic(err) } return registration } func mustFilterHoldRequirementRolling(channel string, kinds []EventKind, runes int) FilterHoldRequirement { requirement, err := NewFilterHoldRequirementRolling(channel, kinds, runes) if err != nil { panic(err) } return requirement } func mustAttemptBinding(t *testing.T, id string, source NormalizedEventSource, controller AttemptController) AttemptBinding { t.Helper() binding, err := NewAttemptBinding(id, "gpt-4", "openai", "primary", source, controller) if err != nil { t.Fatalf("NewAttemptBinding(%s): %v", id, err) } return binding } func assertControllerAbortCount(t *testing.T, controller *fixtureController, want int) { t.Helper() controller.mu.Lock() defer controller.mu.Unlock() if controller.abortCount != want { t.Fatalf("controller %q abort count = %d, want %d", controller.name, controller.abortCount, want) } } func releaseTexts(t *testing.T, events []ReleaseEvent) []string { t.Helper() texts := make([]string, 0, len(events)) for _, event := range events { text, err := event.AsTextDelta() if err != nil { t.Fatalf("ReleaseEvent.AsTextDelta: %v", err) } texts = append(texts, text) } return texts } func newViolationFilter(id string, strategy RecoveryStrategy, directive RecoveryDirective) *customMockFilter { return &customMockFilter{ id: id, evaluateFn: func(ctx context.Context, fc FilterContext, batch EvidenceBatch) (FilterDecision, error) { kind := batch.Events()[0].Kind() evidence := mustSanitizedEvidence(kind, "default", "rule1", "violation", FixedFingerprint{1}, 1, 0, FilterOutcomeKindEvaluated, testNow) intent := mustRecoveryIntent(strategy, directive, "rule_violation", 10) return NewFilterDecision(FilterDecisionKindViolation, "consumer1", id, "rule1", evidence, &intent) }, } } func runtimeOptionsWithRecoveryCap(t *testing.T, cap int) RuntimeOptions { t.Helper() options, err := NewRuntimeOptions(1, 64, 1024, cap, GateCoordinatorOptions{}, RecoveryCoordinatorOptions{}) if err != nil { t.Fatalf("NewRuntimeOptions: %v", err) } return options } func TestRequestRuntimeTerminalFailureFidelity(t *testing.T) { t.Run("success_timestamp", func(t *testing.T) { sink := newFixtureSink() snapshot := createTestRuntimeSnapshot(t, nil, &fixtureDispatcher{}, &fixtureRebuilder{}, sink) occurredAt := testNow.Add(7 * time.Second) initial := mustAttemptBinding(t, "att-1", newSliceEventSource([]NormalizedEvent{mustTerminal("default", occurredAt)}), &fixtureController{}) runtime, err := NewRequestRuntime(snapshot, "group-a", initial) if err != nil { t.Fatalf("NewRequestRuntime: %v", err) } if err := runtime.Run(context.Background()); err != nil { t.Fatalf("Run: %v", err) } if len(sink.terminals) != 1 || !sink.terminals[0].Success() || !sink.terminals[0].OccurredAt().Equal(occurredAt) { t.Fatalf("success terminal = %+v, want exact event timestamp %v", sink.terminals, occurredAt) } }) t.Run("provider_identity_and_cause_cap", func(t *testing.T) { sink := newFixtureSink() snapshot := createTestRuntimeSnapshot(t, nil, &fixtureDispatcher{}, &fixtureRebuilder{}, sink) descriptor, err := NewExternalDescriptor("provider_error", "rate_limit_exceeded", "rate_limit_reached", "requests") if err != nil { t.Fatalf("NewExternalDescriptor: %v", err) } causeValues := make([]FailureCause, 0, 5) for _, code := range []string{"cause1", "cause2", "cause3", "cause4", "cause5"} { cause, causeErr := NewFailureCause("provider", code, "consumer1", "filter1", "rule1") if causeErr != nil { t.Fatalf("NewFailureCause(%s): %v", code, causeErr) } causeValues = append(causeValues, cause) } causes, err := NewFailureCauseChain(causeValues) if err != nil { t.Fatalf("NewFailureCauseChain: %v", err) } occurredAt := testNow.Add(8 * time.Second) providerEvent, err := NewProviderErrorEvent("default", descriptor, causes, occurredAt) if err != nil { t.Fatalf("NewProviderErrorEvent: %v", err) } initial := mustAttemptBinding(t, "att-1", newSliceEventSource([]NormalizedEvent{providerEvent}), &fixtureController{}) runtime, err := NewRequestRuntime(snapshot, "group-a", initial) if err != nil { t.Fatalf("NewRequestRuntime: %v", err) } if err := runtime.Run(context.Background()); err != nil { t.Fatalf("Run: %v", err) } if len(sink.terminals) != 1 { t.Fatalf("terminal count = %d, want 1", len(sink.terminals)) } terminal := sink.terminals[0] gotDescriptor := terminal.ExternalDesc() if gotDescriptor == nil || gotDescriptor.Type() != "provider_error" || gotDescriptor.Code() != "rate_limit_exceeded" || gotDescriptor.Message() != "rate_limit_reached" || gotDescriptor.Param() != "requests" { t.Fatalf("provider descriptor = %+v", gotDescriptor) } if !terminal.OccurredAt().Equal(occurredAt) || terminal.FailureCauses().Len() != MaxFailureCauses { t.Fatalf("provider timestamp/cause count = %v/%d", terminal.OccurredAt(), terminal.FailureCauses().Len()) } if terminal.FailureCauses().All()[0].Code() != "cause2" || terminal.FailureCauses().All()[3].Code() != "cause5" { t.Fatalf("provider cause cap = %+v, want latest four", terminal.FailureCauses().All()) } }) t.Run("fatal_filter_identity", func(t *testing.T) { sink := newFixtureSink() filter := &customMockFilter{ id: "fatal-filter", evaluateFn: func(ctx context.Context, fc FilterContext, batch EvidenceBatch) (FilterDecision, error) { evidence := mustSanitizedEvidence(EventKindTextDelta, "default", "fatal-rule", "fatal_violation", FixedFingerprint{2}, 1, 0, FilterOutcomeKindEvaluated, testNow) return NewFilterDecision(FilterDecisionKindFatal, "consumer1", "fatal-filter", "fatal-rule", evidence, nil) }, } reg := mustFilterRegistration(filter, "cap1", true, FilterEnforcementBlocking, 5*time.Second, 10) snapshot := createTestRuntimeSnapshot(t, []FilterRegistration{reg}, &fixtureDispatcher{}, &fixtureRebuilder{}, sink) initial := mustAttemptBinding(t, "att-1", newSliceEventSource([]NormalizedEvent{mustTextDelta("default", "bad", testNow)}), &fixtureController{}) runtime, err := NewRequestRuntime(snapshot, "group-a", initial) if err != nil { t.Fatalf("NewRequestRuntime: %v", err) } if err := runtime.Run(context.Background()); err != nil { t.Fatalf("Run: %v", err) } if len(sink.terminals) != 1 || sink.terminals[0].Success() { t.Fatalf("fatal terminals = %+v", sink.terminals) } causes := sink.terminals[0].FailureCauses().All() if len(causes) != 1 || causes[0].Consumer() != "" || causes[0].Filter() != "fatal-filter" || causes[0].RuleID() != "fatal-rule" { t.Fatalf("fatal causes = %+v", causes) } }) } func TestRequestRuntimeAtomicInstallFailure(t *testing.T) { for _, abortFails := range []bool{false, true} { name := "prepare_abort_succeeds" if abortFails { name = "prepare_abort_fails" } t.Run(name, func(t *testing.T) { trace := &runtimeTrace{} sink := newFixtureSink() sink.trace = trace dispatcher := &fixtureDispatcher{trace: trace} rebuilder := &fixtureRebuilder{trace: trace} oldController := &fixtureController{name: "old", trace: trace} newController := &fixtureController{name: "new", trace: trace} if abortFails { newController.abortErr = errors.New("raw controller failure") } filter := newViolationFilter("install-filter", RecoveryStrategyExactReplay, mustRecoveryDirectiveExact("req-ref-1")) filter.holdReqFn = func(fc FilterContext) FilterHoldRequirement { if fc.AttemptID() == "att-2" { return FilterHoldRequirement{} } return mustFilterHoldRequirementRolling("default", []EventKind{EventKindTextDelta, EventKindTerminal}, 1) } reg := mustFilterRegistration(filter, "cap1", true, FilterEnforcementBlocking, 5*time.Second, 10) dispatcher.handler = func(ctx context.Context, request RebuiltRequest) (AttemptBinding, error) { return NewAttemptBinding("att-2", "gpt-4", "openai", "primary", newSliceEventSource(nil), newController) } snapshot := createTestRuntimeSnapshot(t, []FilterRegistration{reg}, dispatcher, rebuilder, sink) initial := mustAttemptBinding(t, "att-1", newSliceEventSource([]NormalizedEvent{mustTextDelta("default", "bad", testNow)}), oldController) runtime, err := NewRequestRuntime(snapshot, "group-a", initial) if err != nil { t.Fatalf("NewRequestRuntime: %v", err) } if err := runtime.Run(context.Background()); err != nil { t.Fatalf("Run: %v", err) } assertControllerAbortCount(t, oldController, 1) assertControllerAbortCount(t, newController, 1) if len(sink.terminals) != 1 || sink.terminals[0].Success() { t.Fatalf("terminals = %+v", sink.terminals) } causeCodes := make([]string, 0, sink.terminals[0].FailureCauses().Len()) for _, cause := range sink.terminals[0].FailureCauses().All() { causeCodes = append(causeCodes, cause.Code()) } wantCauses := []string{"install_prepare_failed"} if abortFails { wantCauses = append(wantCauses, "attempt_abort_failed") } if !reflect.DeepEqual(causeCodes, wantCauses) { t.Fatalf("cause codes = %v, want %v", causeCodes, wantCauses) } wantTrace := []string{"abort:old", "rebuild", "dispatch", "abort:new", "terminal:error"} if got := trace.snapshot(); !reflect.DeepEqual(got, wantTrace) { t.Fatalf("trace = %v, want %v", got, wantTrace) } }) } t.Run("continuation_commit_failure_aborts_dispatched_binding", func(t *testing.T) { trace := &runtimeTrace{} sink := newFixtureSink() sink.trace = trace dispatcher := &fixtureDispatcher{trace: trace} rebuilder := &fixtureRebuilder{trace: trace} oldController := &fixtureController{name: "old", trace: trace} newController := &fixtureController{name: "new", trace: trace} filter := &customMockFilter{ id: "commit-filter", evaluateFn: func(ctx context.Context, fc FilterContext, batch EvidenceBatch) (FilterDecision, error) { kind := batch.Events()[0].Kind() evidence := mustSanitizedEvidence(kind, "default", "rule1", "commit_check", FixedFingerprint{1}, 1, 0, FilterOutcomeKindEvaluated, testNow) text := "" if kind == EventKindTextDelta { var textErr error text, textErr = batch.Events()[0].AsTextDelta() if textErr != nil { return FilterDecision{}, textErr } } if text == "bad" { directive := mustRecoveryDirectiveContinuation(1, "snapshot.one") intent := mustRecoveryIntent(RecoveryStrategyContinuationRepair, directive, "continuation_required", 10) return NewFilterDecision(FilterDecisionKindViolation, "consumer1", "commit-filter", "rule1", evidence, &intent) } return NewFilterDecision(FilterDecisionKindPass, "consumer1", "commit-filter", "rule1", evidence, nil) }, } reg := mustFilterRegistration(filter, "cap1", true, FilterEnforcementBlocking, 5*time.Second, 10) dispatcher.handler = func(ctx context.Context, request RebuiltRequest) (AttemptBinding, error) { return NewAttemptBinding("att-1", "gpt-4", "openai", "primary", newSliceEventSource(nil), newController) } snapshot := createTestRuntimeSnapshot(t, []FilterRegistration{reg}, dispatcher, rebuilder, sink) initial := mustAttemptBinding(t, "att-1", newSliceEventSource([]NormalizedEvent{ mustResponseStart("default", 200, nil, testNow), mustTextDelta("default", "prefix", testNow), mustTextDelta("default", "bad", testNow), }), oldController) runtime, err := NewRequestRuntime(snapshot, "group-a", initial) if err != nil { t.Fatalf("NewRequestRuntime: %v", err) } if err := runtime.Run(context.Background()); err != nil { t.Fatalf("Run: %v", err) } assertControllerAbortCount(t, oldController, 1) assertControllerAbortCount(t, newController, 1) if len(sink.terminals) != 1 || sink.terminals[0].FailureCauses().All()[0].Code() != "install_commit_failed" { t.Fatalf("commit failure terminal = %+v", sink.terminals) } wantTrace := []string{"start", "release", "abort:old", "rebuild", "dispatch", "abort:new", "terminal:error"} if got := trace.snapshot(); !reflect.DeepEqual(got, wantTrace) { t.Fatalf("trace = %v, want %v", got, wantTrace) } }) } func TestRequestRuntimeRecoveryFailureOrder(t *testing.T) { for _, tc := range []struct { name string controllerErr error rebuildErr error dispatchErr error wantTrace []string }{ {name: "controller_failure", controllerErr: errors.New("raw abort failure"), wantTrace: []string{"abort:old", "abort:old", "terminal:error"}}, {name: "rebuilder_failure", rebuildErr: errors.New("raw rebuild failure"), wantTrace: []string{"abort:old", "rebuild", "terminal:error"}}, {name: "dispatcher_failure", dispatchErr: errors.New("raw dispatch failure"), wantTrace: []string{"abort:old", "rebuild", "dispatch", "terminal:error"}}, } { t.Run(tc.name, func(t *testing.T) { trace := &runtimeTrace{} sink := newFixtureSink() sink.trace = trace rebuilder := &fixtureRebuilder{err: tc.rebuildErr, trace: trace} dispatcher := &fixtureDispatcher{trace: trace} if tc.dispatchErr != nil { dispatcher.handler = func(ctx context.Context, request RebuiltRequest) (AttemptBinding, error) { return AttemptBinding{}, tc.dispatchErr } } filter := newViolationFilter("failure-filter", RecoveryStrategyExactReplay, mustRecoveryDirectiveExact("req-ref-1")) reg := mustFilterRegistration(filter, "cap1", true, FilterEnforcementBlocking, 5*time.Second, 10) controller := &fixtureController{name: "old", abortErr: tc.controllerErr, trace: trace} snapshot := createTestRuntimeSnapshot(t, []FilterRegistration{reg}, dispatcher, rebuilder, sink) initial := mustAttemptBinding(t, "att-1", newSliceEventSource([]NormalizedEvent{mustTextDelta("default", "bad", testNow)}), controller) runtime, err := NewRequestRuntime(snapshot, "group-a", initial) if err != nil { t.Fatalf("NewRequestRuntime: %v", err) } if err := runtime.Run(context.Background()); err != nil { t.Fatalf("Run: %v", err) } wantAbortCount := 1 if tc.controllerErr != nil { wantAbortCount = 2 } assertControllerAbortCount(t, controller, wantAbortCount) if len(sink.terminals) != 1 || sink.terminals[0].Success() { t.Fatalf("terminals = %+v", sink.terminals) } if got := trace.snapshot(); !reflect.DeepEqual(got, tc.wantTrace) { t.Fatalf("trace = %v, want %v", got, tc.wantTrace) } }) } for _, tc := range []struct { name string prepareErr error releaseErr error wantTrace []string wantSuccess bool }{ {name: "preparation_snapshot_released_before_rebuild", wantTrace: []string{"snapshot:create", "abort:old", "prepare", "snapshot:release", "rebuild", "dispatch", "terminal:success"}, wantSuccess: true}, {name: "preparer_failure_releases_snapshot", prepareErr: errors.New("raw preparation failure"), wantTrace: []string{"snapshot:create", "abort:old", "prepare", "snapshot:release", "terminal:error"}}, {name: "preparation_snapshot_release_failure_stops_cycle", releaseErr: errors.New("raw snapshot release failure"), wantTrace: []string{"snapshot:create", "abort:old", "prepare", "snapshot:release", "terminal:error"}}, } { t.Run(tc.name, func(t *testing.T) { trace := &runtimeTrace{} sink := newFixtureSink() sink.trace = trace rebuilder := &fixtureRebuilder{trace: trace} dispatcher := &fixtureDispatcher{trace: trace} oldController := &fixtureController{name: "old", trace: trace} preparedSnapshot := &testPrepSnapshot{ref: "prep-ref-1", trace: trace} preparedSnapshot.releaseErr = tc.releaseErr factory := &testPrepFactory{prepared: preparedSnapshot, trace: trace} preparer := &testPreparer{err: tc.prepareErr, trace: trace} filter := &customMockFilter{ id: "prepared-filter", evaluateFn: func(ctx context.Context, fc FilterContext, batch EvidenceBatch) (FilterDecision, error) { kind := batch.Events()[0].Kind() evidence := mustSanitizedEvidence(kind, "default", "rule1", "prepared_check", FixedFingerprint{1}, 1, 0, FilterOutcomeKindEvaluated, testNow) if fc.AttemptID() == "att-2" { return NewFilterDecision(FilterDecisionKindPass, "consumer1", "prepared-filter", "rule1", evidence, nil) } intent := mustRecoveryIntent(RecoveryStrategyExactReplay, mustRecoveryDirectiveExact("req-ref-1"), "rule_violation", 10) return NewFilterDecision(FilterDecisionKindViolation, "consumer1", "prepared-filter", "rule1", evidence, &intent) }, } reg := mustFilterRegistration(filter, "cap1", true, FilterEnforcementBlocking, 5*time.Second, 10) dispatcher.handler = func(ctx context.Context, request RebuiltRequest) (AttemptBinding, error) { return NewAttemptBinding("att-2", "gpt-4", "openai", "primary", newSliceEventSource([]NormalizedEvent{mustTerminal("default", testNow)}), &fixtureController{}) } snapshot := createTestRuntimeSnapshotWithPreparer(t, []FilterRegistration{reg}, dispatcher, rebuilder, preparer, factory, sink) initial := mustAttemptBinding(t, "att-1", newSliceEventSource([]NormalizedEvent{mustTextDelta("default", "bad", testNow)}), oldController) runtime, err := NewRequestRuntime(snapshot, "group-a", initial) if err != nil { t.Fatalf("NewRequestRuntime: %v", err) } if err := runtime.Run(context.Background()); err != nil { t.Fatalf("Run: %v", err) } assertControllerAbortCount(t, oldController, 1) preparedSnapshot.mu.Lock() releaseCount := preparedSnapshot.releaseCount preparedSnapshot.mu.Unlock() if releaseCount != 1 { t.Fatalf("snapshot release count = %d, want 1", releaseCount) } if got := trace.snapshot(); !reflect.DeepEqual(got, tc.wantTrace) { t.Fatalf("trace = %v, want %v", got, tc.wantTrace) } if len(sink.terminals) != 1 || sink.terminals[0].Success() != tc.wantSuccess { t.Fatalf("terminal = %+v, wantSuccess=%v", sink.terminals, tc.wantSuccess) } }) } } // newReplacementFilter returns a blocking filter that substitutes the given // proposal for a single original text delta and passes every other event. func newReplacementFilter(id, originalText string, proposal *ReplacementProposal) *customMockFilter { return &customMockFilter{ id: id, evaluateFn: func(ctx context.Context, fc FilterContext, batch EvidenceBatch) (FilterDecision, error) { event := batch.Events()[0] evidence := mustSanitizedEvidence(event.Kind(), "default", "rule1", "replacement_check", FixedFingerprint{3}, 1, 0, FilterOutcomeKindEvaluated, testNow) if event.Kind() != EventKindTextDelta { return NewFilterDecision(FilterDecisionKindPass, "consumer1", id, "rule1", evidence, nil) } text, err := event.AsTextDelta() if err != nil { return FilterDecision{}, err } if text != originalText { return NewFilterDecision(FilterDecisionKindPass, "consumer1", id, "rule1", evidence, nil) } return NewFilterDecisionWithReplacement(FilterDecisionKindReplacement, "consumer1", id, "rule1", evidence, nil, proposal) }, } } // altHeldText is the unrelated-channel event that is already pending when the // replacement settles; altReleaseText then crosses that channel's 100-rune // rolling threshold so both entries release together afterwards. const altHeldText = "alt-held" func TestRequestRuntimeReplacementProposalRelease(t *testing.T) { altReleaseText := repeatRune('z', 100) // The runtime fixture drives two blocking channels from one event source: // "default" carries the replaced content, and "alt" holds unrelated pending // evidence across the replacement because its threshold is not yet met. newReplacementRuntime := func(t *testing.T, trace *runtimeTrace, sink *fixtureSink) *RequestRuntime { t.Helper() proposal := mustReplacementProposal(t, []NormalizedEvent{ mustTextDelta("default", "replacement-1", testNow), mustTextDelta("default", "replacement-2", testNow), }) filter := newReplacementFilter("replacement-filter", "original", proposal) reg := mustFilterRegistration(filter, "cap1", true, FilterEnforcementBlocking, 5*time.Second, 10) altReq := mustFilterHoldRequirementRolling("alt", []EventKind{EventKindTextDelta}, 100) altReg := mustFilterRegistration( &customMockFilter{id: "alt-filter", holdReq: &altReq}, "cap2", true, FilterEnforcementBlocking, 5*time.Second, 5, ) snapshot := createTestRuntimeSnapshot(t, []FilterRegistration{reg, altReg}, &fixtureDispatcher{trace: trace}, &fixtureRebuilder{trace: trace}, sink) initial := mustAttemptBinding(t, "att-1", newSliceEventSource([]NormalizedEvent{ mustResponseStart("default", 200, map[string]string{"X-Attempt": "one"}, testNow), mustTextDelta("alt", altHeldText, testNow), mustTextDelta("default", "original", testNow), mustTextDelta("alt", altReleaseText, testNow), mustTerminal("default", testNow), }), &fixtureController{name: "att-1", trace: trace}) runtime, err := NewRequestRuntime(snapshot, "group-a", initial) if err != nil { t.Fatalf("NewRequestRuntime: %v", err) } return runtime } t.Run("substitutes_ordered_replacement", func(t *testing.T) { trace := &runtimeTrace{} sink := newFixtureSink() sink.trace = trace runtime := newReplacementRuntime(t, trace, sink) if err := runtime.Run(context.Background()); err != nil { t.Fatalf("Run: %v", err) } // The replacement substitutes only the default channel's held content; // the unrelated channel's pending evidence survives and releases after. wantTexts := []string{"replacement-1", "replacement-2", altHeldText, altReleaseText} if got := releaseTexts(t, sink.events); !reflect.DeepEqual(got, wantTexts) { t.Fatalf("released texts = %v, want %v", got, wantTexts) } if len(sink.starts) != 1 || sink.starts[0].Status() != 200 || sink.starts[0].Headers()["X-Attempt"] != "one" { t.Fatalf("starts = %+v, want exactly one 200/one start", sink.starts) } if len(sink.terminals) != 1 || !sink.terminals[0].Success() { t.Fatalf("terminals = %+v, want exactly one success terminal", sink.terminals) } wantTrace := []string{"start", "release", "release", "release", "release", "terminal:success"} if got := trace.snapshot(); !reflect.DeepEqual(got, wantTrace) { t.Fatalf("trace = %v, want %v", got, wantTrace) } }) t.Run("sink_release_failure_stops_output", func(t *testing.T) { trace := &runtimeTrace{} sink := newFixtureSink() sink.trace = trace sink.releaseErr = errors.New("raw sink release failure") runtime := newReplacementRuntime(t, trace, sink) runErr := runtime.Run(context.Background()) if runErr == nil { t.Fatal("Run error = nil, want sink release failure") } wantTexts := []string{"replacement-1"} if got := releaseTexts(t, sink.events); !reflect.DeepEqual(got, wantTexts) { t.Fatalf("released texts = %v, want %v", got, wantTexts) } if len(sink.terminals) != 0 { t.Fatalf("terminals = %+v, want none after replacement release failure", sink.terminals) } wantTrace := []string{"start", "release"} if got := trace.snapshot(); !reflect.DeepEqual(got, wantTrace) { t.Fatalf("trace = %v, want %v", got, wantTrace) } // A failed replacement must not take the unrelated channel's pending // evidence down with it. if got := pendingTexts(t, runtime.tail, "alt"); !reflect.DeepEqual(got, []string{altHeldText}) { t.Fatalf("pending on alt = %v, want [%s]", got, altHeldText) } if got := pendingTexts(t, runtime.tail, "default"); len(got) != 0 { t.Fatalf("pending on default = %v, want none after the replacement consumed it", got) } }) } func TestRequestRuntimeReplaceAttemptStagedStartIsolation(t *testing.T) { type observedBatchStart struct { attemptID string start *ResponseStart } for _, tc := range []struct { name string newAttemptEvents []NormalizedEvent wantNewStart bool }{ { name: "new_attempt_without_response_start", newAttemptEvents: []NormalizedEvent{ mustTextDelta("default", "clean", testNow), mustTerminal("default", testNow), }, }, { name: "new_attempt_with_new_response_start", newAttemptEvents: []NormalizedEvent{ mustResponseStart("default", 200, map[string]string{"X-Attempt": "new"}, testNow), mustTextDelta("default", "clean", testNow), mustTerminal("default", testNow), }, wantNewStart: true, }, } { t.Run(tc.name, func(t *testing.T) { trace := &runtimeTrace{} sink := newFixtureSink() sink.trace = trace dispatcher := &fixtureDispatcher{trace: trace} rebuilder := &fixtureRebuilder{trace: trace} oldController := &fixtureController{name: "att-1", trace: trace} newController := &fixtureController{name: "att-2", trace: trace} var observedMu sync.Mutex var observed []observedBatchStart filter := &customMockFilter{ id: "start-isolation-filter", evaluateFn: func(ctx context.Context, fc FilterContext, batch EvidenceBatch) (FilterDecision, error) { observedMu.Lock() observed = append(observed, observedBatchStart{attemptID: fc.AttemptID(), start: batch.StagedResponseStart()}) observedMu.Unlock() evidence := mustSanitizedEvidence(batch.Events()[0].Kind(), "default", "rule1", "start_isolation", FixedFingerprint{5}, 1, 0, FilterOutcomeKindEvaluated, testNow) if fc.AttemptID() != "att-1" { return NewFilterDecision(FilterDecisionKindPass, "consumer1", "start-isolation-filter", "rule1", evidence, nil) } intent := mustRecoveryIntent(RecoveryStrategyExactReplay, mustRecoveryDirectiveExact("req-ref-1"), "rule_violation", 10) return NewFilterDecision(FilterDecisionKindViolation, "consumer1", "start-isolation-filter", "rule1", evidence, &intent) }, } reg := mustFilterRegistration(filter, "cap1", true, FilterEnforcementBlocking, 5*time.Second, 10) dispatcher.handler = func(ctx context.Context, request RebuiltRequest) (AttemptBinding, error) { return NewAttemptBinding("att-2", "gpt-4", "openai", "primary", newSliceEventSource(tc.newAttemptEvents), newController) } snapshot := createTestRuntimeSnapshot(t, []FilterRegistration{reg}, dispatcher, rebuilder, sink) initial := mustAttemptBinding(t, "att-1", newSliceEventSource([]NormalizedEvent{ mustResponseStart("default", 418, map[string]string{"X-Attempt": "old"}, testNow), mustTextDelta("default", "bad", testNow), }), oldController) runtime, err := NewRequestRuntime(snapshot, "group-a", initial) if err != nil { t.Fatalf("NewRequestRuntime: %v", err) } if err := runtime.Run(context.Background()); err != nil { t.Fatalf("Run: %v", err) } observedMu.Lock() defer observedMu.Unlock() oldStartObserved := false newStartObserved := 0 for _, obs := range observed { if obs.attemptID == "att-1" { if obs.start != nil && obs.start.Status() == 418 && obs.start.Headers()["X-Attempt"] == "old" { oldStartObserved = true } continue } if obs.start == nil { continue } if obs.start.Status() != 200 || obs.start.Headers()["X-Attempt"] != "new" { t.Fatalf("attempt %s batch staged start = %d/%v, want no stale old start", obs.attemptID, obs.start.Status(), obs.start.Headers()) } newStartObserved++ } if !oldStartObserved { t.Fatal("att-1 never observed the old staged start; fixture does not exercise the isolation path") } if tc.wantNewStart && newStartObserved != 1 { t.Fatalf("att-2 batches carrying the new staged start = %d, want 1", newStartObserved) } if !tc.wantNewStart && newStartObserved != 0 { t.Fatalf("att-2 batches carrying a staged start = %d, want 0", newStartObserved) } wantStarts := 0 if tc.wantNewStart { wantStarts = 1 } if len(sink.starts) != wantStarts { t.Fatalf("sink starts = %+v, want %d", sink.starts, wantStarts) } for _, start := range sink.starts { if start.Status() != 200 || start.Headers()["X-Attempt"] != "new" { t.Fatalf("sink start = %d/%v, want only the new attempt start", start.Status(), start.Headers()) } } if got := releaseTexts(t, sink.events); !reflect.DeepEqual(got, []string{"clean"}) { t.Fatalf("released texts = %v, want [clean]", got) } if len(sink.terminals) != 1 || !sink.terminals[0].Success() { t.Fatalf("terminals = %+v, want exactly one success terminal", sink.terminals) } if len(dispatcher.dispatched) != 1 { t.Fatalf("dispatch count = %d, want 1", len(dispatcher.dispatched)) } assertControllerAbortCount(t, oldController, 1) assertControllerAbortCount(t, newController, 0) }) } } func TestRequestRuntimeLifecycleMatrix(t *testing.T) { t.Run("recovery_caps_0_1_3", func(t *testing.T) { for _, cap := range []int{0, 1, 3} { t.Run(fmt.Sprintf("cap_%d", cap), func(t *testing.T) { trace := &runtimeTrace{} sink := newFixtureSink() sink.trace = trace dispatcher := &fixtureDispatcher{trace: trace} rebuilder := &fixtureRebuilder{trace: trace} controllers := []*fixtureController{{name: "att-1", trace: trace}} filter := newViolationFilter("cap-filter", RecoveryStrategyExactReplay, mustRecoveryDirectiveExact("req-ref-1")) reg := mustFilterRegistration(filter, "cap1", true, FilterEnforcementBlocking, 5*time.Second, 10) dispatcher.handler = func(ctx context.Context, request RebuiltRequest) (AttemptBinding, error) { attemptNumber := len(controllers) + 1 controller := &fixtureController{name: fmt.Sprintf("att-%d", attemptNumber), trace: trace} controllers = append(controllers, controller) return NewAttemptBinding(fmt.Sprintf("att-%d", attemptNumber), "gpt-4", "openai", "primary", newSliceEventSource([]NormalizedEvent{mustTextDelta("default", "bad", testNow)}), controller) } options := runtimeOptionsWithRecoveryCap(t, cap) snapshot := createTestRuntimeSnapshotWithOptions(t, []FilterRegistration{reg}, dispatcher, rebuilder, options, sink) initial := mustAttemptBinding(t, "att-1", newSliceEventSource([]NormalizedEvent{mustTextDelta("default", "bad", testNow)}), controllers[0]) runtime, err := NewRequestRuntime(snapshot, "group-a", initial) if err != nil { t.Fatalf("NewRequestRuntime: %v", err) } if err := runtime.Run(context.Background()); err != nil { t.Fatalf("Run: %v", err) } if len(dispatcher.dispatched) != cap || rebuilder.planCount != cap || len(controllers) != cap+1 { t.Fatalf("dispatch/rebuild/controllers = %d/%d/%d, want %d/%d/%d", len(dispatcher.dispatched), rebuilder.planCount, len(controllers), cap, cap, cap+1) } for _, controller := range controllers { assertControllerAbortCount(t, controller, 1) } if len(sink.starts) != 0 || len(sink.events) != 0 || len(sink.terminals) != 1 || sink.terminals[0].Success() { t.Fatalf("sink counts starts/events/terminals = %d/%d/%d", len(sink.starts), len(sink.events), len(sink.terminals)) } wantTrace := make([]string, 0, cap*3+2) for i := 1; i <= cap; i++ { wantTrace = append(wantTrace, fmt.Sprintf("abort:att-%d", i), "rebuild", "dispatch") } wantTrace = append(wantTrace, fmt.Sprintf("abort:att-%d", cap+1), "terminal:error") if got := trace.snapshot(); !reflect.DeepEqual(got, wantTrace) { t.Fatalf("trace = %v, want %v", got, wantTrace) } }) } }) for _, enforcement := range []FilterEnforcement{FilterEnforcementBlocking, FilterEnforcementObserveOnly} { name := string(enforcement) + "_evaluation_error" t.Run(name, func(t *testing.T) { sink := newFixtureSink() filter := &customMockFilter{ id: "error-filter", evaluateFn: func(ctx context.Context, fc FilterContext, batch EvidenceBatch) (FilterDecision, error) { return FilterDecision{}, errors.New("raw filter failure") }, } reg := mustFilterRegistration(filter, "cap1", true, enforcement, 5*time.Second, 10) snapshot := createTestRuntimeSnapshot(t, []FilterRegistration{reg}, &fixtureDispatcher{}, &fixtureRebuilder{}, sink) initial := mustAttemptBinding(t, "att-1", newSliceEventSource([]NormalizedEvent{ mustResponseStart("default", 200, nil, testNow), mustTextDelta("default", "text", testNow), mustTerminal("default", testNow), }), &fixtureController{}) runtime, err := NewRequestRuntime(snapshot, "group-a", initial) if err != nil { t.Fatalf("NewRequestRuntime: %v", err) } if err := runtime.Run(context.Background()); err != nil { t.Fatalf("Run: %v", err) } if len(sink.terminals) != 1 { t.Fatalf("terminal count = %d, want 1", len(sink.terminals)) } if enforcement == FilterEnforcementBlocking { if sink.terminals[0].Success() || len(sink.events) != 0 { t.Fatalf("blocking evaluation error exposed output: events=%d terminal=%+v", len(sink.events), sink.terminals[0]) } } else if !sink.terminals[0].Success() || len(sink.events) != 1 { t.Fatalf("observe-only evaluation error did not pass through: events=%d terminal=%+v", len(sink.events), sink.terminals[0]) } }) } t.Run("cancellation_during_evaluation", func(t *testing.T) { entered := make(chan struct{}) filter := &customMockFilter{ id: "cancel-filter", evaluateFn: func(ctx context.Context, fc FilterContext, batch EvidenceBatch) (FilterDecision, error) { close(entered) <-ctx.Done() return FilterDecision{}, ctx.Err() }, } reg := mustFilterRegistration(filter, "cap1", true, FilterEnforcementBlocking, 5*time.Second, 10) sink := newFixtureSink() snapshot := createTestRuntimeSnapshot(t, []FilterRegistration{reg}, &fixtureDispatcher{}, &fixtureRebuilder{}, sink) initial := mustAttemptBinding(t, "att-1", newSliceEventSource([]NormalizedEvent{mustTextDelta("default", "text", testNow)}), &fixtureController{}) runtime, err := NewRequestRuntime(snapshot, "group-a", initial) if err != nil { t.Fatalf("NewRequestRuntime: %v", err) } ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) go func() { done <- runtime.Run(ctx) }() <-entered cancel() if runErr := <-done; !errors.Is(runErr, context.Canceled) { t.Fatalf("Run error = %v, want context.Canceled", runErr) } if len(sink.starts) != 0 || len(sink.events) != 0 || len(sink.terminals) != 0 { t.Fatalf("canceled run exposed sink calls: %d/%d/%d", len(sink.starts), len(sink.events), len(sink.terminals)) } }) t.Run("backpressure_blocks_next_ingress", func(t *testing.T) { entered := make(chan struct{}) release := make(chan struct{}) filter := &customMockFilter{ id: "backpressure-filter", evaluateFn: func(ctx context.Context, fc FilterContext, batch EvidenceBatch) (FilterDecision, error) { event := batch.Events()[0] evidence := mustSanitizedEvidence(event.Kind(), "default", "rule1", "backpressure", FixedFingerprint{4}, 1, 0, FilterOutcomeKindEvaluated, testNow) if event.Kind() == EventKindTextDelta { close(entered) <-release } return NewFilterDecision(FilterDecisionKindPass, "consumer1", "backpressure-filter", "rule1", evidence, nil) }, } reg := mustFilterRegistration(filter, "cap1", true, FilterEnforcementBlocking, 5*time.Second, 10) sink := newFixtureSink() source := newCountingEventSource([]NormalizedEvent{ mustTextDelta("default", "hold", testNow), mustTerminal("default", testNow), }) snapshot := createTestRuntimeSnapshot(t, []FilterRegistration{reg}, &fixtureDispatcher{}, &fixtureRebuilder{}, sink) initial := mustAttemptBinding(t, "att-1", source, &fixtureController{}) runtime, err := NewRequestRuntime(snapshot, "group-a", initial) if err != nil { t.Fatalf("NewRequestRuntime: %v", err) } done := make(chan error, 1) go func() { done <- runtime.Run(context.Background()) }() // The filter barrier is entered, so the owner loop is inside evaluation. // Reading the source call count after this happens-before edge proves the // next ingress read has not started without sleeping on a timer. <-entered callsDuringEvaluation := source.callCount() close(release) if runErr := <-done; runErr != nil { t.Fatalf("Run: %v", runErr) } if callsDuringEvaluation != 1 { t.Fatalf("NextEvent calls during evaluation = %d, want 1", callsDuringEvaluation) } if got := source.callCount(); got != 2 { t.Fatalf("NextEvent calls after release = %d, want 2", got) } if got := releaseTexts(t, sink.events); !reflect.DeepEqual(got, []string{"hold"}) { t.Fatalf("released texts = %v, want [hold]", got) } if len(sink.terminals) != 1 || !sink.terminals[0].Success() { t.Fatalf("terminals = %+v, want exactly one success terminal", sink.terminals) } }) t.Run("simultaneous_violation_single_winner", func(t *testing.T) { trace := &runtimeTrace{} sink := newFixtureSink() sink.trace = trace dispatcher := &fixtureDispatcher{trace: trace} rebuilder := &fixtureRebuilder{trace: trace} oldController := &fixtureController{name: "att-1", trace: trace} newController := &fixtureController{name: "att-2", trace: trace} makeFilter := func(id, rule string, cost, priority int) FilterRegistration { filter := &customMockFilter{ id: id, evaluateFn: func(ctx context.Context, fc FilterContext, batch EvidenceBatch) (FilterDecision, error) { evidence := mustSanitizedEvidence(EventKindTextDelta, "default", rule, "simultaneous_violation", FixedFingerprint{byte(cost)}, 1, 0, FilterOutcomeKindEvaluated, testNow) if fc.AttemptID() == "att-2" { return NewFilterDecision(FilterDecisionKindPass, id, id, rule, evidence, nil) } intent := mustRecoveryIntent(RecoveryStrategyExactReplay, mustRecoveryDirectiveExact("req-ref-1"), "rule_violation", cost) return NewFilterDecision(FilterDecisionKindViolation, id, id, rule, evidence, &intent) }, } return mustFilterRegistration(filter, id+"-cap", true, FilterEnforcementBlocking, 5*time.Second, priority) } regs := []FilterRegistration{makeFilter("filter1", "rule1", 20, 20), makeFilter("filter2", "rule2", 10, 10)} dispatcher.handler = func(ctx context.Context, request RebuiltRequest) (AttemptBinding, error) { return NewAttemptBinding("att-2", "gpt-4", "openai", "primary", newSliceEventSource([]NormalizedEvent{mustTerminal("default", testNow)}), newController) } snapshot := createTestRuntimeSnapshot(t, regs, dispatcher, rebuilder, sink) initial := mustAttemptBinding(t, "att-1", newSliceEventSource([]NormalizedEvent{mustTextDelta("default", "bad", testNow)}), oldController) runtime, err := NewRequestRuntime(snapshot, "group-a", initial) if err != nil { t.Fatalf("NewRequestRuntime: %v", err) } if err := runtime.Run(context.Background()); err != nil { t.Fatalf("Run: %v", err) } if len(dispatcher.dispatched) != 1 || len(sink.terminals) != 1 { t.Fatalf("dispatch/terminal count = %d/%d, want 1/1", len(dispatcher.dispatched), len(sink.terminals)) } if !sink.terminals[0].Success() || len(sink.starts) != 0 || len(sink.events) != 0 { t.Fatalf("sink starts/events/terminal = %d/%d/%+v, want 0/0/success", len(sink.starts), len(sink.events), sink.terminals[0]) } plan, hasPlan := rebuilder.lastPlan() if !hasPlan { t.Fatal("rebuilder never received a recovery plan") } contributors := plan.Contributors() if len(contributors) != 1 { t.Fatalf("plan contributors = %+v, want exactly one winner", contributors) } // filter1 wins on higher effective priority; filter2 must not contribute. if contributors[0].FilterID() != "filter1" || contributors[0].RuleID() != "rule1" || contributors[0].ConsumerID() != "filter1" { t.Fatalf("winner contributor = %s/%s/%s, want filter1/rule1/filter1", contributors[0].ConsumerID(), contributors[0].FilterID(), contributors[0].RuleID()) } if plan.Priority() != 20 || plan.Strategy() != RecoveryStrategyExactReplay { t.Fatalf("winner plan priority/strategy = %d/%s, want 20/%s", plan.Priority(), plan.Strategy(), RecoveryStrategyExactReplay) } if plan.ResumeMode() != RecoveryResumeModeReplaceAttempt { t.Fatalf("winner plan resume mode = %s, want %s", plan.ResumeMode(), RecoveryResumeModeReplaceAttempt) } wantTrace := []string{"abort:att-1", "rebuild", "dispatch", "terminal:success"} if got := trace.snapshot(); !reflect.DeepEqual(got, wantTrace) { t.Fatalf("trace = %v, want %v", got, wantTrace) } assertControllerAbortCount(t, oldController, 1) assertControllerAbortCount(t, newController, 0) }) }