toki/iop
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
iop/packages/go/streamgate/runtime_test.go
toki 7634ca8962 feat(stream-evidence-gate-core): complete stream evidence gate core implementation 
- OpenAI request rebuilder with tool validation and provider tunnel
- Edge config runtime refresh for stream evidence gate
- Filter observation contract and runtime with sink/correlation
- Stream gate dispatcher, release sink, and vertical slice
- Recovery coordinator for evidence tail
- Parallel evaluation and commit boundary
- E2E test script for OpenAI vLLM
- Archive completed task groups to archive/2026/07
2026-07-28 04:08:12 +09:00

2222 lines
81 KiB
Go
Raw Blame History

package streamgate
import (
"context"
"errors"
"fmt"
"reflect"
"sync"
"testing"
"time"
)
type runtimeTrace struct {
mu sync.Mutex
steps []string
}
func (r *runtimeTrace) add(step string) {
if r == nil {
return
}
r.mu.Lock()
r.steps = append(r.steps, step)
r.mu.Unlock()
}
func (r *runtimeTrace) snapshot() []string {
r.mu.Lock()
defer r.mu.Unlock()
return append([]string(nil), r.steps...)
}
type sliceEventSource struct {
mu sync.Mutex
events []NormalizedEvent
err error
index int
}
func newSliceEventSource(events []NormalizedEvent) *sliceEventSource {
return &sliceEventSource{events: events}
}
func newErrorEventSource(err error) *sliceEventSource {
return &sliceEventSource{err: err}
}
func (s *sliceEventSource) NextEvent(ctx context.Context) (NormalizedEvent, error) {
s.mu.Lock()
defer s.mu.Unlock()
if s.err != nil {
return NormalizedEvent{}, s.err
}
if s.index >= len(s.events) {
return NormalizedEvent{}, errors.New("EOF")
}
ev := s.events[s.index]
s.index++
return ev, nil
}
type fixtureController struct {
mu sync.Mutex
abortCount int
abortErr error
name string
trace *runtimeTrace
}
func (c *fixtureController) AbortAttempt(ctx context.Context) error {
c.mu.Lock()
defer c.mu.Unlock()
c.abortCount++
c.trace.add("abort:" + c.name)
return c.abortErr
}
type fixtureDispatcher struct {
mu sync.Mutex
handler func(ctx context.Context, request RebuiltRequest) (AttemptBinding, error)
dispatched []RebuiltRequest
trace *runtimeTrace
}
func (d *fixtureDispatcher) DispatchAttempt(ctx context.Context, request RebuiltRequest) (AttemptBinding, error) {
d.mu.Lock()
d.dispatched = append(d.dispatched, request)
h := d.handler
d.mu.Unlock()
d.trace.add("dispatch")
if h != nil {
return h(ctx, request)
}
return AttemptBinding{}, errors.New("dispatcher error")
}
type fixtureRebuilder struct {
mu sync.Mutex
planCount int
plans []RecoveryPlan
err error
trace *runtimeTrace
}
func (r *fixtureRebuilder) RebuildRequest(ctx context.Context, snapshot RecoveryRequestSnapshotRef, plan RecoveryPlan) (RebuiltRequestDraft, error) {
r.mu.Lock()
defer r.mu.Unlock()
r.planCount++
r.plans = append(r.plans, plan)
r.trace.add("rebuild")
if r.err != nil {
return RebuiltRequestDraft{}, r.err
}
return NewRebuiltRequestDraftWithIdempotency(plan.PlanID(), plan.IdempotencyKey(), "req-ref-1", "ep", "fam", 10, 20, 100, 5, nil)
}
// lastPlan returns the most recent recovery plan handed to the rebuilder.
func (r *fixtureRebuilder) lastPlan() (RecoveryPlan, bool) {
r.mu.Lock()
defer r.mu.Unlock()
if len(r.plans) == 0 {
return RecoveryPlan{}, false
}
return r.plans[len(r.plans)-1], true
}
// countingEventSource records how many times the runtime owner loop pulled the
// next ingress event so backpressure can be observed without sleeping.
type countingEventSource struct {
mu sync.Mutex
events []NormalizedEvent
index int
calls int
}
func newCountingEventSource(events []NormalizedEvent) *countingEventSource {
return &countingEventSource{events: events}
}
func (s *countingEventSource) NextEvent(ctx context.Context) (NormalizedEvent, error) {
s.mu.Lock()
defer s.mu.Unlock()
s.calls++
if s.index >= len(s.events) {
return NormalizedEvent{}, errors.New("EOF")
}
ev := s.events[s.index]
s.index++
return ev, nil
}
func (s *countingEventSource) callCount() int {
s.mu.Lock()
defer s.mu.Unlock()
return s.calls
}
type fixtureSink struct {
mu sync.Mutex
starts []ResponseStart
events []ReleaseEvent
terminals []TerminalResult
state CommitState
startErr error
releaseErr error
terminalErr error
trace *runtimeTrace
}
func newFixtureSink() *fixtureSink {
return &fixtureSink{state: CommitStateTransportUncommitted}
}
func (s *fixtureSink) CommitResponseStart(ctx context.Context, rs ResponseStart) (CommitState, error) {
s.mu.Lock()
defer s.mu.Unlock()
s.starts = append(s.starts, rs)
s.trace.add("start")
if s.startErr != nil {
return s.state, s.startErr
}
s.state = CommitStateStreamOpen
return CommitStateStreamOpen, nil
}
func (s *fixtureSink) Release(ctx context.Context, ev ReleaseEvent) (CommitState, error) {
s.mu.Lock()
defer s.mu.Unlock()
s.events = append(s.events, ev)
s.trace.add("release")
if s.releaseErr != nil {
return s.state, s.releaseErr
}
s.state = CommitStateStreamOpen
return CommitStateStreamOpen, nil
}
func (s *fixtureSink) CommitTerminal(ctx context.Context, tr TerminalResult) (CommitState, error) {
s.mu.Lock()
defer s.mu.Unlock()
s.terminals = append(s.terminals, tr)
if tr.Success() {
s.trace.add("terminal:success")
} else {
s.trace.add("terminal:error")
}
if s.terminalErr != nil {
return s.state, s.terminalErr
}
s.state = CommitStateTerminalCommitted
return CommitStateTerminalCommitted, nil
}
type customMockFilter struct {
id string
holdReq *FilterHoldRequirement
holdReqFn func(FilterContext) FilterHoldRequirement
appliesFn func(FilterContext) bool
evaluateFn func(context.Context, FilterContext, EvidenceBatch) (FilterDecision, error)
evalCount int
mu sync.Mutex
}
func (m *customMockFilter) ID() string { return m.id }
func (m *customMockFilter) Applies(c FilterContext) bool {
if m.appliesFn != nil {
return m.appliesFn(c)
}
return true
}
func (m *customMockFilter) HoldRequirement(c FilterContext) FilterHoldRequirement {
if m.holdReqFn != nil {
return m.holdReqFn(c)
}
if m.holdReq != nil {
return *m.holdReq
}
return mustFilterHoldRequirementRolling("default", []EventKind{EventKindTextDelta, EventKindTerminal}, 1)
}
func (m *customMockFilter) Evaluate(ctx context.Context, fc FilterContext, batch EvidenceBatch) (FilterDecision, error) {
m.mu.Lock()
m.evalCount++
m.mu.Unlock()
if m.evaluateFn != nil {
return m.evaluateFn(ctx, fc, batch)
}
kind := EventKindTextDelta
if len(batch.Events()) > 0 {
kind = batch.Events()[0].Kind()
}
fp := FixedFingerprint{1}
ev := mustSanitizedEvidence(kind, "default", "rule1", "desc", fp, 1, 0, FilterOutcomeKindEvaluated, time.Now())
return NewFilterDecision(FilterDecisionKindPass, "consumer1", m.id, "rule1", ev, nil)
}
func createTestRuntimeSnapshot(t *testing.T, regs []FilterRegistration, disp AttemptDispatcher, rebuilder RequestRebuilder, sink ReleaseSink) RequestRuntimeSnapshot {
return createTestRuntimeSnapshotWithPreparer(t, regs, disp, rebuilder, nil, nil, sink)
}
func createTestRuntimeSnapshotWithOptions(t *testing.T, regs []FilterRegistration, disp AttemptDispatcher, rebuilder RequestRebuilder, opts RuntimeOptions, sink ReleaseSink) RequestRuntimeSnapshot {
t.Helper()
regSnap, err := NewFilterRegistrySnapshot("gen-1", regs, nil)
if err != nil {
t.Fatalf("NewFilterRegistrySnapshot: %v", err)
}
snapRef, err := NewRecoveryRequestSnapshotRef("snap.ref.1", 100, 200, 1024)
if err != nil {
t.Fatalf("NewRecoveryRequestSnapshotRef: %v", err)
}
snap, err := NewRequestRuntimeSnapshot(
"req-123", "gen-1", "test", "ep", "fam", opts, regSnap, nil, snapRef,
disp, rebuilder, nil, nil, sink,
)
if err != nil {
t.Fatalf("NewRequestRuntimeSnapshot: %v", err)
}
return snap
}
func createTestRuntimeSnapshotWithPreparer(
t *testing.T,
regs []FilterRegistration,
disp AttemptDispatcher,
rebuilder RequestRebuilder,
preparer RecoveryPlanPreparer,
factory RecoveryPreparationSnapshotFactory,
sink ReleaseSink,
) RequestRuntimeSnapshot {
regSnap, err := NewFilterRegistrySnapshot("gen-1", regs, nil)
if err != nil {
t.Fatalf("NewFilterRegistrySnapshot: %v", err)
}
snapRef, err := NewRecoveryRequestSnapshotRef("snap.ref.1", 100, 200, 1024)
if err != nil {
t.Fatalf("NewRecoveryRequestSnapshotRef: %v", err)
}
opts := DefaultRuntimeOptions()
snap, err := NewRequestRuntimeSnapshot(
"req-123",
"gen-1",
"test",
"ep",
"fam",
opts,
regSnap,
nil,
snapRef,
disp,
rebuilder,
preparer,
factory,
sink,
)
if err != nil {
t.Fatalf("NewRequestRuntimeSnapshot: %v", err)
}
return snap
}
type testPrepSnapshot struct {
mu sync.Mutex
ref string
releaseCount int
releaseErr error
trace *runtimeTrace
}
func (s *testPrepSnapshot) SnapshotRef() string { return s.ref }
func (s *testPrepSnapshot) Release() error {
s.mu.Lock()
defer s.mu.Unlock()
s.releaseCount++
s.trace.add("snapshot:release")
return s.releaseErr
}
type testPrepFactory struct {
mu sync.Mutex
fail bool
callCount int
prepared *testPrepSnapshot
trace *runtimeTrace
}
func (f *testPrepFactory) CreatePreparationSnapshot(ctx context.Context, snapshot RequestRuntimeSnapshot, binding AttemptBinding) (RecoveryPreparationSnapshot, error) {
f.mu.Lock()
defer f.mu.Unlock()
f.callCount++
f.trace.add("snapshot:create")
if f.fail {
return nil, errors.New("prep factory failure")
}
if f.prepared != nil {
return f.prepared, nil
}
return &testPrepSnapshot{ref: "prep-ref-1", trace: f.trace}, nil
}
type testPreparer struct {
mu sync.Mutex
callCount int
err error
trace *runtimeTrace
}
func (p *testPreparer) PrepareRecoveryPlan(ctx context.Context, plan RecoveryPlan, snapshot RecoveryPreparationSnapshot) (RecoveryDirective, error) {
p.mu.Lock()
defer p.mu.Unlock()
p.callCount++
p.trace.add("prepare")
if p.err != nil {
return RecoveryDirective{}, p.err
}
return NewRecoveryDirectiveExact("req-ref-1")
}
func TestRequestRuntimeDisabledAndPass(t *testing.T) {
t.Run("DisabledAndPass", func(t *testing.T) {
sink := newFixtureSink()
disp := &fixtureDispatcher{}
rebuilder := &fixtureRebuilder{}
passF := &customMockFilter{id: "pass-filter"}
regPass, err := NewFilterRegistration(passF, "cap1", true, FilterEnforcementBlocking, 5*time.Second, 10)
if err != nil {
t.Fatalf("NewFilterRegistration pass: %v", err)
}
disabledF := &customMockFilter{id: "disabled-filter"}
regDisabled, err := NewFilterRegistration(disabledF, "cap2", false, FilterEnforcementBlocking, 5*time.Second, 10)
if err != nil {
t.Fatalf("NewFilterRegistration disabled: %v", err)
}
snap := createTestRuntimeSnapshot(t, []FilterRegistration{regPass, regDisabled}, disp, rebuilder, sink)
rsEv, err := NewResponseStartEvent("default", 200, map[string]string{"content-type": "text/event-stream"}, time.Now())
if err != nil {
t.Fatalf("NewResponseStartEvent: %v", err)
}
txtEv, err := NewTextDeltaEvent("default", "hello world", time.Now())
if err != nil {
t.Fatalf("NewTextDeltaEvent: %v", err)
}
termEv, err := NewTerminalEvent("default", time.Now())
if err != nil {
t.Fatalf("NewTerminalEvent: %v", err)
}
src := newSliceEventSource([]NormalizedEvent{rsEv, txtEv, termEv})
ctrl := &fixtureController{}
initialBinding, err := NewAttemptBinding("att-1", "gpt-4", "openai", "primary", src, ctrl)
if err != nil {
t.Fatalf("NewAttemptBinding: %v", err)
}
rt, err := NewRequestRuntime(snap, "group-a", initialBinding)
if err != nil {
t.Fatalf("NewRequestRuntime: %v", err)
}
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
if err := rt.Run(ctx); err != nil {
t.Fatalf("Run: %v", err)
}
sink.mu.Lock()
defer sink.mu.Unlock()
if len(sink.starts) != 1 {
t.Errorf("expected 1 response start committed, got %d", len(sink.starts))
}
if len(sink.events) != 1 {
t.Errorf("expected 1 release event, got %d", len(sink.events))
}
if len(sink.terminals) != 1 {
t.Fatalf("expected 1 terminal result, got %d", len(sink.terminals))
}
if !sink.terminals[0].Success() {
t.Logf("terminal: %+v", sink.terminals[0])
t.Errorf("expected success terminal result")
}
disabledF.mu.Lock()
if disabledF.evalCount != 0 {
t.Errorf("disabled filter should not be evaluated, count = %d", disabledF.evalCount)
}
disabledF.mu.Unlock()
})
t.Run("ProviderErrorTerminal", func(t *testing.T) {
sink := newFixtureSink()
disp := &fixtureDispatcher{}
rebuilder := &fixtureRebuilder{}
passF := &customMockFilter{id: "pass-filter"}
regPass, err := NewFilterRegistration(passF, "cap1", true, FilterEnforcementBlocking, 5*time.Second, 10)
if err != nil {
t.Fatalf("NewFilterRegistration: %v", err)
}
snap := createTestRuntimeSnapshot(t, []FilterRegistration{regPass}, disp, rebuilder, sink)
rsEv, err := NewResponseStartEvent("default", 200, nil, time.Now())
if err != nil {
t.Fatalf("NewResponseStartEvent: %v", err)
}
desc, errDesc := NewExternalDescriptor("error", "provider_error", "internal_error", "")
if errDesc != nil {
t.Fatalf("NewExternalDescriptor: %v", errDesc)
}
cause, errCause := NewFailureCause("provider", "internal_error", "", "", "")
if errCause != nil {
t.Fatalf("NewFailureCause: %v", errCause)
}
causes, errCauses := NewFailureCauseChain([]FailureCause{cause})
if errCauses != nil {
t.Fatalf("NewFailureCauseChain: %v", errCauses)
}
provErr, err := NewProviderErrorEvent("default", desc, causes, time.Now())
if err != nil {
t.Fatalf("NewProviderErrorEvent: %v", err)
}
src := newSliceEventSource([]NormalizedEvent{rsEv, provErr})
ctrl := &fixtureController{}
binding, err := NewAttemptBinding("att-1", "gpt-4", "openai", "primary", src, ctrl)
if err != nil {
t.Fatalf("NewAttemptBinding: %v", err)
}
rt, err := NewRequestRuntime(snap, "group-a", binding)
if err != nil {
t.Fatalf("NewRequestRuntime: %v", err)
}
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
if err := rt.Run(ctx); err != nil {
t.Fatalf("Run: %v", err)
}
sink.mu.Lock()
defer sink.mu.Unlock()
if len(sink.terminals) != 1 {
t.Fatalf("expected 1 terminal result for provider error, got %d", len(sink.terminals))
}
if sink.terminals[0].Success() {
t.Errorf("provider error should produce error terminal result")
}
})
t.Run("SourceReadFailure", func(t *testing.T) {
sink := newFixtureSink()
disp := &fixtureDispatcher{}
rebuilder := &fixtureRebuilder{}
passF := &customMockFilter{id: "pass-filter"}
regPass, err := NewFilterRegistration(passF, "cap1", true, FilterEnforcementBlocking, 5*time.Second, 10)
if err != nil {
t.Fatalf("NewFilterRegistration: %v", err)
}
snap := createTestRuntimeSnapshot(t, []FilterRegistration{regPass}, disp, rebuilder, sink)
src := newErrorEventSource(errors.New("stream abrupt disconnect"))
ctrl := &fixtureController{}
binding, err := NewAttemptBinding("att-1", "gpt-4", "openai", "primary", src, ctrl)
if err != nil {
t.Fatalf("NewAttemptBinding: %v", err)
}
rt, err := NewRequestRuntime(snap, "group-a", binding)
if err != nil {
t.Fatalf("NewRequestRuntime: %v", err)
}
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
if err := rt.Run(ctx); err != nil {
t.Fatalf("Run: %v", err)
}
sink.mu.Lock()
defer sink.mu.Unlock()
if len(sink.terminals) != 1 {
t.Fatalf("expected 1 terminal result on source failure, got %d", len(sink.terminals))
}
if sink.terminals[0].Success() {
t.Errorf("source failure should produce error terminal result")
}
})
}
func TestRequestRuntimeRecoveryLifecycle(t *testing.T) {
t.Run("ReplaceAttemptRecovery", func(t *testing.T) {
sink := newFixtureSink()
rebuilder := &fixtureRebuilder{}
disp := &fixtureDispatcher{}
filterID := "violating-filter"
var evalCount int
var mu sync.Mutex
vF := &customMockFilter{
id: filterID,
evaluateFn: func(ctx context.Context, fc FilterContext, batch EvidenceBatch) (FilterDecision, error) {
mu.Lock()
count := evalCount
evalCount++
mu.Unlock()
fp := FixedFingerprint{1}
ev := mustSanitizedEvidence(EventKindTextDelta, "default", "rule1", "desc", fp, 1, 0, FilterOutcomeKindEvaluated, time.Now())
if count == 0 {
dir := mustRecoveryDirectiveExact("req-ref-1")
intent := mustRecoveryIntent(RecoveryStrategyExactReplay, dir, "rule_violation", 10)
return NewFilterDecision(FilterDecisionKindViolation, "consumer1", filterID, "rule1", ev, &intent)
}
return NewFilterDecision(FilterDecisionKindPass, "consumer1", filterID, "rule1", ev, nil)
},
}
reg, err := NewFilterRegistration(vF, "cap1", true, FilterEnforcementBlocking, 5*time.Second, 10)
if err != nil {
t.Fatalf("NewFilterRegistration: %v", err)
}
txtEv2, err := NewTextDeltaEvent("default", "recovered text", time.Now())
if err != nil {
t.Fatalf("NewTextDeltaEvent: %v", err)
}
termEv2, err := NewTerminalEvent("default", time.Now())
if err != nil {
t.Fatalf("NewTerminalEvent: %v", err)
}
src2 := newSliceEventSource([]NormalizedEvent{txtEv2, termEv2})
ctrl2 := &fixtureController{}
disp.handler = func(ctx context.Context, request RebuiltRequest) (AttemptBinding, error) {
return NewAttemptBinding("att-2", "gpt-4", "openai", "primary", src2, ctrl2)
}
snap := createTestRuntimeSnapshot(t, []FilterRegistration{reg}, disp, rebuilder, sink)
txtEv1, err := NewTextDeltaEvent("default", "violating text", time.Now())
if err != nil {
t.Fatalf("NewTextDeltaEvent: %v", err)
}
termEv1, err := NewTerminalEvent("default", time.Now())
if err != nil {
t.Fatalf("NewTerminalEvent: %v", err)
}
src1 := newSliceEventSource([]NormalizedEvent{txtEv1, termEv1})
ctrl1 := &fixtureController{}
initialBinding, err := NewAttemptBinding("att-1", "gpt-4", "openai", "primary", src1, ctrl1)
if err != nil {
t.Fatalf("NewAttemptBinding: %v", err)
}
rt, err := NewRequestRuntime(snap, "group-a", initialBinding)
if err != nil {
t.Fatalf("NewRequestRuntime: %v", err)
}
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
if err := rt.Run(ctx); err != nil {
t.Fatalf("Run: %v", err)
}
sink.mu.Lock()
defer sink.mu.Unlock()
if len(sink.terminals) != 1 {
t.Fatalf("expected exactly 1 terminal result, got %d", len(sink.terminals))
}
if !sink.terminals[0].Success() {
t.Errorf("expected successful terminal result after recovery")
}
ctrl1.mu.Lock()
if ctrl1.abortCount != 1 {
t.Errorf("expected attempt 1 aborted once, got %d", ctrl1.abortCount)
}
ctrl1.mu.Unlock()
})
t.Run("ProviderAndPathSwitching", func(t *testing.T) {
sink := newFixtureSink()
rebuilder := &fixtureRebuilder{}
disp := &fixtureDispatcher{}
filterID := "switch-filter"
var evaluatedProviders []string
var mu sync.Mutex
vF := &customMockFilter{
id: filterID,
evaluateFn: func(ctx context.Context, fc FilterContext, batch EvidenceBatch) (FilterDecision, error) {
mu.Lock()
evaluatedProviders = append(evaluatedProviders, fc.ActualProvider())
mu.Unlock()
kind := EventKindTextDelta
if len(batch.Events()) > 0 {
kind = batch.Events()[0].Kind()
}
fp := FixedFingerprint{1}
ev := mustSanitizedEvidence(kind, "default", "rule1", "desc", fp, 1, 0, FilterOutcomeKindEvaluated, time.Now())
if fc.AttemptID() == "att-2" {
return NewFilterDecision(FilterDecisionKindPass, "consumer1", filterID, "rule1", ev, nil)
}
dir := mustRecoveryDirectiveExact("req-ref-1")
intent := mustRecoveryIntent(RecoveryStrategyExactReplay, dir, "rule_violation", 10)
return NewFilterDecision(FilterDecisionKindViolation, "consumer1", filterID, "rule1", ev, &intent)
},
}
reg, err := NewFilterRegistration(vF, "cap1", true, FilterEnforcementBlocking, 5*time.Second, 10)
if err != nil {
t.Fatalf("NewFilterRegistration: %v", err)
}
txtEv2 := mustTextDelta("default", "fallback text", time.Now())
termEv2 := mustTerminal("default", time.Now())
src2 := newSliceEventSource([]NormalizedEvent{txtEv2, termEv2})
ctrl2 := &fixtureController{}
disp.handler = func(ctx context.Context, request RebuiltRequest) (AttemptBinding, error) {
// Switch to anthropic / fallback
return NewAttemptBinding("att-2", "claude-3-5-sonnet", "anthropic", "fallback", src2, ctrl2)
}
snap := createTestRuntimeSnapshot(t, []FilterRegistration{reg}, disp, rebuilder, sink)
txtEv1 := mustTextDelta("default", "primary text", time.Now())
termEv1 := mustTerminal("default", time.Now())
src1 := newSliceEventSource([]NormalizedEvent{txtEv1, termEv1})
ctrl1 := &fixtureController{}
initialBinding, err := NewAttemptBinding("att-1", "gpt-4", "openai", "primary", src1, ctrl1)
if err != nil {
t.Fatalf("NewAttemptBinding: %v", err)
}
rt, err := NewRequestRuntime(snap, "group-a", initialBinding)
if err != nil {
t.Fatalf("NewRequestRuntime: %v", err)
}
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
if err := rt.Run(ctx); err != nil {
t.Fatalf("Run: %v", err)
}
mu.Lock()
defer mu.Unlock()
if len(evaluatedProviders) != 3 {
t.Fatalf("expected 3 evaluations across provider switch, got %d", len(evaluatedProviders))
}
if evaluatedProviders[0] != "openai" || evaluatedProviders[1] != "anthropic" || evaluatedProviders[2] != "anthropic" {
t.Errorf("evaluated providers = %v, want [openai, anthropic, anthropic]", evaluatedProviders)
}
})
t.Run("PreparerSuccessAndFailure", func(t *testing.T) {
t.Run("PreparerSuccess", func(t *testing.T) {
sink := newFixtureSink()
rebuilder := &fixtureRebuilder{}
disp := &fixtureDispatcher{}
vF := &customMockFilter{
id: "prep-filter",
evaluateFn: func(ctx context.Context, fc FilterContext, batch EvidenceBatch) (FilterDecision, error) {
kind := EventKindTextDelta
if len(batch.Events()) > 0 {
kind = batch.Events()[0].Kind()
}
fp := FixedFingerprint{1}
ev := mustSanitizedEvidence(kind, "default", "rule1", "desc", fp, 1, 0, FilterOutcomeKindEvaluated, time.Now())
if fc.AttemptID() == "att-2" {
return NewFilterDecision(FilterDecisionKindPass, "consumer1", "prep-filter", "rule1", ev, nil)
}
dir := mustRecoveryDirectiveExact("req-ref-1")
intent := mustRecoveryIntent(RecoveryStrategyExactReplay, dir, "rule_violation", 10)
return NewFilterDecision(FilterDecisionKindViolation, "consumer1", "prep-filter", "rule1", ev, &intent)
},
}
reg, err := NewFilterRegistration(vF, "cap1", true, FilterEnforcementBlocking, 5*time.Second, 10)
if err != nil {
t.Fatalf("NewFilterRegistration: %v", err)
}
txtEv2 := mustTextDelta("default", "prep text", time.Now())
termEv2 := mustTerminal("default", time.Now())
src2 := newSliceEventSource([]NormalizedEvent{txtEv2, termEv2})
ctrl2 := &fixtureController{}
disp.handler = func(ctx context.Context, request RebuiltRequest) (AttemptBinding, error) {
return NewAttemptBinding("att-2", "gpt-4", "openai", "primary", src2, ctrl2)
}
prep := &testPreparer{}
factory := &testPrepFactory{}
snap := createTestRuntimeSnapshotWithPreparer(t, []FilterRegistration{reg}, disp, rebuilder, prep, factory, sink)
txtEv1 := mustTextDelta("default", "prep text 1", time.Now())
src1 := newSliceEventSource([]NormalizedEvent{txtEv1})
ctrl1 := &fixtureController{}
initialBinding, err := NewAttemptBinding("att-1", "gpt-4", "openai", "primary", src1, ctrl1)
if err != nil {
t.Fatalf("NewAttemptBinding: %v", err)
}
rt, err := NewRequestRuntime(snap, "group-a", initialBinding)
if err != nil {
t.Fatalf("NewRequestRuntime: %v", err)
}
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
if err := rt.Run(ctx); err != nil {
t.Fatalf("Run: %v", err)
}
factory.mu.Lock()
if factory.callCount != 1 {
t.Errorf("expected factory called 1 time, got %d", factory.callCount)
}
factory.mu.Unlock()
prep.mu.Lock()
if prep.callCount != 1 {
t.Errorf("expected preparer called 1 time, got %d", prep.callCount)
}
prep.mu.Unlock()
})
t.Run("FactoryFailure", func(t *testing.T) {
sink := newFixtureSink()
rebuilder := &fixtureRebuilder{}
disp := &fixtureDispatcher{}
vF := &customMockFilter{
id: "prep-filter",
evaluateFn: func(ctx context.Context, fc FilterContext, batch EvidenceBatch) (FilterDecision, error) {
fp := FixedFingerprint{1}
ev := mustSanitizedEvidence(EventKindTextDelta, "default", "rule1", "desc", fp, 1, 0, FilterOutcomeKindEvaluated, time.Now())
dir := mustRecoveryDirectiveExact("req-ref-1")
intent := mustRecoveryIntent(RecoveryStrategyExactReplay, dir, "rule_violation", 10)
return NewFilterDecision(FilterDecisionKindViolation, "consumer1", "prep-filter", "rule1", ev, &intent)
},
}
reg, err := NewFilterRegistration(vF, "cap1", true, FilterEnforcementBlocking, 5*time.Second, 10)
if err != nil {
t.Fatalf("NewFilterRegistration: %v", err)
}
prep := &testPreparer{}
factory := &testPrepFactory{fail: true}
snap := createTestRuntimeSnapshotWithPreparer(t, []FilterRegistration{reg}, disp, rebuilder, prep, factory, sink)
txtEv1 := mustTextDelta("default", "prep text 1", time.Now())
src1 := newSliceEventSource([]NormalizedEvent{txtEv1})
ctrl1 := &fixtureController{}
initialBinding, err := NewAttemptBinding("att-1", "gpt-4", "openai", "primary", src1, ctrl1)
if err != nil {
t.Fatalf("NewAttemptBinding: %v", err)
}
rt, err := NewRequestRuntime(snap, "group-a", initialBinding)
if err != nil {
t.Fatalf("NewRequestRuntime: %v", err)
}
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
if err := rt.Run(ctx); err != nil {
t.Fatalf("Run: %v", err)
}
sink.mu.Lock()
defer sink.mu.Unlock()
if len(sink.terminals) != 1 {
t.Fatalf("expected 1 terminal result on factory failure, got %d", len(sink.terminals))
}
if sink.terminals[0].Success() {
t.Errorf("factory failure should produce error terminal result")
}
})
})
}
func TestRequestRuntimeFailureMatrix(t *testing.T) {
t.Run("FatalViolation", func(t *testing.T) {
sink := newFixtureSink()
disp := &fixtureDispatcher{}
rebuilder := &fixtureRebuilder{}
fatalF := &customMockFilter{
id: "fatal-filter",
evaluateFn: func(ctx context.Context, fc FilterContext, batch EvidenceBatch) (FilterDecision, error) {
fp := FixedFingerprint{1}
ev := mustSanitizedEvidence(EventKindTextDelta, "default", "rule1", "fatal_desc", fp, 1, 0, FilterOutcomeKindEvaluated, time.Now())
return NewFilterDecision(FilterDecisionKindFatal, "consumer1", "fatal-filter", "rule1", ev, nil)
},
}
reg, err := NewFilterRegistration(fatalF, "cap1", true, FilterEnforcementBlocking, 5*time.Second, 10)
if err != nil {
t.Fatalf("NewFilterRegistration: %v", err)
}
snap := createTestRuntimeSnapshot(t, []FilterRegistration{reg}, disp, rebuilder, sink)
txtEv := mustTextDelta("default", "bad text", time.Now())
src := newSliceEventSource([]NormalizedEvent{txtEv})
ctrl := &fixtureController{}
initialBinding, err := NewAttemptBinding("att-1", "gpt-4", "openai", "primary", src, ctrl)
if err != nil {
t.Fatalf("NewAttemptBinding: %v", err)
}
rt, err := NewRequestRuntime(snap, "group-a", initialBinding)
if err != nil {
t.Fatalf("NewRequestRuntime: %v", err)
}
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
if err := rt.Run(ctx); err != nil {
t.Fatalf("Run: %v", err)
}
sink.mu.Lock()
defer sink.mu.Unlock()
if len(sink.terminals) != 1 {
t.Fatalf("expected 1 terminal result for fatal violation, got %d", len(sink.terminals))
}
if sink.terminals[0].Success() {
t.Errorf("expected error terminal result for fatal violation")
}
})
t.Run("ExhaustedRecoveryBudget", func(t *testing.T) {
sink := newFixtureSink()
disp := &fixtureDispatcher{}
rebuilder := &fixtureRebuilder{}
vF := &customMockFilter{
id: "violating-filter",
evaluateFn: func(ctx context.Context, fc FilterContext, batch EvidenceBatch) (FilterDecision, error) {
fp := FixedFingerprint{1}
ev := mustSanitizedEvidence(EventKindTextDelta, "default", "rule1", "desc", fp, 1, 0, FilterOutcomeKindEvaluated, time.Now())
dir := mustRecoveryDirectiveExact("req-ref-1")
intent := mustRecoveryIntent(RecoveryStrategyExactReplay, dir, "rule_violation", 10)
return NewFilterDecision(FilterDecisionKindViolation, "consumer1", "violating-filter", "rule1", ev, &intent)
},
}
reg, err := NewFilterRegistration(vF, "cap1", true, FilterEnforcementBlocking, 5*time.Second, 10)
if err != nil {
t.Fatalf("NewFilterRegistration: %v", err)
}
disp.handler = func(ctx context.Context, request RebuiltRequest) (AttemptBinding, error) {
txtEv := mustTextDelta("default", "violating text", time.Now())
src := newSliceEventSource([]NormalizedEvent{txtEv})
ctrl := &fixtureController{}
return NewAttemptBinding("att-next", "gpt-4", "openai", "primary", src, ctrl)
}
snap := createTestRuntimeSnapshot(t, []FilterRegistration{reg}, disp, rebuilder, sink)
txtEv := mustTextDelta("default", "violating text 1", time.Now())
src := newSliceEventSource([]NormalizedEvent{txtEv})
ctrl := &fixtureController{}
initialBinding, err := NewAttemptBinding("att-1", "gpt-4", "openai", "primary", src, ctrl)
if err != nil {
t.Fatalf("NewAttemptBinding: %v", err)
}
rt, err := NewRequestRuntime(snap, "group-a", initialBinding)
if err != nil {
t.Fatalf("NewRequestRuntime: %v", err)
}
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
if err := rt.Run(ctx); err != nil {
t.Fatalf("Run: %v", err)
}
sink.mu.Lock()
defer sink.mu.Unlock()
if len(sink.terminals) != 1 {
t.Fatalf("expected single terminal result on budget exhaustion, got %d", len(sink.terminals))
}
if sink.terminals[0].Success() {
t.Errorf("expected error terminal result on budget exhaustion")
}
})
t.Run("CallerCancellation", func(t *testing.T) {
sink := newFixtureSink()
disp := &fixtureDispatcher{}
rebuilder := &fixtureRebuilder{}
passF := &customMockFilter{id: "pass-filter"}
reg, err := NewFilterRegistration(passF, "cap1", true, FilterEnforcementBlocking, 5*time.Second, 10)
if err != nil {
t.Fatalf("NewFilterRegistration: %v", err)
}
snap := createTestRuntimeSnapshot(t, []FilterRegistration{reg}, disp, rebuilder, sink)
ctx, cancel := context.WithCancel(context.Background())
cancel()
txtEv, err := NewTextDeltaEvent("default", "text", time.Now())
if err != nil {
t.Fatalf("NewTextDeltaEvent: %v", err)
}
src := newSliceEventSource([]NormalizedEvent{txtEv})
ctrl := &fixtureController{}
initialBinding, err := NewAttemptBinding("att-1", "gpt-4", "openai", "primary", src, ctrl)
if err != nil {
t.Fatalf("NewAttemptBinding: %v", err)
}
rt, err := NewRequestRuntime(snap, "group-a", initialBinding)
if err != nil {
t.Fatalf("NewRequestRuntime: %v", err)
}
errRun := rt.Run(ctx)
if !errors.Is(errRun, context.Canceled) {
t.Errorf("expected context.Canceled, got %v", errRun)
}
})
}
func TestRequestRuntimePassThroughMatrix(t *testing.T) {
ctx := context.Background()
t.Run("NoFilters", func(t *testing.T) {
sink := newFixtureSink()
disp := &fixtureDispatcher{}
rebuilder := &fixtureRebuilder{}
snap := createTestRuntimeSnapshot(t, nil, disp, rebuilder, sink)
rsEv, err := NewResponseStartEvent("default", 200, nil, time.Now())
if err != nil {
t.Fatalf("NewResponseStartEvent: %v", err)
}
txtEv, err := NewTextDeltaEvent("default", "hello", time.Now())
if err != nil {
t.Fatalf("NewTextDeltaEvent: %v", err)
}
termEv, err := NewTerminalEvent("default", time.Now())
if err != nil {
t.Fatalf("NewTerminalEvent: %v", err)
}
src := newSliceEventSource([]NormalizedEvent{rsEv, txtEv, termEv})
ctrl := &fixtureController{}
initialBinding, err := NewAttemptBinding("att-1", "gpt-4", "openai", "primary", src, ctrl)
if err != nil {
t.Fatalf("NewAttemptBinding: %v", err)
}
rt, err := NewRequestRuntime(snap, "group-a", initialBinding)
if err != nil {
t.Fatalf("NewRequestRuntime: %v", err)
}
if err := rt.Run(ctx); err != nil {
t.Fatalf("Run: %v", err)
}
sink.mu.Lock()
defer sink.mu.Unlock()
if len(sink.starts) != 1 {
t.Errorf("starts = %d, want 1", len(sink.starts))
}
if len(sink.events) != 1 {
t.Errorf("events = %d, want 1", len(sink.events))
}
if len(sink.terminals) != 1 || !sink.terminals[0].Success() {
t.Errorf("terminals = %+v, want 1 success terminal", sink.terminals)
}
})
t.Run("AllDisabled", func(t *testing.T) {
sink := newFixtureSink()
disp := &fixtureDispatcher{}
rebuilder := &fixtureRebuilder{}
disF := &customMockFilter{id: "disabled-1"}
regDisabled, err := NewFilterRegistration(disF, "cap1", false, FilterEnforcementBlocking, 5*time.Second, 10)
if err != nil {
t.Fatalf("NewFilterRegistration: %v", err)
}
snap := createTestRuntimeSnapshot(t, []FilterRegistration{regDisabled}, disp, rebuilder, sink)
rsEv := mustResponseStart("default", 200, nil, time.Now())
txtEv := mustTextDelta("default", "hello", time.Now())
termEv := mustTerminal("default", time.Now())
src := newSliceEventSource([]NormalizedEvent{rsEv, txtEv, termEv})
ctrl := &fixtureController{}
initialBinding, err := NewAttemptBinding("att-1", "gpt-4", "openai", "primary", src, ctrl)
if err != nil {
t.Fatalf("NewAttemptBinding: %v", err)
}
rt, err := NewRequestRuntime(snap, "group-a", initialBinding)
if err != nil {
t.Fatalf("NewRequestRuntime: %v", err)
}
if err := rt.Run(ctx); err != nil {
t.Fatalf("Run: %v", err)
}
disF.mu.Lock()
if disF.evalCount != 0 {
t.Errorf("disabled filter evalCount = %d, want 0", disF.evalCount)
}
disF.mu.Unlock()
sink.mu.Lock()
defer sink.mu.Unlock()
if len(sink.starts) != 1 || len(sink.events) != 1 || len(sink.terminals) != 1 {
t.Errorf("sink counts mismatch: starts=%d, events=%d, term=%d", len(sink.starts), len(sink.events), len(sink.terminals))
}
})
t.Run("ObserveOnly", func(t *testing.T) {
sink := newFixtureSink()
disp := &fixtureDispatcher{}
rebuilder := &fixtureRebuilder{}
obsF := &customMockFilter{id: "observe-1"}
regObserve, err := NewFilterRegistration(obsF, "cap1", true, FilterEnforcementObserveOnly, 5*time.Second, 10)
if err != nil {
t.Fatalf("NewFilterRegistration: %v", err)
}
snap := createTestRuntimeSnapshot(t, []FilterRegistration{regObserve}, disp, rebuilder, sink)
rsEv := mustResponseStart("default", 200, nil, time.Now())
txtEv := mustTextDelta("default", "hello", time.Now())
termEv := mustTerminal("default", time.Now())
src := newSliceEventSource([]NormalizedEvent{rsEv, txtEv, termEv})
ctrl := &fixtureController{}
initialBinding, err := NewAttemptBinding("att-1", "gpt-4", "openai", "primary", src, ctrl)
if err != nil {
t.Fatalf("NewAttemptBinding: %v", err)
}
rt, err := NewRequestRuntime(snap, "group-a", initialBinding)
if err != nil {
t.Fatalf("NewRequestRuntime: %v", err)
}
if err := rt.Run(ctx); err != nil {
t.Fatalf("Run: %v", err)
}
sink.mu.Lock()
defer sink.mu.Unlock()
if len(sink.starts) != 1 || len(sink.events) != 1 || len(sink.terminals) != 1 {
t.Errorf("observe-only passthrough sink counts mismatch")
}
})
t.Run("BlockingFilterNonSubscribedEvent", func(t *testing.T) {
sink := newFixtureSink()
requirement := mustFilterHoldRequirementRolling("default", []EventKind{EventKindReasoningDelta}, 1)
filter := &customMockFilter{id: "reasoning-only", holdReq: &requirement}
reg := mustFilterRegistration(filter, "cap1", true, FilterEnforcementBlocking, 5*time.Second, 10)
snapshot := createTestRuntimeSnapshot(t, []FilterRegistration{reg}, &fixtureDispatcher{}, &fixtureRebuilder{}, sink)
initial := mustAttemptBinding(t, "att-1", newSliceEventSource([]NormalizedEvent{
mustResponseStart("default", 200, nil, testNow),
mustTextDelta("default", "unsubscribed", testNow),
mustTerminal("default", testNow),
}), &fixtureController{})
runtime, err := NewRequestRuntime(snapshot, "group-a", initial)
if err != nil {
t.Fatalf("NewRequestRuntime: %v", err)
}
if err := runtime.Run(ctx); err != nil {
t.Fatalf("Run: %v", err)
}
filter.mu.Lock()
evalCount := filter.evalCount
filter.mu.Unlock()
if evalCount != 0 {
t.Fatalf("non-subscribed filter eval count = %d, want 0", evalCount)
}
if len(sink.starts) != 1 || len(sink.events) != 1 || len(sink.terminals) != 1 || !sink.terminals[0].Success() {
t.Fatalf("sink counts starts/events/terminals = %d/%d/%d", len(sink.starts), len(sink.events), len(sink.terminals))
}
})
}
func TestRequestRuntimeContinuationLifecycle(t *testing.T) {
ctx := context.Background()
trace := &runtimeTrace{}
sink := newFixtureSink()
sink.trace = trace
disp := &fixtureDispatcher{trace: trace}
rebuilder := &fixtureRebuilder{trace: trace}
oldController := &fixtureController{name: "att-1", trace: trace}
newController := &fixtureController{name: "att-2", trace: trace}
var capturedLookBehind []string
requirement := mustFilterHoldRequirementRolling("default", []EventKind{EventKindTextDelta, EventKindTerminal}, 6)
filter := &customMockFilter{
id: "continuation-filter",
holdReq: &requirement,
evaluateFn: func(ctx context.Context, fc FilterContext, batch EvidenceBatch) (FilterDecision, error) {
kind := batch.Events()[0].Kind()
evidence := mustSanitizedEvidence(kind, "default", "rule1", "continuation_check", FixedFingerprint{1}, 1, 0, FilterOutcomeKindEvaluated, testNow)
text := ""
if kind == EventKindTextDelta {
var textErr error
text, textErr = batch.Events()[0].AsTextDelta()
if textErr != nil {
return FilterDecision{}, textErr
}
}
if fc.AttemptID() == "att-1" && text == "violation" {
directive := mustRecoveryDirectiveContinuation(1, "snapshot.one")
intent := mustRecoveryIntent(RecoveryStrategyContinuationRepair, directive, "continuation_required", 10)
return NewFilterDecision(FilterDecisionKindViolation, "consumer1", "continuation-filter", "rule1", evidence, &intent)
}
if fc.AttemptID() == "att-2" && text == "suffix" {
for _, ev := range batch.CommittedLookBehind()["default"] {
if ev.Kind() == EventKindTextDelta {
value, textErr := ev.AsTextDelta()
if textErr != nil {
return FilterDecision{}, textErr
}
capturedLookBehind = append(capturedLookBehind, value)
}
}
}
return NewFilterDecision(FilterDecisionKindPass, "consumer1", "continuation-filter", "rule1", evidence, nil)
},
}
reg := mustFilterRegistration(filter, "cap1", true, FilterEnforcementBlocking, 5*time.Second, 10)
disp.handler = func(ctx context.Context, request RebuiltRequest) (AttemptBinding, error) {
source := newSliceEventSource([]NormalizedEvent{
mustResponseStart("default", 200, nil, testNow.Add(time.Second)),
mustTextDelta("default", "suffix", testNow.Add(2*time.Second)),
mustTerminal("default", testNow.Add(3*time.Second)),
})
return NewAttemptBinding("att-2", "gpt-4", "openai", "primary", source, newController)
}
snapshot := createTestRuntimeSnapshot(t, []FilterRegistration{reg}, disp, rebuilder, sink)
initialSource := newSliceEventSource([]NormalizedEvent{
mustResponseStart("default", 200, nil, testNow),
mustTextDelta("default", "prefix", testNow),
mustTextDelta("default", "violation", testNow),
})
initial := mustAttemptBinding(t, "att-1", initialSource, oldController)
runtime, err := NewRequestRuntime(snapshot, "group-a", initial)
if err != nil {
t.Fatalf("NewRequestRuntime: %v", err)
}
if err := runtime.Run(ctx); err != nil {
t.Fatalf("Run: %v", err)
}
if runtime.boundary.CurrentAttempt() != "att-2" || runtime.boundary.State() != CommitStateTerminalCommitted {
t.Fatalf("boundary attempt/state = %q/%q, want att-2/terminal_committed", runtime.boundary.CurrentAttempt(), runtime.boundary.State())
}
assertControllerAbortCount(t, oldController, 1)
assertControllerAbortCount(t, newController, 0)
if len(sink.starts) != 1 || len(sink.events) != 2 || len(sink.terminals) != 1 || !sink.terminals[0].Success() {
t.Fatalf("sink counts starts/events/terminals = %d/%d/%d", len(sink.starts), len(sink.events), len(sink.terminals))
}
if got := releaseTexts(t, sink.events); !reflect.DeepEqual(got, []string{"prefix", "suffix"}) {
t.Fatalf("released texts = %v, want [prefix suffix]", got)
}
if !reflect.DeepEqual(capturedLookBehind, []string{"prefix"}) {
t.Fatalf("continuation look-behind = %v, want [prefix]", capturedLookBehind)
}
if got := runtime.tail.CommittedCursor("default"); got != 2 {
t.Fatalf("committed cursor = %d, want 2", got)
}
wantTrace := []string{"start", "release", "abort:att-1", "rebuild", "dispatch", "release", "terminal:success"}
if got := trace.snapshot(); !reflect.DeepEqual(got, wantTrace) {
t.Fatalf("trace = %v, want %v", got, wantTrace)
}
}
func mustTextDelta(ch, text string, ts time.Time) NormalizedEvent {
ev, err := NewTextDeltaEvent(ch, text, ts)
if err != nil {
panic(err)
}
return ev
}
func mustTerminal(ch string, ts time.Time) NormalizedEvent {
ev, err := NewTerminalEvent(ch, ts)
if err != nil {
panic(err)
}
return ev
}
func mustSanitizedEvidence(kind EventKind, channel, ruleID, desc string, fp FixedFingerprint, seq, offset int, outcome FilterOutcomeKind, ts time.Time) SanitizedEvidence {
ev, err := NewSanitizedEvidence(kind, channel, ruleID, desc, fp, seq, offset, outcome, ts)
if err != nil {
panic(err)
}
return ev
}
func mustRecoveryDirectiveExact(requestRef string) RecoveryDirective {
directive, err := NewRecoveryDirectiveExact(requestRef)
if err != nil {
panic(err)
}
return directive
}
func mustRecoveryDirectiveContinuation(cursor int, snapshotRef string) RecoveryDirective {
directive, err := NewRecoveryDirectiveContinuation(cursor, snapshotRef)
if err != nil {
panic(err)
}
return directive
}
func mustRecoveryIntent(strategy RecoveryStrategy, directive RecoveryDirective, reason string, cost int) RecoveryIntent {
intent, err := NewRecoveryIntent(strategy, directive, reason, cost)
if err != nil {
panic(err)
}
return intent
}
func mustResponseStart(ch string, code int, headers map[string]string, ts time.Time) NormalizedEvent {
ev, err := NewResponseStartEvent(ch, code, headers, ts)
if err != nil {
panic(err)
}
return ev
}
func mustReplacementProposal(t *testing.T, events []NormalizedEvent) *ReplacementProposal {
t.Helper()
proposal, err := NewReplacementProposal(events)
if err != nil {
t.Fatalf("NewReplacementProposal: %v", err)
}
return &proposal
}
func mustFilterRegistration(filter Filter, cap string, enabled bool, enforcement FilterEnforcement, timeout time.Duration, priority int) FilterRegistration {
registration, err := NewFilterRegistration(filter, cap, enabled, enforcement, timeout, priority)
if err != nil {
panic(err)
}
return registration
}
func mustFilterHoldRequirementRolling(channel string, kinds []EventKind, runes int) FilterHoldRequirement {
requirement, err := NewFilterHoldRequirementRolling(channel, kinds, runes)
if err != nil {
panic(err)
}
return requirement
}
func mustAttemptBinding(t *testing.T, id string, source NormalizedEventSource, controller AttemptController) AttemptBinding {
t.Helper()
binding, err := NewAttemptBinding(id, "gpt-4", "openai", "primary", source, controller)
if err != nil {
t.Fatalf("NewAttemptBinding(%s): %v", id, err)
}
return binding
}
func assertControllerAbortCount(t *testing.T, controller *fixtureController, want int) {
t.Helper()
controller.mu.Lock()
defer controller.mu.Unlock()
if controller.abortCount != want {
t.Fatalf("controller %q abort count = %d, want %d", controller.name, controller.abortCount, want)
}
}
func releaseTexts(t *testing.T, events []ReleaseEvent) []string {
t.Helper()
texts := make([]string, 0, len(events))
for _, event := range events {
text, err := event.AsTextDelta()
if err != nil {
t.Fatalf("ReleaseEvent.AsTextDelta: %v", err)
}
texts = append(texts, text)
}
return texts
}
func newViolationFilter(id string, strategy RecoveryStrategy, directive RecoveryDirective) *customMockFilter {
return &customMockFilter{
id: id,
evaluateFn: func(ctx context.Context, fc FilterContext, batch EvidenceBatch) (FilterDecision, error) {
kind := batch.Events()[0].Kind()
evidence := mustSanitizedEvidence(kind, "default", "rule1", "violation", FixedFingerprint{1}, 1, 0, FilterOutcomeKindEvaluated, testNow)
intent := mustRecoveryIntent(strategy, directive, "rule_violation", 10)
return NewFilterDecision(FilterDecisionKindViolation, "consumer1", id, "rule1", evidence, &intent)
},
}
}
func runtimeOptionsWithRecoveryCap(t *testing.T, cap int) RuntimeOptions {
t.Helper()
options, err := NewRuntimeOptions(1, 64, 1024, cap, GateCoordinatorOptions{}, RecoveryCoordinatorOptions{})
if err != nil {
t.Fatalf("NewRuntimeOptions: %v", err)
}
return options
}
func TestRequestRuntimeTerminalFailureFidelity(t *testing.T) {
t.Run("success_timestamp", func(t *testing.T) {
sink := newFixtureSink()
snapshot := createTestRuntimeSnapshot(t, nil, &fixtureDispatcher{}, &fixtureRebuilder{}, sink)
occurredAt := testNow.Add(7 * time.Second)
initial := mustAttemptBinding(t, "att-1", newSliceEventSource([]NormalizedEvent{mustTerminal("default", occurredAt)}), &fixtureController{})
runtime, err := NewRequestRuntime(snapshot, "group-a", initial)
if err != nil {
t.Fatalf("NewRequestRuntime: %v", err)
}
if err := runtime.Run(context.Background()); err != nil {
t.Fatalf("Run: %v", err)
}
if len(sink.terminals) != 1 || !sink.terminals[0].Success() || !sink.terminals[0].OccurredAt().Equal(occurredAt) {
t.Fatalf("success terminal = %+v, want exact event timestamp %v", sink.terminals, occurredAt)
}
})
t.Run("provider_identity_and_cause_cap", func(t *testing.T) {
sink := newFixtureSink()
snapshot := createTestRuntimeSnapshot(t, nil, &fixtureDispatcher{}, &fixtureRebuilder{}, sink)
descriptor, err := NewExternalDescriptor("provider_error", "rate_limit_exceeded", "rate_limit_reached", "requests")
if err != nil {
t.Fatalf("NewExternalDescriptor: %v", err)
}
causeValues := make([]FailureCause, 0, 5)
for _, code := range []string{"cause1", "cause2", "cause3", "cause4", "cause5"} {
cause, causeErr := NewFailureCause("provider", code, "consumer1", "filter1", "rule1")
if causeErr != nil {
t.Fatalf("NewFailureCause(%s): %v", code, causeErr)
}
causeValues = append(causeValues, cause)
}
causes, err := NewFailureCauseChain(causeValues)
if err != nil {
t.Fatalf("NewFailureCauseChain: %v", err)
}
occurredAt := testNow.Add(8 * time.Second)
providerEvent, err := NewProviderErrorEvent("default", descriptor, causes, occurredAt)
if err != nil {
t.Fatalf("NewProviderErrorEvent: %v", err)
}
initial := mustAttemptBinding(t, "att-1", newSliceEventSource([]NormalizedEvent{providerEvent}), &fixtureController{})
runtime, err := NewRequestRuntime(snapshot, "group-a", initial)
if err != nil {
t.Fatalf("NewRequestRuntime: %v", err)
}
if err := runtime.Run(context.Background()); err != nil {
t.Fatalf("Run: %v", err)
}
if len(sink.terminals) != 1 {
t.Fatalf("terminal count = %d, want 1", len(sink.terminals))
}
terminal := sink.terminals[0]
gotDescriptor := terminal.ExternalDesc()
if gotDescriptor == nil || gotDescriptor.Type() != "provider_error" || gotDescriptor.Code() != "rate_limit_exceeded" || gotDescriptor.Message() != "rate_limit_reached" || gotDescriptor.Param() != "requests" {
t.Fatalf("provider descriptor = %+v", gotDescriptor)
}
if !terminal.OccurredAt().Equal(occurredAt) || terminal.FailureCauses().Len() != MaxFailureCauses {
t.Fatalf("provider timestamp/cause count = %v/%d", terminal.OccurredAt(), terminal.FailureCauses().Len())
}
if terminal.FailureCauses().All()[0].Code() != "cause2" || terminal.FailureCauses().All()[3].Code() != "cause5" {
t.Fatalf("provider cause cap = %+v, want latest four", terminal.FailureCauses().All())
}
})
t.Run("fatal_filter_identity", func(t *testing.T) {
sink := newFixtureSink()
filter := &customMockFilter{
id: "fatal-filter",
evaluateFn: func(ctx context.Context, fc FilterContext, batch EvidenceBatch) (FilterDecision, error) {
evidence := mustSanitizedEvidence(EventKindTextDelta, "default", "fatal-rule", "fatal_violation", FixedFingerprint{2}, 1, 0, FilterOutcomeKindEvaluated, testNow)
return NewFilterDecision(FilterDecisionKindFatal, "consumer1", "fatal-filter", "fatal-rule", evidence, nil)
},
}
reg := mustFilterRegistration(filter, "cap1", true, FilterEnforcementBlocking, 5*time.Second, 10)
snapshot := createTestRuntimeSnapshot(t, []FilterRegistration{reg}, &fixtureDispatcher{}, &fixtureRebuilder{}, sink)
initial := mustAttemptBinding(t, "att-1", newSliceEventSource([]NormalizedEvent{mustTextDelta("default", "bad", testNow)}), &fixtureController{})
runtime, err := NewRequestRuntime(snapshot, "group-a", initial)
if err != nil {
t.Fatalf("NewRequestRuntime: %v", err)
}
if err := runtime.Run(context.Background()); err != nil {
t.Fatalf("Run: %v", err)
}
if len(sink.terminals) != 1 || sink.terminals[0].Success() {
t.Fatalf("fatal terminals = %+v", sink.terminals)
}
causes := sink.terminals[0].FailureCauses().All()
if len(causes) != 1 || causes[0].Consumer() != "" || causes[0].Filter() != "fatal-filter" || causes[0].RuleID() != "fatal-rule" {
t.Fatalf("fatal causes = %+v", causes)
}
})
}
func TestRequestRuntimeAtomicInstallFailure(t *testing.T) {
for _, abortFails := range []bool{false, true} {
name := "prepare_abort_succeeds"
if abortFails {
name = "prepare_abort_fails"
}
t.Run(name, func(t *testing.T) {
trace := &runtimeTrace{}
sink := newFixtureSink()
sink.trace = trace
dispatcher := &fixtureDispatcher{trace: trace}
rebuilder := &fixtureRebuilder{trace: trace}
oldController := &fixtureController{name: "old", trace: trace}
newController := &fixtureController{name: "new", trace: trace}
if abortFails {
newController.abortErr = errors.New("raw controller failure")
}
filter := newViolationFilter("install-filter", RecoveryStrategyExactReplay, mustRecoveryDirectiveExact("req-ref-1"))
filter.holdReqFn = func(fc FilterContext) FilterHoldRequirement {
if fc.AttemptID() == "att-2" {
return FilterHoldRequirement{}
}
return mustFilterHoldRequirementRolling("default", []EventKind{EventKindTextDelta, EventKindTerminal}, 1)
}
reg := mustFilterRegistration(filter, "cap1", true, FilterEnforcementBlocking, 5*time.Second, 10)
dispatcher.handler = func(ctx context.Context, request RebuiltRequest) (AttemptBinding, error) {
return NewAttemptBinding("att-2", "gpt-4", "openai", "primary", newSliceEventSource(nil), newController)
}
snapshot := createTestRuntimeSnapshot(t, []FilterRegistration{reg}, dispatcher, rebuilder, sink)
initial := mustAttemptBinding(t, "att-1", newSliceEventSource([]NormalizedEvent{mustTextDelta("default", "bad", testNow)}), oldController)
runtime, err := NewRequestRuntime(snapshot, "group-a", initial)
if err != nil {
t.Fatalf("NewRequestRuntime: %v", err)
}
if err := runtime.Run(context.Background()); err != nil {
t.Fatalf("Run: %v", err)
}
assertControllerAbortCount(t, oldController, 1)
assertControllerAbortCount(t, newController, 1)
if len(sink.terminals) != 1 || sink.terminals[0].Success() {
t.Fatalf("terminals = %+v", sink.terminals)
}
causeCodes := make([]string, 0, sink.terminals[0].FailureCauses().Len())
for _, cause := range sink.terminals[0].FailureCauses().All() {
causeCodes = append(causeCodes, cause.Code())
}
wantCauses := []string{"install_prepare_failed"}
if abortFails {
wantCauses = append(wantCauses, "attempt_abort_failed")
}
if !reflect.DeepEqual(causeCodes, wantCauses) {
t.Fatalf("cause codes = %v, want %v", causeCodes, wantCauses)
}
wantTrace := []string{"abort:old", "rebuild", "dispatch", "abort:new", "terminal:error"}
if got := trace.snapshot(); !reflect.DeepEqual(got, wantTrace) {
t.Fatalf("trace = %v, want %v", got, wantTrace)
}
})
}
t.Run("continuation_commit_failure_aborts_dispatched_binding", func(t *testing.T) {
trace := &runtimeTrace{}
sink := newFixtureSink()
sink.trace = trace
dispatcher := &fixtureDispatcher{trace: trace}
rebuilder := &fixtureRebuilder{trace: trace}
oldController := &fixtureController{name: "old", trace: trace}
newController := &fixtureController{name: "new", trace: trace}
filter := &customMockFilter{
id: "commit-filter",
evaluateFn: func(ctx context.Context, fc FilterContext, batch EvidenceBatch) (FilterDecision, error) {
kind := batch.Events()[0].Kind()
evidence := mustSanitizedEvidence(kind, "default", "rule1", "commit_check", FixedFingerprint{1}, 1, 0, FilterOutcomeKindEvaluated, testNow)
text := ""
if kind == EventKindTextDelta {
var textErr error
text, textErr = batch.Events()[0].AsTextDelta()
if textErr != nil {
return FilterDecision{}, textErr
}
}
if text == "bad" {
directive := mustRecoveryDirectiveContinuation(1, "snapshot.one")
intent := mustRecoveryIntent(RecoveryStrategyContinuationRepair, directive, "continuation_required", 10)
return NewFilterDecision(FilterDecisionKindViolation, "consumer1", "commit-filter", "rule1", evidence, &intent)
}
return NewFilterDecision(FilterDecisionKindPass, "consumer1", "commit-filter", "rule1", evidence, nil)
},
}
reg := mustFilterRegistration(filter, "cap1", true, FilterEnforcementBlocking, 5*time.Second, 10)
dispatcher.handler = func(ctx context.Context, request RebuiltRequest) (AttemptBinding, error) {
return NewAttemptBinding("att-1", "gpt-4", "openai", "primary", newSliceEventSource(nil), newController)
}
snapshot := createTestRuntimeSnapshot(t, []FilterRegistration{reg}, dispatcher, rebuilder, sink)
initial := mustAttemptBinding(t, "att-1", newSliceEventSource([]NormalizedEvent{
mustResponseStart("default", 200, nil, testNow),
mustTextDelta("default", "prefix", testNow),
mustTextDelta("default", "bad", testNow),
}), oldController)
runtime, err := NewRequestRuntime(snapshot, "group-a", initial)
if err != nil {
t.Fatalf("NewRequestRuntime: %v", err)
}
if err := runtime.Run(context.Background()); err != nil {
t.Fatalf("Run: %v", err)
}
assertControllerAbortCount(t, oldController, 1)
assertControllerAbortCount(t, newController, 1)
if len(sink.terminals) != 1 || sink.terminals[0].FailureCauses().All()[0].Code() != "install_commit_failed" {
t.Fatalf("commit failure terminal = %+v", sink.terminals)
}
wantTrace := []string{"start", "release", "abort:old", "rebuild", "dispatch", "abort:new", "terminal:error"}
if got := trace.snapshot(); !reflect.DeepEqual(got, wantTrace) {
t.Fatalf("trace = %v, want %v", got, wantTrace)
}
})
}
func TestRequestRuntimeRecoveryFailureOrder(t *testing.T) {
for _, tc := range []struct {
name string
controllerErr error
rebuildErr error
dispatchErr error
wantTrace []string
}{
{name: "controller_failure", controllerErr: errors.New("raw abort failure"), wantTrace: []string{"abort:old", "abort:old", "terminal:error"}},
{name: "rebuilder_failure", rebuildErr: errors.New("raw rebuild failure"), wantTrace: []string{"abort:old", "rebuild", "terminal:error"}},
{name: "dispatcher_failure", dispatchErr: errors.New("raw dispatch failure"), wantTrace: []string{"abort:old", "rebuild", "dispatch", "terminal:error"}},
} {
t.Run(tc.name, func(t *testing.T) {
trace := &runtimeTrace{}
sink := newFixtureSink()
sink.trace = trace
rebuilder := &fixtureRebuilder{err: tc.rebuildErr, trace: trace}
dispatcher := &fixtureDispatcher{trace: trace}
if tc.dispatchErr != nil {
dispatcher.handler = func(ctx context.Context, request RebuiltRequest) (AttemptBinding, error) {
return AttemptBinding{}, tc.dispatchErr
}
}
filter := newViolationFilter("failure-filter", RecoveryStrategyExactReplay, mustRecoveryDirectiveExact("req-ref-1"))
reg := mustFilterRegistration(filter, "cap1", true, FilterEnforcementBlocking, 5*time.Second, 10)
controller := &fixtureController{name: "old", abortErr: tc.controllerErr, trace: trace}
snapshot := createTestRuntimeSnapshot(t, []FilterRegistration{reg}, dispatcher, rebuilder, sink)
initial := mustAttemptBinding(t, "att-1", newSliceEventSource([]NormalizedEvent{mustTextDelta("default", "bad", testNow)}), controller)
runtime, err := NewRequestRuntime(snapshot, "group-a", initial)
if err != nil {
t.Fatalf("NewRequestRuntime: %v", err)
}
if err := runtime.Run(context.Background()); err != nil {
t.Fatalf("Run: %v", err)
}
wantAbortCount := 1
if tc.controllerErr != nil {
wantAbortCount = 2
}
assertControllerAbortCount(t, controller, wantAbortCount)
if len(sink.terminals) != 1 || sink.terminals[0].Success() {
t.Fatalf("terminals = %+v", sink.terminals)
}
if got := trace.snapshot(); !reflect.DeepEqual(got, tc.wantTrace) {
t.Fatalf("trace = %v, want %v", got, tc.wantTrace)
}
})
}
for _, tc := range []struct {
name string
prepareErr error
releaseErr error
wantTrace []string
wantSuccess bool
}{
{name: "preparation_snapshot_released_before_rebuild", wantTrace: []string{"snapshot:create", "abort:old", "prepare", "snapshot:release", "rebuild", "dispatch", "terminal:success"}, wantSuccess: true},
{name: "preparer_failure_releases_snapshot", prepareErr: errors.New("raw preparation failure"), wantTrace: []string{"snapshot:create", "abort:old", "prepare", "snapshot:release", "terminal:error"}},
{name: "preparation_snapshot_release_failure_stops_cycle", releaseErr: errors.New("raw snapshot release failure"), wantTrace: []string{"snapshot:create", "abort:old", "prepare", "snapshot:release", "terminal:error"}},
} {
t.Run(tc.name, func(t *testing.T) {
trace := &runtimeTrace{}
sink := newFixtureSink()
sink.trace = trace
rebuilder := &fixtureRebuilder{trace: trace}
dispatcher := &fixtureDispatcher{trace: trace}
oldController := &fixtureController{name: "old", trace: trace}
preparedSnapshot := &testPrepSnapshot{ref: "prep-ref-1", trace: trace}
preparedSnapshot.releaseErr = tc.releaseErr
factory := &testPrepFactory{prepared: preparedSnapshot, trace: trace}
preparer := &testPreparer{err: tc.prepareErr, trace: trace}
filter := &customMockFilter{
id: "prepared-filter",
evaluateFn: func(ctx context.Context, fc FilterContext, batch EvidenceBatch) (FilterDecision, error) {
kind := batch.Events()[0].Kind()
evidence := mustSanitizedEvidence(kind, "default", "rule1", "prepared_check", FixedFingerprint{1}, 1, 0, FilterOutcomeKindEvaluated, testNow)
if fc.AttemptID() == "att-2" {
return NewFilterDecision(FilterDecisionKindPass, "consumer1", "prepared-filter", "rule1", evidence, nil)
}
intent := mustRecoveryIntent(RecoveryStrategyExactReplay, mustRecoveryDirectiveExact("req-ref-1"), "rule_violation", 10)
return NewFilterDecision(FilterDecisionKindViolation, "consumer1", "prepared-filter", "rule1", evidence, &intent)
},
}
reg := mustFilterRegistration(filter, "cap1", true, FilterEnforcementBlocking, 5*time.Second, 10)
dispatcher.handler = func(ctx context.Context, request RebuiltRequest) (AttemptBinding, error) {
return NewAttemptBinding("att-2", "gpt-4", "openai", "primary", newSliceEventSource([]NormalizedEvent{mustTerminal("default", testNow)}), &fixtureController{})
}
snapshot := createTestRuntimeSnapshotWithPreparer(t, []FilterRegistration{reg}, dispatcher, rebuilder, preparer, factory, sink)
initial := mustAttemptBinding(t, "att-1", newSliceEventSource([]NormalizedEvent{mustTextDelta("default", "bad", testNow)}), oldController)
runtime, err := NewRequestRuntime(snapshot, "group-a", initial)
if err != nil {
t.Fatalf("NewRequestRuntime: %v", err)
}
if err := runtime.Run(context.Background()); err != nil {
t.Fatalf("Run: %v", err)
}
assertControllerAbortCount(t, oldController, 1)
preparedSnapshot.mu.Lock()
releaseCount := preparedSnapshot.releaseCount
preparedSnapshot.mu.Unlock()
if releaseCount != 1 {
t.Fatalf("snapshot release count = %d, want 1", releaseCount)
}
if got := trace.snapshot(); !reflect.DeepEqual(got, tc.wantTrace) {
t.Fatalf("trace = %v, want %v", got, tc.wantTrace)
}
if len(sink.terminals) != 1 || sink.terminals[0].Success() != tc.wantSuccess {
t.Fatalf("terminal = %+v, wantSuccess=%v", sink.terminals, tc.wantSuccess)
}
})
}
}
// newReplacementFilter returns a blocking filter that substitutes the given
// proposal for a single original text delta and passes every other event.
func newReplacementFilter(id, originalText string, proposal *ReplacementProposal) *customMockFilter {
return &customMockFilter{
id: id,
evaluateFn: func(ctx context.Context, fc FilterContext, batch EvidenceBatch) (FilterDecision, error) {
event := batch.Events()[0]
evidence := mustSanitizedEvidence(event.Kind(), "default", "rule1", "replacement_check", FixedFingerprint{3}, 1, 0, FilterOutcomeKindEvaluated, testNow)
if event.Kind() != EventKindTextDelta {
return NewFilterDecision(FilterDecisionKindPass, "consumer1", id, "rule1", evidence, nil)
}
text, err := event.AsTextDelta()
if err != nil {
return FilterDecision{}, err
}
if text != originalText {
return NewFilterDecision(FilterDecisionKindPass, "consumer1", id, "rule1", evidence, nil)
}
return NewFilterDecisionWithReplacement(FilterDecisionKindReplacement, "consumer1", id, "rule1", evidence, nil, proposal)
},
}
}
// altHeldText is the unrelated-channel event that is already pending when the
// replacement settles; altReleaseText then crosses that channel's 100-rune
// rolling threshold so both entries release together afterwards.
const altHeldText = "alt-held"
func TestRequestRuntimeReplacementProposalRelease(t *testing.T) {
altReleaseText := repeatRune('z', 100)
// The runtime fixture drives two blocking channels from one event source:
// "default" carries the replaced content, and "alt" holds unrelated pending
// evidence across the replacement because its threshold is not yet met.
newReplacementRuntime := func(t *testing.T, trace *runtimeTrace, sink *fixtureSink) *RequestRuntime {
t.Helper()
proposal := mustReplacementProposal(t, []NormalizedEvent{
mustTextDelta("default", "replacement-1", testNow),
mustTextDelta("default", "replacement-2", testNow),
})
filter := newReplacementFilter("replacement-filter", "original", proposal)
reg := mustFilterRegistration(filter, "cap1", true, FilterEnforcementBlocking, 5*time.Second, 10)
altReq := mustFilterHoldRequirementRolling("alt", []EventKind{EventKindTextDelta}, 100)
altReg := mustFilterRegistration(
&customMockFilter{id: "alt-filter", holdReq: &altReq},
"cap2", true, FilterEnforcementBlocking, 5*time.Second, 5,
)
snapshot := createTestRuntimeSnapshot(t, []FilterRegistration{reg, altReg}, &fixtureDispatcher{trace: trace}, &fixtureRebuilder{trace: trace}, sink)
initial := mustAttemptBinding(t, "att-1", newSliceEventSource([]NormalizedEvent{
mustResponseStart("default", 200, map[string]string{"X-Attempt": "one"}, testNow),
mustTextDelta("alt", altHeldText, testNow),
mustTextDelta("default", "original", testNow),
mustTextDelta("alt", altReleaseText, testNow),
mustTerminal("default", testNow),
}), &fixtureController{name: "att-1", trace: trace})
runtime, err := NewRequestRuntime(snapshot, "group-a", initial)
if err != nil {
t.Fatalf("NewRequestRuntime: %v", err)
}
return runtime
}
t.Run("substitutes_ordered_replacement", func(t *testing.T) {
trace := &runtimeTrace{}
sink := newFixtureSink()
sink.trace = trace
runtime := newReplacementRuntime(t, trace, sink)
if err := runtime.Run(context.Background()); err != nil {
t.Fatalf("Run: %v", err)
}
// The replacement substitutes only the default channel's held content;
// the unrelated channel's pending evidence survives and releases after.
wantTexts := []string{"replacement-1", "replacement-2", altHeldText, altReleaseText}
if got := releaseTexts(t, sink.events); !reflect.DeepEqual(got, wantTexts) {
t.Fatalf("released texts = %v, want %v", got, wantTexts)
}
if len(sink.starts) != 1 || sink.starts[0].Status() != 200 || sink.starts[0].Headers()["X-Attempt"] != "one" {
t.Fatalf("starts = %+v, want exactly one 200/one start", sink.starts)
}
if len(sink.terminals) != 1 || !sink.terminals[0].Success() {
t.Fatalf("terminals = %+v, want exactly one success terminal", sink.terminals)
}
wantTrace := []string{"start", "release", "release", "release", "release", "terminal:success"}
if got := trace.snapshot(); !reflect.DeepEqual(got, wantTrace) {
t.Fatalf("trace = %v, want %v", got, wantTrace)
}
})
t.Run("sink_release_failure_stops_output", func(t *testing.T) {
trace := &runtimeTrace{}
sink := newFixtureSink()
sink.trace = trace
sink.releaseErr = errors.New("raw sink release failure")
runtime := newReplacementRuntime(t, trace, sink)
runErr := runtime.Run(context.Background())
if runErr == nil {
t.Fatal("Run error = nil, want sink release failure")
}
wantTexts := []string{"replacement-1"}
if got := releaseTexts(t, sink.events); !reflect.DeepEqual(got, wantTexts) {
t.Fatalf("released texts = %v, want %v", got, wantTexts)
}
if len(sink.terminals) != 0 {
t.Fatalf("terminals = %+v, want none after replacement release failure", sink.terminals)
}
wantTrace := []string{"start", "release"}
if got := trace.snapshot(); !reflect.DeepEqual(got, wantTrace) {
t.Fatalf("trace = %v, want %v", got, wantTrace)
}
// A failed replacement must not take the unrelated channel's pending
// evidence down with it.
if got := pendingTexts(t, runtime.tail, "alt"); !reflect.DeepEqual(got, []string{altHeldText}) {
t.Fatalf("pending on alt = %v, want [%s]", got, altHeldText)
}
if got := pendingTexts(t, runtime.tail, "default"); len(got) != 0 {
t.Fatalf("pending on default = %v, want none after the replacement consumed it", got)
}
})
}
func TestRequestRuntimeReplaceAttemptStagedStartIsolation(t *testing.T) {
type observedBatchStart struct {
attemptID string
start *ResponseStart
}
for _, tc := range []struct {
name string
newAttemptEvents []NormalizedEvent
wantNewStart bool
}{
{
name: "new_attempt_without_response_start",
newAttemptEvents: []NormalizedEvent{
mustTextDelta("default", "clean", testNow),
mustTerminal("default", testNow),
},
},
{
name: "new_attempt_with_new_response_start",
newAttemptEvents: []NormalizedEvent{
mustResponseStart("default", 200, map[string]string{"X-Attempt": "new"}, testNow),
mustTextDelta("default", "clean", testNow),
mustTerminal("default", testNow),
},
wantNewStart: true,
},
} {
t.Run(tc.name, func(t *testing.T) {
trace := &runtimeTrace{}
sink := newFixtureSink()
sink.trace = trace
dispatcher := &fixtureDispatcher{trace: trace}
rebuilder := &fixtureRebuilder{trace: trace}
oldController := &fixtureController{name: "att-1", trace: trace}
newController := &fixtureController{name: "att-2", trace: trace}
var observedMu sync.Mutex
var observed []observedBatchStart
filter := &customMockFilter{
id: "start-isolation-filter",
evaluateFn: func(ctx context.Context, fc FilterContext, batch EvidenceBatch) (FilterDecision, error) {
observedMu.Lock()
observed = append(observed, observedBatchStart{attemptID: fc.AttemptID(), start: batch.StagedResponseStart()})
observedMu.Unlock()
evidence := mustSanitizedEvidence(batch.Events()[0].Kind(), "default", "rule1", "start_isolation", FixedFingerprint{5}, 1, 0, FilterOutcomeKindEvaluated, testNow)
if fc.AttemptID() != "att-1" {
return NewFilterDecision(FilterDecisionKindPass, "consumer1", "start-isolation-filter", "rule1", evidence, nil)
}
intent := mustRecoveryIntent(RecoveryStrategyExactReplay, mustRecoveryDirectiveExact("req-ref-1"), "rule_violation", 10)
return NewFilterDecision(FilterDecisionKindViolation, "consumer1", "start-isolation-filter", "rule1", evidence, &intent)
},
}
reg := mustFilterRegistration(filter, "cap1", true, FilterEnforcementBlocking, 5*time.Second, 10)
dispatcher.handler = func(ctx context.Context, request RebuiltRequest) (AttemptBinding, error) {
return NewAttemptBinding("att-2", "gpt-4", "openai", "primary", newSliceEventSource(tc.newAttemptEvents), newController)
}
snapshot := createTestRuntimeSnapshot(t, []FilterRegistration{reg}, dispatcher, rebuilder, sink)
initial := mustAttemptBinding(t, "att-1", newSliceEventSource([]NormalizedEvent{
mustResponseStart("default", 418, map[string]string{"X-Attempt": "old"}, testNow),
mustTextDelta("default", "bad", testNow),
}), oldController)
runtime, err := NewRequestRuntime(snapshot, "group-a", initial)
if err != nil {
t.Fatalf("NewRequestRuntime: %v", err)
}
if err := runtime.Run(context.Background()); err != nil {
t.Fatalf("Run: %v", err)
}
observedMu.Lock()
defer observedMu.Unlock()
oldStartObserved := false
newStartObserved := 0
for _, obs := range observed {
if obs.attemptID == "att-1" {
if obs.start != nil && obs.start.Status() == 418 && obs.start.Headers()["X-Attempt"] == "old" {
oldStartObserved = true
}
continue
}
if obs.start == nil {
continue
}
if obs.start.Status() != 200 || obs.start.Headers()["X-Attempt"] != "new" {
t.Fatalf("attempt %s batch staged start = %d/%v, want no stale old start", obs.attemptID, obs.start.Status(), obs.start.Headers())
}
newStartObserved++
}
if !oldStartObserved {
t.Fatal("att-1 never observed the old staged start; fixture does not exercise the isolation path")
}
if tc.wantNewStart && newStartObserved != 1 {
t.Fatalf("att-2 batches carrying the new staged start = %d, want 1", newStartObserved)
}
if !tc.wantNewStart && newStartObserved != 0 {
t.Fatalf("att-2 batches carrying a staged start = %d, want 0", newStartObserved)
}
wantStarts := 0
if tc.wantNewStart {
wantStarts = 1
}
if len(sink.starts) != wantStarts {
t.Fatalf("sink starts = %+v, want %d", sink.starts, wantStarts)
}
for _, start := range sink.starts {
if start.Status() != 200 || start.Headers()["X-Attempt"] != "new" {
t.Fatalf("sink start = %d/%v, want only the new attempt start", start.Status(), start.Headers())
}
}
if got := releaseTexts(t, sink.events); !reflect.DeepEqual(got, []string{"clean"}) {
t.Fatalf("released texts = %v, want [clean]", got)
}
if len(sink.terminals) != 1 || !sink.terminals[0].Success() {
t.Fatalf("terminals = %+v, want exactly one success terminal", sink.terminals)
}
if len(dispatcher.dispatched) != 1 {
t.Fatalf("dispatch count = %d, want 1", len(dispatcher.dispatched))
}
assertControllerAbortCount(t, oldController, 1)
assertControllerAbortCount(t, newController, 0)
})
}
}
func TestRequestRuntimeLifecycleMatrix(t *testing.T) {
t.Run("recovery_caps_0_1_3", func(t *testing.T) {
for _, cap := range []int{0, 1, 3} {
t.Run(fmt.Sprintf("cap_%d", cap), func(t *testing.T) {
trace := &runtimeTrace{}
sink := newFixtureSink()
sink.trace = trace
dispatcher := &fixtureDispatcher{trace: trace}
rebuilder := &fixtureRebuilder{trace: trace}
controllers := []*fixtureController{{name: "att-1", trace: trace}}
filter := newViolationFilter("cap-filter", RecoveryStrategyExactReplay, mustRecoveryDirectiveExact("req-ref-1"))
reg := mustFilterRegistration(filter, "cap1", true, FilterEnforcementBlocking, 5*time.Second, 10)
dispatcher.handler = func(ctx context.Context, request RebuiltRequest) (AttemptBinding, error) {
attemptNumber := len(controllers) + 1
controller := &fixtureController{name: fmt.Sprintf("att-%d", attemptNumber), trace: trace}
controllers = append(controllers, controller)
return NewAttemptBinding(fmt.Sprintf("att-%d", attemptNumber), "gpt-4", "openai", "primary", newSliceEventSource([]NormalizedEvent{mustTextDelta("default", "bad", testNow)}), controller)
}
options := runtimeOptionsWithRecoveryCap(t, cap)
snapshot := createTestRuntimeSnapshotWithOptions(t, []FilterRegistration{reg}, dispatcher, rebuilder, options, sink)
initial := mustAttemptBinding(t, "att-1", newSliceEventSource([]NormalizedEvent{mustTextDelta("default", "bad", testNow)}), controllers[0])
runtime, err := NewRequestRuntime(snapshot, "group-a", initial)
if err != nil {
t.Fatalf("NewRequestRuntime: %v", err)
}
if err := runtime.Run(context.Background()); err != nil {
t.Fatalf("Run: %v", err)
}
if len(dispatcher.dispatched) != cap || rebuilder.planCount != cap || len(controllers) != cap+1 {
t.Fatalf("dispatch/rebuild/controllers = %d/%d/%d, want %d/%d/%d", len(dispatcher.dispatched), rebuilder.planCount, len(controllers), cap, cap, cap+1)
}
for _, controller := range controllers {
assertControllerAbortCount(t, controller, 1)
}
if len(sink.starts) != 0 || len(sink.events) != 0 || len(sink.terminals) != 1 || sink.terminals[0].Success() {
t.Fatalf("sink counts starts/events/terminals = %d/%d/%d", len(sink.starts), len(sink.events), len(sink.terminals))
}
wantTrace := make([]string, 0, cap*3+2)
for i := 1; i <= cap; i++ {
wantTrace = append(wantTrace, fmt.Sprintf("abort:att-%d", i), "rebuild", "dispatch")
}
wantTrace = append(wantTrace, fmt.Sprintf("abort:att-%d", cap+1), "terminal:error")
if got := trace.snapshot(); !reflect.DeepEqual(got, wantTrace) {
t.Fatalf("trace = %v, want %v", got, wantTrace)
}
})
}
})
for _, enforcement := range []FilterEnforcement{FilterEnforcementBlocking, FilterEnforcementObserveOnly} {
name := string(enforcement) + "_evaluation_error"
t.Run(name, func(t *testing.T) {
sink := newFixtureSink()
filter := &customMockFilter{
id: "error-filter",
evaluateFn: func(ctx context.Context, fc FilterContext, batch EvidenceBatch) (FilterDecision, error) {
return FilterDecision{}, errors.New("raw filter failure")
},
}
reg := mustFilterRegistration(filter, "cap1", true, enforcement, 5*time.Second, 10)
snapshot := createTestRuntimeSnapshot(t, []FilterRegistration{reg}, &fixtureDispatcher{}, &fixtureRebuilder{}, sink)
initial := mustAttemptBinding(t, "att-1", newSliceEventSource([]NormalizedEvent{
mustResponseStart("default", 200, nil, testNow),
mustTextDelta("default", "text", testNow),
mustTerminal("default", testNow),
}), &fixtureController{})
runtime, err := NewRequestRuntime(snapshot, "group-a", initial)
if err != nil {
t.Fatalf("NewRequestRuntime: %v", err)
}
if err := runtime.Run(context.Background()); err != nil {
t.Fatalf("Run: %v", err)
}
if len(sink.terminals) != 1 {
t.Fatalf("terminal count = %d, want 1", len(sink.terminals))
}
if enforcement == FilterEnforcementBlocking {
if sink.terminals[0].Success() || len(sink.events) != 0 {
t.Fatalf("blocking evaluation error exposed output: events=%d terminal=%+v", len(sink.events), sink.terminals[0])
}
} else if !sink.terminals[0].Success() || len(sink.events) != 1 {
t.Fatalf("observe-only evaluation error did not pass through: events=%d terminal=%+v", len(sink.events), sink.terminals[0])
}
})
}
t.Run("cancellation_during_evaluation", func(t *testing.T) {
entered := make(chan struct{})
filter := &customMockFilter{
id: "cancel-filter",
evaluateFn: func(ctx context.Context, fc FilterContext, batch EvidenceBatch) (FilterDecision, error) {
close(entered)
<-ctx.Done()
return FilterDecision{}, ctx.Err()
},
}
reg := mustFilterRegistration(filter, "cap1", true, FilterEnforcementBlocking, 5*time.Second, 10)
sink := newFixtureSink()
snapshot := createTestRuntimeSnapshot(t, []FilterRegistration{reg}, &fixtureDispatcher{}, &fixtureRebuilder{}, sink)
initial := mustAttemptBinding(t, "att-1", newSliceEventSource([]NormalizedEvent{mustTextDelta("default", "text", testNow)}), &fixtureController{})
runtime, err := NewRequestRuntime(snapshot, "group-a", initial)
if err != nil {
t.Fatalf("NewRequestRuntime: %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- runtime.Run(ctx) }()
<-entered
cancel()
if runErr := <-done; !errors.Is(runErr, context.Canceled) {
t.Fatalf("Run error = %v, want context.Canceled", runErr)
}
if len(sink.starts) != 0 || len(sink.events) != 0 || len(sink.terminals) != 0 {
t.Fatalf("canceled run exposed sink calls: %d/%d/%d", len(sink.starts), len(sink.events), len(sink.terminals))
}
})
t.Run("backpressure_blocks_next_ingress", func(t *testing.T) {
entered := make(chan struct{})
release := make(chan struct{})
filter := &customMockFilter{
id: "backpressure-filter",
evaluateFn: func(ctx context.Context, fc FilterContext, batch EvidenceBatch) (FilterDecision, error) {
event := batch.Events()[0]
evidence := mustSanitizedEvidence(event.Kind(), "default", "rule1", "backpressure", FixedFingerprint{4}, 1, 0, FilterOutcomeKindEvaluated, testNow)
if event.Kind() == EventKindTextDelta {
close(entered)
<-release
}
return NewFilterDecision(FilterDecisionKindPass, "consumer1", "backpressure-filter", "rule1", evidence, nil)
},
}
reg := mustFilterRegistration(filter, "cap1", true, FilterEnforcementBlocking, 5*time.Second, 10)
sink := newFixtureSink()
source := newCountingEventSource([]NormalizedEvent{
mustTextDelta("default", "hold", testNow),
mustTerminal("default", testNow),
})
snapshot := createTestRuntimeSnapshot(t, []FilterRegistration{reg}, &fixtureDispatcher{}, &fixtureRebuilder{}, sink)
initial := mustAttemptBinding(t, "att-1", source, &fixtureController{})
runtime, err := NewRequestRuntime(snapshot, "group-a", initial)
if err != nil {
t.Fatalf("NewRequestRuntime: %v", err)
}
done := make(chan error, 1)
go func() { done <- runtime.Run(context.Background()) }()
// The filter barrier is entered, so the owner loop is inside evaluation.
// Reading the source call count after this happens-before edge proves the
// next ingress read has not started without sleeping on a timer.
<-entered
callsDuringEvaluation := source.callCount()
close(release)
if runErr := <-done; runErr != nil {
t.Fatalf("Run: %v", runErr)
}
if callsDuringEvaluation != 1 {
t.Fatalf("NextEvent calls during evaluation = %d, want 1", callsDuringEvaluation)
}
if got := source.callCount(); got != 2 {
t.Fatalf("NextEvent calls after release = %d, want 2", got)
}
if got := releaseTexts(t, sink.events); !reflect.DeepEqual(got, []string{"hold"}) {
t.Fatalf("released texts = %v, want [hold]", got)
}
if len(sink.terminals) != 1 || !sink.terminals[0].Success() {
t.Fatalf("terminals = %+v, want exactly one success terminal", sink.terminals)
}
})
t.Run("simultaneous_violation_single_winner", func(t *testing.T) {
trace := &runtimeTrace{}
sink := newFixtureSink()
sink.trace = trace
dispatcher := &fixtureDispatcher{trace: trace}
rebuilder := &fixtureRebuilder{trace: trace}
oldController := &fixtureController{name: "att-1", trace: trace}
newController := &fixtureController{name: "att-2", trace: trace}
makeFilter := func(id, rule string, cost, priority int) FilterRegistration {
filter := &customMockFilter{
id: id,
evaluateFn: func(ctx context.Context, fc FilterContext, batch EvidenceBatch) (FilterDecision, error) {
evidence := mustSanitizedEvidence(EventKindTextDelta, "default", rule, "simultaneous_violation", FixedFingerprint{byte(cost)}, 1, 0, FilterOutcomeKindEvaluated, testNow)
if fc.AttemptID() == "att-2" {
return NewFilterDecision(FilterDecisionKindPass, id, id, rule, evidence, nil)
}
intent := mustRecoveryIntent(RecoveryStrategyExactReplay, mustRecoveryDirectiveExact("req-ref-1"), "rule_violation", cost)
return NewFilterDecision(FilterDecisionKindViolation, id, id, rule, evidence, &intent)
},
}
return mustFilterRegistration(filter, id+"-cap", true, FilterEnforcementBlocking, 5*time.Second, priority)
}
regs := []FilterRegistration{makeFilter("filter1", "rule1", 20, 20), makeFilter("filter2", "rule2", 10, 10)}
dispatcher.handler = func(ctx context.Context, request RebuiltRequest) (AttemptBinding, error) {
return NewAttemptBinding("att-2", "gpt-4", "openai", "primary", newSliceEventSource([]NormalizedEvent{mustTerminal("default", testNow)}), newController)
}
snapshot := createTestRuntimeSnapshot(t, regs, dispatcher, rebuilder, sink)
initial := mustAttemptBinding(t, "att-1", newSliceEventSource([]NormalizedEvent{mustTextDelta("default", "bad", testNow)}), oldController)
runtime, err := NewRequestRuntime(snapshot, "group-a", initial)
if err != nil {
t.Fatalf("NewRequestRuntime: %v", err)
}
if err := runtime.Run(context.Background()); err != nil {
t.Fatalf("Run: %v", err)
}
if len(dispatcher.dispatched) != 1 || len(sink.terminals) != 1 {
t.Fatalf("dispatch/terminal count = %d/%d, want 1/1", len(dispatcher.dispatched), len(sink.terminals))
}
if !sink.terminals[0].Success() || len(sink.starts) != 0 || len(sink.events) != 0 {
t.Fatalf("sink starts/events/terminal = %d/%d/%+v, want 0/0/success", len(sink.starts), len(sink.events), sink.terminals[0])
}
plan, hasPlan := rebuilder.lastPlan()
if !hasPlan {
t.Fatal("rebuilder never received a recovery plan")
}
contributors := plan.Contributors()
if len(contributors) != 1 {
t.Fatalf("plan contributors = %+v, want exactly one winner", contributors)
}
// filter1 wins on higher effective priority; filter2 must not contribute.
if contributors[0].FilterID() != "filter1" || contributors[0].RuleID() != "rule1" || contributors[0].ConsumerID() != "filter1" {
t.Fatalf("winner contributor = %s/%s/%s, want filter1/rule1/filter1", contributors[0].ConsumerID(), contributors[0].FilterID(), contributors[0].RuleID())
}
if plan.Priority() != 20 || plan.Strategy() != RecoveryStrategyExactReplay {
t.Fatalf("winner plan priority/strategy = %d/%s, want 20/%s", plan.Priority(), plan.Strategy(), RecoveryStrategyExactReplay)
}
if plan.ResumeMode() != RecoveryResumeModeReplaceAttempt {
t.Fatalf("winner plan resume mode = %s, want %s", plan.ResumeMode(), RecoveryResumeModeReplaceAttempt)
}
wantTrace := []string{"abort:att-1", "rebuild", "dispatch", "terminal:success"}
if got := trace.snapshot(); !reflect.DeepEqual(got, wantTrace) {
t.Fatalf("trace = %v, want %v", got, wantTrace)
}
assertControllerAbortCount(t, oldController, 1)
assertControllerAbortCount(t, newController, 0)
})
}
Reference in a new issue View git blame Copy permalink