418 lines
13 KiB
Go
418 lines
13 KiB
Go
package streamgate
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
type sliceEventSource struct {
|
|
mu sync.Mutex
|
|
events []NormalizedEvent
|
|
index int
|
|
}
|
|
|
|
func newSliceEventSource(events []NormalizedEvent) *sliceEventSource {
|
|
return &sliceEventSource{events: events}
|
|
}
|
|
|
|
func (s *sliceEventSource) NextEvent(ctx context.Context) (NormalizedEvent, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
if s.index >= len(s.events) {
|
|
return NormalizedEvent{}, errors.New("EOF")
|
|
}
|
|
ev := s.events[s.index]
|
|
s.index++
|
|
return ev, nil
|
|
}
|
|
|
|
type fixtureController struct {
|
|
mu sync.Mutex
|
|
abortCount int
|
|
}
|
|
|
|
func (c *fixtureController) AbortAttempt(ctx context.Context) error {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
c.abortCount++
|
|
return nil
|
|
}
|
|
|
|
type fixtureDispatcher struct {
|
|
mu sync.Mutex
|
|
handler func(ctx context.Context, request RebuiltRequest) (AttemptBinding, error)
|
|
dispatched []RebuiltRequest
|
|
}
|
|
|
|
func (d *fixtureDispatcher) DispatchAttempt(ctx context.Context, request RebuiltRequest) (AttemptBinding, error) {
|
|
d.mu.Lock()
|
|
d.dispatched = append(d.dispatched, request)
|
|
h := d.handler
|
|
d.mu.Unlock()
|
|
|
|
if h != nil {
|
|
return h(ctx, request)
|
|
}
|
|
return AttemptBinding{}, errors.New("dispatcher error")
|
|
}
|
|
|
|
type fixtureRebuilder struct {
|
|
mu sync.Mutex
|
|
planCount int
|
|
}
|
|
|
|
func (r *fixtureRebuilder) RebuildRequest(ctx context.Context, snapshot RecoveryRequestSnapshotRef, plan RecoveryPlan) (RebuiltRequestDraft, error) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
r.planCount++
|
|
return NewRebuiltRequestDraftWithIdempotency(plan.PlanID(), plan.IdempotencyKey(), "req-ref-1", "ep", "fam", 10, 20, 100, 5, nil)
|
|
}
|
|
|
|
type fixtureSink struct {
|
|
mu sync.Mutex
|
|
starts []ResponseStart
|
|
events []ReleaseEvent
|
|
terminals []TerminalResult
|
|
state CommitState
|
|
}
|
|
|
|
func newFixtureSink() *fixtureSink {
|
|
return &fixtureSink{state: CommitStateTransportUncommitted}
|
|
}
|
|
|
|
func (s *fixtureSink) CommitResponseStart(ctx context.Context, rs ResponseStart) (CommitState, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
s.starts = append(s.starts, rs)
|
|
s.state = CommitStateStreamOpen
|
|
return CommitStateStreamOpen, nil
|
|
}
|
|
|
|
func (s *fixtureSink) Release(ctx context.Context, ev ReleaseEvent) (CommitState, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
s.events = append(s.events, ev)
|
|
s.state = CommitStateStreamOpen
|
|
return CommitStateStreamOpen, nil
|
|
}
|
|
|
|
func (s *fixtureSink) CommitTerminal(ctx context.Context, tr TerminalResult) (CommitState, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
s.terminals = append(s.terminals, tr)
|
|
s.state = CommitStateTerminalCommitted
|
|
return CommitStateTerminalCommitted, nil
|
|
}
|
|
|
|
type customMockFilter struct {
|
|
id string
|
|
holdReq *FilterHoldRequirement
|
|
appliesFn func(FilterContext) bool
|
|
evaluateFn func(context.Context, FilterContext, EvidenceBatch) (FilterDecision, error)
|
|
}
|
|
|
|
func (m *customMockFilter) ID() string { return m.id }
|
|
func (m *customMockFilter) Applies(c FilterContext) bool {
|
|
if m.appliesFn != nil {
|
|
return m.appliesFn(c)
|
|
}
|
|
return true
|
|
}
|
|
func (m *customMockFilter) HoldRequirement(c FilterContext) FilterHoldRequirement {
|
|
if m.holdReq != nil {
|
|
return *m.holdReq
|
|
}
|
|
req, _ := NewFilterHoldRequirementRolling("default", []EventKind{EventKindTextDelta, EventKindTerminal}, 1)
|
|
return req
|
|
}
|
|
func (m *customMockFilter) Evaluate(ctx context.Context, fc FilterContext, batch EvidenceBatch) (FilterDecision, error) {
|
|
if m.evaluateFn != nil {
|
|
return m.evaluateFn(ctx, fc, batch)
|
|
}
|
|
fp := FixedFingerprint{1}
|
|
ev, _ := NewSanitizedEvidence(EventKindTextDelta, "default", "rule1", "desc", fp, 1, 0, FilterOutcomeKindEvaluated, time.Now())
|
|
return NewFilterDecision(FilterDecisionKindPass, "consumer1", m.id, "rule1", ev, nil)
|
|
}
|
|
|
|
func createTestRuntimeSnapshot(t *testing.T, regs []FilterRegistration, disp AttemptDispatcher, rebuilder RequestRebuilder, sink ReleaseSink) RequestRuntimeSnapshot {
|
|
regSnap, err := NewFilterRegistrySnapshot("gen-1", regs, nil)
|
|
if err != nil {
|
|
t.Fatalf("NewFilterRegistrySnapshot: %v", err)
|
|
}
|
|
snapRef, err := NewRecoveryRequestSnapshotRef("snap.ref.1", 100, 200, 1024)
|
|
if err != nil {
|
|
t.Fatalf("NewRecoveryRequestSnapshotRef: %v", err)
|
|
}
|
|
opts := DefaultRuntimeOptions()
|
|
snap, err := NewRequestRuntimeSnapshot(
|
|
"req-123",
|
|
"gen-1",
|
|
"test",
|
|
"ep",
|
|
"fam",
|
|
opts,
|
|
regSnap,
|
|
nil,
|
|
snapRef,
|
|
disp,
|
|
rebuilder,
|
|
nil,
|
|
sink,
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("NewRequestRuntimeSnapshot: %v", err)
|
|
}
|
|
return snap
|
|
}
|
|
|
|
func TestRequestRuntimeDisabledAndPass(t *testing.T) {
|
|
sink := newFixtureSink()
|
|
disp := &fixtureDispatcher{}
|
|
rebuilder := &fixtureRebuilder{}
|
|
|
|
passF := &customMockFilter{id: "pass-filter"}
|
|
regPass, err := NewFilterRegistration(passF, "cap1", true, FilterEnforcementBlocking, 5*time.Second, 10)
|
|
if err != nil {
|
|
t.Fatalf("NewFilterRegistration pass: %v", err)
|
|
}
|
|
disabledF := &customMockFilter{id: "disabled-filter"}
|
|
regDisabled, err := NewFilterRegistration(disabledF, "cap2", false, FilterEnforcementBlocking, 5*time.Second, 10)
|
|
if err != nil {
|
|
t.Fatalf("NewFilterRegistration disabled: %v", err)
|
|
}
|
|
|
|
snap := createTestRuntimeSnapshot(t, []FilterRegistration{regPass, regDisabled}, disp, rebuilder, sink)
|
|
|
|
rsEv, _ := NewResponseStartEvent("default", 200, map[string]string{"content-type": "text/event-stream"}, time.Now())
|
|
txtEv, _ := NewTextDeltaEvent("default", "hello world", time.Now())
|
|
termEv, _ := NewTerminalEvent("default", time.Now())
|
|
|
|
src := newSliceEventSource([]NormalizedEvent{rsEv, txtEv, termEv})
|
|
ctrl := &fixtureController{}
|
|
initialBinding, err := NewAttemptBinding("att-1", "gpt-4", "openai", "primary", src, ctrl)
|
|
if err != nil {
|
|
t.Fatalf("NewAttemptBinding: %v", err)
|
|
}
|
|
|
|
rt, err := NewRequestRuntime(snap, "group-a", initialBinding)
|
|
if err != nil {
|
|
t.Fatalf("NewRequestRuntime: %v", err)
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
|
defer cancel()
|
|
|
|
if err := rt.Run(ctx); err != nil {
|
|
t.Fatalf("Run: %v", err)
|
|
}
|
|
|
|
sink.mu.Lock()
|
|
defer sink.mu.Unlock()
|
|
|
|
if len(sink.starts) != 1 {
|
|
t.Errorf("expected 1 response start committed, got %d", len(sink.starts))
|
|
}
|
|
if len(sink.events) != 1 {
|
|
t.Errorf("expected 1 release event, got %d", len(sink.events))
|
|
}
|
|
if len(sink.terminals) != 1 {
|
|
t.Errorf("expected 1 terminal result, got %d", len(sink.terminals))
|
|
}
|
|
}
|
|
|
|
func TestRequestRuntimeRecoveryLifecycle(t *testing.T) {
|
|
sink := newFixtureSink()
|
|
rebuilder := &fixtureRebuilder{}
|
|
disp := &fixtureDispatcher{}
|
|
|
|
filterID := "violating-filter"
|
|
var evalCount int
|
|
var mu sync.Mutex
|
|
|
|
vF := &customMockFilter{
|
|
id: filterID,
|
|
evaluateFn: func(ctx context.Context, fc FilterContext, batch EvidenceBatch) (FilterDecision, error) {
|
|
mu.Lock()
|
|
count := evalCount
|
|
evalCount++
|
|
mu.Unlock()
|
|
|
|
fp := FixedFingerprint{1}
|
|
ev, _ := NewSanitizedEvidence(EventKindTextDelta, "default", "rule1", "desc", fp, 1, 0, FilterOutcomeKindEvaluated, time.Now())
|
|
|
|
if count == 0 {
|
|
dir, _ := NewRecoveryDirectiveExact("req-ref-1")
|
|
intent, _ := NewRecoveryIntent(RecoveryStrategyExactReplay, dir, "rule_violation", 10)
|
|
return NewFilterDecision(FilterDecisionKindViolation, "consumer1", filterID, "rule1", ev, &intent)
|
|
}
|
|
return NewFilterDecision(FilterDecisionKindPass, "consumer1", filterID, "rule1", ev, nil)
|
|
},
|
|
}
|
|
|
|
reg, err := NewFilterRegistration(vF, "cap1", true, FilterEnforcementBlocking, 5*time.Second, 10)
|
|
if err != nil {
|
|
t.Fatalf("NewFilterRegistration: %v", err)
|
|
}
|
|
|
|
// Prepare second attempt source
|
|
txtEv2, _ := NewTextDeltaEvent("default", "recovered text", time.Now())
|
|
termEv2, _ := NewTerminalEvent("default", time.Now())
|
|
src2 := newSliceEventSource([]NormalizedEvent{txtEv2, termEv2})
|
|
ctrl2 := &fixtureController{}
|
|
|
|
disp.handler = func(ctx context.Context, request RebuiltRequest) (AttemptBinding, error) {
|
|
return NewAttemptBinding("att-2", "gpt-4", "openai", "primary", src2, ctrl2)
|
|
}
|
|
|
|
snap := createTestRuntimeSnapshot(t, []FilterRegistration{reg}, disp, rebuilder, sink)
|
|
|
|
txtEv1, _ := NewTextDeltaEvent("default", "violating text", time.Now())
|
|
termEv1, _ := NewTerminalEvent("default", time.Now())
|
|
src1 := newSliceEventSource([]NormalizedEvent{txtEv1, termEv1})
|
|
ctrl1 := &fixtureController{}
|
|
|
|
initialBinding, err := NewAttemptBinding("att-1", "gpt-4", "openai", "primary", src1, ctrl1)
|
|
if err != nil {
|
|
t.Fatalf("NewAttemptBinding: %v", err)
|
|
}
|
|
|
|
rt, err := NewRequestRuntime(snap, "group-a", initialBinding)
|
|
if err != nil {
|
|
t.Fatalf("NewRequestRuntime: %v", err)
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
|
defer cancel()
|
|
|
|
if err := rt.Run(ctx); err != nil {
|
|
t.Fatalf("Run: %v", err)
|
|
}
|
|
|
|
sink.mu.Lock()
|
|
defer sink.mu.Unlock()
|
|
|
|
if len(sink.terminals) != 1 {
|
|
t.Fatalf("expected exactly 1 terminal result, got %d", len(sink.terminals))
|
|
}
|
|
if !sink.terminals[0].Success() {
|
|
t.Errorf("expected successful terminal result after recovery")
|
|
}
|
|
}
|
|
|
|
func TestRequestRuntimeFailureMatrix(t *testing.T) {
|
|
t.Run("FatalViolation", func(t *testing.T) {
|
|
sink := newFixtureSink()
|
|
disp := &fixtureDispatcher{}
|
|
rebuilder := &fixtureRebuilder{}
|
|
|
|
fatalF := &customMockFilter{
|
|
id: "fatal-filter",
|
|
evaluateFn: func(ctx context.Context, fc FilterContext, batch EvidenceBatch) (FilterDecision, error) {
|
|
fp := FixedFingerprint{1}
|
|
ev, _ := NewSanitizedEvidence(EventKindTextDelta, "default", "rule1", "fatal desc", fp, 1, 0, FilterOutcomeKindEvaluated, time.Now())
|
|
return NewFilterDecision(FilterDecisionKindFatal, "consumer1", "fatal-filter", "rule1", ev, nil)
|
|
},
|
|
}
|
|
|
|
reg, err := NewFilterRegistration(fatalF, "cap1", true, FilterEnforcementBlocking, 5*time.Second, 10)
|
|
if err != nil {
|
|
t.Fatalf("NewFilterRegistration: %v", err)
|
|
}
|
|
|
|
snap := createTestRuntimeSnapshot(t, []FilterRegistration{reg}, disp, rebuilder, sink)
|
|
txtEv, _ := NewTextDeltaEvent("default", "bad text", time.Now())
|
|
src := newSliceEventSource([]NormalizedEvent{txtEv})
|
|
ctrl := &fixtureController{}
|
|
initialBinding, _ := NewAttemptBinding("att-1", "gpt-4", "openai", "primary", src, ctrl)
|
|
|
|
rt, _ := NewRequestRuntime(snap, "group-a", initialBinding)
|
|
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
|
defer cancel()
|
|
|
|
_ = rt.Run(ctx)
|
|
|
|
sink.mu.Lock()
|
|
defer sink.mu.Unlock()
|
|
if len(sink.terminals) != 1 {
|
|
t.Fatalf("expected 1 terminal result for fatal violation, got %d", len(sink.terminals))
|
|
}
|
|
if sink.terminals[0].Success() {
|
|
t.Errorf("expected error terminal result for fatal violation")
|
|
}
|
|
})
|
|
|
|
t.Run("ExhaustedRecoveryBudget", func(t *testing.T) {
|
|
sink := newFixtureSink()
|
|
disp := &fixtureDispatcher{}
|
|
rebuilder := &fixtureRebuilder{}
|
|
|
|
// Always violate filter
|
|
vF := &customMockFilter{
|
|
id: "violating-filter",
|
|
evaluateFn: func(ctx context.Context, fc FilterContext, batch EvidenceBatch) (FilterDecision, error) {
|
|
fp := FixedFingerprint{1}
|
|
ev, _ := NewSanitizedEvidence(EventKindTextDelta, "default", "rule1", "desc", fp, 1, 0, FilterOutcomeKindEvaluated, time.Now())
|
|
dir, _ := NewRecoveryDirectiveExact("req-ref-1")
|
|
intent, _ := NewRecoveryIntent(RecoveryStrategyExactReplay, dir, "rule_violation", 10)
|
|
return NewFilterDecision(FilterDecisionKindViolation, "consumer1", "violating-filter", "rule1", ev, &intent)
|
|
},
|
|
}
|
|
|
|
reg, _ := NewFilterRegistration(vF, "cap1", true, FilterEnforcementBlocking, 5*time.Second, 10)
|
|
|
|
disp.handler = func(ctx context.Context, request RebuiltRequest) (AttemptBinding, error) {
|
|
txtEv, _ := NewTextDeltaEvent("default", "violating text", time.Now())
|
|
src := newSliceEventSource([]NormalizedEvent{txtEv})
|
|
ctrl := &fixtureController{}
|
|
return NewAttemptBinding("att-next", "gpt-4", "openai", "primary", src, ctrl)
|
|
}
|
|
|
|
snap := createTestRuntimeSnapshot(t, []FilterRegistration{reg}, disp, rebuilder, sink)
|
|
txtEv, _ := NewTextDeltaEvent("default", "violating text 1", time.Now())
|
|
src := newSliceEventSource([]NormalizedEvent{txtEv})
|
|
ctrl := &fixtureController{}
|
|
initialBinding, _ := NewAttemptBinding("att-1", "gpt-4", "openai", "primary", src, ctrl)
|
|
|
|
rt, _ := NewRequestRuntime(snap, "group-a", initialBinding)
|
|
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
|
defer cancel()
|
|
|
|
_ = rt.Run(ctx)
|
|
|
|
sink.mu.Lock()
|
|
defer sink.mu.Unlock()
|
|
if len(sink.terminals) != 1 {
|
|
t.Fatalf("expected single terminal result on budget exhaustion, got %d", len(sink.terminals))
|
|
}
|
|
if sink.terminals[0].Success() {
|
|
t.Errorf("expected error terminal result on budget exhaustion")
|
|
}
|
|
})
|
|
|
|
t.Run("CallerCancellation", func(t *testing.T) {
|
|
sink := newFixtureSink()
|
|
disp := &fixtureDispatcher{}
|
|
rebuilder := &fixtureRebuilder{}
|
|
passF := &customMockFilter{id: "pass-filter"}
|
|
reg, _ := NewFilterRegistration(passF, "cap1", true, FilterEnforcementBlocking, 5*time.Second, 10)
|
|
|
|
snap := createTestRuntimeSnapshot(t, []FilterRegistration{reg}, disp, rebuilder, sink)
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel() // cancel immediately
|
|
|
|
txtEv, _ := NewTextDeltaEvent("default", "text", time.Now())
|
|
src := newSliceEventSource([]NormalizedEvent{txtEv})
|
|
ctrl := &fixtureController{}
|
|
initialBinding, _ := NewAttemptBinding("att-1", "gpt-4", "openai", "primary", src, ctrl)
|
|
|
|
rt, _ := NewRequestRuntime(snap, "group-a", initialBinding)
|
|
err := rt.Run(ctx)
|
|
if !errors.Is(err, context.Canceled) {
|
|
t.Errorf("expected context.Canceled, got %v", err)
|
|
}
|
|
})
|
|
}
|