package streamgate import ( "context" "errors" "strings" "sync" "sync/atomic" "testing" "time" ) // testNow is a deterministic timestamp used in tests. var testNow = time.Date(2026, 7, 24, 12, 0, 0, 0, time.UTC) // testSink is a configurable test-only ReleaseSink that records calls and // can be programmed to fail specific calls. type testSink struct { mu sync.Mutex responseStarts []ResponseStart releases []ReleaseEvent terminals []TerminalResult state CommitState failAfterSuccesses int // >0: fail after N successes; 0: no failure failAll bool // if true, all releases fail successCount int // how many successful releases have occurred (only tracked when failAfterSuccesses > 0) onRelease func(ReleaseEvent) onTerminal func(TerminalResult) } func (s *testSink) CommitResponseStart(ctx context.Context, rs ResponseStart) (CommitState, error) { s.mu.Lock() defer s.mu.Unlock() s.responseStarts = append(s.responseStarts, rs) s.state = CommitStateStreamOpen return CommitStateStreamOpen, nil } func (s *testSink) Release(ctx context.Context, ev ReleaseEvent) (CommitState, error) { s.mu.Lock() if s.onRelease != nil { fn := s.onRelease s.mu.Unlock() fn(ev) s.mu.Lock() } defer s.mu.Unlock() if s.failAll { return "", errors.New("test: forced release failure") } if s.failAfterSuccesses > 0 { s.successCount++ if s.successCount > s.failAfterSuccesses { return "", errors.New("test: forced release failure") } } s.releases = append(s.releases, ev) return s.state, nil } func (s *testSink) CommitTerminal(ctx context.Context, tr TerminalResult) (CommitState, error) { s.mu.Lock() if s.onTerminal != nil { fn := s.onTerminal s.mu.Unlock() fn(tr) s.mu.Lock() } defer s.mu.Unlock() s.terminals = append(s.terminals, tr) s.state = CommitStateTerminalCommitted return CommitStateTerminalCommitted, nil } // --------------------------------------------------------------------------- // REVIEW_API-1: Request-scope staging and monotonic sink state // --------------------------------------------------------------------------- // TestCommitBoundaryStagesUntilFirstSafeRelease verifies that stage calls // produce zero sink invocations and the first release commits start→event // order with a single start. func TestCommitBoundaryStagesUntilFirstSafeRelease(t *testing.T) { ctx := context.Background() sink := &testSink{} b, err := NewCommitBoundary(sink) if err != nil { t.Fatalf("NewCommitBoundary: %v", err) } rs, _ := NewResponseStart("ch", 200, map[string]string{"x-test": "v"}, testNow) if err := b.BeginAttempt("a1"); err != nil { t.Fatalf("BeginAttempt: %v", err) } if err := b.StageResponseStart("a1", rs); err != nil { t.Fatalf("StageResponseStart: %v", err) } if got := b.State(); got != CommitStateTransportUncommitted { t.Errorf("state before release = %q, want %q", got, CommitStateTransportUncommitted) } ev1, _ := NewReleaseTextDeltaEvent("ch", "hello ", testNow) ev2, _ := NewReleaseTextDeltaEvent("ch", "world", testNow) prog, err := b.ReleaseSafe(ctx, "a1", []ReleaseEvent{ev1, ev2}) if err != nil { t.Fatalf("ReleaseSafe: %v", err) } if len(sink.responseStarts) != 1 { t.Errorf("responseStarts = %d, want 1", len(sink.responseStarts)) } if len(sink.releases) != 2 { t.Errorf("releases = %d, want 2", len(sink.releases)) } if prog.ReleasedEvents() != 2 { t.Errorf("ReleasedEvents = %d, want 2", prog.ReleasedEvents()) } if got := b.State(); got != CommitStateStreamOpen { t.Errorf("state after release = %q, want %q", got, CommitStateStreamOpen) } // Second release still works (start already consumed). _, err = b.ReleaseSafe(ctx, "a1", []ReleaseEvent{ev1}) if err != nil { t.Fatalf("second ReleaseSafe: %v", err) } if len(sink.releases) != 3 { t.Errorf("total releases after second call = %d, want 3", len(sink.releases)) } if len(sink.responseStarts) != 1 { t.Errorf("total responseStarts = %d, want 1", len(sink.responseStarts)) } } // TestCommitBoundaryPreservesProgressAndMonotonicState verifies the state // transition table for start/body/error and that successful event count is // preserved across failures. func TestCommitBoundaryPreservesProgressAndMonotonicState(t *testing.T) { ctx := context.Background() t.Run("success_steps_preserve_count", func(t *testing.T) { sink := &testSink{} b, _ := NewCommitBoundary(sink) b.BeginAttempt("a1") rs, _ := NewResponseStart("ch", 200, nil, testNow) b.StageResponseStart("a1", rs) evs := make([]ReleaseEvent, 5) for i := range evs { e, _ := NewReleaseTextDeltaEvent("ch", "chunk", testNow) evs[i] = e } prog, err := b.ReleaseSafe(ctx, "a1", evs) if err != nil { t.Fatalf("ReleaseSafe: %v", err) } if prog.ReleasedEvents() != 5 { t.Errorf("ReleasedEvents = %d, want 5", prog.ReleasedEvents()) } if len(sink.releases) != 5 { t.Errorf("releases = %d, want 5", len(sink.releases)) } if got := b.State(); got != CommitStateStreamOpen { t.Errorf("state = %q, want stream_open", got) } }) t.Run("sink_error_preserves_progress", func(t *testing.T) { sink := &testSink{failAfterSuccesses: 1} b, _ := NewCommitBoundary(sink) b.BeginAttempt("a1") rs, _ := NewResponseStart("ch", 200, nil, testNow) b.StageResponseStart("a1", rs) evs := make([]ReleaseEvent, 3) for i := range evs { e, _ := NewReleaseTextDeltaEvent("ch", "chunk", testNow) evs[i] = e } prog, err := b.ReleaseSafe(ctx, "a1", evs) if err == nil { t.Error("expected error from ReleaseSafe") } // First event succeeded before the forced failure. if prog.ReleasedEvents() != 1 { t.Errorf("ReleasedEvents = %d, want 1", prog.ReleasedEvents()) } if len(sink.releases) != 1 { t.Errorf("releases = %d, want 1", len(sink.releases)) } if got := b.State(); got != CommitStateStreamOpen { t.Errorf("state = %q, want stream_open (preserved)", got) } }) t.Run("begin_after_stream_open_rejected", func(t *testing.T) { sink := &testSink{} b, _ := NewCommitBoundary(sink) b.BeginAttempt("a1") rs, _ := NewResponseStart("ch", 200, nil, testNow) b.StageResponseStart("a1", rs) ev, _ := NewReleaseTextDeltaEvent("ch", "x", testNow) b.ReleaseSafe(ctx, "a1", []ReleaseEvent{ev}) err := b.BeginAttempt("a2") if err != ErrBoundaryUncommitted { t.Errorf("BeginAttempt error = %v, want ErrBoundaryUncommitted", err) } }) t.Run("zero_release_events_rejected", func(t *testing.T) { sink := &testSink{} b, _ := NewCommitBoundary(sink) b.BeginAttempt("a1") _, err := b.ReleaseSafe(ctx, "a1", nil) if err == nil { t.Error("expected error for empty events") } }) } // TestCommitBoundaryCommitsEmptySuccessAndTerminalOnce verifies that success // terminal without body commits start first, terminal exactly once, and // State() does not deadlock during callbacks. func TestCommitBoundaryCommitsEmptySuccessAndTerminalOnce(t *testing.T) { ctx := context.Background() t.Run("success_terminal_no_body", func(t *testing.T) { sink := &testSink{} b, _ := NewCommitBoundary(sink) b.BeginAttempt("a1") rs, _ := NewResponseStart("ch", 200, nil, testNow) b.StageResponseStart("a1", rs) tr, _ := NewSuccessTerminalResult("ch", testNow) if err := b.CommitTerminal(ctx, "a1", tr); err != nil { t.Fatalf("CommitTerminal: %v", err) } if len(sink.responseStarts) != 1 { t.Errorf("responseStarts = %d, want 1", len(sink.responseStarts)) } if len(sink.terminals) != 1 { t.Errorf("terminals = %d, want 1", len(sink.terminals)) } if got := b.State(); got != CommitStateTerminalCommitted { t.Errorf("state = %q, want terminal_committed", got) } }) t.Run("second_terminal_rejected", func(t *testing.T) { sink := &testSink{} b, _ := NewCommitBoundary(sink) b.BeginAttempt("a1") tr, _ := NewSuccessTerminalResult("ch", testNow) b.CommitTerminal(ctx, "a1", tr) err := b.CommitTerminal(ctx, "a1", tr) if err != ErrBoundaryAlreadyTerminal { t.Errorf("second CommitTerminal error = %v, want ErrBoundaryAlreadyTerminal", err) } }) t.Run("state_no_deadlock_during_callback", func(t *testing.T) { sink := &testSink{} b, _ := NewCommitBoundary(sink) b.BeginAttempt("a1") rs, _ := NewResponseStart("ch", 200, nil, testNow) b.StageResponseStart("a1", rs) done := make(chan struct{}) go func() { defer close(done) for i := 0; i < 100; i++ { _ = b.State() } }() evs := make([]ReleaseEvent, 50) for i := range evs { e, _ := NewReleaseTextDeltaEvent("ch", "x", testNow) evs[i] = e } _, err := b.ReleaseSafe(ctx, "a1", evs) if err != nil { t.Fatalf("ReleaseSafe: %v", err) } <-done }) t.Run("error_terminal_discards_staged_start", func(t *testing.T) { sink := &testSink{} b, _ := NewCommitBoundary(sink) b.BeginAttempt("a1") rs, _ := NewResponseStart("ch", 200, nil, testNow) b.StageResponseStart("a1", rs) desc, _ := NewExternalDescriptor("generic", "internal_error", "error", "") causes, _ := NewFailureCauseChain([]FailureCause{}) tr, _ := NewErrorTerminalResult("ch", desc, causes, testNow) if err := b.CommitTerminal(ctx, "a1", tr); err != nil { t.Fatalf("CommitTerminal: %v", err) } if len(sink.responseStarts) != 0 { t.Errorf("responseStarts = %d, want 0 (error discards start)", len(sink.responseStarts)) } if len(sink.terminals) != 1 { t.Errorf("terminals = %d, want 1", len(sink.terminals)) } }) } // --------------------------------------------------------------------------- // REVIEW_API-3: Attempt replacement and recovery strategy matrix // --------------------------------------------------------------------------- // TestCommitBoundaryReplacesOnlyUncommittedAttempt verifies that replacement // discards the old attempt and staged start, allowing a new attempt to begin. func TestCommitBoundaryReplacesOnlyUncommittedAttempt(t *testing.T) { ctx := context.Background() t.Run("replacement_discards_old_and_allows_new", func(t *testing.T) { sink := &testSink{} b, _ := NewCommitBoundary(sink) // First attempt with staged start. b.BeginAttempt("old") rs, _ := NewResponseStart("ch", 200, nil, testNow) b.StageResponseStart("old", rs) // Replace. if err := b.ReplaceUncommittedAttempt("old", "new"); err != nil { t.Fatalf("ReplaceUncommittedAttempt: %v", err) } // Old attempt should not work (no current attempt). err := b.StageResponseStart("old", rs) if err == nil { // Actually after replace, there's no current attempt, so // StageResponseStart should work since state is still uncommitted. // But the old attempt ID is not special anymore. } // New attempt should work. b.BeginAttempt("new") rs2, _ := NewResponseStart("ch", 200, nil, testNow) b.StageResponseStart("new", rs2) ev, _ := NewReleaseTextDeltaEvent("ch", "data", testNow) _, err = b.ReleaseSafe(ctx, "new", []ReleaseEvent{ev}) if err != nil { t.Fatalf("ReleaseSafe: %v", err) } // Exactly one start from new attempt. if len(sink.responseStarts) != 1 { t.Errorf("responseStarts = %d, want 1 (only new attempt)", len(sink.responseStarts)) } }) t.Run("post_open_replacement_rejected", func(t *testing.T) { sink := &testSink{} b, _ := NewCommitBoundary(sink) b.BeginAttempt("a1") rs, _ := NewResponseStart("ch", 200, nil, testNow) b.StageResponseStart("a1", rs) ev, _ := NewReleaseTextDeltaEvent("ch", "x", testNow) b.ReleaseSafe(ctx, "a1", []ReleaseEvent{ev}) // After stream_open, we cannot begin a new attempt. err := b.BeginAttempt("a2") if err != ErrBoundaryUncommitted { t.Errorf("BeginAttempt after open error = %v, want ErrBoundaryUncommitted", err) } }) } // TestCommitBoundaryCommitStrategyMatrix verifies the allowed strategy table // for each commit state. func TestCommitBoundaryCommitStrategyMatrix(t *testing.T) { ctx := context.Background() strategies := []RecoveryStrategy{ RecoveryStrategyExactReplay, RecoveryStrategyContinuationRepair, RecoveryStrategySchemaRepair, } t.Run("transport_uncommitted", func(t *testing.T) { sink := &testSink{} b, _ := NewCommitBoundary(sink) b.BeginAttempt("a1") expectations := map[RecoveryStrategy]bool{ RecoveryStrategyExactReplay: true, RecoveryStrategySchemaRepair: true, RecoveryStrategyContinuationRepair: false, } for _, s := range strategies { got := b.CommitAllowsStrategy(s) want := expectations[s] if got != want { t.Errorf("CommitAllowsStrategy(%s) = %v, want %v", s, got, want) } } }) t.Run("stream_open", func(t *testing.T) { sink := &testSink{} b, _ := NewCommitBoundary(sink) b.BeginAttempt("a1") rs, _ := NewResponseStart("ch", 200, nil, testNow) b.StageResponseStart("a1", rs) ev, _ := NewReleaseTextDeltaEvent("ch", "x", testNow) b.ReleaseSafe(ctx, "a1", []ReleaseEvent{ev}) expectations := map[RecoveryStrategy]bool{ RecoveryStrategyExactReplay: false, RecoveryStrategySchemaRepair: false, RecoveryStrategyContinuationRepair: true, } for _, s := range strategies { got := b.CommitAllowsStrategy(s) want := expectations[s] if got != want { t.Errorf("CommitAllowsStrategy(%s) = %v, want %v", s, got, want) } } }) t.Run("terminal_committed", func(t *testing.T) { sink := &testSink{} b, _ := NewCommitBoundary(sink) b.BeginAttempt("a1") tr, _ := NewSuccessTerminalResult("ch", testNow) b.CommitTerminal(ctx, "a1", tr) for _, s := range strategies { got := b.CommitAllowsStrategy(s) if got { t.Errorf("CommitAllowsStrategy(%s) = true, want false after terminal", s) } } }) } func TestCommitBoundaryPreservesReturnedStateAcrossSinkErrors(t *testing.T) { ctx := context.Background() t.Run("sink_error_with_forward_state_preserved", func(t *testing.T) { sink := &testSink{failAfterSuccesses: 1} b, _ := NewCommitBoundary(sink) _ = b.BeginAttempt("a1") rs, _ := NewResponseStart("ch", 200, nil, testNow) _ = b.StageResponseStart("a1", rs) evs := make([]ReleaseEvent, 3) for i := range evs { e, _ := NewReleaseTextDeltaEvent("ch", "chunk", testNow) evs[i] = e } prog, err := b.ReleaseSafe(ctx, "a1", evs) if err == nil { t.Fatal("expected sink error") } if prog.ReleasedEvents() != 1 { t.Fatalf("ReleasedEvents = %d, want 1", prog.ReleasedEvents()) } if b.State() != CommitStateStreamOpen { t.Fatalf("state = %v, want stream_open", b.State()) } }) t.Run("empty_returned_state_on_success_rejected", func(t *testing.T) { customSink := &customStateSink{ startState: "", } b, _ := NewCommitBoundary(customSink) _ = b.BeginAttempt("a1") rs, _ := NewResponseStart("ch", 200, nil, testNow) _ = b.StageResponseStart("a1", rs) ev, _ := NewReleaseTextDeltaEvent("ch", "x", testNow) _, err := b.ReleaseSafe(ctx, "a1", []ReleaseEvent{ev}) if !errors.Is(err, ErrInvalidStateTransition) { t.Fatalf("expected ErrInvalidStateTransition for empty state on success, got %v", err) } }) t.Run("invalid_state_regression_rejected", func(t *testing.T) { customSink := &customStateSink{ startState: CommitStateStreamOpen, releaseState: CommitStateTransportUncommitted, // invalid transition stream_open -> uncommitted } b, _ := NewCommitBoundary(customSink) _ = b.BeginAttempt("a1") rs, _ := NewResponseStart("ch", 200, nil, testNow) _ = b.StageResponseStart("a1", rs) ev, _ := NewReleaseTextDeltaEvent("ch", "x", testNow) _, err := b.ReleaseSafe(ctx, "a1", []ReleaseEvent{ev}) if !errors.Is(err, ErrInvalidStateTransition) { t.Fatalf("expected ErrInvalidStateTransition for state regression, got %v", err) } }) t.Run("terminal_returns_non_terminal_state_rejected", func(t *testing.T) { customSink := &customStateSink{ terminalState: CommitStateStreamOpen, // wrong: terminal should return terminal_committed } b, _ := NewCommitBoundary(customSink) _ = b.BeginAttempt("a1") tr, _ := NewSuccessTerminalResult("ch", testNow) err := b.CommitTerminal(ctx, "a1", tr) if !errors.Is(err, ErrInvalidStateTransition) { t.Fatalf("expected ErrInvalidStateTransition for terminal returning %q, got %v", CommitStateStreamOpen, err) } // Role mismatch does not prevent advancing to a valid forward state. // The callback returned stream_open which is a valid forward transition // from transport_uncommitted, so the state is preserved. if b.State() != CommitStateStreamOpen { t.Fatalf("state = %v, want stream_open (forward state preserved despite role mismatch)", b.State()) } }) t.Run("response_start_returns_non_stream_open_rejected", func(t *testing.T) { customSink := &customStateSink{ startState: CommitStateTerminalCommitted, // wrong: response-start should return stream_open } b, _ := NewCommitBoundary(customSink) _ = b.BeginAttempt("a1") rs, _ := NewResponseStart("ch", 200, nil, testNow) _ = b.StageResponseStart("a1", rs) ev, _ := NewReleaseTextDeltaEvent("ch", "x", testNow) _, err := b.ReleaseSafe(ctx, "a1", []ReleaseEvent{ev}) if !errors.Is(err, ErrInvalidStateTransition) { t.Fatalf("expected ErrInvalidStateTransition for response-start returning %q, got %v", CommitStateTerminalCommitted, err) } // Role mismatch does not prevent advancing to a valid forward state. // The callback returned terminal_committed which is a valid forward // transition from transport_uncommitted, so the state is preserved. if b.State() != CommitStateTerminalCommitted { t.Fatalf("state = %v, want terminal_committed (forward state preserved despite role mismatch)", b.State()) } }) t.Run("release_returns_non_stream_open_rejected", func(t *testing.T) { customSink := &customStateSink{ startState: CommitStateStreamOpen, releaseState: CommitStateTerminalCommitted, // wrong: release should return stream_open } b, _ := NewCommitBoundary(customSink) _ = b.BeginAttempt("a1") rs, _ := NewResponseStart("ch", 200, nil, testNow) _ = b.StageResponseStart("a1", rs) ev, _ := NewReleaseTextDeltaEvent("ch", "x", testNow) _, err := b.ReleaseSafe(ctx, "a1", []ReleaseEvent{ev}) if !errors.Is(err, ErrInvalidStateTransition) { t.Fatalf("expected ErrInvalidStateTransition for release returning %q, got %v", CommitStateTerminalCommitted, err) } // Role mismatch does not prevent advancing to a valid forward state. // Response-start advanced to stream_open, and release returned // terminal_committed which is a valid forward transition, so state // is preserved at terminal_committed. if b.State() != CommitStateTerminalCommitted { t.Fatalf("state = %v, want terminal_committed (forward state preserved despite role mismatch)", b.State()) } }) t.Run("response_start_error_short_circuits_release", func(t *testing.T) { customSink := &customStateSink{ startErr: errors.New("test: start failed"), } b, _ := NewCommitBoundary(customSink) _ = b.BeginAttempt("a1") rs, _ := NewResponseStart("ch", 200, nil, testNow) _ = b.StageResponseStart("a1", rs) ev, _ := NewReleaseTextDeltaEvent("ch", "x", testNow) _, err := b.ReleaseSafe(ctx, "a1", []ReleaseEvent{ev}) if err == nil { t.Fatal("expected error from response-start short-circuit") } }) t.Run("terminal_success_with_terminal_committed_allows_no_reentry", func(t *testing.T) { customSink := &customStateSink{ terminalState: CommitStateTerminalCommitted, } b, _ := NewCommitBoundary(customSink) _ = b.BeginAttempt("a1") tr, _ := NewSuccessTerminalResult("ch", testNow) if err := b.CommitTerminal(ctx, "a1", tr); err != nil { t.Fatalf("CommitTerminal: %v", err) } if b.State() != CommitStateTerminalCommitted { t.Fatalf("state = %v, want terminal_committed", b.State()) } // Re-entry must be rejected. err := b.CommitTerminal(ctx, "a1", tr) if !errors.Is(err, ErrBoundaryAlreadyTerminal) { t.Fatalf("re-entry error = %v, want ErrBoundaryAlreadyTerminal", err) } }) t.Run("terminal_error_with_terminal_committed_preserves_state", func(t *testing.T) { customSink := &customStateSink{ terminalState: CommitStateTerminalCommitted, terminalErr: errors.New("test: terminal network error"), } b, _ := NewCommitBoundary(customSink) _ = b.BeginAttempt("a1") tr, _ := NewSuccessTerminalResult("ch", testNow) err := b.CommitTerminal(ctx, "a1", tr) if err == nil { t.Fatal("expected error from terminal with error + terminal_committed") } // State should still be terminal_committed. if b.State() != CommitStateTerminalCommitted { t.Fatalf("state = %v, want terminal_committed (preserved)", b.State()) } // Re-entry must still be rejected. err2 := b.CommitTerminal(ctx, "a1", tr) if !errors.Is(err2, ErrBoundaryAlreadyTerminal) { t.Fatalf("re-entry error = %v, want ErrBoundaryAlreadyTerminal", err2) } }) t.Run("callback_exact_count", func(t *testing.T) { var callCounts struct { starts, releases, terminals int } customSink := &countingTestSink{ inner: &testSink{}, onStart: func() { callCounts.starts++ }, onRelease: func() { callCounts.releases++ }, onTerminal: func() { callCounts.terminals++ }, } b, _ := NewCommitBoundary(customSink) _ = b.BeginAttempt("a1") rs, _ := NewResponseStart("ch", 200, nil, testNow) _ = b.StageResponseStart("a1", rs) evs := make([]ReleaseEvent, 2) for i := range evs { e, _ := NewReleaseTextDeltaEvent("ch", "chunk", testNow) evs[i] = e } _, err := b.ReleaseSafe(ctx, "a1", evs) if err != nil { t.Fatalf("ReleaseSafe: %v", err) } if callCounts.starts != 1 { t.Errorf("starts = %d, want 1", callCounts.starts) } if callCounts.releases != 2 { t.Errorf("releases = %d, want 2", callCounts.releases) } tr, _ := NewSuccessTerminalResult("ch", testNow) if err := b.CommitTerminal(ctx, "a1", tr); err != nil { t.Fatalf("CommitTerminal: %v", err) } if callCounts.terminals != 1 { t.Errorf("terminals = %d, want 1", callCounts.terminals) } // Second terminal should be rejected without calling sink. _ = b.CommitTerminal(ctx, "a1", tr) if callCounts.terminals != 1 { t.Errorf("terminals after re-entry = %d, want 1", callCounts.terminals) } }) t.Run("attempt_mismatch_rejected_with_identity_error", func(t *testing.T) { // Defense-in-depth: verify that applySinkOutcomeLockedWithOpLocked // rejects a callback when res.attemptID != b.attemptID. // We trigger this by using a custom sink that, during a release // callback, changes b.attemptID via ReplaceUncommittedAttempt. // Since ReplaceUncommittedAttempt returns ErrBoundarySerializerConflict // when an active reservation exists, we instead directly verify the // identity check by calling applySinkOutcomeLockedWithOpLocked with // a reservation whose attemptID differs from the boundary's. b, _ := NewCommitBoundary(&testSink{}) _ = b.BeginAttempt("a1") // Manually create a reservation with a different attemptID. b.mu.Lock() b.generation++ fakeRes := &reservation{ op: boundaryOpRelease, attemptID: "wrong-attempt", generation: b.generation, } b.activeReservation = fakeRes b.mu.Unlock() // Call applySinkOutcomeLockedWithOpLocked with the correct op. // The attemptID mismatch should produce ErrBoundaryInvalidAttempt. applyErr := b.applySinkOutcomeLockedWithOpLocked( fakeRes, boundaryOpRelease, CommitStateStreamOpen, nil, ) if !errors.Is(applyErr, ErrBoundaryInvalidAttempt) { t.Fatalf("applySinkOutcomeLockedWithOpLocked error = %v, want ErrBoundaryInvalidAttempt", applyErr) } // State should still be preserved (forward state applied despite identity error). if b.State() != CommitStateStreamOpen { t.Fatalf("state = %v, want stream_open (forward state preserved)", b.State()) } }) t.Run("release_reservation_rejects_terminal_callback", func(t *testing.T) { // Defense-in-depth: verify that a release reservation rejects a terminal callback. b, _ := NewCommitBoundary(&testSink{}) _ = b.BeginAttempt("a1") b.mu.Lock() b.generation++ releaseRes := &reservation{ op: boundaryOpRelease, attemptID: "a1", generation: b.generation, } b.activeReservation = releaseRes b.mu.Unlock() // Call applySinkOutcomeLockedWithOpLocked with terminal op (mismatch). applyErr := b.applySinkOutcomeLockedWithOpLocked( releaseRes, boundaryOpTerminal, CommitStateTerminalCommitted, nil, ) if !errors.Is(applyErr, ErrBoundaryInvalidCallbackOp) { t.Fatalf("applySinkOutcomeLockedWithOpLocked error = %v, want ErrBoundaryInvalidCallbackOp", applyErr) } // Forward state should still be preserved. if b.State() != CommitStateTerminalCommitted { t.Fatalf("state = %v, want terminal_committed (forward state preserved)", b.State()) } }) t.Run("terminal_reservation_rejects_release_callback", func(t *testing.T) { // Defense-in-depth: verify that a terminal reservation rejects a release callback. b, _ := NewCommitBoundary(&testSink{}) _ = b.BeginAttempt("a1") b.mu.Lock() b.generation++ terminalRes := &reservation{ op: boundaryOpTerminal, attemptID: "a1", generation: b.generation, } b.activeReservation = terminalRes b.mu.Unlock() // Call applySinkOutcomeLockedWithOpLocked with release op (mismatch). applyErr := b.applySinkOutcomeLockedWithOpLocked( terminalRes, boundaryOpRelease, CommitStateStreamOpen, nil, ) if !errors.Is(applyErr, ErrBoundaryInvalidCallbackOp) { t.Fatalf("applySinkOutcomeLockedWithOpLocked error = %v, want ErrBoundaryInvalidCallbackOp", applyErr) } }) t.Run("terminal_state_blocks_callback_reentry", func(t *testing.T) { // Verify that after terminal, ReleaseSafe and CommitTerminal return // ErrBoundaryAlreadyTerminal and sink counts do not increase. var startCount, relCount, termCount int countingSink := &countingTestSink{ inner: &testSink{}, onStart: func() { startCount++ }, onRelease: func() { relCount++ }, onTerminal: func() { termCount++ }, } b, _ := NewCommitBoundary(countingSink) _ = b.BeginAttempt("a1") rs, _ := NewResponseStart("ch", 200, nil, testNow) _ = b.StageResponseStart("a1", rs) ev, _ := NewReleaseTextDeltaEvent("ch", "x", testNow) _, err := b.ReleaseSafe(ctx, "a1", []ReleaseEvent{ev}) if err != nil { t.Fatalf("ReleaseSafe: %v", err) } tr, _ := NewSuccessTerminalResult("ch", testNow) err = b.CommitTerminal(ctx, "a1", tr) if err != nil { t.Fatalf("CommitTerminal: %v", err) } if b.State() != CommitStateTerminalCommitted { t.Fatalf("state = %v, want terminal_committed", b.State()) } // Re-entry via ReleaseSafe. _, err = b.ReleaseSafe(ctx, "a1", []ReleaseEvent{ev}) if !errors.Is(err, ErrBoundaryAlreadyTerminal) { t.Fatalf("ReleaseSafe re-entry error = %v, want ErrBoundaryAlreadyTerminal", err) } if relCount != 1 { t.Fatalf("releases after ReleaseSafe re-entry = %d, want 1 (no increase)", relCount) } // Re-entry via CommitTerminal. err = b.CommitTerminal(ctx, "a1", tr) if !errors.Is(err, ErrBoundaryAlreadyTerminal) { t.Fatalf("CommitTerminal re-entry error = %v, want ErrBoundaryAlreadyTerminal", err) } if termCount != 1 { t.Fatalf("terminals after CommitTerminal re-entry = %d, want 1 (no increase)", termCount) } // Verify exact counts: 1 start, 1 release, 1 terminal. if startCount != 1 { t.Errorf("starts = %d, want 1", startCount) } if relCount != 1 { t.Errorf("releases = %d, want 1", relCount) } if termCount != 1 { t.Errorf("terminals = %d, want 1", termCount) } }) } type customStateSink struct { startState CommitState startErr error releaseState CommitState releaseErr error terminalState CommitState terminalErr error } func (s *customStateSink) CommitResponseStart(ctx context.Context, rs ResponseStart) (CommitState, error) { return s.startState, s.startErr } func (s *customStateSink) Release(ctx context.Context, ev ReleaseEvent) (CommitState, error) { st := s.releaseState if st == "" { st = s.startState } return st, s.releaseErr } func (s *customStateSink) CommitTerminal(ctx context.Context, tr TerminalResult) (CommitState, error) { st := s.terminalState if st == "" { st = CommitStateTerminalCommitted } return st, s.terminalErr } func TestCommitBoundaryRejectsWrongOrMissingAttempt(t *testing.T) { ctx := context.Background() sink := &testSink{} b, _ := NewCommitBoundary(sink) ev, _ := NewReleaseTextDeltaEvent("ch", "x", testNow) rs, _ := NewResponseStart("ch", 200, nil, testNow) tr, _ := NewSuccessTerminalResult("ch", testNow) // StageResponseStart checks if err := b.StageResponseStart("", rs); err == nil { t.Fatal("expected error for empty attempt on stage") } if err := b.StageResponseStart("a1", rs); err != ErrBoundaryNoCurrentAttempt { t.Fatalf("StageResponseStart without current attempt err = %v, want ErrBoundaryNoCurrentAttempt", err) } _ = b.BeginAttempt("a1") if err := b.StageResponseStart("wrong", rs); err != ErrBoundaryInvalidAttempt { t.Fatalf("StageResponseStart wrong attempt err = %v, want ErrBoundaryInvalidAttempt", err) } // ReleaseSafe checks _, err := b.ReleaseSafe(ctx, "", []ReleaseEvent{ev}) if err == nil { t.Fatal("expected error for empty attempt on release") } _, err = b.ReleaseSafe(ctx, "wrong", []ReleaseEvent{ev}) if err != ErrBoundaryInvalidAttempt { t.Fatalf("ReleaseSafe wrong attempt err = %v, want ErrBoundaryInvalidAttempt", err) } // CommitTerminal checks err = b.CommitTerminal(ctx, "", tr) if err == nil { t.Fatal("expected error for empty attempt on terminal") } err = b.CommitTerminal(ctx, "wrong", tr) if err != ErrBoundaryInvalidAttempt { t.Fatalf("CommitTerminal wrong attempt err = %v, want ErrBoundaryInvalidAttempt", err) } // ReplaceUncommittedAttempt checks err = b.ReplaceUncommittedAttempt("", "a2") if err != ErrBoundaryInvalidAttempt { t.Fatalf("ReplaceUncommittedAttempt empty oldAttemptID err = %v, want ErrBoundaryInvalidAttempt", err) } err = b.ReplaceUncommittedAttempt("wrong", "a2") if err != ErrBoundaryInvalidAttempt { t.Fatalf("ReplaceUncommittedAttempt wrong oldAttemptID err = %v, want ErrBoundaryInvalidAttempt", err) } } func TestCommitBoundarySerializesConcurrentStartAndTerminal(t *testing.T) { ctx := context.Background() callCounts := &sinkCallCounts{starts: new(int32), releases: new(int32), terminals: new(int32)} barrierSink := &barrierTestSink{ enterBarrier: make(chan struct{}), blockChan: make(chan struct{}), callCounts: callCounts, } b, _ := NewCommitBoundary(barrierSink) _ = b.BeginAttempt("a1") rs, _ := NewResponseStart("ch", 200, nil, testNow) _ = b.StageResponseStart("a1", rs) ev, _ := NewReleaseTextDeltaEvent("ch", "x", testNow) tr, _ := NewSuccessTerminalResult("ch", testNow) var wg sync.WaitGroup var relErr, termErr error wg.Add(1) go func() { defer wg.Done() _, relErr = b.ReleaseSafe(ctx, "a1", []ReleaseEvent{ev}) }() // Wait until ReleaseSafe enters sink.Release and holds active reservation select { case <-barrierSink.enterBarrier: case <-time.After(time.Second): t.Fatal("timeout waiting for ReleaseSafe sink entry") } // Concurrent CommitTerminal must fail with ErrBoundarySerializerConflict termErr = b.CommitTerminal(ctx, "a1", tr) if !errors.Is(termErr, ErrBoundarySerializerConflict) { t.Fatalf("concurrent CommitTerminal error = %v, want ErrBoundarySerializerConflict", termErr) } // Unblock first release close(barrierSink.blockChan) wg.Wait() if relErr != nil { t.Fatalf("ReleaseSafe failed: %v", relErr) } if b.State() != CommitStateStreamOpen { t.Fatalf("state = %v, want stream_open", b.State()) } // Verify exact call counts after release. if atomic.LoadInt32(callCounts.starts) != 1 { t.Errorf("starts after release = %d, want 1", atomic.LoadInt32(callCounts.starts)) } if atomic.LoadInt32(callCounts.releases) != 1 { t.Errorf("releases after release = %d, want 1", atomic.LoadInt32(callCounts.releases)) } if atomic.LoadInt32(callCounts.terminals) != 0 { t.Errorf("terminals after release = %d, want 0", atomic.LoadInt32(callCounts.terminals)) } // CommitTerminal after release succeeds. if err := b.CommitTerminal(ctx, "a1", tr); err != nil { t.Fatalf("subsequent CommitTerminal failed: %v", err) } if b.State() != CommitStateTerminalCommitted { t.Fatalf("state = %v, want terminal_committed", b.State()) } // Verify exact call counts after terminal. if atomic.LoadInt32(callCounts.starts) != 1 { t.Errorf("starts after terminal = %d, want 1", atomic.LoadInt32(callCounts.starts)) } if atomic.LoadInt32(callCounts.releases) != 1 { t.Errorf("releases after terminal = %d, want 1", atomic.LoadInt32(callCounts.releases)) } if atomic.LoadInt32(callCounts.terminals) != 1 { t.Errorf("terminals after terminal = %d, want 1", atomic.LoadInt32(callCounts.terminals)) } } // countingTestSink wraps a ReleaseSink and counts calls to each method. // It is used to verify exact callback counts in tests. type countingTestSink struct { inner ReleaseSink onStart func() onRelease func() onTerminal func() } func (s *countingTestSink) CommitResponseStart(ctx context.Context, rs ResponseStart) (CommitState, error) { if s.onStart != nil { s.onStart() } return s.inner.CommitResponseStart(ctx, rs) } func (s *countingTestSink) Release(ctx context.Context, ev ReleaseEvent) (CommitState, error) { if s.onRelease != nil { s.onRelease() } return s.inner.Release(ctx, ev) } func (s *countingTestSink) CommitTerminal(ctx context.Context, tr TerminalResult) (CommitState, error) { if s.onTerminal != nil { s.onTerminal() } return s.inner.CommitTerminal(ctx, tr) } // barrierTestSink is a test sink with an enter barrier and block channel // for testing concurrent reservation conflict. type barrierTestSink struct { enterBarrier chan struct{} blockChan chan struct{} mu sync.Mutex starts int releases int terminals int callCounts *sinkCallCounts } type sinkCallCounts struct { starts *int32 releases *int32 terminals *int32 } func (s *barrierTestSink) CommitResponseStart(ctx context.Context, rs ResponseStart) (CommitState, error) { s.mu.Lock() s.starts++ if s.callCounts != nil { atomic.AddInt32(s.callCounts.starts, 1) } s.mu.Unlock() return CommitStateStreamOpen, nil } func (s *barrierTestSink) Release(ctx context.Context, ev ReleaseEvent) (CommitState, error) { s.mu.Lock() s.releases++ if s.callCounts != nil { atomic.AddInt32(s.callCounts.releases, 1) } s.mu.Unlock() s.enterBarrier <- struct{}{} <-s.blockChan return CommitStateStreamOpen, nil } func (s *barrierTestSink) CommitTerminal(ctx context.Context, tr TerminalResult) (CommitState, error) { s.mu.Lock() s.terminals++ if s.callCounts != nil { atomic.AddInt32(s.callCounts.terminals, 1) } s.mu.Unlock() return CommitStateTerminalCommitted, nil } // TestCommitBoundaryPreservesTerminalStateOnCallbackError verifies that when // the terminal callback returns terminal_committed with an error, the state // is preserved and re-entry is rejected (exactly-once terminal). func TestCommitBoundaryPreservesTerminalStateOnCallbackError(t *testing.T) { ctx := context.Background() t.Run("terminal_error_preserves_terminal_state", func(t *testing.T) { var termCount int countingSink := &countingTestSink{ inner: &customStateSink{ terminalState: CommitStateTerminalCommitted, terminalErr: errors.New("test: terminal network error"), }, onTerminal: func() { termCount++ }, } b, _ := NewCommitBoundary(countingSink) _ = b.BeginAttempt("a1") tr, _ := NewSuccessTerminalResult("ch", testNow) err := b.CommitTerminal(ctx, "a1", tr) if err == nil { t.Fatal("expected error from terminal with network error") } if b.State() != CommitStateTerminalCommitted { t.Fatalf("state = %v, want terminal_committed (preserved despite error)", b.State()) } // Re-entry must be rejected without calling sink. err2 := b.CommitTerminal(ctx, "a1", tr) if !errors.Is(err2, ErrBoundaryAlreadyTerminal) { t.Fatalf("re-entry error = %v, want ErrBoundaryAlreadyTerminal", err2) } if termCount != 1 { t.Fatalf("terminals = %d, want 1 (exactly-once)", termCount) } }) t.Run("terminal_error_with_role_mismatch_preserves_state", func(t *testing.T) { var termCount int countingSink := &countingTestSink{ inner: &customStateSink{ terminalState: CommitStateStreamOpen, terminalErr: errors.New("test: terminal partial error"), }, onTerminal: func() { termCount++ }, } b, _ := NewCommitBoundary(countingSink) _ = b.BeginAttempt("a1") tr, _ := NewSuccessTerminalResult("ch", testNow) err := b.CommitTerminal(ctx, "a1", tr) if err == nil { t.Fatal("expected error from terminal with role mismatch + error") } // Verify joined error contains both sink and role errors. errStr := err.Error() if !strings.Contains(errStr, "test: terminal partial error") { t.Errorf("expected sink error in joined error, got: %v", err) } if !strings.Contains(errStr, ErrInvalidStateTransition.Error()) { t.Errorf("expected role error in joined error, got: %v", err) } // State should be preserved at stream_open (valid forward state). if b.State() != CommitStateStreamOpen { t.Fatalf("state = %v, want stream_open (forward state preserved)", b.State()) } if termCount != 1 { t.Fatalf("terminals = %d, want 1", termCount) } }) t.Run("response_start_role_mismatch_advances_state", func(t *testing.T) { var startCount int countingSink := &countingTestSink{ inner: &customStateSink{ startState: CommitStateTerminalCommitted, }, onStart: func() { startCount++ }, } b, _ := NewCommitBoundary(countingSink) _ = b.BeginAttempt("a1") rs, _ := NewResponseStart("ch", 200, nil, testNow) _ = b.StageResponseStart("a1", rs) ev, _ := NewReleaseTextDeltaEvent("ch", "x", testNow) _, err := b.ReleaseSafe(ctx, "a1", []ReleaseEvent{ev}) if !errors.Is(err, ErrInvalidStateTransition) { t.Fatalf("expected ErrInvalidStateTransition, got %v", err) } if b.State() != CommitStateTerminalCommitted { t.Fatalf("state = %v, want terminal_committed (forward state preserved)", b.State()) } // Staged start should be cleared because state advanced. if startCount != 1 { t.Fatalf("responseStarts = %d, want 1", startCount) } }) t.Run("release_role_mismatch_advances_to_terminal", func(t *testing.T) { var relCount int countingSink := &countingTestSink{ inner: &customStateSink{ startState: CommitStateStreamOpen, releaseState: CommitStateTerminalCommitted, }, onRelease: func() { relCount++ }, } b, _ := NewCommitBoundary(countingSink) _ = b.BeginAttempt("a1") rs, _ := NewResponseStart("ch", 200, nil, testNow) _ = b.StageResponseStart("a1", rs) ev, _ := NewReleaseTextDeltaEvent("ch", "x", testNow) _, err := b.ReleaseSafe(ctx, "a1", []ReleaseEvent{ev}) if !errors.Is(err, ErrInvalidStateTransition) { t.Fatalf("expected ErrInvalidStateTransition, got %v", err) } if b.State() != CommitStateTerminalCommitted { t.Fatalf("state = %v, want terminal_committed (forward state preserved)", b.State()) } // No further releases should be attempted. if relCount != 1 { t.Fatalf("releases = %d, want 1 (short-circuited after role mismatch)", relCount) } }) t.Run("sink_error_with_valid_forward_state_joins_errors", func(t *testing.T) { customSink := &customStateSink{ terminalState: CommitStateTerminalCommitted, terminalErr: errors.New("test: terminal error"), } b, _ := NewCommitBoundary(customSink) _ = b.BeginAttempt("a1") tr, _ := NewSuccessTerminalResult("ch", testNow) err := b.CommitTerminal(ctx, "a1", tr) if err == nil { t.Fatal("expected error") } // Should be a joined error with the sink error. if !strings.Contains(err.Error(), "test: terminal error") { t.Fatalf("expected error to contain 'test: terminal error', got %v", err) } if b.State() != CommitStateTerminalCommitted { t.Fatalf("state = %v, want terminal_committed", b.State()) } }) t.Run("exact_callback_count_after_terminal_preservation", func(t *testing.T) { var termCount int countingSink := &countingTestSink{ inner: &customStateSink{ terminalState: CommitStateTerminalCommitted, terminalErr: errors.New("test: terminal failure"), }, onTerminal: func() { termCount++ }, } b, _ := NewCommitBoundary(countingSink) _ = b.BeginAttempt("a1") tr, _ := NewSuccessTerminalResult("ch", testNow) err := b.CommitTerminal(ctx, "a1", tr) if err == nil { t.Fatal("expected error") } if termCount != 1 { t.Fatalf("terminals = %d, want 1", termCount) } // Re-entry should be rejected without calling sink. _ = b.CommitTerminal(ctx, "a1", tr) if termCount != 1 { t.Fatalf("terminals after re-entry = %d, want 1 (exactly-once)", termCount) } if b.State() != CommitStateTerminalCommitted { t.Fatalf("state = %v, want terminal_committed", b.State()) } }) } func TestCommitBoundaryRejectsPostOpenReplacement(t *testing.T) { ctx := context.Background() sink := &testSink{} b, _ := NewCommitBoundary(sink) _ = b.BeginAttempt("a1") rs, _ := NewResponseStart("ch", 200, nil, testNow) _ = b.StageResponseStart("a1", rs) // Valid replacement prior to release if err := b.ReplaceUncommittedAttempt("a1", "a2"); err != nil { t.Fatalf("valid ReplaceUncommittedAttempt failed: %v", err) } if b.CurrentAttempt() != "a2" { t.Fatalf("CurrentAttempt = %q, want a2", b.CurrentAttempt()) } _ = b.StageResponseStart("a2", rs) ev, _ := NewReleaseTextDeltaEvent("ch", "x", testNow) _, _ = b.ReleaseSafe(ctx, "a2", []ReleaseEvent{ev}) // After stream_open, replacement is rejected and selected attempt is preserved err := b.ReplaceUncommittedAttempt("a2", "a3") if err != ErrBoundaryUncommitted { t.Fatalf("ReplaceUncommittedAttempt after open err = %v, want ErrBoundaryUncommitted", err) } if b.CurrentAttempt() != "a2" { t.Fatalf("CurrentAttempt = %q, want preserved a2", b.CurrentAttempt()) } } func mustReleaseTextDelta(channel, text string, ts time.Time) ReleaseEvent { ev, err := NewReleaseTextDeltaEvent(channel, text, ts) if err != nil { panic(err) } return ev }