iop/packages/go/streamgate/commit_boundary_test.go

1302 lines
41 KiB
Go

package streamgate
import (
"context"
"errors"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
)
// testNow is a deterministic timestamp used in tests.
var testNow = time.Date(2026, 7, 24, 12, 0, 0, 0, time.UTC)
// testSink is a configurable test-only ReleaseSink that records calls and
// can be programmed to fail specific calls.
type testSink struct {
mu sync.Mutex
responseStarts []ResponseStart
releases []ReleaseEvent
terminals []TerminalResult
state CommitState
failAfterSuccesses int // >0: fail after N successes; 0: no failure
failAll bool // if true, all releases fail
successCount int // how many successful releases have occurred (only tracked when failAfterSuccesses > 0)
onRelease func(ReleaseEvent)
onTerminal func(TerminalResult)
}
func (s *testSink) CommitResponseStart(ctx context.Context, rs ResponseStart) (CommitState, error) {
s.mu.Lock()
defer s.mu.Unlock()
s.responseStarts = append(s.responseStarts, rs)
s.state = CommitStateStreamOpen
return CommitStateStreamOpen, nil
}
func (s *testSink) Release(ctx context.Context, ev ReleaseEvent) (CommitState, error) {
s.mu.Lock()
if s.onRelease != nil {
fn := s.onRelease
s.mu.Unlock()
fn(ev)
s.mu.Lock()
}
defer s.mu.Unlock()
if s.failAll {
return "", errors.New("test: forced release failure")
}
if s.failAfterSuccesses > 0 {
s.successCount++
if s.successCount > s.failAfterSuccesses {
return "", errors.New("test: forced release failure")
}
}
s.releases = append(s.releases, ev)
return s.state, nil
}
func (s *testSink) CommitTerminal(ctx context.Context, tr TerminalResult) (CommitState, error) {
s.mu.Lock()
if s.onTerminal != nil {
fn := s.onTerminal
s.mu.Unlock()
fn(tr)
s.mu.Lock()
}
defer s.mu.Unlock()
s.terminals = append(s.terminals, tr)
s.state = CommitStateTerminalCommitted
return CommitStateTerminalCommitted, nil
}
// ---------------------------------------------------------------------------
// REVIEW_API-1: Request-scope staging and monotonic sink state
// ---------------------------------------------------------------------------
// TestCommitBoundaryStagesUntilFirstSafeRelease verifies that stage calls
// produce zero sink invocations and the first release commits start→event
// order with a single start.
func TestCommitBoundaryStagesUntilFirstSafeRelease(t *testing.T) {
ctx := context.Background()
sink := &testSink{}
b, err := NewCommitBoundary(sink)
if err != nil {
t.Fatalf("NewCommitBoundary: %v", err)
}
rs, _ := NewResponseStart("ch", 200, map[string]string{"x-test": "v"}, testNow)
if err := b.BeginAttempt("a1"); err != nil {
t.Fatalf("BeginAttempt: %v", err)
}
if err := b.StageResponseStart("a1", rs); err != nil {
t.Fatalf("StageResponseStart: %v", err)
}
if got := b.State(); got != CommitStateTransportUncommitted {
t.Errorf("state before release = %q, want %q", got, CommitStateTransportUncommitted)
}
ev1, _ := NewReleaseTextDeltaEvent("ch", "hello ", testNow)
ev2, _ := NewReleaseTextDeltaEvent("ch", "world", testNow)
prog, err := b.ReleaseSafe(ctx, "a1", []ReleaseEvent{ev1, ev2})
if err != nil {
t.Fatalf("ReleaseSafe: %v", err)
}
if len(sink.responseStarts) != 1 {
t.Errorf("responseStarts = %d, want 1", len(sink.responseStarts))
}
if len(sink.releases) != 2 {
t.Errorf("releases = %d, want 2", len(sink.releases))
}
if prog.ReleasedEvents() != 2 {
t.Errorf("ReleasedEvents = %d, want 2", prog.ReleasedEvents())
}
if got := b.State(); got != CommitStateStreamOpen {
t.Errorf("state after release = %q, want %q", got, CommitStateStreamOpen)
}
// Second release still works (start already consumed).
_, err = b.ReleaseSafe(ctx, "a1", []ReleaseEvent{ev1})
if err != nil {
t.Fatalf("second ReleaseSafe: %v", err)
}
if len(sink.releases) != 3 {
t.Errorf("total releases after second call = %d, want 3", len(sink.releases))
}
if len(sink.responseStarts) != 1 {
t.Errorf("total responseStarts = %d, want 1", len(sink.responseStarts))
}
}
// TestCommitBoundaryPreservesProgressAndMonotonicState verifies the state
// transition table for start/body/error and that successful event count is
// preserved across failures.
func TestCommitBoundaryPreservesProgressAndMonotonicState(t *testing.T) {
ctx := context.Background()
t.Run("success_steps_preserve_count", func(t *testing.T) {
sink := &testSink{}
b, _ := NewCommitBoundary(sink)
b.BeginAttempt("a1")
rs, _ := NewResponseStart("ch", 200, nil, testNow)
b.StageResponseStart("a1", rs)
evs := make([]ReleaseEvent, 5)
for i := range evs {
e, _ := NewReleaseTextDeltaEvent("ch", "chunk", testNow)
evs[i] = e
}
prog, err := b.ReleaseSafe(ctx, "a1", evs)
if err != nil {
t.Fatalf("ReleaseSafe: %v", err)
}
if prog.ReleasedEvents() != 5 {
t.Errorf("ReleasedEvents = %d, want 5", prog.ReleasedEvents())
}
if len(sink.releases) != 5 {
t.Errorf("releases = %d, want 5", len(sink.releases))
}
if got := b.State(); got != CommitStateStreamOpen {
t.Errorf("state = %q, want stream_open", got)
}
})
t.Run("sink_error_preserves_progress", func(t *testing.T) {
sink := &testSink{failAfterSuccesses: 1}
b, _ := NewCommitBoundary(sink)
b.BeginAttempt("a1")
rs, _ := NewResponseStart("ch", 200, nil, testNow)
b.StageResponseStart("a1", rs)
evs := make([]ReleaseEvent, 3)
for i := range evs {
e, _ := NewReleaseTextDeltaEvent("ch", "chunk", testNow)
evs[i] = e
}
prog, err := b.ReleaseSafe(ctx, "a1", evs)
if err == nil {
t.Error("expected error from ReleaseSafe")
}
// First event succeeded before the forced failure.
if prog.ReleasedEvents() != 1 {
t.Errorf("ReleasedEvents = %d, want 1", prog.ReleasedEvents())
}
if len(sink.releases) != 1 {
t.Errorf("releases = %d, want 1", len(sink.releases))
}
if got := b.State(); got != CommitStateStreamOpen {
t.Errorf("state = %q, want stream_open (preserved)", got)
}
})
t.Run("begin_after_stream_open_rejected", func(t *testing.T) {
sink := &testSink{}
b, _ := NewCommitBoundary(sink)
b.BeginAttempt("a1")
rs, _ := NewResponseStart("ch", 200, nil, testNow)
b.StageResponseStart("a1", rs)
ev, _ := NewReleaseTextDeltaEvent("ch", "x", testNow)
b.ReleaseSafe(ctx, "a1", []ReleaseEvent{ev})
err := b.BeginAttempt("a2")
if err != ErrBoundaryUncommitted {
t.Errorf("BeginAttempt error = %v, want ErrBoundaryUncommitted", err)
}
})
t.Run("zero_release_events_rejected", func(t *testing.T) {
sink := &testSink{}
b, _ := NewCommitBoundary(sink)
b.BeginAttempt("a1")
_, err := b.ReleaseSafe(ctx, "a1", nil)
if err == nil {
t.Error("expected error for empty events")
}
})
}
// TestCommitBoundaryCommitsEmptySuccessAndTerminalOnce verifies that success
// terminal without body commits start first, terminal exactly once, and
// State() does not deadlock during callbacks.
func TestCommitBoundaryCommitsEmptySuccessAndTerminalOnce(t *testing.T) {
ctx := context.Background()
t.Run("success_terminal_no_body", func(t *testing.T) {
sink := &testSink{}
b, _ := NewCommitBoundary(sink)
b.BeginAttempt("a1")
rs, _ := NewResponseStart("ch", 200, nil, testNow)
b.StageResponseStart("a1", rs)
tr, _ := NewSuccessTerminalResult("ch", testNow)
if err := b.CommitTerminal(ctx, "a1", tr); err != nil {
t.Fatalf("CommitTerminal: %v", err)
}
if len(sink.responseStarts) != 1 {
t.Errorf("responseStarts = %d, want 1", len(sink.responseStarts))
}
if len(sink.terminals) != 1 {
t.Errorf("terminals = %d, want 1", len(sink.terminals))
}
if got := b.State(); got != CommitStateTerminalCommitted {
t.Errorf("state = %q, want terminal_committed", got)
}
})
t.Run("second_terminal_rejected", func(t *testing.T) {
sink := &testSink{}
b, _ := NewCommitBoundary(sink)
b.BeginAttempt("a1")
tr, _ := NewSuccessTerminalResult("ch", testNow)
b.CommitTerminal(ctx, "a1", tr)
err := b.CommitTerminal(ctx, "a1", tr)
if err != ErrBoundaryAlreadyTerminal {
t.Errorf("second CommitTerminal error = %v, want ErrBoundaryAlreadyTerminal", err)
}
})
t.Run("state_no_deadlock_during_callback", func(t *testing.T) {
sink := &testSink{}
b, _ := NewCommitBoundary(sink)
b.BeginAttempt("a1")
rs, _ := NewResponseStart("ch", 200, nil, testNow)
b.StageResponseStart("a1", rs)
done := make(chan struct{})
go func() {
defer close(done)
for i := 0; i < 100; i++ {
_ = b.State()
}
}()
evs := make([]ReleaseEvent, 50)
for i := range evs {
e, _ := NewReleaseTextDeltaEvent("ch", "x", testNow)
evs[i] = e
}
_, err := b.ReleaseSafe(ctx, "a1", evs)
if err != nil {
t.Fatalf("ReleaseSafe: %v", err)
}
<-done
})
t.Run("error_terminal_discards_staged_start", func(t *testing.T) {
sink := &testSink{}
b, _ := NewCommitBoundary(sink)
b.BeginAttempt("a1")
rs, _ := NewResponseStart("ch", 200, nil, testNow)
b.StageResponseStart("a1", rs)
desc, _ := NewExternalDescriptor("generic", "internal_error", "error", "")
causes, _ := NewFailureCauseChain([]FailureCause{})
tr, _ := NewErrorTerminalResult("ch", desc, causes, testNow)
if err := b.CommitTerminal(ctx, "a1", tr); err != nil {
t.Fatalf("CommitTerminal: %v", err)
}
if len(sink.responseStarts) != 0 {
t.Errorf("responseStarts = %d, want 0 (error discards start)", len(sink.responseStarts))
}
if len(sink.terminals) != 1 {
t.Errorf("terminals = %d, want 1", len(sink.terminals))
}
})
}
// ---------------------------------------------------------------------------
// REVIEW_API-3: Attempt replacement and recovery strategy matrix
// ---------------------------------------------------------------------------
// TestCommitBoundaryReplacesOnlyUncommittedAttempt verifies that replacement
// discards the old attempt and staged start, allowing a new attempt to begin.
func TestCommitBoundaryReplacesOnlyUncommittedAttempt(t *testing.T) {
ctx := context.Background()
t.Run("replacement_discards_old_and_allows_new", func(t *testing.T) {
sink := &testSink{}
b, _ := NewCommitBoundary(sink)
// First attempt with staged start.
b.BeginAttempt("old")
rs, _ := NewResponseStart("ch", 200, nil, testNow)
b.StageResponseStart("old", rs)
// Replace.
if err := b.ReplaceUncommittedAttempt("old", "new"); err != nil {
t.Fatalf("ReplaceUncommittedAttempt: %v", err)
}
// Old attempt should not work (no current attempt).
err := b.StageResponseStart("old", rs)
if err == nil {
// Actually after replace, there's no current attempt, so
// StageResponseStart should work since state is still uncommitted.
// But the old attempt ID is not special anymore.
}
// New attempt should work.
b.BeginAttempt("new")
rs2, _ := NewResponseStart("ch", 200, nil, testNow)
b.StageResponseStart("new", rs2)
ev, _ := NewReleaseTextDeltaEvent("ch", "data", testNow)
_, err = b.ReleaseSafe(ctx, "new", []ReleaseEvent{ev})
if err != nil {
t.Fatalf("ReleaseSafe: %v", err)
}
// Exactly one start from new attempt.
if len(sink.responseStarts) != 1 {
t.Errorf("responseStarts = %d, want 1 (only new attempt)", len(sink.responseStarts))
}
})
t.Run("post_open_replacement_rejected", func(t *testing.T) {
sink := &testSink{}
b, _ := NewCommitBoundary(sink)
b.BeginAttempt("a1")
rs, _ := NewResponseStart("ch", 200, nil, testNow)
b.StageResponseStart("a1", rs)
ev, _ := NewReleaseTextDeltaEvent("ch", "x", testNow)
b.ReleaseSafe(ctx, "a1", []ReleaseEvent{ev})
// After stream_open, we cannot begin a new attempt.
err := b.BeginAttempt("a2")
if err != ErrBoundaryUncommitted {
t.Errorf("BeginAttempt after open error = %v, want ErrBoundaryUncommitted", err)
}
})
}
// TestCommitBoundaryCommitStrategyMatrix verifies the allowed strategy table
// for each commit state.
func TestCommitBoundaryCommitStrategyMatrix(t *testing.T) {
ctx := context.Background()
strategies := []RecoveryStrategy{
RecoveryStrategyExactReplay,
RecoveryStrategyContinuationRepair,
RecoveryStrategySchemaRepair,
}
t.Run("transport_uncommitted", func(t *testing.T) {
sink := &testSink{}
b, _ := NewCommitBoundary(sink)
b.BeginAttempt("a1")
expectations := map[RecoveryStrategy]bool{
RecoveryStrategyExactReplay: true,
RecoveryStrategySchemaRepair: true,
RecoveryStrategyContinuationRepair: false,
}
for _, s := range strategies {
got := b.CommitAllowsStrategy(s)
want := expectations[s]
if got != want {
t.Errorf("CommitAllowsStrategy(%s) = %v, want %v", s, got, want)
}
}
})
t.Run("stream_open", func(t *testing.T) {
sink := &testSink{}
b, _ := NewCommitBoundary(sink)
b.BeginAttempt("a1")
rs, _ := NewResponseStart("ch", 200, nil, testNow)
b.StageResponseStart("a1", rs)
ev, _ := NewReleaseTextDeltaEvent("ch", "x", testNow)
b.ReleaseSafe(ctx, "a1", []ReleaseEvent{ev})
expectations := map[RecoveryStrategy]bool{
RecoveryStrategyExactReplay: false,
RecoveryStrategySchemaRepair: false,
RecoveryStrategyContinuationRepair: true,
}
for _, s := range strategies {
got := b.CommitAllowsStrategy(s)
want := expectations[s]
if got != want {
t.Errorf("CommitAllowsStrategy(%s) = %v, want %v", s, got, want)
}
}
})
t.Run("terminal_committed", func(t *testing.T) {
sink := &testSink{}
b, _ := NewCommitBoundary(sink)
b.BeginAttempt("a1")
tr, _ := NewSuccessTerminalResult("ch", testNow)
b.CommitTerminal(ctx, "a1", tr)
for _, s := range strategies {
got := b.CommitAllowsStrategy(s)
if got {
t.Errorf("CommitAllowsStrategy(%s) = true, want false after terminal", s)
}
}
})
}
func TestCommitBoundaryPreservesReturnedStateAcrossSinkErrors(t *testing.T) {
ctx := context.Background()
t.Run("sink_error_with_forward_state_preserved", func(t *testing.T) {
sink := &testSink{failAfterSuccesses: 1}
b, _ := NewCommitBoundary(sink)
_ = b.BeginAttempt("a1")
rs, _ := NewResponseStart("ch", 200, nil, testNow)
_ = b.StageResponseStart("a1", rs)
evs := make([]ReleaseEvent, 3)
for i := range evs {
e, _ := NewReleaseTextDeltaEvent("ch", "chunk", testNow)
evs[i] = e
}
prog, err := b.ReleaseSafe(ctx, "a1", evs)
if err == nil {
t.Fatal("expected sink error")
}
if prog.ReleasedEvents() != 1 {
t.Fatalf("ReleasedEvents = %d, want 1", prog.ReleasedEvents())
}
if b.State() != CommitStateStreamOpen {
t.Fatalf("state = %v, want stream_open", b.State())
}
})
t.Run("empty_returned_state_on_success_rejected", func(t *testing.T) {
customSink := &customStateSink{
startState: "",
}
b, _ := NewCommitBoundary(customSink)
_ = b.BeginAttempt("a1")
rs, _ := NewResponseStart("ch", 200, nil, testNow)
_ = b.StageResponseStart("a1", rs)
ev, _ := NewReleaseTextDeltaEvent("ch", "x", testNow)
_, err := b.ReleaseSafe(ctx, "a1", []ReleaseEvent{ev})
if !errors.Is(err, ErrInvalidStateTransition) {
t.Fatalf("expected ErrInvalidStateTransition for empty state on success, got %v", err)
}
})
t.Run("invalid_state_regression_rejected", func(t *testing.T) {
customSink := &customStateSink{
startState: CommitStateStreamOpen,
releaseState: CommitStateTransportUncommitted, // invalid transition stream_open -> uncommitted
}
b, _ := NewCommitBoundary(customSink)
_ = b.BeginAttempt("a1")
rs, _ := NewResponseStart("ch", 200, nil, testNow)
_ = b.StageResponseStart("a1", rs)
ev, _ := NewReleaseTextDeltaEvent("ch", "x", testNow)
_, err := b.ReleaseSafe(ctx, "a1", []ReleaseEvent{ev})
if !errors.Is(err, ErrInvalidStateTransition) {
t.Fatalf("expected ErrInvalidStateTransition for state regression, got %v", err)
}
})
t.Run("terminal_returns_non_terminal_state_rejected", func(t *testing.T) {
customSink := &customStateSink{
terminalState: CommitStateStreamOpen, // wrong: terminal should return terminal_committed
}
b, _ := NewCommitBoundary(customSink)
_ = b.BeginAttempt("a1")
tr, _ := NewSuccessTerminalResult("ch", testNow)
err := b.CommitTerminal(ctx, "a1", tr)
if !errors.Is(err, ErrInvalidStateTransition) {
t.Fatalf("expected ErrInvalidStateTransition for terminal returning %q, got %v", CommitStateStreamOpen, err)
}
// Role mismatch does not prevent advancing to a valid forward state.
// The callback returned stream_open which is a valid forward transition
// from transport_uncommitted, so the state is preserved.
if b.State() != CommitStateStreamOpen {
t.Fatalf("state = %v, want stream_open (forward state preserved despite role mismatch)", b.State())
}
})
t.Run("response_start_returns_non_stream_open_rejected", func(t *testing.T) {
customSink := &customStateSink{
startState: CommitStateTerminalCommitted, // wrong: response-start should return stream_open
}
b, _ := NewCommitBoundary(customSink)
_ = b.BeginAttempt("a1")
rs, _ := NewResponseStart("ch", 200, nil, testNow)
_ = b.StageResponseStart("a1", rs)
ev, _ := NewReleaseTextDeltaEvent("ch", "x", testNow)
_, err := b.ReleaseSafe(ctx, "a1", []ReleaseEvent{ev})
if !errors.Is(err, ErrInvalidStateTransition) {
t.Fatalf("expected ErrInvalidStateTransition for response-start returning %q, got %v", CommitStateTerminalCommitted, err)
}
// Role mismatch does not prevent advancing to a valid forward state.
// The callback returned terminal_committed which is a valid forward
// transition from transport_uncommitted, so the state is preserved.
if b.State() != CommitStateTerminalCommitted {
t.Fatalf("state = %v, want terminal_committed (forward state preserved despite role mismatch)", b.State())
}
})
t.Run("release_returns_non_stream_open_rejected", func(t *testing.T) {
customSink := &customStateSink{
startState: CommitStateStreamOpen,
releaseState: CommitStateTerminalCommitted, // wrong: release should return stream_open
}
b, _ := NewCommitBoundary(customSink)
_ = b.BeginAttempt("a1")
rs, _ := NewResponseStart("ch", 200, nil, testNow)
_ = b.StageResponseStart("a1", rs)
ev, _ := NewReleaseTextDeltaEvent("ch", "x", testNow)
_, err := b.ReleaseSafe(ctx, "a1", []ReleaseEvent{ev})
if !errors.Is(err, ErrInvalidStateTransition) {
t.Fatalf("expected ErrInvalidStateTransition for release returning %q, got %v", CommitStateTerminalCommitted, err)
}
// Role mismatch does not prevent advancing to a valid forward state.
// Response-start advanced to stream_open, and release returned
// terminal_committed which is a valid forward transition, so state
// is preserved at terminal_committed.
if b.State() != CommitStateTerminalCommitted {
t.Fatalf("state = %v, want terminal_committed (forward state preserved despite role mismatch)", b.State())
}
})
t.Run("response_start_error_short_circuits_release", func(t *testing.T) {
customSink := &customStateSink{
startErr: errors.New("test: start failed"),
}
b, _ := NewCommitBoundary(customSink)
_ = b.BeginAttempt("a1")
rs, _ := NewResponseStart("ch", 200, nil, testNow)
_ = b.StageResponseStart("a1", rs)
ev, _ := NewReleaseTextDeltaEvent("ch", "x", testNow)
_, err := b.ReleaseSafe(ctx, "a1", []ReleaseEvent{ev})
if err == nil {
t.Fatal("expected error from response-start short-circuit")
}
})
t.Run("terminal_success_with_terminal_committed_allows_no_reentry", func(t *testing.T) {
customSink := &customStateSink{
terminalState: CommitStateTerminalCommitted,
}
b, _ := NewCommitBoundary(customSink)
_ = b.BeginAttempt("a1")
tr, _ := NewSuccessTerminalResult("ch", testNow)
if err := b.CommitTerminal(ctx, "a1", tr); err != nil {
t.Fatalf("CommitTerminal: %v", err)
}
if b.State() != CommitStateTerminalCommitted {
t.Fatalf("state = %v, want terminal_committed", b.State())
}
// Re-entry must be rejected.
err := b.CommitTerminal(ctx, "a1", tr)
if !errors.Is(err, ErrBoundaryAlreadyTerminal) {
t.Fatalf("re-entry error = %v, want ErrBoundaryAlreadyTerminal", err)
}
})
t.Run("terminal_error_with_terminal_committed_preserves_state", func(t *testing.T) {
customSink := &customStateSink{
terminalState: CommitStateTerminalCommitted,
terminalErr: errors.New("test: terminal network error"),
}
b, _ := NewCommitBoundary(customSink)
_ = b.BeginAttempt("a1")
tr, _ := NewSuccessTerminalResult("ch", testNow)
err := b.CommitTerminal(ctx, "a1", tr)
if err == nil {
t.Fatal("expected error from terminal with error + terminal_committed")
}
// State should still be terminal_committed.
if b.State() != CommitStateTerminalCommitted {
t.Fatalf("state = %v, want terminal_committed (preserved)", b.State())
}
// Re-entry must still be rejected.
err2 := b.CommitTerminal(ctx, "a1", tr)
if !errors.Is(err2, ErrBoundaryAlreadyTerminal) {
t.Fatalf("re-entry error = %v, want ErrBoundaryAlreadyTerminal", err2)
}
})
t.Run("callback_exact_count", func(t *testing.T) {
var callCounts struct {
starts, releases, terminals int
}
customSink := &countingTestSink{
inner: &testSink{},
onStart: func() { callCounts.starts++ },
onRelease: func() { callCounts.releases++ },
onTerminal: func() { callCounts.terminals++ },
}
b, _ := NewCommitBoundary(customSink)
_ = b.BeginAttempt("a1")
rs, _ := NewResponseStart("ch", 200, nil, testNow)
_ = b.StageResponseStart("a1", rs)
evs := make([]ReleaseEvent, 2)
for i := range evs {
e, _ := NewReleaseTextDeltaEvent("ch", "chunk", testNow)
evs[i] = e
}
_, err := b.ReleaseSafe(ctx, "a1", evs)
if err != nil {
t.Fatalf("ReleaseSafe: %v", err)
}
if callCounts.starts != 1 {
t.Errorf("starts = %d, want 1", callCounts.starts)
}
if callCounts.releases != 2 {
t.Errorf("releases = %d, want 2", callCounts.releases)
}
tr, _ := NewSuccessTerminalResult("ch", testNow)
if err := b.CommitTerminal(ctx, "a1", tr); err != nil {
t.Fatalf("CommitTerminal: %v", err)
}
if callCounts.terminals != 1 {
t.Errorf("terminals = %d, want 1", callCounts.terminals)
}
// Second terminal should be rejected without calling sink.
_ = b.CommitTerminal(ctx, "a1", tr)
if callCounts.terminals != 1 {
t.Errorf("terminals after re-entry = %d, want 1", callCounts.terminals)
}
})
t.Run("attempt_mismatch_rejected_with_identity_error", func(t *testing.T) {
// Defense-in-depth: verify that applySinkOutcomeLockedWithOpLocked
// rejects a callback when res.attemptID != b.attemptID.
// We trigger this by using a custom sink that, during a release
// callback, changes b.attemptID via ReplaceUncommittedAttempt.
// Since ReplaceUncommittedAttempt returns ErrBoundarySerializerConflict
// when an active reservation exists, we instead directly verify the
// identity check by calling applySinkOutcomeLockedWithOpLocked with
// a reservation whose attemptID differs from the boundary's.
b, _ := NewCommitBoundary(&testSink{})
_ = b.BeginAttempt("a1")
// Manually create a reservation with a different attemptID.
b.mu.Lock()
b.generation++
fakeRes := &reservation{
op: boundaryOpRelease,
attemptID: "wrong-attempt",
generation: b.generation,
}
b.activeReservation = fakeRes
b.mu.Unlock()
// Call applySinkOutcomeLockedWithOpLocked with the correct op.
// The attemptID mismatch should produce ErrBoundaryInvalidAttempt.
applyErr := b.applySinkOutcomeLockedWithOpLocked(
fakeRes, boundaryOpRelease, CommitStateStreamOpen, nil,
)
if !errors.Is(applyErr, ErrBoundaryInvalidAttempt) {
t.Fatalf("applySinkOutcomeLockedWithOpLocked error = %v, want ErrBoundaryInvalidAttempt", applyErr)
}
// State should still be preserved (forward state applied despite identity error).
if b.State() != CommitStateStreamOpen {
t.Fatalf("state = %v, want stream_open (forward state preserved)", b.State())
}
})
t.Run("release_reservation_rejects_terminal_callback", func(t *testing.T) {
// Defense-in-depth: verify that a release reservation rejects a terminal callback.
b, _ := NewCommitBoundary(&testSink{})
_ = b.BeginAttempt("a1")
b.mu.Lock()
b.generation++
releaseRes := &reservation{
op: boundaryOpRelease,
attemptID: "a1",
generation: b.generation,
}
b.activeReservation = releaseRes
b.mu.Unlock()
// Call applySinkOutcomeLockedWithOpLocked with terminal op (mismatch).
applyErr := b.applySinkOutcomeLockedWithOpLocked(
releaseRes, boundaryOpTerminal, CommitStateTerminalCommitted, nil,
)
if !errors.Is(applyErr, ErrBoundaryInvalidCallbackOp) {
t.Fatalf("applySinkOutcomeLockedWithOpLocked error = %v, want ErrBoundaryInvalidCallbackOp", applyErr)
}
// Forward state should still be preserved.
if b.State() != CommitStateTerminalCommitted {
t.Fatalf("state = %v, want terminal_committed (forward state preserved)", b.State())
}
})
t.Run("terminal_reservation_rejects_release_callback", func(t *testing.T) {
// Defense-in-depth: verify that a terminal reservation rejects a release callback.
b, _ := NewCommitBoundary(&testSink{})
_ = b.BeginAttempt("a1")
b.mu.Lock()
b.generation++
terminalRes := &reservation{
op: boundaryOpTerminal,
attemptID: "a1",
generation: b.generation,
}
b.activeReservation = terminalRes
b.mu.Unlock()
// Call applySinkOutcomeLockedWithOpLocked with release op (mismatch).
applyErr := b.applySinkOutcomeLockedWithOpLocked(
terminalRes, boundaryOpRelease, CommitStateStreamOpen, nil,
)
if !errors.Is(applyErr, ErrBoundaryInvalidCallbackOp) {
t.Fatalf("applySinkOutcomeLockedWithOpLocked error = %v, want ErrBoundaryInvalidCallbackOp", applyErr)
}
})
t.Run("terminal_state_blocks_callback_reentry", func(t *testing.T) {
// Verify that after terminal, ReleaseSafe and CommitTerminal return
// ErrBoundaryAlreadyTerminal and sink counts do not increase.
var startCount, relCount, termCount int
countingSink := &countingTestSink{
inner: &testSink{},
onStart: func() { startCount++ },
onRelease: func() { relCount++ },
onTerminal: func() { termCount++ },
}
b, _ := NewCommitBoundary(countingSink)
_ = b.BeginAttempt("a1")
rs, _ := NewResponseStart("ch", 200, nil, testNow)
_ = b.StageResponseStart("a1", rs)
ev, _ := NewReleaseTextDeltaEvent("ch", "x", testNow)
_, err := b.ReleaseSafe(ctx, "a1", []ReleaseEvent{ev})
if err != nil {
t.Fatalf("ReleaseSafe: %v", err)
}
tr, _ := NewSuccessTerminalResult("ch", testNow)
err = b.CommitTerminal(ctx, "a1", tr)
if err != nil {
t.Fatalf("CommitTerminal: %v", err)
}
if b.State() != CommitStateTerminalCommitted {
t.Fatalf("state = %v, want terminal_committed", b.State())
}
// Re-entry via ReleaseSafe.
_, err = b.ReleaseSafe(ctx, "a1", []ReleaseEvent{ev})
if !errors.Is(err, ErrBoundaryAlreadyTerminal) {
t.Fatalf("ReleaseSafe re-entry error = %v, want ErrBoundaryAlreadyTerminal", err)
}
if relCount != 1 {
t.Fatalf("releases after ReleaseSafe re-entry = %d, want 1 (no increase)", relCount)
}
// Re-entry via CommitTerminal.
err = b.CommitTerminal(ctx, "a1", tr)
if !errors.Is(err, ErrBoundaryAlreadyTerminal) {
t.Fatalf("CommitTerminal re-entry error = %v, want ErrBoundaryAlreadyTerminal", err)
}
if termCount != 1 {
t.Fatalf("terminals after CommitTerminal re-entry = %d, want 1 (no increase)", termCount)
}
// Verify exact counts: 1 start, 1 release, 1 terminal.
if startCount != 1 {
t.Errorf("starts = %d, want 1", startCount)
}
if relCount != 1 {
t.Errorf("releases = %d, want 1", relCount)
}
if termCount != 1 {
t.Errorf("terminals = %d, want 1", termCount)
}
})
}
type customStateSink struct {
startState CommitState
startErr error
releaseState CommitState
releaseErr error
terminalState CommitState
terminalErr error
}
func (s *customStateSink) CommitResponseStart(ctx context.Context, rs ResponseStart) (CommitState, error) {
return s.startState, s.startErr
}
func (s *customStateSink) Release(ctx context.Context, ev ReleaseEvent) (CommitState, error) {
st := s.releaseState
if st == "" {
st = s.startState
}
return st, s.releaseErr
}
func (s *customStateSink) CommitTerminal(ctx context.Context, tr TerminalResult) (CommitState, error) {
st := s.terminalState
if st == "" {
st = CommitStateTerminalCommitted
}
return st, s.terminalErr
}
func TestCommitBoundaryRejectsWrongOrMissingAttempt(t *testing.T) {
ctx := context.Background()
sink := &testSink{}
b, _ := NewCommitBoundary(sink)
ev, _ := NewReleaseTextDeltaEvent("ch", "x", testNow)
rs, _ := NewResponseStart("ch", 200, nil, testNow)
tr, _ := NewSuccessTerminalResult("ch", testNow)
// StageResponseStart checks
if err := b.StageResponseStart("", rs); err == nil {
t.Fatal("expected error for empty attempt on stage")
}
if err := b.StageResponseStart("a1", rs); err != ErrBoundaryNoCurrentAttempt {
t.Fatalf("StageResponseStart without current attempt err = %v, want ErrBoundaryNoCurrentAttempt", err)
}
_ = b.BeginAttempt("a1")
if err := b.StageResponseStart("wrong", rs); err != ErrBoundaryInvalidAttempt {
t.Fatalf("StageResponseStart wrong attempt err = %v, want ErrBoundaryInvalidAttempt", err)
}
// ReleaseSafe checks
_, err := b.ReleaseSafe(ctx, "", []ReleaseEvent{ev})
if err == nil {
t.Fatal("expected error for empty attempt on release")
}
_, err = b.ReleaseSafe(ctx, "wrong", []ReleaseEvent{ev})
if err != ErrBoundaryInvalidAttempt {
t.Fatalf("ReleaseSafe wrong attempt err = %v, want ErrBoundaryInvalidAttempt", err)
}
// CommitTerminal checks
err = b.CommitTerminal(ctx, "", tr)
if err == nil {
t.Fatal("expected error for empty attempt on terminal")
}
err = b.CommitTerminal(ctx, "wrong", tr)
if err != ErrBoundaryInvalidAttempt {
t.Fatalf("CommitTerminal wrong attempt err = %v, want ErrBoundaryInvalidAttempt", err)
}
// ReplaceUncommittedAttempt checks
err = b.ReplaceUncommittedAttempt("", "a2")
if err != ErrBoundaryInvalidAttempt {
t.Fatalf("ReplaceUncommittedAttempt empty oldAttemptID err = %v, want ErrBoundaryInvalidAttempt", err)
}
err = b.ReplaceUncommittedAttempt("wrong", "a2")
if err != ErrBoundaryInvalidAttempt {
t.Fatalf("ReplaceUncommittedAttempt wrong oldAttemptID err = %v, want ErrBoundaryInvalidAttempt", err)
}
}
func TestCommitBoundarySerializesConcurrentStartAndTerminal(t *testing.T) {
ctx := context.Background()
callCounts := &sinkCallCounts{starts: new(int32), releases: new(int32), terminals: new(int32)}
barrierSink := &barrierTestSink{
enterBarrier: make(chan struct{}),
blockChan: make(chan struct{}),
callCounts: callCounts,
}
b, _ := NewCommitBoundary(barrierSink)
_ = b.BeginAttempt("a1")
rs, _ := NewResponseStart("ch", 200, nil, testNow)
_ = b.StageResponseStart("a1", rs)
ev, _ := NewReleaseTextDeltaEvent("ch", "x", testNow)
tr, _ := NewSuccessTerminalResult("ch", testNow)
var wg sync.WaitGroup
var relErr, termErr error
wg.Add(1)
go func() {
defer wg.Done()
_, relErr = b.ReleaseSafe(ctx, "a1", []ReleaseEvent{ev})
}()
// Wait until ReleaseSafe enters sink.Release and holds active reservation
select {
case <-barrierSink.enterBarrier:
case <-time.After(time.Second):
t.Fatal("timeout waiting for ReleaseSafe sink entry")
}
// Concurrent CommitTerminal must fail with ErrBoundarySerializerConflict
termErr = b.CommitTerminal(ctx, "a1", tr)
if !errors.Is(termErr, ErrBoundarySerializerConflict) {
t.Fatalf("concurrent CommitTerminal error = %v, want ErrBoundarySerializerConflict", termErr)
}
// Unblock first release
close(barrierSink.blockChan)
wg.Wait()
if relErr != nil {
t.Fatalf("ReleaseSafe failed: %v", relErr)
}
if b.State() != CommitStateStreamOpen {
t.Fatalf("state = %v, want stream_open", b.State())
}
// Verify exact call counts after release.
if atomic.LoadInt32(callCounts.starts) != 1 {
t.Errorf("starts after release = %d, want 1", atomic.LoadInt32(callCounts.starts))
}
if atomic.LoadInt32(callCounts.releases) != 1 {
t.Errorf("releases after release = %d, want 1", atomic.LoadInt32(callCounts.releases))
}
if atomic.LoadInt32(callCounts.terminals) != 0 {
t.Errorf("terminals after release = %d, want 0", atomic.LoadInt32(callCounts.terminals))
}
// CommitTerminal after release succeeds.
if err := b.CommitTerminal(ctx, "a1", tr); err != nil {
t.Fatalf("subsequent CommitTerminal failed: %v", err)
}
if b.State() != CommitStateTerminalCommitted {
t.Fatalf("state = %v, want terminal_committed", b.State())
}
// Verify exact call counts after terminal.
if atomic.LoadInt32(callCounts.starts) != 1 {
t.Errorf("starts after terminal = %d, want 1", atomic.LoadInt32(callCounts.starts))
}
if atomic.LoadInt32(callCounts.releases) != 1 {
t.Errorf("releases after terminal = %d, want 1", atomic.LoadInt32(callCounts.releases))
}
if atomic.LoadInt32(callCounts.terminals) != 1 {
t.Errorf("terminals after terminal = %d, want 1", atomic.LoadInt32(callCounts.terminals))
}
}
// countingTestSink wraps a ReleaseSink and counts calls to each method.
// It is used to verify exact callback counts in tests.
type countingTestSink struct {
inner ReleaseSink
onStart func()
onRelease func()
onTerminal func()
}
func (s *countingTestSink) CommitResponseStart(ctx context.Context, rs ResponseStart) (CommitState, error) {
if s.onStart != nil {
s.onStart()
}
return s.inner.CommitResponseStart(ctx, rs)
}
func (s *countingTestSink) Release(ctx context.Context, ev ReleaseEvent) (CommitState, error) {
if s.onRelease != nil {
s.onRelease()
}
return s.inner.Release(ctx, ev)
}
func (s *countingTestSink) CommitTerminal(ctx context.Context, tr TerminalResult) (CommitState, error) {
if s.onTerminal != nil {
s.onTerminal()
}
return s.inner.CommitTerminal(ctx, tr)
}
// barrierTestSink is a test sink with an enter barrier and block channel
// for testing concurrent reservation conflict.
type barrierTestSink struct {
enterBarrier chan struct{}
blockChan chan struct{}
mu sync.Mutex
starts int
releases int
terminals int
callCounts *sinkCallCounts
}
type sinkCallCounts struct {
starts *int32
releases *int32
terminals *int32
}
func (s *barrierTestSink) CommitResponseStart(ctx context.Context, rs ResponseStart) (CommitState, error) {
s.mu.Lock()
s.starts++
if s.callCounts != nil {
atomic.AddInt32(s.callCounts.starts, 1)
}
s.mu.Unlock()
return CommitStateStreamOpen, nil
}
func (s *barrierTestSink) Release(ctx context.Context, ev ReleaseEvent) (CommitState, error) {
s.mu.Lock()
s.releases++
if s.callCounts != nil {
atomic.AddInt32(s.callCounts.releases, 1)
}
s.mu.Unlock()
s.enterBarrier <- struct{}{}
<-s.blockChan
return CommitStateStreamOpen, nil
}
func (s *barrierTestSink) CommitTerminal(ctx context.Context, tr TerminalResult) (CommitState, error) {
s.mu.Lock()
s.terminals++
if s.callCounts != nil {
atomic.AddInt32(s.callCounts.terminals, 1)
}
s.mu.Unlock()
return CommitStateTerminalCommitted, nil
}
// TestCommitBoundaryPreservesTerminalStateOnCallbackError verifies that when
// the terminal callback returns terminal_committed with an error, the state
// is preserved and re-entry is rejected (exactly-once terminal).
func TestCommitBoundaryPreservesTerminalStateOnCallbackError(t *testing.T) {
ctx := context.Background()
t.Run("terminal_error_preserves_terminal_state", func(t *testing.T) {
var termCount int
countingSink := &countingTestSink{
inner: &customStateSink{
terminalState: CommitStateTerminalCommitted,
terminalErr: errors.New("test: terminal network error"),
},
onTerminal: func() { termCount++ },
}
b, _ := NewCommitBoundary(countingSink)
_ = b.BeginAttempt("a1")
tr, _ := NewSuccessTerminalResult("ch", testNow)
err := b.CommitTerminal(ctx, "a1", tr)
if err == nil {
t.Fatal("expected error from terminal with network error")
}
if b.State() != CommitStateTerminalCommitted {
t.Fatalf("state = %v, want terminal_committed (preserved despite error)", b.State())
}
// Re-entry must be rejected without calling sink.
err2 := b.CommitTerminal(ctx, "a1", tr)
if !errors.Is(err2, ErrBoundaryAlreadyTerminal) {
t.Fatalf("re-entry error = %v, want ErrBoundaryAlreadyTerminal", err2)
}
if termCount != 1 {
t.Fatalf("terminals = %d, want 1 (exactly-once)", termCount)
}
})
t.Run("terminal_error_with_role_mismatch_preserves_state", func(t *testing.T) {
var termCount int
countingSink := &countingTestSink{
inner: &customStateSink{
terminalState: CommitStateStreamOpen,
terminalErr: errors.New("test: terminal partial error"),
},
onTerminal: func() { termCount++ },
}
b, _ := NewCommitBoundary(countingSink)
_ = b.BeginAttempt("a1")
tr, _ := NewSuccessTerminalResult("ch", testNow)
err := b.CommitTerminal(ctx, "a1", tr)
if err == nil {
t.Fatal("expected error from terminal with role mismatch + error")
}
// Verify joined error contains both sink and role errors.
errStr := err.Error()
if !strings.Contains(errStr, "test: terminal partial error") {
t.Errorf("expected sink error in joined error, got: %v", err)
}
if !strings.Contains(errStr, ErrInvalidStateTransition.Error()) {
t.Errorf("expected role error in joined error, got: %v", err)
}
// State should be preserved at stream_open (valid forward state).
if b.State() != CommitStateStreamOpen {
t.Fatalf("state = %v, want stream_open (forward state preserved)", b.State())
}
if termCount != 1 {
t.Fatalf("terminals = %d, want 1", termCount)
}
})
t.Run("response_start_role_mismatch_advances_state", func(t *testing.T) {
var startCount int
countingSink := &countingTestSink{
inner: &customStateSink{
startState: CommitStateTerminalCommitted,
},
onStart: func() { startCount++ },
}
b, _ := NewCommitBoundary(countingSink)
_ = b.BeginAttempt("a1")
rs, _ := NewResponseStart("ch", 200, nil, testNow)
_ = b.StageResponseStart("a1", rs)
ev, _ := NewReleaseTextDeltaEvent("ch", "x", testNow)
_, err := b.ReleaseSafe(ctx, "a1", []ReleaseEvent{ev})
if !errors.Is(err, ErrInvalidStateTransition) {
t.Fatalf("expected ErrInvalidStateTransition, got %v", err)
}
if b.State() != CommitStateTerminalCommitted {
t.Fatalf("state = %v, want terminal_committed (forward state preserved)", b.State())
}
// Staged start should be cleared because state advanced.
if startCount != 1 {
t.Fatalf("responseStarts = %d, want 1", startCount)
}
})
t.Run("release_role_mismatch_advances_to_terminal", func(t *testing.T) {
var relCount int
countingSink := &countingTestSink{
inner: &customStateSink{
startState: CommitStateStreamOpen,
releaseState: CommitStateTerminalCommitted,
},
onRelease: func() { relCount++ },
}
b, _ := NewCommitBoundary(countingSink)
_ = b.BeginAttempt("a1")
rs, _ := NewResponseStart("ch", 200, nil, testNow)
_ = b.StageResponseStart("a1", rs)
ev, _ := NewReleaseTextDeltaEvent("ch", "x", testNow)
_, err := b.ReleaseSafe(ctx, "a1", []ReleaseEvent{ev})
if !errors.Is(err, ErrInvalidStateTransition) {
t.Fatalf("expected ErrInvalidStateTransition, got %v", err)
}
if b.State() != CommitStateTerminalCommitted {
t.Fatalf("state = %v, want terminal_committed (forward state preserved)", b.State())
}
// No further releases should be attempted.
if relCount != 1 {
t.Fatalf("releases = %d, want 1 (short-circuited after role mismatch)", relCount)
}
})
t.Run("sink_error_with_valid_forward_state_joins_errors", func(t *testing.T) {
customSink := &customStateSink{
terminalState: CommitStateTerminalCommitted,
terminalErr: errors.New("test: terminal error"),
}
b, _ := NewCommitBoundary(customSink)
_ = b.BeginAttempt("a1")
tr, _ := NewSuccessTerminalResult("ch", testNow)
err := b.CommitTerminal(ctx, "a1", tr)
if err == nil {
t.Fatal("expected error")
}
// Should be a joined error with the sink error.
if !strings.Contains(err.Error(), "test: terminal error") {
t.Fatalf("expected error to contain 'test: terminal error', got %v", err)
}
if b.State() != CommitStateTerminalCommitted {
t.Fatalf("state = %v, want terminal_committed", b.State())
}
})
t.Run("exact_callback_count_after_terminal_preservation", func(t *testing.T) {
var termCount int
countingSink := &countingTestSink{
inner: &customStateSink{
terminalState: CommitStateTerminalCommitted,
terminalErr: errors.New("test: terminal failure"),
},
onTerminal: func() { termCount++ },
}
b, _ := NewCommitBoundary(countingSink)
_ = b.BeginAttempt("a1")
tr, _ := NewSuccessTerminalResult("ch", testNow)
err := b.CommitTerminal(ctx, "a1", tr)
if err == nil {
t.Fatal("expected error")
}
if termCount != 1 {
t.Fatalf("terminals = %d, want 1", termCount)
}
// Re-entry should be rejected without calling sink.
_ = b.CommitTerminal(ctx, "a1", tr)
if termCount != 1 {
t.Fatalf("terminals after re-entry = %d, want 1 (exactly-once)", termCount)
}
if b.State() != CommitStateTerminalCommitted {
t.Fatalf("state = %v, want terminal_committed", b.State())
}
})
}
func TestCommitBoundaryRejectsPostOpenReplacement(t *testing.T) {
ctx := context.Background()
sink := &testSink{}
b, _ := NewCommitBoundary(sink)
_ = b.BeginAttempt("a1")
rs, _ := NewResponseStart("ch", 200, nil, testNow)
_ = b.StageResponseStart("a1", rs)
// Valid replacement prior to release
if err := b.ReplaceUncommittedAttempt("a1", "a2"); err != nil {
t.Fatalf("valid ReplaceUncommittedAttempt failed: %v", err)
}
if b.CurrentAttempt() != "a2" {
t.Fatalf("CurrentAttempt = %q, want a2", b.CurrentAttempt())
}
_ = b.StageResponseStart("a2", rs)
ev, _ := NewReleaseTextDeltaEvent("ch", "x", testNow)
_, _ = b.ReleaseSafe(ctx, "a2", []ReleaseEvent{ev})
// After stream_open, replacement is rejected and selected attempt is preserved
err := b.ReplaceUncommittedAttempt("a2", "a3")
if err != ErrBoundaryUncommitted {
t.Fatalf("ReplaceUncommittedAttempt after open err = %v, want ErrBoundaryUncommitted", err)
}
if b.CurrentAttempt() != "a2" {
t.Fatalf("CurrentAttempt = %q, want preserved a2", b.CurrentAttempt())
}
}
func mustReleaseTextDelta(channel, text string, ts time.Time) ReleaseEvent {
ev, err := NewReleaseTextDeltaEvent(channel, text, ts)
if err != nil {
panic(err)
}
return ev
}