iop/apps/edge/internal/service/single_request_observation_test.go

package service

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"strings"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	toki "git.toki-labs.com/toki/proto-socket/go"

	iop "iop/proto/gen/iop"
)

// capturingObserver captures every emitted DTO for test assertions.
type capturingObserver struct {
	mu       sync.Mutex
	events   []singleRequestDTO
	failures int64
	hookErr  error
}

func (c *capturingObserver) Emit(dto singleRequestDTO) error {
	c.mu.Lock()
	c.events = append(c.events, dto)
	c.mu.Unlock()
	return c.hookErr
}

func (c *capturingObserver) snapshot() []singleRequestDTO {
	c.mu.Lock()
	defer c.mu.Unlock()
	out := make([]singleRequestDTO, len(c.events))
	copy(out, c.events)
	return out
}

func (c *capturingObserver) count() int {
	c.mu.Lock()
	defer c.mu.Unlock()
	return len(c.events)
}

func assertSingleRequestCorrelation(t *testing.T, events []singleRequestDTO, forbidden ...string) string {
	t.Helper()
	if len(events) == 0 {
		t.Fatal("expected lifecycle observation events")
	}
	correlation := events[0].Correlation
	if correlation == "" || len(correlation) > 64 {
		t.Fatalf("invalid correlation %q", correlation)
	}
	for _, event := range events {
		if event.Correlation != correlation {
			t.Fatalf("event correlation=%q, want request correlation %q: %#v", event.Correlation, correlation, event)
		}
		if singleRequestContainsSecretSentinel(event.Correlation) {
			t.Fatalf("correlation leaked secret sentinel: %q", event.Correlation)
		}
		for _, value := range forbidden {
			if value != "" && strings.Contains(event.Correlation, value) {
				t.Fatalf("correlation leaked caller-controlled value %q: %q", value, event.Correlation)
			}
		}
	}
	return correlation
}

// panickingObserver always panics on Emit to test failure isolation.
type panickingObserver struct{}

func (panickingObserver) Emit(_ singleRequestDTO) error {
	panic("observer panic")
}

func TestSingleRequestObservationClosedEnums(t *testing.T) {
	// Unknown enum values normalize to empty and are dropped.
	if singleRequestEventClassIsValid("unknown") {
		t.Fatal("unknown event class should be invalid")
	}
	if singleRequestStageIsValid("unknown") {
		t.Fatal("unknown stage should be invalid")
	}
	if singleRequestOperationIsValid("unknown") {
		t.Fatal("unknown operation should be invalid")
	}
	if singleRequestOutcomeIsValid("unknown") {
		t.Fatal("unknown outcome should be invalid")
	}
	if singleRequestErrorClassIsValid("unknown") {
		t.Fatal("unknown error class should be invalid")
	}

	// Known values are valid.
	if !singleRequestEventClassIsValid(singleRequestEventClassRequest) {
		t.Fatal("request event class should be valid")
	}
	if !singleRequestStageIsValid(singleRequestStagePlan) {
		t.Fatal("plan stage should be valid")
	}
	if !singleRequestOperationIsValid(singleRequestOperationTotal) {
		t.Fatal("total operation should be valid")
	}
	if !singleRequestOutcomeIsValid(singleRequestOutcomeSuccess) {
		t.Fatal("success outcome should be valid")
	}
	if !singleRequestErrorClassIsValid(singleRequestErrorClassTimeout) {
		t.Fatal("timeout error class should be valid")
	}

	// Normalizers return empty for unknown.
	if singleRequestNormalizeStage("unknown") != "" {
		t.Fatal("normalizer should return empty for unknown")
	}
	if singleRequestNormalizeOperation("unknown") != "" {
		t.Fatal("normalizer should return empty for unknown")
	}
	if singleRequestNormalizeOutcome("unknown") != "" {
		t.Fatal("normalizer should return empty for unknown")
	}
	if singleRequestNormalizeErrorClass("unknown") != "" {
		t.Fatal("normalizer should return empty for unknown")
	}
}

func TestSingleRequestObservationSanitization(t *testing.T) {
	tests := []struct {
		name     string
		input    string
		expected string
	}{
		{"clean", "hello world", "hello world"},
		{"truncated", strings.Repeat("x", 100), strings.Repeat("x", 64)},
		{"secret", "my secret token", ""},
		{"bearer", "bearer abc123", ""},
		{"api_key", "api_key=xyz", ""},
		{"null_byte", "hello\x00world", ""},
		{"case_insensitive", "SECRET here", ""},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := singleRequestSanitizeString(tt.input)
			if got != tt.expected {
				t.Fatalf("sanitize(%q) = %q, want %q", tt.input, got, tt.expected)
			}
		})
	}

	if !singleRequestContainsSecretSentinel("my secret") {
		t.Fatal("should detect secret")
	}
	if !singleRequestContainsSecretSentinel("bearer abc") {
		t.Fatal("should detect bearer")
	}
	if !singleRequestContainsSecretSentinel("api_key=xyz") {
		t.Fatal("should detect api_key")
	}
	if !singleRequestContainsSecretSentinel("has\x00null") {
		t.Fatal("should detect null byte")
	}
	if singleRequestContainsSecretSentinel("clean text") {
		t.Fatal("should not flag clean text")
	}
}

func TestSingleRequestObservationDeterministicTiming(t *testing.T) {
	clock := newSingleRequestManualClock(time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC))
	captured := &capturingObserver{}
	acc := newSingleRequestTimingAccumulator(clock, captured)

	// Request event.
	acc.onRequest()

	// Stage enter: plan.
	acc.onStageEnter()
	clock.Advance(100 * time.Millisecond)

	// Stage exit: plan -> success.
	acc.onStageExit(singleRequestStagePlan, singleRequestOutcomeSuccess, "")

	// Stage enter: work.
	acc.onStageEnter()
	clock.Advance(200 * time.Millisecond)

	// Tool enter/exit (should be excluded from stage active time).
	acc.onToolEnter()
	clock.Advance(50 * time.Millisecond)
	acc.onToolExit(singleRequestOutcomeSuccess, "")

	// Continue work.
	clock.Advance(150 * time.Millisecond)

	// Stage exit: work -> success.
	acc.onStageExit(singleRequestStageWork, singleRequestOutcomeSuccess, "")

	// Stage enter: review.
	acc.onStageEnter()
	clock.Advance(80 * time.Millisecond)

	// Stage exit: review -> success.
	acc.onStageExit(singleRequestStageReview, singleRequestOutcomeSuccess, "")

	// Cleanup (after all stages, before terminal).
	acc.onCleanupEnter()
	clock.Advance(30 * time.Millisecond)
	acc.onCleanupExit(singleRequestOutcomeSuccess, "")

	// Terminal.
	acc.onTerminal(singleRequestOutcomeSuccess, "", true)

	events := captured.snapshot()

	// Expected events include the actual internal tool outcome.
	if got := len(events); got != 7 {
		t.Fatalf("event count = %d, want 7", got)
	}

	// Verify event classes.
	expectedClasses := []singleRequestEventClass{
		singleRequestEventClassRequest,
		singleRequestEventClassStage,
		singleRequestEventClassTool,
		singleRequestEventClassStage,
		singleRequestEventClassStage,
		singleRequestEventClassCleanup,
		singleRequestEventClassTerminal,
	}
	for i, expected := range expectedClasses {
		if events[i].EventClass != expected {
			t.Fatalf("event[%d].EventClass = %s, want %s", i, events[i].EventClass, expected)
		}
	}

	// Verify timing math: stage_active + tool + cleanup <= total.
	// plan=100ms, work=200+150=350ms (tool 50ms excluded), review=80ms => stageActive=530ms
	// ToolMs=50ms, CleanupMs=30ms, Total=610ms (100+200+50+150+80+30)
	snap := acc.timingSnapshot()
	if snap.StageActiveMs != 530 {
		t.Fatalf("stageActiveMs = %d, want 530", snap.StageActiveMs)
	}
	if snap.ToolMs != 50 {
		t.Fatalf("toolMs = %d, want 50", snap.ToolMs)
	}
	if snap.CleanupMs != 30 {
		t.Fatalf("cleanupMs = %d, want 30", snap.CleanupMs)
	}
	if snap.TotalMs != 610 {
		t.Fatalf("totalMs = %d, want 610", snap.TotalMs)
	}

	// Invariant: stage_active + tool + cleanup <= total.
	if snap.StageActiveMs+snap.ToolMs+snap.CleanupMs > snap.TotalMs {
		t.Fatalf("timing invariant violated: stage(%d) + tool(%d) + cleanup(%d) > total(%d)",
			snap.StageActiveMs, snap.ToolMs, snap.CleanupMs, snap.TotalMs)
	}
}

func TestSingleRequestObservationTerminalRacesExactlyOnce(t *testing.T) {
	clock := newSingleRequestManualClock(time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC))
	captured := &capturingObserver{}
	acc := newSingleRequestTimingAccumulator(clock, captured)

	// Simulate multiple concurrent terminal calls.
	var wg sync.WaitGroup
	for i := 0; i < 20; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			acc.onTerminal(singleRequestOutcomeSuccess, "", true)
		}()
	}
	wg.Wait()

	// Should have exactly one terminal event.
	terminalCount := 0
	for _, e := range captured.snapshot() {
		if e.EventClass == singleRequestEventClassTerminal {
			terminalCount++
		}
	}
	if terminalCount != 1 {
		t.Fatalf("terminal event count = %d, want 1", terminalCount)
	}
}

func TestSingleRequestObservationObserverPanicIsolation(t *testing.T) {
	clock := newSingleRequestManualClock(time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC))
	// Panicking observer should not affect request lifecycle.
	acc := newSingleRequestTimingAccumulator(clock, panickingObserver{})

	// These should all complete without panic propagating.
	acc.onRequest()
	acc.onStageEnter()
	clock.Advance(10 * time.Millisecond)
	acc.onStageExit(singleRequestStagePlan, singleRequestOutcomeSuccess, "")
	acc.onTerminal(singleRequestOutcomeSuccess, "", false)

	// Verify events were still emitted (safe observer captures them).
	if acc.observer.failureCount() == 0 {
		t.Fatal("expected isolated failures from panicking observer")
	}
}

func TestSingleRequestObservationObserverErrorIsolation(t *testing.T) {
	clock := newSingleRequestManualClock(time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC))
	expectedErr := errors.New("observer write failure")
	captured := &capturingObserver{hookErr: expectedErr}
	acc := newSingleRequestTimingAccumulator(clock, captured)

	// Emit should not propagate the error.
	acc.onRequest()
	acc.onStageEnter()
	clock.Advance(10 * time.Millisecond)
	acc.onStageExit(singleRequestStagePlan, singleRequestOutcomeSuccess, "")
	acc.onTerminal(singleRequestOutcomeSuccess, "", false)

	// Verify failures were counted.
	if acc.observer.failureCount() == 0 {
		t.Fatal("expected isolated failures from erroring observer")
	}
}

func TestSingleRequestObservationSuccessLifecycle(t *testing.T) {
	executor := &channelFakeExecutor{fn: func(_ context.Context, req SingleRequestRequest, ctrl SingleRequestController) error {
		return submitToFinalizing(req, ctrl, &SingleRequestResult{Output: "success"})
	}}
	handle := startTestExecution(t, executor)
	waitForState(t, handle, SingleRequestStateFinalizing)
	if err := handle.AcknowledgeTerminal(true); err != nil {
		t.Fatalf("AcknowledgeTerminal: %v", err)
	}
	_, err := waitForExecution(t, handle)
	if err != nil {
		t.Fatalf("Wait error = %v", err)
	}
}

func TestSingleRequestObservationErrorLifecycle(t *testing.T) {
	executor := &channelFakeExecutor{fn: func(_ context.Context, req SingleRequestRequest, ctrl SingleRequestController) error {
		return errors.New("provider failure")
	}}
	handle := startTestExecution(t, executor)
	_, err := waitForExecution(t, handle)
	if err == nil || handle.State() != SingleRequestStateFailed {
		t.Fatalf("Wait=(%v, state=%s), want failed", err, handle.State())
	}
}

func TestSingleRequestObservationCancelLifecycle(t *testing.T) {
	release := make(chan struct{})
	executor := &channelFakeExecutor{fn: func(ctx context.Context, _ SingleRequestRequest, _ SingleRequestController) error {
		<-ctx.Done()
		return ctx.Err()
	}}
	handle := startTestExecution(t, executor)
	handle.Cancel()
	close(release)
	_, err := waitForExecution(t, handle)
	if !errors.Is(err, ErrSingleRequestCancelled) {
		t.Fatalf("Wait error = %v, want ErrSingleRequestCancelled", err)
	}
}

func TestSingleRequestObservationToolTimingExcludedFromStage(t *testing.T) {
	// This test verifies that tool execution time is excluded from stage active time.
	// We test the accumulator directly with a simulated lifecycle.
	clock := newSingleRequestManualClock(time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC))
	captured := &capturingObserver{}

	acc := newSingleRequestTimingAccumulator(clock, captured)

	// Simulate: plan stage with tool call inside.
	acc.onStageEnter()
	clock.Advance(100 * time.Millisecond)

	// Tool enter/exit.
	acc.onToolEnter()
	clock.Advance(50 * time.Millisecond)
	acc.onToolExit(singleRequestOutcomeSuccess, "")

	// Continue plan.
	clock.Advance(50 * time.Millisecond)
	acc.onStageExit(singleRequestStagePlan, singleRequestOutcomeSuccess, "")

	// Emit terminal to set total.
	acc.onTerminal(singleRequestOutcomeSuccess, "", false)
	snap := acc.timingSnapshot()
	// Stage active should be 100 + 50 = 150ms (tool 50ms excluded).
	if snap.StageActiveMs != 150 {
		t.Fatalf("stageActiveMs = %d, want 150 (tool time excluded)", snap.StageActiveMs)
	}
	// Tool time should be 50ms.
	if snap.ToolMs != 50 {
		t.Fatalf("toolMs = %d, want 50", snap.ToolMs)
	}
	// Invariant: stage + tool <= total.
	if snap.StageActiveMs+snap.ToolMs > snap.TotalMs {
		t.Fatalf("invariant violated: stage(%d) + tool(%d) > total(%d)",
			snap.StageActiveMs, snap.ToolMs, snap.TotalMs)
	}
}

func TestSingleRequestObservationSentinelExclusion(t *testing.T) {
	// Verify that DTOs with secret sentinels in correlation are sanitized.
	clock := newSingleRequestManualClock(time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC))
	captured := &capturingObserver{}
	acc := newSingleRequestTimingAccumulator(clock, captured)

	acc.onRequest()

	events := captured.snapshot()
	if len(events) != 1 {
		t.Fatalf("event count = %d, want 1", len(events))
	}

	// Correlation should be sanitized (no secrets).
	if singleRequestContainsSecretSentinel(events[0].Correlation) {
		t.Fatal("correlation should not contain secret sentinels")
	}
}

func TestSingleRequestObservationNoopObserver(t *testing.T) {
	// nil observer should become noop.
	clock := newSingleRequestManualClock(time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC))
	acc := newSingleRequestTimingAccumulator(clock, nil)

	// Should not panic.
	acc.onRequest()
	acc.onStageEnter()
	clock.Advance(10 * time.Millisecond)
	acc.onStageExit(singleRequestStagePlan, singleRequestOutcomeSuccess, "")
	acc.onTerminal(singleRequestOutcomeSuccess, "", false)
}

func TestSingleRequestObservationTimingInvariant(t *testing.T) {
	// Verify that stage_active + tool + cleanup <= total for various scenarios.
	scenarios := []struct {
		name       string
		stageMs    int64
		toolMs     int64
		cleanupMs  int64
		terminalMs int64
	}{
		{"minimal", 10, 5, 3, 20},
		{"no_tool", 100, 0, 10, 115},
		{"no_cleanup", 50, 20, 0, 75},
		{"heavy_tool", 30, 200, 5, 240},
	}
	for _, sc := range scenarios {
		t.Run(sc.name, func(t *testing.T) {
			clock := newSingleRequestManualClock(time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC))
			captured := &capturingObserver{}
			acc := newSingleRequestTimingAccumulator(clock, captured)

			acc.onRequest()
			acc.onStageEnter()
			clock.Advance(time.Duration(sc.stageMs) * time.Millisecond)
			acc.onStageExit(singleRequestStagePlan, singleRequestOutcomeSuccess, "")

			if sc.toolMs > 0 {
				acc.onToolEnter()
				clock.Advance(time.Duration(sc.toolMs) * time.Millisecond)
				acc.onToolExit(singleRequestOutcomeSuccess, "")
			}

			if sc.cleanupMs > 0 {
				acc.onCleanupEnter()
				clock.Advance(time.Duration(sc.cleanupMs) * time.Millisecond)
				acc.onCleanupExit(singleRequestOutcomeSuccess, "")
			}

			remaining := sc.terminalMs - sc.stageMs - sc.toolMs - sc.cleanupMs
			if remaining > 0 {
				clock.Advance(time.Duration(remaining) * time.Millisecond)
			}
			acc.onTerminal(singleRequestOutcomeSuccess, "", true)

			snap := acc.timingSnapshot()
			if snap.StageActiveMs+snap.ToolMs+snap.CleanupMs > snap.TotalMs {
				t.Fatalf("invariant violated: stage(%d) + tool(%d) + cleanup(%d) > total(%d)",
					snap.StageActiveMs, snap.ToolMs, snap.CleanupMs, snap.TotalMs)
			}
		})
	}
}

func TestSingleRequestObservationTerminalOutcome(t *testing.T) {
	clock := newSingleRequestManualClock(time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC))
	captured := &capturingObserver{}
	acc := newSingleRequestTimingAccumulator(clock, captured)

	// Request event.
	acc.onRequest()

	// Success terminal (first terminal should be recorded).
	acc.onTerminal(singleRequestOutcomeSuccess, "", true)

	// Error terminal (should be ignored, already have terminal).
	acc.onTerminal(singleRequestOutcomeError, singleRequestErrorClassProvider, false)

	// Cancel terminal (should be ignored, already have terminal).
	acc.onTerminal(singleRequestOutcomeCancel, singleRequestErrorClassCancel, false)

	// Only the first terminal event should be recorded.
	terminalCount := 0
	for _, e := range captured.snapshot() {
		if e.EventClass == singleRequestEventClassTerminal {
			terminalCount++
		}
	}
	if terminalCount != 1 {
		t.Fatalf("terminal event count = %d, want 1", terminalCount)
	}

	// Verify the first (and only) terminal's outcome.
	events := captured.snapshot()
	if len(events) < 2 {
		t.Fatal("expected at least request and terminal events")
	}
	firstTerminal := events[1] // index 0 is request event
	if firstTerminal.Outcome != singleRequestOutcomeSuccess {
		t.Fatalf("first terminal outcome = %s, want success", firstTerminal.Outcome)
	}
}

func TestSingleRequestObservationDTOValidation(t *testing.T) {
	// Invalid DTO should be dropped by emitSafe.
	clock := newSingleRequestManualClock(time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC))
	captured := &capturingObserver{}
	acc := newSingleRequestTimingAccumulator(clock, captured)

	// Direct call to emitSafe with invalid DTO.
	acc.emitSafe(singleRequestDTO{EventClass: "invalid"})

	if captured.count() != 0 {
		t.Fatalf("invalid DTO should be dropped, got %d events", captured.count())
	}
}

func TestSingleRequestObservationCorrelationID(t *testing.T) {
	first := newSingleRequestCorrelationID()
	second := newSingleRequestCorrelationID()
	if first == "" || second == "" {
		t.Fatal("correlations should not be empty")
	}
	if first == second {
		t.Fatalf("separate correlations must differ: %q", first)
	}
	for _, correlation := range []string{first, second} {
		if len(correlation) > 64 {
			t.Fatalf("correlation exceeds bound: %q", correlation)
		}
		if strings.Contains(correlation, "request-secret-sentinel") || singleRequestContainsSecretSentinel(correlation) {
			t.Fatalf("correlation includes caller or secret material: %q", correlation)
		}
		for _, character := range correlation {
			if !(character >= 'a' && character <= 'z' || character >= '0' && character <= '9' || character == '-') {
				t.Fatalf("correlation contains disallowed character %q in %q", character, correlation)
			}
		}
	}
}

func TestSingleRequestObservationAccumulatorCorrelationsAreDistinct(t *testing.T) {
	const accumulatorCount = 32
	correlations := make(chan string, accumulatorCount)
	var group sync.WaitGroup
	for range accumulatorCount {
		group.Add(1)
		go func() {
			defer group.Done()
			observer := &capturingObserver{}
			accumulator := newSingleRequestTimingAccumulator(nil, observer)
			accumulator.onRequest()
			correlations <- assertSingleRequestCorrelation(t, observer.snapshot(), "request-secret-sentinel")
		}()
	}
	group.Wait()
	close(correlations)
	seen := make(map[string]struct{}, accumulatorCount)
	for correlation := range correlations {
		if _, duplicate := seen[correlation]; duplicate {
			t.Fatalf("duplicate accumulator correlation %q", correlation)
		}
		seen[correlation] = struct{}{}
	}
}

func TestSingleRequestObservationSafeObserverConcurrency(t *testing.T) {
	// Verify that safe observer is concurrent-safe.
	inner := &capturingObserver{}
	safe := &singleRequestSafeObserver{inner: inner}

	var wg sync.WaitGroup
	for i := 0; i < 100; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			safe.Emit(singleRequestDTO{EventClass: singleRequestEventClassRequest})
		}()
	}
	wg.Wait()

	if inner.count() != 100 {
		t.Fatalf("event count = %d, want 100", inner.count())
	}
}

func TestSingleRequestObservationSafeObserverNilInner(t *testing.T) {
	safe := &singleRequestSafeObserver{}
	// Should not panic with nil inner.
	err := safe.Emit(singleRequestDTO{EventClass: singleRequestEventClassRequest})
	if err != nil {
		t.Fatalf("emit with nil inner should return nil, got %v", err)
	}
}

func TestSingleRequestObservationManualClock(t *testing.T) {
	clock := newSingleRequestManualClock(time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC))

	if !clock.Now().Equal(time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)) {
		t.Fatal("initial time mismatch")
	}

	clock.Advance(100 * time.Millisecond)
	if since := clock.Since(time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)); since != 100*time.Millisecond {
		t.Fatalf("since = %v, want 100ms", since)
	}

	clock.Advance(50 * time.Millisecond)
	if since := clock.Since(time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)); since != 150*time.Millisecond {
		t.Fatalf("since = %v, want 150ms", since)
	}
}

func TestSingleRequestObservationRealClock(t *testing.T) {
	clock := singleRequestRealClock{}
	now := clock.Now()
	if now.IsZero() {
		t.Fatal("now should not be zero")
	}
	since := clock.Since(now)
	if since < 0 {
		t.Fatalf("since should be non-negative, got %v", since)
	}
}

func TestSingleRequestObservationTimingSnapshotCopySafe(t *testing.T) {
	clock := newSingleRequestManualClock(time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC))
	captured := &capturingObserver{}
	acc := newSingleRequestTimingAccumulator(clock, captured)

	acc.onRequest()
	acc.onStageEnter()
	clock.Advance(100 * time.Millisecond)
	acc.onStageExit(singleRequestStagePlan, singleRequestOutcomeSuccess, "")
	acc.onTerminal(singleRequestOutcomeSuccess, "", true)

	snap1 := acc.timingSnapshot()
	clock.Advance(50 * time.Millisecond)
	acc.onTerminal(singleRequestOutcomeSuccess, "", false) // ignored, already terminal
	snap2 := acc.timingSnapshot()

	// Snapshots should be independent.
	if snap1.StageActiveMs != snap2.StageActiveMs {
		t.Fatal("snapshots should be independent")
	}
}

func TestSingleRequestObservationObserverFailureHook(t *testing.T) {
	var hookCalled atomic.Bool
	var hookDTO singleRequestDTO
	var hookErr error

	inner := panickingObserver{}
	hook := func(dto singleRequestDTO, err error) {
		hookCalled.Store(true)
		hookDTO = dto
		hookErr = err
	}

	safe := &singleRequestSafeObserver{inner: inner, onFailure: hook}
	safe.Emit(singleRequestDTO{EventClass: singleRequestEventClassRequest})

	if !hookCalled.Load() {
		t.Fatal("hook should have been called")
	}
	if hookDTO.EventClass != singleRequestEventClassRequest {
		t.Fatalf("hook DTO event class = %s, want request", hookDTO.EventClass)
	}
	if hookErr == nil {
		t.Fatal("hook error should not be nil")
	}
	if hookErr == nil || !strings.Contains(hookErr.Error(), "panic") {
		t.Fatalf("hook error = %v, want observer panic", hookErr)
	}
}

func TestSingleRequestObservationHookPanicIsolation(t *testing.T) {
	// Hook that panics should not affect the safe observer.
	inner := &capturingObserver{}
	hook := func(_ singleRequestDTO, _ error) {
		panic("hook panic")
	}
	safe := &singleRequestSafeObserver{inner: inner, onFailure: hook}

	// Should not panic.
	safe.Emit(singleRequestDTO{EventClass: singleRequestEventClassRequest})

	// Inner should still receive the event.
	if inner.count() != 1 {
		t.Fatalf("inner event count = %d, want 1", inner.count())
	}
}

type observationToolExecutor struct {
	clock   *singleRequestManualClock
	results chan InternalWorkspaceToolResult
}

func (e *observationToolExecutor) ExecuteSingleRequest(ctx context.Context, req SingleRequestRequest, ctrl SingleRequestController) error {
	if err := ctrl.SubmitEnvelope(testEnvelope(req.RequestID, 1, SingleRequestStatePlanning)); err != nil {
		return err
	}
	e.clock.Advance(10 * time.Millisecond)
	if err := ctrl.SubmitEnvelope(SingleRequestEnvelope{
		RequestID: req.RequestID, Sequence: 2,
		Stage: SingleRequestStateInternalTool, SavedStage: SingleRequestStatePlanning,
		ToolCall: &InternalWorkspaceToolCall{
			RequestID: req.RequestID, StageID: "plan", ToolCallID: "observed-tool",
			Name: InternalWorkspaceToolRead, Arguments: json.RawMessage(`{"relative_path":"README.md"}`),
		},
	}); err != nil {
		return err
	}
	select {
	case <-e.results:
	case <-ctx.Done():
		return ctx.Err()
	}
	if err := ctrl.SubmitEnvelope(SingleRequestEnvelope{
		RequestID: req.RequestID, Sequence: 3,
		Stage: SingleRequestStatePlanning, SavedStage: SingleRequestStatePlanning,
	}); err != nil {
		return err
	}
	e.clock.Advance(7 * time.Millisecond)
	if err := ctrl.SubmitEnvelope(testEnvelope(req.RequestID, 4, SingleRequestStateWorking)); err != nil {
		return err
	}
	e.clock.Advance(11 * time.Millisecond)
	if err := ctrl.SubmitEnvelope(testEnvelope(req.RequestID, 5, SingleRequestStateReviewing)); err != nil {
		return err
	}
	e.clock.Advance(13 * time.Millisecond)
	return ctrl.SubmitEnvelope(SingleRequestEnvelope{
		RequestID: req.RequestID, Sequence: 6, Stage: SingleRequestStateFinalizing,
		Result: &SingleRequestResult{Output: "final result"},
	})
}

func (e *observationToolExecutor) ContinueInternalTool(_ context.Context, result InternalWorkspaceToolResult) error {
	e.results <- result.Clone()
	return nil
}

func TestSingleRequestObservationLifecycleIntegration(t *testing.T) {
	t.Run("observer panic cannot alter service result", func(t *testing.T) {
		service, _ := newInternalToolLoopService(t, &channelFakeExecutor{fn: func(_ context.Context, req SingleRequestRequest, ctrl SingleRequestController) error {
			return submitToFinalizing(req, ctrl, &SingleRequestResult{Output: "success"})
		}})
		service.SetSingleRequestObserver(panickingObserver{})
		handle, err := service.StartSingleRequest(context.Background(), internalLoopRequest(t, nil))
		if err != nil {
			t.Fatalf("StartSingleRequest: %v", err)
		}
		waitForState(t, handle, SingleRequestStateFinalizing)
		if err := handle.AcknowledgeTerminal(true); err != nil {
			t.Fatalf("AcknowledgeTerminal: %v", err)
		}
		if result, err := waitForExecution(t, handle); err != nil || result.Output != "success" {
			t.Fatalf("Wait=(%q, %v)", result.Output, err)
		}
		if failures := handle.(*singleRequestHandle).timing.observer.failureCount(); failures == 0 {
			t.Fatal("panicking observer failure was not isolated and recorded")
		}
	})

	t.Run("service terminal race emits once", func(t *testing.T) {
		observer := &capturingObserver{}
		service, _ := newInternalToolLoopService(t, &channelFakeExecutor{fn: func(_ context.Context, req SingleRequestRequest, ctrl SingleRequestController) error {
			return submitToFinalizing(req, ctrl, &SingleRequestResult{Output: "candidate"})
		}})
		service.SetSingleRequestObserver(observer)
		handle, err := service.StartSingleRequest(context.Background(), internalLoopRequest(t, nil))
		if err != nil {
			t.Fatalf("StartSingleRequest: %v", err)
		}
		waitForState(t, handle, SingleRequestStateFinalizing)
		var group sync.WaitGroup
		group.Add(2)
		go func() { defer group.Done(); _ = handle.AcknowledgeTerminal(true) }()
		go func() { defer group.Done(); handle.Cancel() }()
		group.Wait()
		_, _ = waitForExecution(t, handle)
		terminalCount := 0
		for _, event := range observer.snapshot() {
			if event.EventClass == singleRequestEventClassTerminal {
				terminalCount++
			}
		}
		if terminalCount != 1 {
			t.Fatalf("terminal events=%d, want 1", terminalCount)
		}
	})

	t.Run("service tool pause cleanup and terminal", func(t *testing.T) {
		clock := newSingleRequestManualClock(time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC))
		observer := &capturingObserver{}
		executor := &observationToolExecutor{clock: clock, results: make(chan InternalWorkspaceToolResult, 1)}
		service, node := newInternalToolLoopService(t, executor)
		service.SetSingleRequestClock(clock)
		service.SetSingleRequestObserver(observer)
		var openCount atomic.Int32
		toki.AddRequestListenerTyped[*iop.WorkspaceOpenRequest, *iop.WorkspaceOpenResponse](&node.Communicator, func(req *iop.WorkspaceOpenRequest) (*iop.WorkspaceOpenResponse, error) {
			openCount.Add(1)
			return &iop.WorkspaceOpenResponse{RequestId: req.GetRequestId(), WorkspaceRef: req.GetWorkspaceRef(), Status: iop.WorkspaceStatus_WORKSPACE_STATUS_SUCCESS}, nil
		})
		toki.AddRequestListenerTyped[*iop.WorkspaceToolRequest, *iop.WorkspaceToolResponse](&node.Communicator, func(req *iop.WorkspaceToolRequest) (*iop.WorkspaceToolResponse, error) {
			clock.Advance(50 * time.Millisecond)
			return &iop.WorkspaceToolResponse{RequestId: req.GetRequestId(), StageId: req.GetStageId(), ToolCallId: req.GetToolCallId(), Status: iop.WorkspaceStatus_WORKSPACE_STATUS_SUCCESS, Content: []byte("redacted")}, nil
		})
		toki.AddRequestListenerTyped[*iop.WorkspaceCleanupRequest, *iop.WorkspaceCleanupResponse](&node.Communicator, func(req *iop.WorkspaceCleanupRequest) (*iop.WorkspaceCleanupResponse, error) {
			clock.Advance(20 * time.Millisecond)
			return &iop.WorkspaceCleanupResponse{RequestId: req.GetRequestId(), Status: iop.WorkspaceStatus_WORKSPACE_STATUS_SUCCESS}, nil
		})

		handle, err := service.StartSingleRequest(context.Background(), internalLoopRequest(t, nil))
		if err != nil {
			t.Fatalf("StartSingleRequest: %v", err)
		}
		waitForState(t, handle, SingleRequestStateFinalizing)
		waitForSingleRequestCleanup(t, handle)
		clock.Advance(5 * time.Millisecond)
		if err := handle.AcknowledgeTerminal(true); err != nil {
			t.Fatalf("AcknowledgeTerminal: %v", err)
		}
		if _, err := waitForExecution(t, handle); err != nil {
			t.Fatalf("Wait: %v", err)
		}

		events := observer.snapshot()
		wantClasses := []singleRequestEventClass{
			singleRequestEventClassRequest, singleRequestEventClassTool,
			singleRequestEventClassStage, singleRequestEventClassStage, singleRequestEventClassStage,
			singleRequestEventClassCleanup, singleRequestEventClassTerminal,
		}
		if len(events) != len(wantClasses) {
			t.Fatalf("event count=%d, want %d: %#v", len(events), len(wantClasses), events)
		}
		for index, want := range wantClasses {
			if events[index].EventClass != want {
				t.Fatalf("event[%d].EventClass=%q, want %q", index, events[index].EventClass, want)
			}
			if singleRequestContainsSecretSentinel(events[index].Correlation) {
				t.Fatalf("event[%d] correlation leaked a sentinel: %q", index, events[index].Correlation)
			}
		}
		assertSingleRequestCorrelation(t, events, "request-loop", "observed-tool", "redacted")
		if events[1].DurationMS != 50 || events[1].Outcome != singleRequestOutcomeSuccess {
			t.Fatalf("tool event=%+v, want one successful 50ms tool", events[1])
		}
		if events[2].Stage != singleRequestStagePlan || events[2].DurationMS != 17 || events[2].ToolCount != 1 {
			t.Fatalf("plan stage=%+v, want 17ms with one tool", events[2])
		}
		if events[5].DurationMS != 20 || events[5].Outcome != singleRequestOutcomeSuccess {
			t.Fatalf("cleanup event=%+v", events[5])
		}
		if events[6].Outcome != singleRequestOutcomeSuccess || events[6].DurationMS != 116 {
			t.Fatalf("terminal event=%+v, want successful 116ms terminal", events[6])
		}
		if openCount.Load() != 1 {
			t.Fatalf("workspace open count=%d, want 1", openCount.Load())
		}
	})

	t.Run("cleanup failure closes service observation", func(t *testing.T) {
		clock := newSingleRequestManualClock(time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC))
		observer := &capturingObserver{}
		executor := &observationToolExecutor{clock: clock, results: make(chan InternalWorkspaceToolResult, 1)}
		service, node := newInternalToolLoopService(t, executor)
		service.SetSingleRequestClock(clock)
		service.SetSingleRequestObserver(observer)
		toki.AddRequestListenerTyped[*iop.WorkspaceOpenRequest, *iop.WorkspaceOpenResponse](&node.Communicator, func(req *iop.WorkspaceOpenRequest) (*iop.WorkspaceOpenResponse, error) {
			return &iop.WorkspaceOpenResponse{RequestId: req.GetRequestId(), WorkspaceRef: req.GetWorkspaceRef(), Status: iop.WorkspaceStatus_WORKSPACE_STATUS_SUCCESS}, nil
		})
		toki.AddRequestListenerTyped[*iop.WorkspaceToolRequest, *iop.WorkspaceToolResponse](&node.Communicator, func(req *iop.WorkspaceToolRequest) (*iop.WorkspaceToolResponse, error) {
			return &iop.WorkspaceToolResponse{RequestId: req.GetRequestId(), StageId: req.GetStageId(), ToolCallId: req.GetToolCallId(), Status: iop.WorkspaceStatus_WORKSPACE_STATUS_SUCCESS}, nil
		})
		toki.AddRequestListenerTyped[*iop.WorkspaceCleanupRequest, *iop.WorkspaceCleanupResponse](&node.Communicator, func(req *iop.WorkspaceCleanupRequest) (*iop.WorkspaceCleanupResponse, error) {
			clock.Advance(9 * time.Millisecond)
			return &iop.WorkspaceCleanupResponse{RequestId: req.GetRequestId(), Status: iop.WorkspaceStatus_WORKSPACE_STATUS_ERROR}, nil
		})
		handle, err := service.StartSingleRequest(context.Background(), internalLoopRequest(t, nil))
		if err != nil {
			t.Fatalf("StartSingleRequest: %v", err)
		}
		if _, err := waitForExecution(t, handle); !errors.Is(err, ErrSingleRequestWorkspaceCleanup) {
			t.Fatalf("Wait error=%v, want cleanup failure", err)
		}
		events := observer.snapshot()
		if len(events) != 7 || events[5].EventClass != singleRequestEventClassCleanup ||
			events[5].Outcome != singleRequestOutcomeError || events[5].ErrorClass != singleRequestErrorClassWorkspaceCleanup ||
			events[6].EventClass != singleRequestEventClassTerminal || events[6].Outcome != singleRequestOutcomeError {
			t.Fatalf("cleanup failure observations=%#v", events)
		}
		if events[6].ErrorClass != singleRequestErrorClassWorkspaceCleanup {
			t.Fatalf("cleanup conversion terminal class=%q, want workspace_cleanup", events[6].ErrorClass)
		}
		assertSingleRequestCorrelation(t, events, "request-loop")
	})

	for _, test := range []struct {
		name    string
		execute func(*singleRequestManualClock, chan struct{}) SingleRequestExecutor
		outcome singleRequestOutcome
	}{
		{
			name: "provider failure",
			execute: func(clock *singleRequestManualClock, _ chan struct{}) SingleRequestExecutor {
				return &channelFakeExecutor{fn: func(_ context.Context, req SingleRequestRequest, ctrl SingleRequestController) error {
					if err := ctrl.SubmitEnvelope(testEnvelope(req.RequestID, 1, SingleRequestStatePlanning)); err != nil {
						return err
					}
					clock.Advance(4 * time.Millisecond)
					return errors.New("provider failure")
				}}
			}, outcome: singleRequestOutcomeError,
		},
		{
			name: "caller cancellation",
			execute: func(_ *singleRequestManualClock, started chan struct{}) SingleRequestExecutor {
				return &channelFakeExecutor{fn: func(ctx context.Context, req SingleRequestRequest, ctrl SingleRequestController) error {
					if err := ctrl.SubmitEnvelope(testEnvelope(req.RequestID, 1, SingleRequestStatePlanning)); err != nil {
						return err
					}
					close(started)
					<-ctx.Done()
					return ctx.Err()
				}}
			}, outcome: singleRequestOutcomeCancel,
		},
	} {
		t.Run(test.name, func(t *testing.T) {
			clock := newSingleRequestManualClock(time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC))
			observer := &capturingObserver{}
			started := make(chan struct{})
			service, _ := newInternalToolLoopService(t, test.execute(clock, started))
			service.SetSingleRequestClock(clock)
			service.SetSingleRequestObserver(observer)
			handle, err := service.StartSingleRequest(context.Background(), internalLoopRequest(t, nil))
			if err != nil {
				t.Fatalf("StartSingleRequest: %v", err)
			}
			if test.outcome == singleRequestOutcomeCancel {
				<-started
				clock.Advance(6 * time.Millisecond)
				handle.Cancel()
			}
			if _, err := waitForExecution(t, handle); err == nil {
				t.Fatal("Wait error=nil, want terminal failure")
			}
			events := observer.snapshot()
			if len(events) != 4 {
				t.Fatalf("event count=%d, want request/stage/cleanup/terminal", len(events))
			}
			if events[1].EventClass != singleRequestEventClassStage || events[1].Outcome != test.outcome ||
				events[3].EventClass != singleRequestEventClassTerminal || events[3].Outcome != test.outcome {
				t.Fatalf("unexpected terminal lifecycle events: %#v", events)
			}
		})
	}
}

func TestSingleRequestObservationInFlightToolTerminalOrdering(t *testing.T) {
	for _, test := range []struct {
		name              string
		toolResponder     func(*iop.WorkspaceToolRequest) (*iop.WorkspaceToolResponse, error)
		cancel            bool
		wantOutcome       singleRequestOutcome
		wantTerminalClass singleRequestErrorClass
		wantWait          error
	}{
		{
			name: "tool failure preserves primary class across cleanup failure",
			toolResponder: func(req *iop.WorkspaceToolRequest) (*iop.WorkspaceToolResponse, error) {
				return &iop.WorkspaceToolResponse{RequestId: req.GetRequestId(), StageId: req.GetStageId(), ToolCallId: req.GetToolCallId(), Status: iop.WorkspaceStatus_WORKSPACE_STATUS_ERROR}, nil
			},
			wantOutcome:       singleRequestOutcomeError,
			wantTerminalClass: singleRequestErrorClassInternalToolFailed,
			wantWait:          ErrSingleRequestInternalToolFailed,
		},
		{
			name: "caller cancellation settles in-flight tool before stage",
			toolResponder: func(req *iop.WorkspaceToolRequest) (*iop.WorkspaceToolResponse, error) {
				return &iop.WorkspaceToolResponse{RequestId: req.GetRequestId(), StageId: req.GetStageId(), ToolCallId: req.GetToolCallId(), Status: iop.WorkspaceStatus_WORKSPACE_STATUS_SUCCESS}, nil
			},
			cancel:            true,
			wantOutcome:       singleRequestOutcomeCancel,
			wantTerminalClass: singleRequestErrorClassCancel,
			wantWait:          ErrSingleRequestCancelled,
		},
	} {
		t.Run(test.name, func(t *testing.T) {
			observer := &capturingObserver{}
			clock := newSingleRequestManualClock(time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC))
			executor := &observationToolExecutor{clock: clock, results: make(chan InternalWorkspaceToolResult, 1)}
			service, node := newInternalToolLoopService(t, executor)
			service.SetSingleRequestClock(clock)
			service.SetSingleRequestObserver(observer)
			toki.AddRequestListenerTyped[*iop.WorkspaceOpenRequest, *iop.WorkspaceOpenResponse](&node.Communicator, func(req *iop.WorkspaceOpenRequest) (*iop.WorkspaceOpenResponse, error) {
				return &iop.WorkspaceOpenResponse{RequestId: req.GetRequestId(), WorkspaceRef: req.GetWorkspaceRef(), Status: iop.WorkspaceStatus_WORKSPACE_STATUS_SUCCESS}, nil
			})
			toolStarted := make(chan struct{})
			toolRelease := make(chan struct{})
			var toolStartOnce sync.Once
			toki.AddRequestListenerTyped[*iop.WorkspaceToolRequest, *iop.WorkspaceToolResponse](&node.Communicator, func(req *iop.WorkspaceToolRequest) (*iop.WorkspaceToolResponse, error) {
				toolStartOnce.Do(func() { close(toolStarted) })
				if test.cancel {
					<-toolRelease
				}
				return test.toolResponder(req)
			})
			toki.AddRequestListenerTyped[*iop.WorkspaceCleanupRequest, *iop.WorkspaceCleanupResponse](&node.Communicator, func(req *iop.WorkspaceCleanupRequest) (*iop.WorkspaceCleanupResponse, error) {
				return &iop.WorkspaceCleanupResponse{RequestId: req.GetRequestId(), Status: iop.WorkspaceStatus_WORKSPACE_STATUS_ERROR}, nil
			})

			handle, err := service.StartSingleRequest(context.Background(), internalLoopRequest(t, nil))
			if err != nil {
				t.Fatalf("StartSingleRequest: %v", err)
			}
			if test.cancel {
				select {
				case <-toolStarted:
				case <-time.After(2 * time.Second):
					t.Fatal("tool did not start")
				}
				handle.Cancel()
				close(toolRelease)
			}
			if _, err := waitForExecution(t, handle); !errors.Is(err, test.wantWait) {
				t.Fatalf("Wait error=%v, want %v", err, test.wantWait)
			}
			events := observer.snapshot()
			if len(events) != 5 {
				t.Fatalf("event count=%d, want request/tool/stage/cleanup/terminal: %#v", len(events), events)
			}
			if events[1].EventClass != singleRequestEventClassTool || events[1].Outcome != test.wantOutcome ||
				events[2].EventClass != singleRequestEventClassStage || events[2].Outcome != test.wantOutcome || events[2].ToolCount != 1 {
				t.Fatalf("tool/stage ordering=%#v", events)
			}
			if events[4].EventClass != singleRequestEventClassTerminal || events[4].Outcome != test.wantOutcome || events[4].ErrorClass != test.wantTerminalClass {
				t.Fatalf("terminal=%#v, want outcome=%q class=%q", events[4], test.wantOutcome, test.wantTerminalClass)
			}
			assertSingleRequestCorrelation(t, events, "request-loop", "observed-tool")
		})
	}
}

type admissionRaceExecutor struct {
	planningStarted chan struct{}
	submitTool      chan struct{}
}

func (e *admissionRaceExecutor) ExecuteSingleRequest(ctx context.Context, req SingleRequestRequest, ctrl SingleRequestController) error {
	if err := ctrl.SubmitEnvelope(testEnvelope(req.RequestID, 1, SingleRequestStatePlanning)); err != nil {
		return err
	}
	close(e.planningStarted)
	select {
	case <-e.submitTool:
	case <-ctx.Done():
		return ctx.Err()
	}
	err := ctrl.SubmitEnvelope(SingleRequestEnvelope{
		RequestID: req.RequestID, Sequence: 2,
		Stage: SingleRequestStateInternalTool, SavedStage: SingleRequestStatePlanning,
		ToolCall: &InternalWorkspaceToolCall{
			RequestID: req.RequestID, StageID: "plan", ToolCallID: "admission-race-tool",
			Name: InternalWorkspaceToolRead, Arguments: json.RawMessage(`{"relative_path":"README.md"}`),
		},
	})
	if err != nil {
		return err
	}
	<-ctx.Done()
	return ctx.Err()
}

func (e *admissionRaceExecutor) ContinueInternalTool(_ context.Context, _ InternalWorkspaceToolResult) error {
	return nil
}

func TestSingleRequestObservationDeadlineClassifications(t *testing.T) {
	terminalEvent := func(t *testing.T, observer *capturingObserver) singleRequestDTO {
		t.Helper()
		for _, event := range observer.snapshot() {
			if event.EventClass == singleRequestEventClassTerminal {
				return event
			}
		}
		t.Fatal("terminal observation was not emitted")
		return singleRequestDTO{}
	}

	t.Run("request wall-clock expiry preserves the budget sentinel", func(t *testing.T) {
		observer := &capturingObserver{}
		service, _ := newInternalToolLoopService(t, &channelFakeExecutor{fn: func(ctx context.Context, _ SingleRequestRequest, _ SingleRequestController) error {
			<-ctx.Done()
			return ctx.Err()
		}})
		service.SetSingleRequestObserver(observer)
		handle, err := service.StartSingleRequest(context.Background(), internalLoopRequest(t, func(binding *SingleRequestBinding) {
			binding.Limits.WallClockMS = 50
			binding.Limits.StageTimeoutMS = 50
		}))
		if err != nil {
			t.Fatalf("StartSingleRequest: %v", err)
		}
		if _, err := waitForExecution(t, handle); !errors.Is(err, ErrSingleRequestInternalToolBudget) {
			t.Fatalf("Wait error=%v, want internal tool budget sentinel", err)
		}
		if terminal := terminalEvent(t, observer); terminal.ErrorClass != singleRequestErrorClassTimeout {
			t.Fatalf("terminal error class=%q, want timeout", terminal.ErrorClass)
		}
	})

	t.Run("stage timer expiry is observed as timeout", func(t *testing.T) {
		observer := &capturingObserver{}
		service, _ := newInternalToolLoopService(t, &channelFakeExecutor{fn: func(ctx context.Context, req SingleRequestRequest, ctrl SingleRequestController) error {
			if err := ctrl.SubmitEnvelope(testEnvelope(req.RequestID, 1, SingleRequestStatePlanning)); err != nil {
				return err
			}
			<-ctx.Done()
			return ctx.Err()
		}})
		service.SetSingleRequestObserver(observer)
		handle, err := service.StartSingleRequest(context.Background(), internalLoopRequest(t, func(binding *SingleRequestBinding) {
			binding.Limits.WallClockMS = 250
			binding.Limits.StageTimeoutMS = 50
		}))
		if err != nil {
			t.Fatalf("StartSingleRequest: %v", err)
		}
		if _, err := waitForExecution(t, handle); !errors.Is(err, ErrSingleRequestInternalToolBudget) {
			t.Fatalf("Wait error=%v, want internal tool budget sentinel", err)
		}
		var stage singleRequestDTO
		for _, event := range observer.snapshot() {
			if event.EventClass == singleRequestEventClassStage {
				stage = event
			}
		}
		if stage.ErrorClass != singleRequestErrorClassTimeout {
			t.Fatalf("stage error class=%q, want timeout", stage.ErrorClass)
		}
		if terminal := terminalEvent(t, observer); terminal.ErrorClass != singleRequestErrorClassTimeout {
			t.Fatalf("terminal error class=%q, want timeout", terminal.ErrorClass)
		}
	})

	t.Run("in-flight tool deadline wins before cleanup failure", func(t *testing.T) {
		observer := &capturingObserver{}
		executor := newScriptedInternalToolExecutor(InternalWorkspaceToolCall{
			ToolCallID: "deadline-tool", Name: InternalWorkspaceToolRead,
			Arguments: json.RawMessage(`{"relative_path":"README.md"}`),
		})
		service, node := newInternalToolLoopService(t, executor)
		service.SetSingleRequestObserver(observer)
		var openCount atomic.Int32
		toki.AddRequestListenerTyped[*iop.WorkspaceOpenRequest, *iop.WorkspaceOpenResponse](&node.Communicator, func(req *iop.WorkspaceOpenRequest) (*iop.WorkspaceOpenResponse, error) {
			openCount.Add(1)
			return &iop.WorkspaceOpenResponse{RequestId: req.GetRequestId(), WorkspaceRef: req.GetWorkspaceRef(), Status: iop.WorkspaceStatus_WORKSPACE_STATUS_SUCCESS}, nil
		})
		toolStarted := make(chan struct{})
		toolRelease := make(chan struct{})
		var toolStartOnce sync.Once
		toki.AddRequestListenerTyped[*iop.WorkspaceToolRequest, *iop.WorkspaceToolResponse](&node.Communicator, func(req *iop.WorkspaceToolRequest) (*iop.WorkspaceToolResponse, error) {
			toolStartOnce.Do(func() { close(toolStarted) })
			<-toolRelease
			return &iop.WorkspaceToolResponse{RequestId: req.GetRequestId(), StageId: req.GetStageId(), ToolCallId: req.GetToolCallId(), Status: iop.WorkspaceStatus_WORKSPACE_STATUS_CANCELLED}, nil
		})
		toki.AddRequestListenerTyped[*iop.WorkspaceCleanupRequest, *iop.WorkspaceCleanupResponse](&node.Communicator, func(req *iop.WorkspaceCleanupRequest) (*iop.WorkspaceCleanupResponse, error) {
			return &iop.WorkspaceCleanupResponse{RequestId: req.GetRequestId(), Status: iop.WorkspaceStatus_WORKSPACE_STATUS_ERROR}, nil
		})

		handle, err := service.StartSingleRequest(context.Background(), internalLoopRequest(t, func(binding *SingleRequestBinding) {
			binding.Limits.WallClockMS = 250
			binding.Limits.StageTimeoutMS = 50
		}))
		if err != nil {
			t.Fatalf("StartSingleRequest: %v", err)
		}
		select {
		case <-toolStarted:
		case <-time.After(2 * time.Second):
			t.Fatal("tool did not reach Node")
		}
		waitForState(t, handle, SingleRequestStateFailed)
		close(toolRelease)
		if _, err := waitForExecution(t, handle); !errors.Is(err, ErrSingleRequestInternalToolBudget) || !errors.Is(err, ErrSingleRequestWorkspaceCleanup) {
			t.Fatalf("Wait error=%v, want deadline sentinel joined with cleanup failure", err)
		}
		events := observer.snapshot()
		if len(events) != 5 || events[1].EventClass != singleRequestEventClassTool || events[1].ErrorClass != singleRequestErrorClassTimeout ||
			events[2].EventClass != singleRequestEventClassStage || events[2].ErrorClass != singleRequestErrorClassTimeout || events[2].ToolCount != 1 ||
			events[3].EventClass != singleRequestEventClassCleanup || events[3].ErrorClass != singleRequestErrorClassWorkspaceCleanup {
			t.Fatalf("tool deadline lifecycle=%#v", events)
		}
		if terminal := terminalEvent(t, observer); terminal.ErrorClass != singleRequestErrorClassTimeout {
			t.Fatalf("terminal error class=%q, want timeout", terminal.ErrorClass)
		}
		if openCount.Load() != 1 {
			t.Fatalf("workspace open count=%d, want 1", openCount.Load())
		}
	})

	t.Run("iteration exhaustion remains an internal tool budget", func(t *testing.T) {
		observer := &capturingObserver{}
		executor := newScriptedInternalToolExecutor(
			InternalWorkspaceToolCall{ToolCallID: "budget-tool-1", Name: InternalWorkspaceToolRead, Arguments: json.RawMessage(`{"relative_path":"README.md"}`)},
			InternalWorkspaceToolCall{ToolCallID: "budget-tool-2", Name: InternalWorkspaceToolRead, Arguments: json.RawMessage(`{"relative_path":"README.md"}`)},
		)
		service, node := newInternalToolLoopService(t, executor)
		service.SetSingleRequestObserver(observer)
		var openCount atomic.Int32
		installInternalLoopOpenResponder(node, &openCount)
		toki.AddRequestListenerTyped[*iop.WorkspaceToolRequest, *iop.WorkspaceToolResponse](&node.Communicator, func(req *iop.WorkspaceToolRequest) (*iop.WorkspaceToolResponse, error) {
			return &iop.WorkspaceToolResponse{RequestId: req.GetRequestId(), StageId: req.GetStageId(), ToolCallId: req.GetToolCallId(), Status: iop.WorkspaceStatus_WORKSPACE_STATUS_SUCCESS}, nil
		})
		handle, err := service.StartSingleRequest(context.Background(), internalLoopRequest(t, func(binding *SingleRequestBinding) {
			binding.Limits.MaxToolIterations = 1
		}))
		if err != nil {
			t.Fatalf("StartSingleRequest: %v", err)
		}
		if _, err := waitForExecution(t, handle); !errors.Is(err, ErrSingleRequestInternalToolBudget) {
			t.Fatalf("Wait error=%v, want internal tool budget", err)
		}
		if terminal := terminalEvent(t, observer); terminal.ErrorClass != singleRequestErrorClassInternalToolBudget {
			t.Fatalf("terminal error class=%q, want internal_tool_budget", terminal.ErrorClass)
		}
		if openCount.Load() != 1 {
			t.Fatalf("workspace open count=%d, want 1", openCount.Load())
		}
	})

	t.Run("expired stage deadline at tool admission is observed as timeout", func(t *testing.T) {
		observer := &capturingObserver{}
		executor := &admissionRaceExecutor{
			planningStarted: make(chan struct{}),
			submitTool:      make(chan struct{}),
		}
		service, _ := newInternalToolLoopService(t, executor)
		service.SetSingleRequestObserver(observer)
		handle, err := service.StartSingleRequest(context.Background(), internalLoopRequest(t, func(binding *SingleRequestBinding) {
			binding.Limits.WallClockMS = 250
			binding.Limits.StageTimeoutMS = 30
		}))
		if err != nil {
			t.Fatalf("StartSingleRequest: %v", err)
		}
		select {
		case <-executor.planningStarted:
		case <-time.After(2 * time.Second):
			t.Fatal("planning state did not start")
		}
		h := handle.(*singleRequestHandle)
		h.mu.Lock()
		close(executor.submitTool)
		time.Sleep(50 * time.Millisecond)
		h.mu.Unlock()

		if _, err := waitForExecution(t, handle); !errors.Is(err, ErrSingleRequestInternalToolBudget) {
			t.Fatalf("Wait error=%v, want internal tool budget sentinel", err)
		}
		if terminal := terminalEvent(t, observer); terminal.ErrorClass != singleRequestErrorClassTimeout {
			t.Fatalf("terminal error class=%q, want timeout", terminal.ErrorClass)
		}
	})
}

func TestSingleRequestObservationErrorClassMapping(t *testing.T) {
	for _, test := range []struct {
		name string
		err  error
		want singleRequestErrorClass
	}{
		{name: "provider", err: errors.New("provider failure"), want: singleRequestErrorClassProvider},
		{name: "validation", err: fmt.Errorf("wrapped: %w", ErrSingleRequestIdentityMismatch), want: singleRequestErrorClassValidation},
		{name: "budget", err: fmt.Errorf("wrapped: %w", ErrSingleRequestInternalToolBudget), want: singleRequestErrorClassInternalToolBudget},
		{name: "tool", err: fmt.Errorf("wrapped: %w", ErrSingleRequestInternalToolFailed), want: singleRequestErrorClassInternalToolFailed},
		{name: "cleanup", err: fmt.Errorf("wrapped: %w", ErrSingleRequestWorkspaceCleanup), want: singleRequestErrorClassWorkspaceCleanup},
		{name: "timeout", err: context.DeadlineExceeded, want: singleRequestErrorClassTimeout},
		{name: "cancel", err: context.Canceled, want: singleRequestErrorClassCancel},
	} {
		t.Run(test.name, func(t *testing.T) {
			if got := singleRequestErrorClassFromErr(test.err); got != test.want {
				t.Fatalf("error class=%q, want %q", got, test.want)
			}
		})
	}
}