iop/packages/go/agenttask/test_support_test.go

package agenttask

import (
	"context"
	"fmt"
	"io"
	"maps"
	"os"
	"os/exec"
	"path/filepath"
	"reflect"
	"slices"
	"strconv"
	"sync"
	"testing"
	"time"

	"iop/packages/go/agentguard"
)

type memoryStore struct {
	mu       sync.Mutex
	revision uint64
	state    ManagerState
}

func newMemoryStore() *memoryStore {
	return &memoryStore{state: ManagerState{SchemaVersion: currentSchemaVersion}}
}

func (s *memoryStore) Load(context.Context) (ManagerState, StateRevision, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	return cloneState(s.state), StateRevision(strconv.FormatUint(s.revision, 10)), nil
}

func (s *memoryStore) CompareAndSwap(
	_ context.Context,
	expected StateRevision,
	next ManagerState,
) (StateRevision, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	if expected != StateRevision(strconv.FormatUint(s.revision, 10)) {
		return "", ErrRevisionConflict
	}
	s.revision++
	s.state = cloneState(next)
	return StateRevision(strconv.FormatUint(s.revision, 10)), nil
}

func (s *memoryStore) edit(change func(*ManagerState)) {
	s.mu.Lock()
	defer s.mu.Unlock()
	next := cloneState(s.state)
	change(&next)
	s.revision++
	s.state = next
}

func (s *memoryStore) snapshot() ManagerState {
	s.mu.Lock()
	defer s.mu.Unlock()
	return cloneState(s.state)
}

type fixedClock struct {
	now time.Time
}

func (c fixedClock) Now() time.Time { return c.now }

type advancingClock struct {
	mu  sync.Mutex
	now time.Time
}

func (c *advancingClock) Now() time.Time {
	c.mu.Lock()
	defer c.mu.Unlock()
	return c.now
}

func (c *advancingClock) Advance(d time.Duration) {
	c.mu.Lock()
	c.now = c.now.Add(d)
	c.mu.Unlock()
}

type fakeWorkflow struct {
	mu        sync.Mutex
	snapshots map[ProjectID]ProjectWorkflowSnapshot
	errors    map[ProjectID]error
}

func (f *fakeWorkflow) RegisteredProjects(context.Context) ([]ProjectID, error) {
	f.mu.Lock()
	defer f.mu.Unlock()
	ids := make([]ProjectID, 0, len(f.snapshots)+len(f.errors))
	seen := make(map[ProjectID]struct{})
	for id := range f.snapshots {
		ids = append(ids, id)
		seen[id] = struct{}{}
	}
	for id := range f.errors {
		if _, ok := seen[id]; !ok {
			ids = append(ids, id)
		}
	}
	return ids, nil
}

func (f *fakeWorkflow) Snapshot(
	_ context.Context,
	projectID ProjectID,
) (ProjectWorkflowSnapshot, error) {
	f.mu.Lock()
	defer f.mu.Unlock()
	if err := f.errors[projectID]; err != nil {
		return ProjectWorkflowSnapshot{}, err
	}
	snapshot, ok := f.snapshots[projectID]
	if !ok {
		return ProjectWorkflowSnapshot{}, fmt.Errorf("missing project %s", projectID)
	}
	return cloneWorkflow(snapshot), nil
}

type fakeSelector struct {
	capacity   int
	providerID string
}

func (s fakeSelector) Select(
	_ context.Context,
	request SelectionRequest,
) (ExecutionTarget, error) {
	providerID := s.providerID
	if providerID == "" {
		providerID = "provider"
	}
	return ExecutionTarget{
		ProviderID:      providerID,
		ModelID:         "model",
		ProfileID:       "profile",
		ProfileRevision: "profile-r1",
		ConfigRevision:  request.Project.Intent.ConfigRevision,
		Capacity:        s.capacity,
	}, nil
}

type fakeIsolation struct {
	t            *testing.T
	root         string
	mu           sync.Mutex
	baseRoots    map[WorkspaceID]string
	taskRoots    map[string]string
	preparations []string
	proofs       map[string]*fakeConfinement
	startErr     error
	nilStarted   bool
	invalidStart bool
}

type fakeConfinement struct {
	binding ConfinementBinding
	state   *fakeConfinementState
}

type fakeConfinementState struct {
	mu           sync.Mutex
	startErr     error
	nilStarted   bool
	invalidStart bool
	starts       int
	commands     []ConfinementCommand
	children     []*exec.Cmd
	handles      []StartedConfinement
	order        []string
}

type fakeStartedConfinement struct {
	child  *exec.Cmd
	stdin  *os.File
	stdout *os.File
	stderr *os.File

	invalid   bool
	abortOnce sync.Once
	mu        sync.Mutex
	aborts    int
}

func (started *fakeStartedConfinement) Child() *exec.Cmd {
	if started == nil {
		return nil
	}
	return started.child
}

func (started *fakeStartedConfinement) Stdin() io.WriteCloser {
	if started == nil || started.invalid {
		return nil
	}
	return started.stdin
}

func (started *fakeStartedConfinement) Stdout() io.ReadCloser {
	if started == nil {
		return nil
	}
	return started.stdout
}

func (started *fakeStartedConfinement) Stderr() io.ReadCloser {
	if started == nil {
		return nil
	}
	return started.stderr
}

func (started *fakeStartedConfinement) Abort() error {
	if started == nil {
		return nil
	}
	started.abortOnce.Do(func() {
		for _, endpoint := range []*os.File{started.stdin, started.stdout, started.stderr} {
			if endpoint != nil {
				_ = endpoint.Close()
			}
		}
		if started.child != nil &&
			started.child.Process != nil &&
			started.child.ProcessState == nil {
			_ = started.child.Process.Kill()
			_ = started.child.Wait()
		}
		started.mu.Lock()
		started.aborts++
		started.mu.Unlock()
	})
	return nil
}

func (started *fakeStartedConfinement) abortCount() int {
	if started == nil {
		return 0
	}
	started.mu.Lock()
	defer started.mu.Unlock()
	return started.aborts
}

func (proof *fakeConfinement) Revision() string {
	if proof == nil {
		return ""
	}
	return proof.binding.Revision
}

func (proof *fakeConfinement) Binding() ConfinementBinding {
	if proof == nil {
		return ConfinementBinding{}
	}
	binding := proof.binding
	binding.WritableRoots = slices.Clone(binding.WritableRoots)
	return binding
}

func (proof *fakeConfinement) Validate(expected ConfinementBinding) error {
	if proof == nil || !reflect.DeepEqual(proof.Binding(), expected) {
		return fmt.Errorf("fake confinement identity mismatch")
	}
	return nil
}

func (proof *fakeConfinement) Start(
	ctx context.Context,
	spec ConfinementCommand,
) (StartedConfinement, error) {
	if err := proof.Validate(proof.Binding()); err != nil {
		return nil, err
	}
	var nilStarted bool
	var invalidStart bool
	if proof.state != nil {
		proof.state.mu.Lock()
		proof.state.starts++
		proof.state.commands = append(proof.state.commands, spec)
		proof.state.order = append(proof.state.order, "proof-start")
		startErr := proof.state.startErr
		nilStarted = proof.state.nilStarted
		invalidStart = proof.state.invalidStart
		proof.state.mu.Unlock()
		if startErr != nil {
			return nil, startErr
		}
	}
	if nilStarted {
		return nil, nil
	}
	command := exec.CommandContext(ctx, spec.Name, spec.Args...)
	command.Env = slices.Clone(spec.Env)
	stdinChild, stdinParent, err := os.Pipe()
	if err != nil {
		return nil, err
	}
	stdoutParent, stdoutChild, err := os.Pipe()
	if err != nil {
		_ = stdinChild.Close()
		_ = stdinParent.Close()
		return nil, err
	}
	stderrParent, stderrChild, err := os.Pipe()
	if err != nil {
		for _, endpoint := range []*os.File{
			stdinChild, stdinParent, stdoutParent, stdoutChild,
		} {
			_ = endpoint.Close()
		}
		return nil, err
	}
	command.Stdin = stdinChild
	command.Stdout = stdoutChild
	command.Stderr = stderrChild
	if err := command.Start(); err != nil {
		for _, endpoint := range []*os.File{
			stdinChild, stdinParent,
			stdoutParent, stdoutChild,
			stderrParent, stderrChild,
		} {
			_ = endpoint.Close()
		}
		return nil, err
	}
	for _, endpoint := range []*os.File{stdinChild, stdoutChild, stderrChild} {
		_ = endpoint.Close()
	}
	started := &fakeStartedConfinement{
		child: command, stdin: stdinParent, stdout: stdoutParent, stderr: stderrParent,
		invalid: invalidStart,
	}
	if proof.state != nil {
		proof.state.mu.Lock()
		proof.state.children = append(proof.state.children, command)
		proof.state.handles = append(proof.state.handles, started)
		proof.state.mu.Unlock()
	}
	return started, nil
}

func (proof *fakeConfinement) startCount() int {
	if proof == nil || proof.state == nil {
		return 0
	}
	proof.state.mu.Lock()
	defer proof.state.mu.Unlock()
	return proof.state.starts
}

func (proof *fakeConfinement) startCommands() []ConfinementCommand {
	if proof == nil || proof.state == nil {
		return nil
	}
	proof.state.mu.Lock()
	defer proof.state.mu.Unlock()
	return slices.Clone(proof.state.commands)
}

func (proof *fakeConfinement) launchOrder() []string {
	if proof == nil || proof.state == nil {
		return nil
	}
	proof.state.mu.Lock()
	defer proof.state.mu.Unlock()
	return slices.Clone(proof.state.order)
}

func (proof *fakeConfinement) startedChildren() []*exec.Cmd {
	if proof == nil || proof.state == nil {
		return nil
	}
	proof.state.mu.Lock()
	defer proof.state.mu.Unlock()
	return slices.Clone(proof.state.children)
}

func (proof *fakeConfinement) startedHandles() []StartedConfinement {
	if proof == nil || proof.state == nil {
		return nil
	}
	proof.state.mu.Lock()
	defer proof.state.mu.Unlock()
	return slices.Clone(proof.state.handles)
}

func newFakeIsolation(t *testing.T) *fakeIsolation {
	t.Helper()
	return &fakeIsolation{
		t: t, root: t.TempDir(),
		baseRoots: make(map[WorkspaceID]string),
		taskRoots: make(map[string]string),
		proofs:    make(map[string]*fakeConfinement),
	}
}

func (f *fakeIsolation) Prepare(
	_ context.Context,
	request IsolationRequest,
) (PreparedIsolation, error) {
	f.mu.Lock()
	defer f.mu.Unlock()
	workspaceID := request.Project.WorkspaceID
	baseRoot := f.baseRoots[workspaceID]
	if baseRoot == "" {
		baseRoot = filepath.Join(f.root, "base-"+string(workspaceID))
		if err := os.MkdirAll(baseRoot, 0o755); err != nil {
			f.t.Fatalf("create base root: %v", err)
		}
		f.baseRoots[workspaceID] = baseRoot
	}
	key := request.IdempotencyKey
	taskRoot := f.taskRoots[key]
	if taskRoot == "" {
		taskRoot = filepath.Join(
			f.root,
			"task-"+string(request.Project.ProjectID)+"-"+string(request.Work.AttemptID),
		)
		if err := os.MkdirAll(taskRoot, 0o755); err != nil {
			f.t.Fatalf("create task root: %v", err)
		}
		f.taskRoots[key] = taskRoot
	}
	viewRoot := filepath.Join(taskRoot, "view")
	tempRoot := filepath.Join(taskRoot, "temp")
	cacheRoot := filepath.Join(taskRoot, "cache")
	snapshotRoot := filepath.Join(f.root, "snapshots", "base-r1")
	for _, root := range []string{viewRoot, tempRoot, cacheRoot, snapshotRoot} {
		if err := os.MkdirAll(root, 0o755); err != nil {
			f.t.Fatalf("create isolation root: %v", err)
		}
	}
	writableRoots := []string{viewRoot, tempRoot, cacheRoot}
	binding := ConfinementBinding{
		Revision:           "confinement-r1",
		IsolationID:        request.IdempotencyKey,
		IsolationRevision:  "isolation-r1",
		PinnedBaseRevision: "base-r1",
		ConfigRevision:     string(request.Project.Intent.ConfigRevision),
		GrantRevision:      string(request.Project.Intent.GrantRevision),
		ProfileRevision:    request.Target.ProfileRevision,
		BaseRoot:           baseRoot,
		RuntimeRoot:        f.root,
		SnapshotRoot:       snapshotRoot,
		TaskRoot:           taskRoot,
		WorkingDir:         viewRoot,
		WritableRoots:      slices.Clone(writableRoots),
	}
	proof := &fakeConfinement{
		binding: binding,
		state: &fakeConfinementState{
			startErr:     f.startErr,
			nilStarted:   f.nilStarted,
			invalidStart: f.invalidStart,
		},
	}
	f.preparations = append(f.preparations, key)
	f.proofs[key] = proof
	return PreparedIsolation{
		Grant: &agentguard.WorkspaceGrant{
			ProjectID: string(request.Project.ProjectID), WorkspaceID: string(workspaceID),
			Root: baseRoot, Revision: string(request.Project.Intent.GrantRevision),
		},
		Descriptor: &agentguard.IsolationDescriptor{
			ID: request.IdempotencyKey, Revision: "isolation-r1",
			Mode: request.Work.Unit.IsolationMode, BaseRoot: baseRoot, TaskRoot: taskRoot,
			WorkingDir: viewRoot, WritableRoots: slices.Clone(writableRoots),
			PinnedBaseRevision: "base-r1", ConfinementRevision: "confinement-r1",
		},
		Profile: agentguard.ProviderProfile{
			ProviderID: request.Target.ProviderID, ModelID: request.Target.ModelID,
			ProfileID: request.Target.ProfileID, Revision: request.Target.ProfileRevision,
			Unattended: true, ApprovalBypass: true, WritableRootConfinement: true,
		},
		Confinement: proof,
	}, nil
}

func (f *fakeIsolation) proof(key string) *fakeConfinement {
	f.mu.Lock()
	defer f.mu.Unlock()
	return f.proofs[key]
}

type fakeRecovery struct {
	mu           sync.Mutex
	observations map[WorkUnitID]RecoveryObservation
	errors       map[WorkUnitID]error
	actualCalls  []RecoveryRequest
}

func newFakeRecovery() *fakeRecovery {
	return &fakeRecovery{
		observations: make(map[WorkUnitID]RecoveryObservation),
		errors:       make(map[WorkUnitID]error),
	}
}

func (f *fakeRecovery) Inspect(
	_ context.Context,
	request RecoveryRequest,
) (RecoveryObservation, error) {
	f.mu.Lock()
	defer f.mu.Unlock()
	f.actualCalls = append(f.actualCalls, request)
	if err := f.errors[request.Work.Unit.ID]; err != nil {
		return RecoveryObservation{}, err
	}
	if observation, ok := f.observations[request.Work.Unit.ID]; ok {
		return observation, nil
	}
	return RecoveryObservation{
		ProjectID: request.Project.ProjectID, WorkspaceID: request.Project.WorkspaceID,
		WorkUnitID: request.Work.Unit.ID, AttemptID: request.Work.AttemptID,
		Execution: RecoveryExecutionAbsent, Completion: RecoveryCompletionComplete,
	}, nil
}

func (f *fakeRecovery) callCount() int {
	f.mu.Lock()
	defer f.mu.Unlock()
	return len(f.actualCalls)
}

type fakeInvoker struct {
	mu                sync.Mutex
	results           map[string]Submission
	actualCalls       []DispatchRequest
	active            int
	maxActive         int
	delays            map[WorkUnitID]time.Duration
	locators          map[WorkUnitID][]LocatorRecord
	blockUntil        map[string]chan struct{} // keyed by IdempotencyKey
	blockAll          chan struct{}            // if non-nil, blocks all invocations
	cancelled         int64
	prepareErr        error
	bindErr           error
	command           ConfinementCommand
	prepareCalls      int
	bindCalls         int
	launchOrder       []string
	proofStartsAtBind []int
	boundHandles      []StartedConfinement
}

func newFakeInvoker() *fakeInvoker {
	return &fakeInvoker{
		results:    make(map[string]Submission),
		delays:     make(map[WorkUnitID]time.Duration),
		locators:   make(map[WorkUnitID][]LocatorRecord),
		blockUntil: make(map[string]chan struct{}),
	}
}

func (f *fakeInvoker) blockAfterStart(key string, release chan struct{}) {
	f.mu.Lock()
	defer f.mu.Unlock()
	f.blockUntil[key] = release
}

func (f *fakeInvoker) blockAllInvocations(block chan struct{}) {
	f.mu.Lock()
	defer f.mu.Unlock()
	f.blockAll = block
}

func (f *fakeInvoker) cancelCount() int {
	f.mu.Lock()
	defer f.mu.Unlock()
	return int(f.cancelled)
}

func (f *fakeInvoker) Prepare(
	_ context.Context,
	request DispatchRequest,
) (ProviderLaunch, error) {
	f.mu.Lock()
	defer f.mu.Unlock()
	if request.Permit == nil ||
		request.Workspace.TaskRoot == request.Workspace.BaseRoot ||
		request.Confinement == nil ||
		request.Confinement.Revision() != request.Workspace.ConfinementRevision {
		return nil, fmt.Errorf("unsafe dispatch request")
	}
	if err := request.Confinement.Validate(request.Confinement.Binding()); err != nil {
		return nil, fmt.Errorf("unsafe confinement proof: %w", err)
	}
	f.prepareCalls++
	f.launchOrder = append(f.launchOrder, "prepare")
	if f.prepareErr != nil {
		return nil, f.prepareErr
	}
	command := f.command
	if command.Name == "" {
		command.Name = "true"
	}
	return &fakeLaunch{owner: f, request: request, command: command}, nil
}

type fakeLaunch struct {
	owner   *fakeInvoker
	request DispatchRequest
	command ConfinementCommand
}

func (launch *fakeLaunch) Command() ConfinementCommand {
	return launch.command
}

func (launch *fakeLaunch) BindStarted(
	started StartedConfinement,
) (ProviderInvocation, error) {
	if started == nil || started.Child() == nil {
		return nil, fmt.Errorf("cannot bind a nil started handle")
	}
	return launch.owner.bindStarted(launch.request, started)
}

func (f *fakeInvoker) bindStarted(
	request DispatchRequest,
	started StartedConfinement,
) (ProviderInvocation, error) {
	f.mu.Lock()
	f.bindCalls++
	f.launchOrder = append(f.launchOrder, "bind")
	f.boundHandles = append(f.boundHandles, started)
	if proof, ok := request.Confinement.(*fakeConfinement); ok {
		f.proofStartsAtBind = append(f.proofStartsAtBind, proof.startCount())
	}
	if f.bindErr != nil {
		f.mu.Unlock()
		return nil, f.bindErr
	}
	if previous, ok := f.results[request.IdempotencyKey]; ok {
		f.mu.Unlock()
		return &fakeInvocation{
			owner: f, request: request, existing: &previous,
			locators: slices.Clone(previous.Locators), cancelled: make(chan struct{}), started: started,
		}, nil
	}
	delay := f.delays[request.Work.Unit.ID]
	locators := slices.Clone(f.locators[request.Work.Unit.ID])
	if len(locators) == 0 {
		locators = []LocatorRecord{{
			Kind:   LocatorProcess,
			Opaque: "fake-process:" + request.IdempotencyKey, Revision: "process-r1",
			ProjectID: request.Project.ProjectID, WorkspaceID: request.Project.WorkspaceID,
			WorkUnitID: request.Work.Unit.ID, AttemptID: request.Work.AttemptID,
		}}
	}
	var blockChan chan struct{}
	if f.blockAll != nil {
		blockChan = f.blockAll
	} else if ch, ok := f.blockUntil[request.IdempotencyKey]; ok {
		blockChan = ch
	}
	f.mu.Unlock()
	return &fakeInvocation{
		owner: f, request: request, delay: delay, locators: locators,
		cancelled: make(chan struct{}), block: blockChan, started: started,
	}, nil
}

type fakeInvocation struct {
	owner     *fakeInvoker
	request   DispatchRequest
	delay     time.Duration
	locators  []LocatorRecord
	existing  *Submission
	cancelled chan struct{}
	block     chan struct{}
	cancel    sync.Once
	finish    sync.Once
	started   StartedConfinement
	result    Submission
	err       error
}

func (i *fakeInvocation) Locators() []LocatorRecord {
	return slices.Clone(i.locators)
}

func (i *fakeInvocation) Wait(ctx context.Context) (Submission, error) {
	i.finish.Do(func() {
		if i.existing != nil {
			i.finishChild()
			i.result = *i.existing
			i.result.Metadata = maps.Clone(i.existing.Metadata)
			i.result.Locators = slices.Clone(i.existing.Locators)
			return
		}
		i.owner.startInvocation()
		if i.delay > 0 {
			timer := time.NewTimer(i.delay)
			defer timer.Stop()
			select {
			case <-ctx.Done():
				i.abortChild()
				i.err = ctx.Err()
				i.owner.finishInvocation(i.request, Submission{}, false)
				return
			case <-i.cancelled:
				i.abortChild()
				i.err = context.Canceled
				i.owner.finishInvocation(i.request, Submission{}, false)
				return
			case <-timer.C:
			}
		}
		if i.block != nil {
			select {
			case <-ctx.Done():
				i.abortChild()
				i.err = ctx.Err()
				i.owner.finishInvocation(i.request, Submission{}, false)
				return
			case <-i.cancelled:
				i.abortChild()
				i.err = context.Canceled
				i.owner.finishInvocation(i.request, Submission{}, false)
				return
			case <-i.block:
			}
		}
		i.finishChild()
		i.result = Submission{
			ProjectID: i.request.Project.ProjectID, WorkUnitID: i.request.Work.Unit.ID,
			AttemptID:  i.request.Work.AttemptID,
			ArtifactID: ArtifactID("artifact-" + string(i.request.Work.AttemptID)),
			Ready:      true,
			Locators:   slices.Clone(i.locators),
		}
		i.owner.finishInvocation(i.request, i.result, true)
	})
	return i.result, i.err
}

func (i *fakeInvocation) Cancel(context.Context) error {
	i.cancel.Do(func() {
		close(i.cancelled)
	})
	i.finish.Do(func() {
		if i.existing == nil {
			i.owner.startInvocation()
			i.abortChild()
			i.err = context.Canceled
			i.owner.finishInvocation(i.request, Submission{}, false)
		}
	})
	return nil
}

func (i *fakeInvocation) finishChild() {
	if i.started == nil {
		return
	}
	if stdin := i.started.Stdin(); stdin != nil {
		_ = stdin.Close()
	}
	child := i.started.Child()
	if child != nil && child.ProcessState == nil {
		_ = child.Wait()
	}
	for _, endpoint := range []io.Closer{i.started.Stdout(), i.started.Stderr()} {
		if endpoint != nil {
			_ = endpoint.Close()
		}
	}
}

func (i *fakeInvocation) abortChild() {
	if i.started != nil {
		_ = i.started.Abort()
	}
}

func (f *fakeInvoker) finishInvocation(
	request DispatchRequest,
	result Submission,
	success bool,
) {
	f.mu.Lock()
	defer f.mu.Unlock()
	f.active--
	if !success {
		f.cancelled++
	}
	if success {
		f.results[request.IdempotencyKey] = result
		f.actualCalls = append(f.actualCalls, request)
	}
}

func (f *fakeInvoker) startInvocation() {
	f.mu.Lock()
	defer f.mu.Unlock()
	f.active++
	if f.active > f.maxActive {
		f.maxActive = f.active
	}
}

func (f *fakeInvoker) callCount() int {
	f.mu.Lock()
	defer f.mu.Unlock()
	return len(f.actualCalls)
}

func (f *fakeInvoker) launchStats() (int, int, []int) {
	f.mu.Lock()
	defer f.mu.Unlock()
	return f.prepareCalls, f.bindCalls, slices.Clone(f.proofStartsAtBind)
}

func (f *fakeInvoker) startedHandles() []StartedConfinement {
	f.mu.Lock()
	defer f.mu.Unlock()
	return slices.Clone(f.boundHandles)
}

func (f *fakeInvoker) maxConcurrency() int {
	f.mu.Lock()
	defer f.mu.Unlock()
	return f.maxActive
}

func (f *fakeInvoker) activeCount() int {
	f.mu.Lock()
	defer f.mu.Unlock()
	return f.active
}

func (f *fakeInvoker) roots() []string {
	f.mu.Lock()
	defer f.mu.Unlock()
	roots := make([]string, 0, len(f.actualCalls))
	for _, request := range f.actualCalls {
		roots = append(roots, request.Workspace.TaskRoot)
	}
	return roots
}

type fakeReviewer struct {
	mu          sync.Mutex
	sequences   map[WorkUnitID][]ReviewVerdict
	results     map[string]ReviewResult
	actualCalls []ReviewRequest
	blockUntil  map[string]chan struct{} // keyed by IdempotencyKey
	blockAll    chan struct{}            // if non-nil, blocks all reviews
}

func newFakeReviewer() *fakeReviewer {
	return &fakeReviewer{
		sequences:  make(map[WorkUnitID][]ReviewVerdict),
		results:    make(map[string]ReviewResult),
		blockUntil: make(map[string]chan struct{}),
	}
}

func (f *fakeReviewer) blockAfterReview(key string, release chan struct{}) {
	f.mu.Lock()
	defer f.mu.Unlock()
	f.blockUntil[key] = release
}

func (f *fakeReviewer) blockAllReviews(block chan struct{}) {
	f.mu.Lock()
	defer f.mu.Unlock()
	f.blockAll = block
}

func (f *fakeReviewer) Review(
	ctx context.Context,
	request ReviewRequest,
) (ReviewResult, error) {
	f.mu.Lock()
	if previous, ok := f.results[request.IdempotencyKey]; ok {
		f.mu.Unlock()
		return previous, nil
	}
	var blockChan chan struct{}
	if f.blockAll != nil {
		blockChan = f.blockAll
	} else if ch, ok := f.blockUntil[request.IdempotencyKey]; ok {
		blockChan = ch
	}
	f.mu.Unlock()
	if blockChan != nil {
		select {
		case <-blockChan:
		case <-ctx.Done():
			return ReviewResult{}, ctx.Err()
		}
	}
	f.mu.Lock()
	sequence := f.sequences[request.Work.Unit.ID]
	index := int(request.Work.Attempt) - 1
	verdict := ReviewVerdictPass
	if index >= 0 && index < len(sequence) {
		verdict = sequence[index]
	}
	result := ReviewResult{
		ProjectID: request.Project.ProjectID, WorkUnitID: request.Work.Unit.ID,
		AttemptID: request.Work.AttemptID, ArtifactID: request.Submission.ArtifactID,
		Verdict: verdict, Message: string(verdict),
	}
	switch verdict {
	case ReviewVerdictPass:
		result.ChangeSet = &ChangeSetIdentity{
			ID:       ChangeSetID("change-" + string(request.Work.AttemptID)),
			Revision: "change-r1", ArtifactID: request.Submission.ArtifactID,
		}
	case ReviewVerdictWarn, ReviewVerdictFail:
		result.Rework = true
	}
	f.results[request.IdempotencyKey] = result
	f.actualCalls = append(f.actualCalls, request)
	f.mu.Unlock()
	return result, nil
}

func (f *fakeReviewer) callCount() int {
	f.mu.Lock()
	defer f.mu.Unlock()
	return len(f.actualCalls)
}

type fakeIntegrator struct {
	mu          sync.Mutex
	outcomes    map[WorkUnitID]IntegrationOutcome
	results     map[string]IntegrationResult
	actualCalls []IntegrationRequest
	blockUntil  map[string]chan struct{} // keyed by IdempotencyKey
	blockAll    chan struct{}            // if non-nil, blocks all integrations
}

func newFakeIntegrator() *fakeIntegrator {
	return &fakeIntegrator{
		outcomes:   make(map[WorkUnitID]IntegrationOutcome),
		results:    make(map[string]IntegrationResult),
		blockUntil: make(map[string]chan struct{}),
	}
}

func (f *fakeIntegrator) blockAfterIntegrate(key string, release chan struct{}) {
	f.mu.Lock()
	defer f.mu.Unlock()
	f.blockUntil[key] = release
}

func (f *fakeIntegrator) blockAllIntegrations(block chan struct{}) {
	f.mu.Lock()
	defer f.mu.Unlock()
	f.blockAll = block
}

func (f *fakeIntegrator) Integrate(
	ctx context.Context,
	request IntegrationRequest,
) (IntegrationResult, error) {
	f.mu.Lock()
	if previous, ok := f.results[request.IdempotencyKey]; ok {
		f.mu.Unlock()
		return previous, nil
	}
	var blockChan chan struct{}
	if f.blockAll != nil {
		blockChan = f.blockAll
	} else if ch, ok := f.blockUntil[request.IdempotencyKey]; ok {
		blockChan = ch
	}
	f.mu.Unlock()
	if blockChan != nil {
		select {
		case <-blockChan:
		case <-ctx.Done():
			return IntegrationResult{}, ctx.Err()
		}
	}
	f.mu.Lock()
	outcome := f.outcomes[request.Work.Unit.ID]
	if outcome == "" {
		outcome = IntegrationOutcomeIntegrated
	}
	result := IntegrationResult{
		ProjectID: request.Project.ProjectID, WorkUnitID: request.Work.Unit.ID,
		ChangeSet: request.ChangeSet, Ordinal: request.Ordinal, Attempt: request.Attempt,
		Outcome: outcome, BeforeRevision: "before",
	}
	if outcome == IntegrationOutcomeIntegrated {
		result.AfterRevision = "after"
	} else {
		result.Retained = true
		result.Blocker = &Blocker{Code: BlockerIntegrationFailed, Message: "fixture blocker"}
	}
	f.results[request.IdempotencyKey] = result
	f.actualCalls = append(f.actualCalls, request)
	f.mu.Unlock()
	return result, nil
}

func (f *fakeIntegrator) ordinals() []DispatchOrdinal {
	f.mu.Lock()
	defer f.mu.Unlock()
	out := make([]DispatchOrdinal, 0, len(f.actualCalls))
	for _, request := range f.actualCalls {
		out = append(out, request.Ordinal)
	}
	return out
}

func (f *fakeIntegrator) callCount() int {
	f.mu.Lock()
	defer f.mu.Unlock()
	return len(f.actualCalls)
}

type recordingEvents struct {
	mu     sync.Mutex
	events []Event
}

func (r *recordingEvents) Emit(_ context.Context, event Event) error {
	r.mu.Lock()
	defer r.mu.Unlock()
	r.events = append(r.events, event)
	return nil
}

func (r *recordingEvents) snapshot() []Event {
	r.mu.Lock()
	defer r.mu.Unlock()
	return append([]Event(nil), r.events...)
}

type managerHarness struct {
	t          *testing.T
	store      *memoryStore
	workflow   *fakeWorkflow
	isolation  *fakeIsolation
	invoker    *fakeInvoker
	recovery   *fakeRecovery
	evidence   *fakeWorkflowEvidence
	reviewer   *fakeReviewer
	integrator *fakeIntegrator
	events     *recordingEvents
	manager    *Manager
}

func newHarness(
	t *testing.T,
	snapshots map[ProjectID]ProjectWorkflowSnapshot,
	capacity int,
) *managerHarness {
	t.Helper()
	store := newMemoryStore()
	workflow := &fakeWorkflow{snapshots: snapshots, errors: make(map[ProjectID]error)}
	isolation := newFakeIsolation(t)
	invoker := newFakeInvoker()
	recovery := newFakeRecovery()
	evidence := newFakeWorkflowEvidence()
	reviewer := newFakeReviewer()
	integrator := newFakeIntegrator()
	events := &recordingEvents{}
	manager, err := NewManager(
		ManagerConfig{
			OwnerID: "manager-1", LeaseDuration: time.Minute,
			MaxReworkAttempts: 3,
		},
		fixedClock{now: time.Date(2026, 7, 28, 0, 0, 0, 0, time.UTC)},
		store, workflow, fakeSelector{capacity: capacity}, isolation,
		invoker, recovery, evidence, reviewer, integrator, events,
	)
	if err != nil {
		t.Fatalf("NewManager: %v", err)
	}
	return &managerHarness{
		t: t, store: store, workflow: workflow, isolation: isolation,
		invoker: invoker, recovery: recovery, evidence: evidence, reviewer: reviewer, integrator: integrator,
		events: events, manager: manager,
	}
}

func testSnapshot(
	projectID ProjectID,
	workspaceID WorkspaceID,
	units ...WorkUnit,
) ProjectWorkflowSnapshot {
	return ProjectWorkflowSnapshot{
		ProjectID: projectID, WorkspaceID: workspaceID,
		Revision: "workflow-r1", Units: units,
	}
}

func testUnit(id WorkUnitID, kind WriteSetKind) WorkUnit {
	return WorkUnit{
		ID: id, MilestoneID: "milestone", WriteSetKind: kind,
		IsolationMode: agentguard.IsolationModeOverlay,
	}
}

func (h *managerHarness) start(
	projectID ProjectID,
	workspaceID WorkspaceID,
	autoResume *bool,
) {
	h.t.Helper()
	err := h.manager.StartProject(context.Background(), StartRequest{
		CommandID: CommandID("start-" + string(projectID)),
		ProjectID: projectID, WorkspaceID: workspaceID, MilestoneID: "milestone",
		WorkflowRevision: "workflow-r1", ConfigRevision: "config-r1",
		GrantRevision: "grant-r1", AutoResumeInterrupted: autoResume,
	})
	if err != nil {
		h.t.Fatalf("StartProject(%s): %v", projectID, err)
	}
}

// setLeaseDuration overrides the manager's lease duration. The renewal
// supervisor reads this value on each tick so the change takes effect on the
// next renewal attempt without restarting the manager.
func (h *managerHarness) setLeaseDuration(d time.Duration) {
	h.manager.config.LeaseDuration = d
}

func (h *managerHarness) manualRenewals() (chan time.Time, *advancingClock) {
	h.t.Helper()
	ticks := make(chan time.Time, 16)
	clock := &advancingClock{now: h.manager.clock.Now()}
	h.manager.clock = clock
	h.manager.renewalTicks = func(time.Duration) <-chan time.Time { return ticks }
	return ticks, clock
}

func waitFor(t *testing.T, description string, condition func() bool) {
	t.Helper()
	deadline := time.Now().Add(2 * time.Second)
	for !condition() {
		if time.Now().After(deadline) {
			t.Fatalf("timed out waiting for %s", description)
		}
		time.Sleep(time.Millisecond)
	}
}