package agenttask import ( "context" "fmt" "io" "maps" "os" "os/exec" "path/filepath" "reflect" "slices" "strconv" "sync" "testing" "time" "iop/packages/go/agentguard" ) type memoryStore struct { mu sync.Mutex revision uint64 state ManagerState } func newMemoryStore() *memoryStore { return &memoryStore{state: ManagerState{SchemaVersion: currentSchemaVersion}} } func (s *memoryStore) Load(context.Context) (ManagerState, StateRevision, error) { s.mu.Lock() defer s.mu.Unlock() return cloneState(s.state), StateRevision(strconv.FormatUint(s.revision, 10)), nil } func (s *memoryStore) CompareAndSwap( _ context.Context, expected StateRevision, next ManagerState, ) (StateRevision, error) { s.mu.Lock() defer s.mu.Unlock() if expected != StateRevision(strconv.FormatUint(s.revision, 10)) { return "", ErrRevisionConflict } s.revision++ s.state = cloneState(next) return StateRevision(strconv.FormatUint(s.revision, 10)), nil } func (s *memoryStore) edit(change func(*ManagerState)) { s.mu.Lock() defer s.mu.Unlock() next := cloneState(s.state) change(&next) s.revision++ s.state = next } func (s *memoryStore) snapshot() ManagerState { s.mu.Lock() defer s.mu.Unlock() return cloneState(s.state) } type fixedClock struct { now time.Time } func (c fixedClock) Now() time.Time { return c.now } type advancingClock struct { mu sync.Mutex now time.Time } func (c *advancingClock) Now() time.Time { c.mu.Lock() defer c.mu.Unlock() return c.now } func (c *advancingClock) Advance(d time.Duration) { c.mu.Lock() c.now = c.now.Add(d) c.mu.Unlock() } type fakeWorkflow struct { mu sync.Mutex snapshots map[ProjectID]ProjectWorkflowSnapshot errors map[ProjectID]error } func (f *fakeWorkflow) RegisteredProjects(context.Context) ([]ProjectID, error) { f.mu.Lock() defer f.mu.Unlock() ids := make([]ProjectID, 0, len(f.snapshots)+len(f.errors)) seen := make(map[ProjectID]struct{}) for id := range f.snapshots { ids = append(ids, id) seen[id] = struct{}{} } for id := range f.errors { if _, ok := seen[id]; !ok { ids = append(ids, id) } } return ids, nil } func (f *fakeWorkflow) Snapshot( _ context.Context, projectID ProjectID, ) (ProjectWorkflowSnapshot, error) { f.mu.Lock() defer f.mu.Unlock() if err := f.errors[projectID]; err != nil { return ProjectWorkflowSnapshot{}, err } snapshot, ok := f.snapshots[projectID] if !ok { return ProjectWorkflowSnapshot{}, fmt.Errorf("missing project %s", projectID) } return cloneWorkflow(snapshot), nil } type fakeSelector struct { capacity int providerID string } func (s fakeSelector) Select( _ context.Context, request SelectionRequest, ) (ExecutionTarget, error) { providerID := s.providerID if providerID == "" { providerID = "provider" } return ExecutionTarget{ ProviderID: providerID, ModelID: "model", ProfileID: "profile", ProfileRevision: "profile-r1", ConfigRevision: request.Project.Intent.ConfigRevision, Capacity: s.capacity, }, nil } type fakeIsolation struct { t *testing.T root string mu sync.Mutex baseRoots map[WorkspaceID]string taskRoots map[string]string preparations []string proofs map[string]*fakeConfinement startErr error nilStarted bool invalidStart bool } type fakeConfinement struct { binding ConfinementBinding state *fakeConfinementState } type fakeConfinementState struct { mu sync.Mutex startErr error nilStarted bool invalidStart bool starts int commands []ConfinementCommand children []*exec.Cmd handles []StartedConfinement order []string } type fakeStartedConfinement struct { child *exec.Cmd stdin *os.File stdout *os.File stderr *os.File invalid bool abortOnce sync.Once mu sync.Mutex aborts int } func (started *fakeStartedConfinement) Child() *exec.Cmd { if started == nil { return nil } return started.child } func (started *fakeStartedConfinement) Stdin() io.WriteCloser { if started == nil || started.invalid { return nil } return started.stdin } func (started *fakeStartedConfinement) Stdout() io.ReadCloser { if started == nil { return nil } return started.stdout } func (started *fakeStartedConfinement) Stderr() io.ReadCloser { if started == nil { return nil } return started.stderr } func (started *fakeStartedConfinement) Abort() error { if started == nil { return nil } started.abortOnce.Do(func() { for _, endpoint := range []*os.File{started.stdin, started.stdout, started.stderr} { if endpoint != nil { _ = endpoint.Close() } } if started.child != nil && started.child.Process != nil && started.child.ProcessState == nil { _ = started.child.Process.Kill() _ = started.child.Wait() } started.mu.Lock() started.aborts++ started.mu.Unlock() }) return nil } func (started *fakeStartedConfinement) abortCount() int { if started == nil { return 0 } started.mu.Lock() defer started.mu.Unlock() return started.aborts } func (proof *fakeConfinement) Revision() string { if proof == nil { return "" } return proof.binding.Revision } func (proof *fakeConfinement) Binding() ConfinementBinding { if proof == nil { return ConfinementBinding{} } binding := proof.binding binding.WritableRoots = slices.Clone(binding.WritableRoots) return binding } func (proof *fakeConfinement) Validate(expected ConfinementBinding) error { if proof == nil || !reflect.DeepEqual(proof.Binding(), expected) { return fmt.Errorf("fake confinement identity mismatch") } return nil } func (proof *fakeConfinement) Start( ctx context.Context, spec ConfinementCommand, ) (StartedConfinement, error) { if err := proof.Validate(proof.Binding()); err != nil { return nil, err } var nilStarted bool var invalidStart bool if proof.state != nil { proof.state.mu.Lock() proof.state.starts++ proof.state.commands = append(proof.state.commands, spec) proof.state.order = append(proof.state.order, "proof-start") startErr := proof.state.startErr nilStarted = proof.state.nilStarted invalidStart = proof.state.invalidStart proof.state.mu.Unlock() if startErr != nil { return nil, startErr } } if nilStarted { return nil, nil } command := exec.CommandContext(ctx, spec.Name, spec.Args...) command.Env = slices.Clone(spec.Env) stdinChild, stdinParent, err := os.Pipe() if err != nil { return nil, err } stdoutParent, stdoutChild, err := os.Pipe() if err != nil { _ = stdinChild.Close() _ = stdinParent.Close() return nil, err } stderrParent, stderrChild, err := os.Pipe() if err != nil { for _, endpoint := range []*os.File{ stdinChild, stdinParent, stdoutParent, stdoutChild, } { _ = endpoint.Close() } return nil, err } command.Stdin = stdinChild command.Stdout = stdoutChild command.Stderr = stderrChild if err := command.Start(); err != nil { for _, endpoint := range []*os.File{ stdinChild, stdinParent, stdoutParent, stdoutChild, stderrParent, stderrChild, } { _ = endpoint.Close() } return nil, err } for _, endpoint := range []*os.File{stdinChild, stdoutChild, stderrChild} { _ = endpoint.Close() } started := &fakeStartedConfinement{ child: command, stdin: stdinParent, stdout: stdoutParent, stderr: stderrParent, invalid: invalidStart, } if proof.state != nil { proof.state.mu.Lock() proof.state.children = append(proof.state.children, command) proof.state.handles = append(proof.state.handles, started) proof.state.mu.Unlock() } return started, nil } func (proof *fakeConfinement) startCount() int { if proof == nil || proof.state == nil { return 0 } proof.state.mu.Lock() defer proof.state.mu.Unlock() return proof.state.starts } func (proof *fakeConfinement) startCommands() []ConfinementCommand { if proof == nil || proof.state == nil { return nil } proof.state.mu.Lock() defer proof.state.mu.Unlock() return slices.Clone(proof.state.commands) } func (proof *fakeConfinement) launchOrder() []string { if proof == nil || proof.state == nil { return nil } proof.state.mu.Lock() defer proof.state.mu.Unlock() return slices.Clone(proof.state.order) } func (proof *fakeConfinement) startedChildren() []*exec.Cmd { if proof == nil || proof.state == nil { return nil } proof.state.mu.Lock() defer proof.state.mu.Unlock() return slices.Clone(proof.state.children) } func (proof *fakeConfinement) startedHandles() []StartedConfinement { if proof == nil || proof.state == nil { return nil } proof.state.mu.Lock() defer proof.state.mu.Unlock() return slices.Clone(proof.state.handles) } func newFakeIsolation(t *testing.T) *fakeIsolation { t.Helper() return &fakeIsolation{ t: t, root: t.TempDir(), baseRoots: make(map[WorkspaceID]string), taskRoots: make(map[string]string), proofs: make(map[string]*fakeConfinement), } } func (f *fakeIsolation) Prepare( _ context.Context, request IsolationRequest, ) (PreparedIsolation, error) { f.mu.Lock() defer f.mu.Unlock() workspaceID := request.Project.WorkspaceID baseRoot := f.baseRoots[workspaceID] if baseRoot == "" { baseRoot = filepath.Join(f.root, "base-"+string(workspaceID)) if err := os.MkdirAll(baseRoot, 0o755); err != nil { f.t.Fatalf("create base root: %v", err) } f.baseRoots[workspaceID] = baseRoot } key := request.IdempotencyKey taskRoot := f.taskRoots[key] if taskRoot == "" { taskRoot = filepath.Join( f.root, "task-"+string(request.Project.ProjectID)+"-"+string(request.Work.AttemptID), ) if err := os.MkdirAll(taskRoot, 0o755); err != nil { f.t.Fatalf("create task root: %v", err) } f.taskRoots[key] = taskRoot } viewRoot := filepath.Join(taskRoot, "view") tempRoot := filepath.Join(taskRoot, "temp") cacheRoot := filepath.Join(taskRoot, "cache") snapshotRoot := filepath.Join(f.root, "snapshots", "base-r1") for _, root := range []string{viewRoot, tempRoot, cacheRoot, snapshotRoot} { if err := os.MkdirAll(root, 0o755); err != nil { f.t.Fatalf("create isolation root: %v", err) } } writableRoots := []string{viewRoot, tempRoot, cacheRoot} binding := ConfinementBinding{ Revision: "confinement-r1", IsolationID: request.IdempotencyKey, IsolationRevision: "isolation-r1", PinnedBaseRevision: "base-r1", ConfigRevision: string(request.Project.Intent.ConfigRevision), GrantRevision: string(request.Project.Intent.GrantRevision), ProfileRevision: request.Target.ProfileRevision, BaseRoot: baseRoot, RuntimeRoot: f.root, SnapshotRoot: snapshotRoot, TaskRoot: taskRoot, WorkingDir: viewRoot, WritableRoots: slices.Clone(writableRoots), } proof := &fakeConfinement{ binding: binding, state: &fakeConfinementState{ startErr: f.startErr, nilStarted: f.nilStarted, invalidStart: f.invalidStart, }, } f.preparations = append(f.preparations, key) f.proofs[key] = proof return PreparedIsolation{ Grant: &agentguard.WorkspaceGrant{ ProjectID: string(request.Project.ProjectID), WorkspaceID: string(workspaceID), Root: baseRoot, Revision: string(request.Project.Intent.GrantRevision), }, Descriptor: &agentguard.IsolationDescriptor{ ID: request.IdempotencyKey, Revision: "isolation-r1", Mode: request.Work.Unit.IsolationMode, BaseRoot: baseRoot, TaskRoot: taskRoot, WorkingDir: viewRoot, WritableRoots: slices.Clone(writableRoots), PinnedBaseRevision: "base-r1", ConfinementRevision: "confinement-r1", }, Profile: agentguard.ProviderProfile{ ProviderID: request.Target.ProviderID, ModelID: request.Target.ModelID, ProfileID: request.Target.ProfileID, Revision: request.Target.ProfileRevision, Unattended: true, ApprovalBypass: true, WritableRootConfinement: true, }, Confinement: proof, }, nil } func (f *fakeIsolation) proof(key string) *fakeConfinement { f.mu.Lock() defer f.mu.Unlock() return f.proofs[key] } type fakeRecovery struct { mu sync.Mutex observations map[WorkUnitID]RecoveryObservation errors map[WorkUnitID]error actualCalls []RecoveryRequest } func newFakeRecovery() *fakeRecovery { return &fakeRecovery{ observations: make(map[WorkUnitID]RecoveryObservation), errors: make(map[WorkUnitID]error), } } func (f *fakeRecovery) Inspect( _ context.Context, request RecoveryRequest, ) (RecoveryObservation, error) { f.mu.Lock() defer f.mu.Unlock() f.actualCalls = append(f.actualCalls, request) if err := f.errors[request.Work.Unit.ID]; err != nil { return RecoveryObservation{}, err } if observation, ok := f.observations[request.Work.Unit.ID]; ok { return observation, nil } return RecoveryObservation{ ProjectID: request.Project.ProjectID, WorkspaceID: request.Project.WorkspaceID, WorkUnitID: request.Work.Unit.ID, AttemptID: request.Work.AttemptID, Execution: RecoveryExecutionAbsent, Completion: RecoveryCompletionComplete, }, nil } func (f *fakeRecovery) callCount() int { f.mu.Lock() defer f.mu.Unlock() return len(f.actualCalls) } type fakeInvoker struct { mu sync.Mutex results map[string]Submission actualCalls []DispatchRequest active int maxActive int delays map[WorkUnitID]time.Duration locators map[WorkUnitID][]LocatorRecord blockUntil map[string]chan struct{} // keyed by IdempotencyKey blockAll chan struct{} // if non-nil, blocks all invocations cancelled int64 prepareErr error bindErr error command ConfinementCommand prepareCalls int bindCalls int launchOrder []string proofStartsAtBind []int boundHandles []StartedConfinement } func newFakeInvoker() *fakeInvoker { return &fakeInvoker{ results: make(map[string]Submission), delays: make(map[WorkUnitID]time.Duration), locators: make(map[WorkUnitID][]LocatorRecord), blockUntil: make(map[string]chan struct{}), } } func (f *fakeInvoker) blockAfterStart(key string, release chan struct{}) { f.mu.Lock() defer f.mu.Unlock() f.blockUntil[key] = release } func (f *fakeInvoker) blockAllInvocations(block chan struct{}) { f.mu.Lock() defer f.mu.Unlock() f.blockAll = block } func (f *fakeInvoker) cancelCount() int { f.mu.Lock() defer f.mu.Unlock() return int(f.cancelled) } func (f *fakeInvoker) Prepare( _ context.Context, request DispatchRequest, ) (ProviderLaunch, error) { f.mu.Lock() defer f.mu.Unlock() if request.Permit == nil || request.Workspace.TaskRoot == request.Workspace.BaseRoot || request.Confinement == nil || request.Confinement.Revision() != request.Workspace.ConfinementRevision { return nil, fmt.Errorf("unsafe dispatch request") } if err := request.Confinement.Validate(request.Confinement.Binding()); err != nil { return nil, fmt.Errorf("unsafe confinement proof: %w", err) } f.prepareCalls++ f.launchOrder = append(f.launchOrder, "prepare") if f.prepareErr != nil { return nil, f.prepareErr } command := f.command if command.Name == "" { command.Name = "true" } return &fakeLaunch{owner: f, request: request, command: command}, nil } type fakeLaunch struct { owner *fakeInvoker request DispatchRequest command ConfinementCommand } func (launch *fakeLaunch) Command() ConfinementCommand { return launch.command } func (launch *fakeLaunch) BindStarted( started StartedConfinement, ) (ProviderInvocation, error) { if started == nil || started.Child() == nil { return nil, fmt.Errorf("cannot bind a nil started handle") } return launch.owner.bindStarted(launch.request, started) } func (f *fakeInvoker) bindStarted( request DispatchRequest, started StartedConfinement, ) (ProviderInvocation, error) { f.mu.Lock() f.bindCalls++ f.launchOrder = append(f.launchOrder, "bind") f.boundHandles = append(f.boundHandles, started) if proof, ok := request.Confinement.(*fakeConfinement); ok { f.proofStartsAtBind = append(f.proofStartsAtBind, proof.startCount()) } if f.bindErr != nil { f.mu.Unlock() return nil, f.bindErr } if previous, ok := f.results[request.IdempotencyKey]; ok { f.mu.Unlock() return &fakeInvocation{ owner: f, request: request, existing: &previous, locators: slices.Clone(previous.Locators), cancelled: make(chan struct{}), started: started, }, nil } delay := f.delays[request.Work.Unit.ID] locators := slices.Clone(f.locators[request.Work.Unit.ID]) if len(locators) == 0 { locators = []LocatorRecord{{ Kind: LocatorProcess, Opaque: "fake-process:" + request.IdempotencyKey, Revision: "process-r1", ProjectID: request.Project.ProjectID, WorkspaceID: request.Project.WorkspaceID, WorkUnitID: request.Work.Unit.ID, AttemptID: request.Work.AttemptID, }} } var blockChan chan struct{} if f.blockAll != nil { blockChan = f.blockAll } else if ch, ok := f.blockUntil[request.IdempotencyKey]; ok { blockChan = ch } f.mu.Unlock() return &fakeInvocation{ owner: f, request: request, delay: delay, locators: locators, cancelled: make(chan struct{}), block: blockChan, started: started, }, nil } type fakeInvocation struct { owner *fakeInvoker request DispatchRequest delay time.Duration locators []LocatorRecord existing *Submission cancelled chan struct{} block chan struct{} cancel sync.Once finish sync.Once started StartedConfinement result Submission err error } func (i *fakeInvocation) Locators() []LocatorRecord { return slices.Clone(i.locators) } func (i *fakeInvocation) Wait(ctx context.Context) (Submission, error) { i.finish.Do(func() { if i.existing != nil { i.finishChild() i.result = *i.existing i.result.Metadata = maps.Clone(i.existing.Metadata) i.result.Locators = slices.Clone(i.existing.Locators) return } i.owner.startInvocation() if i.delay > 0 { timer := time.NewTimer(i.delay) defer timer.Stop() select { case <-ctx.Done(): i.abortChild() i.err = ctx.Err() i.owner.finishInvocation(i.request, Submission{}, false) return case <-i.cancelled: i.abortChild() i.err = context.Canceled i.owner.finishInvocation(i.request, Submission{}, false) return case <-timer.C: } } if i.block != nil { select { case <-ctx.Done(): i.abortChild() i.err = ctx.Err() i.owner.finishInvocation(i.request, Submission{}, false) return case <-i.cancelled: i.abortChild() i.err = context.Canceled i.owner.finishInvocation(i.request, Submission{}, false) return case <-i.block: } } i.finishChild() i.result = Submission{ ProjectID: i.request.Project.ProjectID, WorkUnitID: i.request.Work.Unit.ID, AttemptID: i.request.Work.AttemptID, ArtifactID: ArtifactID("artifact-" + string(i.request.Work.AttemptID)), Ready: true, Locators: slices.Clone(i.locators), } i.owner.finishInvocation(i.request, i.result, true) }) return i.result, i.err } func (i *fakeInvocation) Cancel(context.Context) error { i.cancel.Do(func() { close(i.cancelled) }) i.finish.Do(func() { if i.existing == nil { i.owner.startInvocation() i.abortChild() i.err = context.Canceled i.owner.finishInvocation(i.request, Submission{}, false) } }) return nil } func (i *fakeInvocation) finishChild() { if i.started == nil { return } if stdin := i.started.Stdin(); stdin != nil { _ = stdin.Close() } child := i.started.Child() if child != nil && child.ProcessState == nil { _ = child.Wait() } for _, endpoint := range []io.Closer{i.started.Stdout(), i.started.Stderr()} { if endpoint != nil { _ = endpoint.Close() } } } func (i *fakeInvocation) abortChild() { if i.started != nil { _ = i.started.Abort() } } func (f *fakeInvoker) finishInvocation( request DispatchRequest, result Submission, success bool, ) { f.mu.Lock() defer f.mu.Unlock() f.active-- if !success { f.cancelled++ } if success { f.results[request.IdempotencyKey] = result f.actualCalls = append(f.actualCalls, request) } } func (f *fakeInvoker) startInvocation() { f.mu.Lock() defer f.mu.Unlock() f.active++ if f.active > f.maxActive { f.maxActive = f.active } } func (f *fakeInvoker) callCount() int { f.mu.Lock() defer f.mu.Unlock() return len(f.actualCalls) } func (f *fakeInvoker) launchStats() (int, int, []int) { f.mu.Lock() defer f.mu.Unlock() return f.prepareCalls, f.bindCalls, slices.Clone(f.proofStartsAtBind) } func (f *fakeInvoker) startedHandles() []StartedConfinement { f.mu.Lock() defer f.mu.Unlock() return slices.Clone(f.boundHandles) } func (f *fakeInvoker) maxConcurrency() int { f.mu.Lock() defer f.mu.Unlock() return f.maxActive } func (f *fakeInvoker) activeCount() int { f.mu.Lock() defer f.mu.Unlock() return f.active } func (f *fakeInvoker) roots() []string { f.mu.Lock() defer f.mu.Unlock() roots := make([]string, 0, len(f.actualCalls)) for _, request := range f.actualCalls { roots = append(roots, request.Workspace.TaskRoot) } return roots } type fakeReviewer struct { mu sync.Mutex sequences map[WorkUnitID][]ReviewVerdict results map[string]ReviewResult actualCalls []ReviewRequest blockUntil map[string]chan struct{} // keyed by IdempotencyKey blockAll chan struct{} // if non-nil, blocks all reviews } func newFakeReviewer() *fakeReviewer { return &fakeReviewer{ sequences: make(map[WorkUnitID][]ReviewVerdict), results: make(map[string]ReviewResult), blockUntil: make(map[string]chan struct{}), } } func (f *fakeReviewer) blockAfterReview(key string, release chan struct{}) { f.mu.Lock() defer f.mu.Unlock() f.blockUntil[key] = release } func (f *fakeReviewer) blockAllReviews(block chan struct{}) { f.mu.Lock() defer f.mu.Unlock() f.blockAll = block } func (f *fakeReviewer) Review( ctx context.Context, request ReviewRequest, ) (ReviewResult, error) { f.mu.Lock() if previous, ok := f.results[request.IdempotencyKey]; ok { f.mu.Unlock() return previous, nil } var blockChan chan struct{} if f.blockAll != nil { blockChan = f.blockAll } else if ch, ok := f.blockUntil[request.IdempotencyKey]; ok { blockChan = ch } f.mu.Unlock() if blockChan != nil { select { case <-blockChan: case <-ctx.Done(): return ReviewResult{}, ctx.Err() } } f.mu.Lock() sequence := f.sequences[request.Work.Unit.ID] index := int(request.Work.Attempt) - 1 verdict := ReviewVerdictPass if index >= 0 && index < len(sequence) { verdict = sequence[index] } result := ReviewResult{ ProjectID: request.Project.ProjectID, WorkUnitID: request.Work.Unit.ID, AttemptID: request.Work.AttemptID, ArtifactID: request.Submission.ArtifactID, Verdict: verdict, Message: string(verdict), } switch verdict { case ReviewVerdictPass: result.ChangeSet = &ChangeSetIdentity{ ID: ChangeSetID("change-" + string(request.Work.AttemptID)), Revision: "change-r1", ArtifactID: request.Submission.ArtifactID, } case ReviewVerdictWarn, ReviewVerdictFail: result.Rework = true } f.results[request.IdempotencyKey] = result f.actualCalls = append(f.actualCalls, request) f.mu.Unlock() return result, nil } func (f *fakeReviewer) callCount() int { f.mu.Lock() defer f.mu.Unlock() return len(f.actualCalls) } type fakeIntegrator struct { mu sync.Mutex outcomes map[WorkUnitID]IntegrationOutcome results map[string]IntegrationResult actualCalls []IntegrationRequest blockUntil map[string]chan struct{} // keyed by IdempotencyKey blockAll chan struct{} // if non-nil, blocks all integrations } func newFakeIntegrator() *fakeIntegrator { return &fakeIntegrator{ outcomes: make(map[WorkUnitID]IntegrationOutcome), results: make(map[string]IntegrationResult), blockUntil: make(map[string]chan struct{}), } } func (f *fakeIntegrator) blockAfterIntegrate(key string, release chan struct{}) { f.mu.Lock() defer f.mu.Unlock() f.blockUntil[key] = release } func (f *fakeIntegrator) blockAllIntegrations(block chan struct{}) { f.mu.Lock() defer f.mu.Unlock() f.blockAll = block } func (f *fakeIntegrator) Integrate( ctx context.Context, request IntegrationRequest, ) (IntegrationResult, error) { f.mu.Lock() if previous, ok := f.results[request.IdempotencyKey]; ok { f.mu.Unlock() return previous, nil } var blockChan chan struct{} if f.blockAll != nil { blockChan = f.blockAll } else if ch, ok := f.blockUntil[request.IdempotencyKey]; ok { blockChan = ch } f.mu.Unlock() if blockChan != nil { select { case <-blockChan: case <-ctx.Done(): return IntegrationResult{}, ctx.Err() } } f.mu.Lock() outcome := f.outcomes[request.Work.Unit.ID] if outcome == "" { outcome = IntegrationOutcomeIntegrated } result := IntegrationResult{ ProjectID: request.Project.ProjectID, WorkUnitID: request.Work.Unit.ID, ChangeSet: request.ChangeSet, Ordinal: request.Ordinal, Attempt: request.Attempt, Outcome: outcome, BeforeRevision: "before", } if outcome == IntegrationOutcomeIntegrated { result.AfterRevision = "after" } else { result.Retained = true result.Blocker = &Blocker{Code: BlockerIntegrationFailed, Message: "fixture blocker"} } f.results[request.IdempotencyKey] = result f.actualCalls = append(f.actualCalls, request) f.mu.Unlock() return result, nil } func (f *fakeIntegrator) ordinals() []DispatchOrdinal { f.mu.Lock() defer f.mu.Unlock() out := make([]DispatchOrdinal, 0, len(f.actualCalls)) for _, request := range f.actualCalls { out = append(out, request.Ordinal) } return out } func (f *fakeIntegrator) callCount() int { f.mu.Lock() defer f.mu.Unlock() return len(f.actualCalls) } type recordingEvents struct { mu sync.Mutex events []Event } func (r *recordingEvents) Emit(_ context.Context, event Event) error { r.mu.Lock() defer r.mu.Unlock() r.events = append(r.events, event) return nil } func (r *recordingEvents) snapshot() []Event { r.mu.Lock() defer r.mu.Unlock() return append([]Event(nil), r.events...) } type managerHarness struct { t *testing.T store *memoryStore workflow *fakeWorkflow isolation *fakeIsolation invoker *fakeInvoker recovery *fakeRecovery evidence *fakeWorkflowEvidence reviewer *fakeReviewer integrator *fakeIntegrator events *recordingEvents manager *Manager } func newHarness( t *testing.T, snapshots map[ProjectID]ProjectWorkflowSnapshot, capacity int, ) *managerHarness { t.Helper() store := newMemoryStore() workflow := &fakeWorkflow{snapshots: snapshots, errors: make(map[ProjectID]error)} isolation := newFakeIsolation(t) invoker := newFakeInvoker() recovery := newFakeRecovery() evidence := newFakeWorkflowEvidence() reviewer := newFakeReviewer() integrator := newFakeIntegrator() events := &recordingEvents{} manager, err := NewManager( ManagerConfig{ OwnerID: "manager-1", LeaseDuration: time.Minute, MaxReworkAttempts: 3, }, fixedClock{now: time.Date(2026, 7, 28, 0, 0, 0, 0, time.UTC)}, store, workflow, fakeSelector{capacity: capacity}, isolation, invoker, recovery, evidence, reviewer, integrator, events, ) if err != nil { t.Fatalf("NewManager: %v", err) } return &managerHarness{ t: t, store: store, workflow: workflow, isolation: isolation, invoker: invoker, recovery: recovery, evidence: evidence, reviewer: reviewer, integrator: integrator, events: events, manager: manager, } } func testSnapshot( projectID ProjectID, workspaceID WorkspaceID, units ...WorkUnit, ) ProjectWorkflowSnapshot { return ProjectWorkflowSnapshot{ ProjectID: projectID, WorkspaceID: workspaceID, Revision: "workflow-r1", Units: units, } } func testUnit(id WorkUnitID, kind WriteSetKind) WorkUnit { return WorkUnit{ ID: id, MilestoneID: "milestone", WriteSetKind: kind, IsolationMode: agentguard.IsolationModeOverlay, } } func (h *managerHarness) start( projectID ProjectID, workspaceID WorkspaceID, autoResume *bool, ) { h.t.Helper() err := h.manager.StartProject(context.Background(), StartRequest{ CommandID: CommandID("start-" + string(projectID)), ProjectID: projectID, WorkspaceID: workspaceID, MilestoneID: "milestone", WorkflowRevision: "workflow-r1", ConfigRevision: "config-r1", GrantRevision: "grant-r1", AutoResumeInterrupted: autoResume, }) if err != nil { h.t.Fatalf("StartProject(%s): %v", projectID, err) } } // setLeaseDuration overrides the manager's lease duration. The renewal // supervisor reads this value on each tick so the change takes effect on the // next renewal attempt without restarting the manager. func (h *managerHarness) setLeaseDuration(d time.Duration) { h.manager.config.LeaseDuration = d } func (h *managerHarness) manualRenewals() (chan time.Time, *advancingClock) { h.t.Helper() ticks := make(chan time.Time, 16) clock := &advancingClock{now: h.manager.clock.Now()} h.manager.clock = clock h.manager.renewalTicks = func(time.Duration) <-chan time.Time { return ticks } return ticks, clock } func waitFor(t *testing.T, description string, condition func() bool) { t.Helper() deadline := time.Now().Add(2 * time.Second) for !condition() { if time.Now().After(deadline) { t.Fatalf("timed out waiting for %s", description) } time.Sleep(time.Millisecond) } }