package bootstrap import ( "context" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" "io" "os" "os/exec" "path/filepath" "reflect" "strings" "sync" "testing" "time" "iop/apps/agent/internal/clientprocess" "iop/apps/agent/internal/host" "iop/apps/agent/internal/localcontrol" "iop/apps/agent/internal/projectlog" "iop/apps/agent/internal/taskloop" "iop/packages/go/agentconfig" "iop/packages/go/agenttask" "iop/packages/go/agentworkspace" iop "iop/proto/gen/iop" ) type testComponent struct { name string trace *traceLog start error stop error once sync.Once } type traceLog struct { mu sync.Mutex events []string } func (l *traceLog) add(event string) { l.mu.Lock() defer l.mu.Unlock() l.events = append(l.events, event) } func (l *traceLog) snapshot() []string { l.mu.Lock() defer l.mu.Unlock() return append([]string(nil), l.events...) } func (c *testComponent) Name() string { return c.name } func (c *testComponent) Start(ctx context.Context) error { c.trace.add("start:" + c.name) return c.start } func (c *testComponent) Stop(ctx context.Context) error { c.trace.add("stop:" + c.name) return c.stop } // nilComponent is a host.Component that is nil, used to test nil rejection. var nilComponent host.Component = nil // typedNilComponent is a *testComponent pointer that is nil but held in a // host.Component interface. It is not equal to nil as an interface value, // so the bootstrap layer must detect it via reflection. var typedNilComponent host.Component = (*testComponent)(nil) func TestNewModuleRejectsInvalidDependencies(t *testing.T) { tests := []struct { name string give []host.Component err string }{ { name: "empty component list", give: nil, err: "at least one component is required", }, { name: "nil component", give: []host.Component{nilComponent, &testComponent{name: "a"}}, err: "component at index 0 is nil", }, { name: "typed-nil component", give: []host.Component{typedNilComponent, &testComponent{name: "a"}}, err: "component at index 0 is nil", }, { name: "duplicate namer names", give: []host.Component{ &testComponent{name: "same"}, &testComponent{name: "same"}, }, err: "duplicate component name", }, { name: "unique names succeed", give: []host.Component{ &testComponent{name: "alpha"}, &testComponent{name: "beta"}, }, err: "", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { _, err := NewModule(tt.give...) if tt.err == "" { if err != nil { t.Fatalf("NewModule() unexpected error: %v", err) } return } if err == nil { t.Fatalf("NewModule() expected error containing %q, got nil", tt.err) } if got := err.Error(); !contains(got, tt.err) { t.Fatalf("NewModule() error = %q, want substring %q", got, tt.err) } }) } } func TestModuleRunDelegatesLifecycle(t *testing.T) { trace := &traceLog{} components := []*testComponent{ {name: "first", trace: trace}, {name: "second", trace: trace}, {name: "third", trace: trace}, } comps := make([]host.Component, len(components)) for i, c := range components { comps[i] = c } m, err := NewModule(comps...) if err != nil { t.Fatalf("NewModule() error = %v", err) } if err := m.Run(context.Background()); err != nil { t.Fatalf("Run() error = %v", err) } // Verify components started in declared order. if got, want := trace.snapshot(), []string{ "start:first", "start:second", "start:third", }; !reflect.DeepEqual(got, want) { t.Fatalf("trace = %v, want %v", got, want) } // Verify status reflects running state. status := m.Status() if !reflect.DeepEqual(status.Started, []string{"first", "second", "third"}) { t.Fatalf("Status().Started = %v, want [first second third]", status.Started) } // Close should stop in reverse order. if err := m.Close(context.Background()); err != nil { t.Fatalf("Close() error = %v", err) } if got, want := trace.snapshot(), []string{ "start:first", "start:second", "start:third", "stop:third", "stop:second", "stop:first", }; !reflect.DeepEqual(got, want) { t.Fatalf("trace = %v, want %v", got, want) } // Idempotent Close. if err := m.Close(context.Background()); err != nil { t.Fatalf("second Close() error = %v, want nil", err) } } func TestModuleStartupFailureRollsBack(t *testing.T) { trace := &traceLog{} launchErr := errors.New("launch failed") rollbackErr := errors.New("rollback failed") first := &testComponent{name: "first", trace: trace, stop: rollbackErr} second := &testComponent{name: "second", trace: trace, start: launchErr} comps := []host.Component{first, second} m, err := NewModule(comps...) if err != nil { t.Fatalf("NewModule() error = %v", err) } err = m.Run(context.Background()) if !errors.Is(err, launchErr) { t.Fatalf("Run() error = %v, want launch error identity", err) } if !errors.Is(err, rollbackErr) { t.Fatalf("Run() error = %v, want rollback error identity", err) } // First component should have been rolled back. if got, want := trace.snapshot(), []string{"start:first", "start:second", "stop:first"}; !reflect.DeepEqual(got, want) { t.Fatalf("trace = %v, want %v", got, want) } // Status should record failure details. status := m.Status() if status.Failed != "second" { t.Fatalf("Status().Failed = %q, want second", status.Failed) } if !errors.Is(status.LaunchErr, launchErr) { t.Fatalf("Status().LaunchErr = %v, want launch error identity", status.LaunchErr) } if !errors.Is(status.StopErr, rollbackErr) { t.Fatalf("Status().StopErr = %v, want rollback error identity", status.StopErr) } if !status.Stopped { t.Fatal("Status().Stopped = false after completed rollback") } // Second Run should be rejected since the host is already stopped. err = m.Run(context.Background()) if err == nil { t.Fatalf("second Run() returned nil, want error") } // Close after failed startup should be safe (idempotent). if err := m.Close(context.Background()); err != nil { t.Fatalf("Close() after failed Run() error = %v, want nil", err) } } func TestNewModuleDuplicateNameMatchesSentinel(t *testing.T) { first := &testComponent{name: "dup"} second := &testComponent{name: "dup"} _, err := NewModule(first, second) if err == nil { t.Fatalf("NewModule() returned nil, want duplicate error") } if !errors.Is(err, ErrDuplicateName) { t.Fatalf("NewModule() error = %v, want errors.Is(err, ErrDuplicateName) to be true", err) } } // nonNamerComponent does not implement host.Namer, so bootstrap must fall // back to the synthetic name. Its Start/Stop still work through the // resolvedComponent wrapper. type nonNamerComponent struct { trace *traceLog start error stop error } func (c *nonNamerComponent) Start(ctx context.Context) error { c.trace.add("start:nonNamer") return c.start } func (c *nonNamerComponent) Stop(ctx context.Context) error { c.trace.add("stop:nonNamer") return c.stop } // duplicateFallbackNamer implements host.Namer but returns a name that // would collide with the synthetic fallback used by bootstrap for // non-Namer components at the same index. type duplicateFallbackNamer struct { trace *traceLog } func (c *duplicateFallbackNamer) Name() string { return "component-x" } func (c *duplicateFallbackNamer) Start(ctx context.Context) error { c.trace.add("start:duplicateFallbackNamer") return nil } func (c *duplicateFallbackNamer) Stop(ctx context.Context) error { c.trace.add("stop:duplicateFallbackNamer") return nil } func TestModulePreservesResolvedComponentNames(t *testing.T) { trace := &traceLog{} // nonNamerComp does not implement host.Namer, so bootstrap assigns // "component-x" via its fallback. The host must see the same name. nonNamerComp := &nonNamerComponent{trace: trace} // dupFallbackComp explicitly names itself "component-x", which is the // same fallback bootstrap assigns to nonNamerComp at index 0. This // must be rejected at construction as a duplicate. dupFallbackComp := &duplicateFallbackNamer{trace: trace} _, err := NewModule(nonNamerComp, dupFallbackComp) if err == nil { t.Fatalf("NewModule() returned nil, want duplicate error") } if !errors.Is(err, ErrDuplicateName) { t.Fatalf("NewModule() error = %v, want errors.Is(err, ErrDuplicateName)", err) } // Now verify a nonNamer component at index 0 coexists with a Namer // component at index 1 that resolves to a distinct name, and that // Status().Started reflects the preserved names. trace2 := &traceLog{} first := &nonNamerComponent{trace: trace2} second := &testComponent{name: "beta", trace: trace2} m, err := NewModule(first, second) if err != nil { t.Fatalf("NewModule() error = %v", err) } if err := m.Run(context.Background()); err != nil { t.Fatalf("Run() error = %v", err) } status := m.Status() if len(status.Started) != 2 { t.Fatalf("Status().Started = %v, want 2 entries", status.Started) } if status.Started[0] != "component-x" { t.Fatalf("Status().Started[0] = %q, want %q (bootstrap fallback preserved)", status.Started[0], "component-x") } if status.Started[1] != "beta" { t.Fatalf("Status().Started[1] = %q, want %q", status.Started[1], "beta") } if err := m.Close(context.Background()); err != nil { t.Fatalf("Close() error = %v", err) } expected := []string{ "start:nonNamer", "start:beta", "stop:beta", "stop:nonNamer", } if got := trace2.snapshot(); !reflect.DeepEqual(got, expected) { t.Fatalf("trace = %v, want %v", got, expected) } } func contains(s, substr string) bool { for i := 0; i+len(substr) <= len(s); i++ { if s[i:i+len(substr)] == substr { return true } } return false } type changingNamerComponent struct { mu sync.Mutex callCount int trace *traceLog } func (c *changingNamerComponent) Name() string { c.mu.Lock() defer c.mu.Unlock() c.callCount++ if c.callCount == 1 { return "first-name" } return "" } func (c *changingNamerComponent) Start(ctx context.Context) error { c.trace.add("start:changingNamer") return nil } func (c *changingNamerComponent) Stop(ctx context.Context) error { c.trace.add("stop:changingNamer") return nil } func TestModuleResolvesComponentNameExactlyOnce(t *testing.T) { trace := &traceLog{} comp := &changingNamerComponent{trace: trace} m, err := NewModule(comp) if err != nil { t.Fatalf("NewModule() error = %v", err) } comp.mu.Lock() calls := comp.callCount comp.mu.Unlock() if calls != 1 { t.Fatalf("Name() call count = %d, want 1", calls) } if err := m.Run(context.Background()); err != nil { t.Fatalf("Run() error = %v", err) } status := m.Status() if !reflect.DeepEqual(status.Started, []string{"first-name"}) { t.Fatalf("Status().Started = %v, want [first-name]", status.Started) } if err := m.Close(context.Background()); err != nil { t.Fatalf("Close() error = %v", err) } } func TestNewDaemonModuleCompositionAndLifecycle(t *testing.T) { stateRoot := t.TempDir() _ = os.Chmod(stateRoot, 0700) overlayRoot := filepath.Join(stateRoot, "overlays") logRoot := filepath.Join(stateRoot, "logs") globalConfig := `version: "1" defaults: default_profile: p1 isolation: default_mode: overlay retention: completed_days: 14 ` localConfig := fmt.Sprintf(`version: "1" device: state_root: %s overlay_root: %s log_root: %s projects: proj-1: workspace: %s enabled: true selected_milestone: m1 clients: flutter: executable: /bin/echo working_directory: /tmp launch_on_start: false `, stateRoot, overlayRoot, logRoot, filepath.Join(stateRoot, "ws1")) snapshot, err := agentconfig.LoadRuntimeConfigBytes([]byte(globalConfig), []byte(localConfig)) if err != nil { t.Fatalf("LoadRuntimeConfigBytes error = %v", err) } backend := &fakeProcessBackend{} daemon, err := NewDaemonModule( context.Background(), snapshot, WithProcessBackend(backend), WithTaskCatalog(bootstrapTestCatalog(snapshot)), ) if err != nil { t.Fatalf("NewDaemonModule() error = %v", err) } if daemon.StateStore == nil { t.Error("expected StateStore to be initialized") } if daemon.ClientManager == nil { t.Error("expected ClientManager to be initialized") } if daemon.TaskRuntime == nil { t.Error("expected TaskRuntime to be initialized") } if len(daemon.ProjectStores) != 1 || daemon.ProjectStores["proj-1"] == nil { t.Errorf("expected ProjectStore for proj-1, got %v", daemon.ProjectStores) } if daemon.ControlServer == nil { t.Error("expected ControlServer to be initialized") } if daemon.Ledger == nil { t.Error("expected Ledger to be initialized") } ctx := context.Background() if err := daemon.Run(ctx); err != nil { t.Fatalf("Run() error = %v", err) } status := daemon.Status() expectedStarted := []string{"project-log", "task-runtime", "client-process", "local-control"} if !reflect.DeepEqual(status.Started, expectedStarted) { t.Fatalf("Status().Started = %v, want %v", status.Started, expectedStarted) } socketPath := daemon.ControlServer.Path() if _, err := os.Lstat(socketPath); err != nil { t.Errorf("expected Unix socket file at %s, got %v", socketPath, err) } if err := daemon.Close(ctx); err != nil { t.Fatalf("Close() error = %v", err) } if _, err := os.Lstat(socketPath); !os.IsNotExist(err) { t.Errorf("expected Unix socket file to be removed after Close(), got err = %v", err) } } func TestDaemonClientExitIsolation(t *testing.T) { stateRoot := t.TempDir() _ = os.Chmod(stateRoot, 0700) overlayRoot := filepath.Join(stateRoot, "overlays") logRoot := filepath.Join(stateRoot, "logs") globalConfig := `version: "1" defaults: default_profile: p1 isolation: default_mode: overlay retention: completed_days: 14 ` localConfig := fmt.Sprintf(`version: "1" device: state_root: %s overlay_root: %s log_root: %s clients: flutter: executable: /bin/echo working_directory: /tmp launch_on_start: false `, stateRoot, overlayRoot, logRoot) snapshot, err := agentconfig.LoadRuntimeConfigBytes([]byte(globalConfig), []byte(localConfig)) if err != nil { t.Fatalf("LoadRuntimeConfigBytes error = %v", err) } backend := &fakeProcessBackend{} daemon, err := NewDaemonModule( context.Background(), snapshot, WithProcessBackend(backend), WithTaskCatalog(bootstrapTestCatalog(snapshot)), ) if err != nil { t.Fatalf("NewDaemonModule() error = %v", err) } ctx := context.Background() if err := daemon.Run(ctx); err != nil { t.Fatalf("Run() error = %v", err) } defer daemon.Close(ctx) res, err := daemon.ClientManager.Start(ctx, clientprocess.ClientFlutter, "cmd-1") if err != nil { t.Fatalf("Start client error = %v", err) } if res.Record.State != clientprocess.StateStarting && res.Record.State != clientprocess.StateConnected { t.Fatalf("unexpected client state: %v", res.Record.State) } proc := backend.lastProcess if proc != nil { _ = proc.Abort() } status := daemon.Status() if status.Stopped { t.Fatal("host stopped unexpectedly after client process exited") } if len(status.Started) != 4 { t.Fatalf("host components lost after client exit: %v", status.Started) } } type fakeProcessBackend struct { mu sync.Mutex lastProcess *fakeOwnedProcess } func (b *fakeProcessBackend) Start(ctx context.Context, kind clientprocess.ClientKind, spec agentconfig.ClientProcessSpec) (clientprocess.OwnedProcess, error) { b.mu.Lock() defer b.mu.Unlock() p := &fakeOwnedProcess{ identity: clientprocess.ProcessIdentity{PID: 1234, StartToken: "token-1234"}, done: make(chan struct{}), } b.lastProcess = p return p, nil } func (b *fakeProcessBackend) Inspect(ctx context.Context, identity clientprocess.ProcessIdentity) (clientprocess.IdentityObservation, error) { return clientprocess.IdentityObservation{State: clientprocess.IdentityLive}, nil } func (b *fakeProcessBackend) Signal(ctx context.Context, identity clientprocess.ProcessIdentity, sig os.Signal) error { return nil } func (b *fakeProcessBackend) Kill(ctx context.Context, identity clientprocess.ProcessIdentity) error { return nil } func (b *fakeProcessBackend) Focus(ctx context.Context, kind clientprocess.ClientKind, spec agentconfig.ClientProcessSpec) error { return nil } type fakeOwnedProcess struct { identity clientprocess.ProcessIdentity done chan struct{} once sync.Once } func (p *fakeOwnedProcess) Identity() clientprocess.ProcessIdentity { return p.identity } func (p *fakeOwnedProcess) Wait() error { <-p.done return nil } func (p *fakeOwnedProcess) Abort() error { p.once.Do(func() { close(p.done) }) return nil } func bootstrapTestCatalog(snapshot agentconfig.RuntimeSnapshot) agentconfig.Catalog { cfg := snapshot.Config() profileID := cfg.Defaults.DefaultProfile if profileID == "" { profileID = "p1" } providerID := cfg.Selection.Default.Provider if providerID == "" { providerID = "test-provider" } modelID := cfg.Selection.Default.Model if modelID == "" { modelID = "test-model" } capabilities := []string{ "approval_bypass", "run", "unattended", "writable_root_confinement", } return agentconfig.Catalog{ Version: agentconfig.SchemaVersion, Providers: []agentconfig.Provider{{ ID: providerID, Command: "provider-must-not-run-in-tests", VersionProbe: agentconfig.CommandProbe{Args: []string{"--version"}}, Authentication: agentconfig.AuthenticationProbe{Args: []string{"auth", "status"}}, Capabilities: capabilities, }}, Models: []agentconfig.Model{{ ID: modelID, Provider: providerID, Target: modelID, }}, Profiles: []agentconfig.Profile{{ ID: profileID, Provider: providerID, Model: modelID, Args: []string{"--model", "{{model}}"}, Capabilities: capabilities, }}, } } func TestDaemonTaskRuntimeIsAuthoritativeForLocalControl(t *testing.T) { stateRoot := t.TempDir() _ = os.Chmod(stateRoot, 0700) overlayRoot := filepath.Join(stateRoot, "overlays") logRoot := filepath.Join(stateRoot, "logs") workspace := filepath.Join(stateRoot, "ws1") taskRoot := filepath.Join(workspace, "agent-task", "m-m1", "1_fixture") if err := os.MkdirAll(taskRoot, 0700); err != nil { t.Fatalf("create task fixture: %v", err) } if err := os.WriteFile(filepath.Join(taskRoot, "PLAN-test.md"), []byte(`# Plan ## Modified Files Summary | File | Purpose | |------|---------| | `+"`README.md`"+` | Exercise local control. | `), 0600); err != nil { t.Fatalf("write PLAN fixture: %v", err) } if err := os.WriteFile( filepath.Join(taskRoot, "CODE_REVIEW-test.md"), []byte("# Code Review Reference\n"), 0600, ); err != nil { t.Fatalf("write review fixture: %v", err) } globalConfig := `version: "1" defaults: default_profile: p1 isolation: default_mode: overlay retention: completed_days: 14 ` localConfig := fmt.Sprintf(`version: "1" device: state_root: %s overlay_root: %s log_root: %s projects: proj-1: workspace: %s enabled: true selected_milestone: m1 clients: flutter: executable: /bin/echo working_directory: /tmp launch_on_start: false `, stateRoot, overlayRoot, logRoot, workspace) snapshot, err := agentconfig.LoadRuntimeConfigBytes([]byte(globalConfig), []byte(localConfig)) if err != nil { t.Fatalf("LoadRuntimeConfigBytes error = %v", err) } backend := &fakeProcessBackend{} daemon, err := NewDaemonModule( context.Background(), snapshot, WithProcessBackend(backend), WithTaskCatalog(bootstrapTestCatalog(snapshot)), ) if err != nil { t.Fatalf("NewDaemonModule() error = %v", err) } ctx := context.Background() bridge := &daemonStateBridge{ snapshot: snapshot, manager: daemon.ClientManager, task: daemon.TaskRuntime, } before, err := bridge.ProjectStatus(ctx, "proj-1") if err != nil || before.State != "observed" { t.Fatalf("ProjectStatus before start = %#v, err = %v", before, err) } blockers, err := bridge.BlockerList(ctx, "proj-1") if err != nil || blockers.Summary != "0 blocker(s)" { t.Fatalf("BlockerList = %#v, err = %v", blockers, err) } workflow, err := daemon.TaskRuntime.WorkflowSnapshot(ctx, "proj-1") if err != nil { t.Fatalf("WorkflowSnapshot error = %v", err) } svc, err := localcontrol.NewService(bridge, bridge, daemon.Ledger) if err != nil { t.Fatalf("NewService error = %v", err) } statusRequest := &iop.AgentLocalEnvelope{ ProtocolVersion: localcontrol.ProtocolVersion, Kind: iop.AgentLocalKind_AGENT_LOCAL_KIND_REQUEST, MessageId: "msg-status-1", Operation: localcontrol.OperationProjectStatus, Payload: &iop.AgentLocalEnvelope_Request{ Request: &iop.AgentLocalRequest{ Payload: &iop.AgentLocalRequest_Read{ Read: &iop.AgentLocalReadRequest{ ProjectId: "proj-1", }, }, }, }, } startRequest := &iop.AgentLocalEnvelope{ ProtocolVersion: localcontrol.ProtocolVersion, Kind: iop.AgentLocalKind_AGENT_LOCAL_KIND_REQUEST, MessageId: "msg-start-1", Operation: localcontrol.OperationProjectStart, Payload: &iop.AgentLocalEnvelope_Request{ Request: &iop.AgentLocalRequest{ CommandId: "cmd-start-1", Payload: &iop.AgentLocalRequest_Project{ Project: &iop.AgentLocalProjectRequest{ ProjectId: "proj-1", WorkspaceId: string(workflow.WorkspaceID), MilestoneId: "m1", }, }, }, }, } statusResponse := svc.Handle(ctx, true, statusRequest) if statusResponse.GetKind() != iop.AgentLocalKind_AGENT_LOCAL_KIND_RESPONSE || statusResponse.GetResponse().GetSnapshot().GetState() != "observed" { t.Fatalf("status response = %#v", statusResponse) } startResponse := svc.Handle(ctx, true, startRequest) if startResponse.GetKind() != iop.AgentLocalKind_AGENT_LOCAL_KIND_RESPONSE || startResponse.GetResponse().GetMutation().GetState() != "started" { t.Fatalf("start response = %#v", startResponse) } after, err := bridge.ProjectStatus(ctx, "proj-1") if err != nil || after.State != "started" { t.Fatalf("ProjectStatus after start = %#v, err = %v", after, err) } cmd := localcontrol.ProjectCommand{ ProjectID: "proj-1", WorkspaceID: string(workflow.WorkspaceID), MilestoneID: "m1", } stopped, err := bridge.StopProject(ctx, cmd) if err != nil || stopped.State != "stopped" { t.Fatalf("StopProject = %#v, err = %v", stopped, err) } resumed, err := bridge.ResumeProject(ctx, cmd) if err != nil || resumed.State != "started" { t.Fatalf("ResumeProject = %#v, err = %v", resumed, err) } events, _, errReplay := daemon.Ledger.Replay(ctx, "iop-agent-daemon", 0) if errReplay != nil { t.Fatalf("Ledger.Replay error = %v", errReplay) } if len(events) != 1 { t.Fatalf("expected 1 retained local-control mutation event, got %d", len(events)) } } func TestDaemonFakeProviderPersistedLifecycleRollbackAndRestart(t *testing.T) { fixture := createDaemonLifecycleFixture(t) provider := &daemonLifecycleProvider{} reviewer := &daemonLifecycleReviewer{} validator := &daemonLifecycleValidator{} ctx := context.Background() daemon := newLifecycleDaemon(t, ctx, fixture, provider, reviewer, validator) if err := daemon.Run(ctx); err != nil { t.Fatalf("Run: %v", err) } bridge := &daemonStateBridge{ snapshot: fixture.snapshot, manager: daemon.ClientManager, task: daemon.TaskRuntime, } workflow, err := daemon.TaskRuntime.WorkflowSnapshot(ctx, "project") if err != nil { t.Fatalf("WorkflowSnapshot: %v", err) } started, err := bridge.StartProject(ctx, localcontrol.ProjectCommand{ ProjectID: "project", WorkspaceID: string(workflow.WorkspaceID), MilestoneID: "m1", }) if err != nil || (started.State != "started" && started.State != "running") { t.Fatalf("StartProject = %#v, err = %v", started, err) } waitDaemonLifecycle(t, daemon.TaskRuntime) if err := daemon.TaskRuntime.Reconcile(ctx); err != nil { t.Fatalf("terminal Reconcile: %v", err) } assertDaemonLifecycle(t, fixture, daemon, bridge, provider, reviewer, validator) if err := daemon.Close(ctx); err != nil { t.Fatalf("Close: %v", err) } restarted := newLifecycleDaemon(t, ctx, fixture, provider, reviewer, validator) if err := restarted.Run(ctx); err != nil { t.Fatalf("Run(restart): %v", err) } waitDaemonLifecycle(t, restarted.TaskRuntime) if err := restarted.TaskRuntime.Reconcile(ctx); err != nil { t.Fatalf("restart terminal Reconcile: %v", err) } restartedBridge := &daemonStateBridge{ snapshot: fixture.snapshot, manager: restarted.ClientManager, task: restarted.TaskRuntime, } blockers, err := restartedBridge.BlockerList(ctx, "project") if err != nil || len(blockers.Entries) != 1 { t.Fatalf("restart blockers = %#v, err = %v", blockers, err) } if provider.Count() != 2 || reviewer.Count() != 2 { t.Fatalf( "restart duplicated work: dispatches=%d reviews=%d", provider.Count(), reviewer.Count(), ) } if err := restarted.Close(ctx); err != nil { t.Fatalf("Close(restart): %v", err) } } type daemonLifecycleFixture struct { snapshot agentconfig.RuntimeSnapshot catalog agentconfig.Catalog workspace string logRoot string } func createDaemonLifecycleFixture(t *testing.T) daemonLifecycleFixture { t.Helper() root, err := filepath.EvalSymlinks(t.TempDir()) if err != nil { t.Fatal(err) } workspace := filepath.Join(root, "workspace") stateRoot := filepath.Join(root, "state") logRoot := filepath.Join(root, "logs") for _, directory := range []string{workspace, stateRoot, logRoot} { if err := os.MkdirAll(directory, 0o700); err != nil { t.Fatal(err) } } writeDaemonLifecycleFile(t, filepath.Join(workspace, "rejected.txt"), "base rejected\n") writeDaemonLifecycleFile(t, filepath.Join(workspace, "sibling.txt"), "base sibling\n") createDaemonLifecycleTask(t, workspace, "1_rejected", "rejected.txt") createDaemonLifecycleTask(t, workspace, "2_sibling", "sibling.txt") runDaemonLifecycleGit(t, workspace, "init", "-q") runDaemonLifecycleGit(t, workspace, "config", "user.email", "daemon@example.invalid") runDaemonLifecycleGit(t, workspace, "config", "user.name", "Daemon Fixture") runDaemonLifecycleGit(t, workspace, "add", "-A") runDaemonLifecycleGit(t, workspace, "commit", "-q", "-m", "base") global := `version: "1" defaults: default_profile: p1 selection: timezone: UTC default: provider: test-provider model: test-model profile: p1 isolation: default_mode: overlay retention: completed_days: 14 blocked_days: 14 ` local := fmt.Sprintf(`version: "1" device: state_root: %s overlay_root: %s log_root: %s projects: project: workspace: %s enabled: true selected_milestone: m1 `, stateRoot, filepath.Join(root, "overlays"), logRoot, workspace) snapshot, err := agentconfig.LoadRuntimeConfigBytes([]byte(global), []byte(local)) if err != nil { t.Fatalf("LoadRuntimeConfigBytes: %v", err) } return daemonLifecycleFixture{ snapshot: snapshot, catalog: bootstrapTestCatalog(snapshot), workspace: workspace, logRoot: logRoot, } } func createDaemonLifecycleTask( t *testing.T, workspace, directory, target string, ) { t.Helper() taskRoot := filepath.Join(workspace, "agent-task", "m-m1", directory) if err := os.MkdirAll(taskRoot, 0o700); err != nil { t.Fatal(err) } reviewRelative := filepath.ToSlash( filepath.Join("agent-task", "m-m1", directory, "CODE_REVIEW-test.md"), ) writeDaemonLifecycleFile(t, filepath.Join(taskRoot, "PLAN-test.md"), fmt.Sprintf(`# Plan ## Modified Files Summary | File | Purpose | |------|---------| | %s | Exercise provider output. | | %s | Record implementation and review. | `, "`"+target+"`", "`"+reviewRelative+"`")) writeDaemonLifecycleFile(t, filepath.Join(taskRoot, "CODE_REVIEW-test.md"), `# Code Review Reference ## Implementation Notes Deterministic daemon evidence is complete. `) } func writeDaemonLifecycleFile(t *testing.T, path, content string) { t.Helper() if err := os.WriteFile(path, []byte(content), 0o600); err != nil { t.Fatal(err) } } func runDaemonLifecycleGit(t *testing.T, root string, args ...string) { t.Helper() command := exec.Command("git", append([]string{"-C", root}, args...)...) if output, err := command.CombinedOutput(); err != nil { t.Fatalf("git %v: %v: %s", args, err, output) } } func newLifecycleDaemon( t *testing.T, ctx context.Context, fixture daemonLifecycleFixture, provider *daemonLifecycleProvider, reviewer *daemonLifecycleReviewer, validator *daemonLifecycleValidator, ) *DaemonModule { t.Helper() daemon, err := NewDaemonModule( ctx, fixture.snapshot, WithTaskCatalog(fixture.catalog), WithTaskRuntimePorts(provider, reviewer, validator.Validate, nil), ) if err != nil { t.Fatalf("NewDaemonModule: %v", err) } return daemon } func waitDaemonLifecycle(t *testing.T, runtime *taskloop.Runtime) { t.Helper() deadline := time.Now().Add(10 * time.Second) for time.Now().Before(deadline) { view, err := runtime.ProjectStatus(context.Background(), "project") if err == nil && len(view.Works) == 2 { terminal := true for _, work := range view.Works { terminal = terminal && work.State.Terminal() } if terminal { return } } time.Sleep(25 * time.Millisecond) } t.Fatal("daemon lifecycle did not reach terminal projections") } func assertDaemonLifecycle( t *testing.T, fixture daemonLifecycleFixture, daemon *DaemonModule, bridge *daemonStateBridge, provider *daemonLifecycleProvider, reviewer *daemonLifecycleReviewer, validator *daemonLifecycleValidator, ) { t.Helper() if provider.Count() != 2 || reviewer.Count() != 2 || validator.Count() != 2 { t.Fatalf( "lifecycle counts dispatch=%d review=%d validation=%d", provider.Count(), reviewer.Count(), validator.Count(), ) } state, _, err := daemon.StateStore.Load(context.Background()) if err != nil { t.Fatal(err) } project := state.Projects["project"] rejected := project.Works["1"] sibling := project.Works["2"] if rejected.State != agenttask.WorkStateTerminalDeferred || rejected.Review == nil || rejected.Review.Verdict != agenttask.ReviewVerdictPass || rejected.ChangeSet == nil || rejected.Blocker == nil || !strings.Contains(rejected.Blocker.Message, "validation") { t.Fatalf("rejected work = %#v", rejected) } if sibling.State != agenttask.WorkStateCompleted || sibling.Review == nil || sibling.Review.Verdict != agenttask.ReviewVerdictPass || sibling.ChangeSet == nil || !sibling.CompletionVerified { t.Fatalf("sibling work = %#v", sibling) } rejectedProjection, err := bridge.IntegrationStatus( context.Background(), "project", "1", ) if err != nil || rejectedProjection.State != string(agenttask.WorkStateTerminalDeferred) || rejectedProjection.Summary != string(agenttask.IntegrationOutcomeTerminalDeferred) { t.Fatalf("rejected projection = %#v, err = %v", rejectedProjection, err) } blockers, err := bridge.BlockerList(context.Background(), "project") if err != nil || len(blockers.Entries) != 1 { t.Fatalf("blocker projection = %#v, err = %v", blockers, err) } assertDaemonLifecycleFile(t, filepath.Join(fixture.workspace, "rejected.txt"), "base rejected\n") assertDaemonLifecycleFile(t, filepath.Join(fixture.workspace, "sibling.txt"), "integrated 2\n") assertDaemonTerminalArchives(t, fixture.logRoot) } func assertDaemonLifecycleFile(t *testing.T, path, want string) { t.Helper() content, err := os.ReadFile(path) if err != nil { t.Fatal(err) } if string(content) != want { t.Fatalf("%s = %q, want %q", path, content, want) } } func assertDaemonTerminalArchives(t *testing.T, root string) { t.Helper() manifests := 0 timelines := make(map[agenttask.WorkUnitID]bool) if err := filepath.Walk(root, func(path string, info os.FileInfo, walkErr error) error { if walkErr != nil { return walkErr } if info.IsDir() { return nil } switch { case strings.HasSuffix(path, ".manifest.json"): manifests++ case strings.HasSuffix(path, ".timeline.jsonl"): content, err := os.ReadFile(path) if err != nil { return err } var last projectlog.ProjectLogRecord var prior uint64 for index, line := range strings.Split(strings.TrimSpace(string(content)), "\n") { var record projectlog.ProjectLogRecord if err := json.Unmarshal([]byte(line), &record); err != nil { return err } if index > 0 && record.Sequence != prior+1 { return errors.New("daemon project log sequence is not monotonic") } prior = record.Sequence last = record } if !last.Terminal { return errors.New("daemon project log timeline is not terminal") } timelines[last.WorkUnitID] = true } return nil }); err != nil { t.Fatal(err) } if manifests != 2 || !timelines["1"] || !timelines["2"] { t.Fatalf("archive evidence manifests=%d timelines=%#v", manifests, timelines) } } type daemonLifecycleProvider struct { mu sync.Mutex requests []agenttask.DispatchRequest } func (provider *daemonLifecycleProvider) Prepare( _ context.Context, request agenttask.DispatchRequest, ) (agenttask.ProviderLaunch, error) { if request.Confinement == nil || request.Permit == nil { return nil, errors.New("daemon fake requires admitted confinement") } binding := request.Confinement.Binding() if err := request.Confinement.Validate(binding); err != nil { return nil, err } provider.mu.Lock() provider.requests = append(provider.requests, request) provider.mu.Unlock() return &daemonLifecycleLaunch{request: request, root: binding.WorkingDir}, nil } func (provider *daemonLifecycleProvider) Count() int { provider.mu.Lock() defer provider.mu.Unlock() return len(provider.requests) } type daemonLifecycleLaunch struct { request agenttask.DispatchRequest root string } func (launch *daemonLifecycleLaunch) Command() agenttask.ConfinementCommand { return agenttask.ConfinementCommand{Name: "true"} } func (launch *daemonLifecycleLaunch) BindStarted( started agenttask.StartedConfinement, ) (agenttask.ProviderInvocation, error) { if started == nil || started.Child() == nil || started.Child().Process == nil { return nil, errors.New("daemon fake child is incomplete") } opaque := fmt.Sprintf("daemon-fake-%d", started.Child().Process.Pid) return &daemonLifecycleInvocation{ request: launch.request, root: launch.root, started: started, locator: agenttask.LocatorRecord{ Kind: agenttask.LocatorProcess, Opaque: opaque, Revision: daemonLifecycleDigest("process", opaque), ProjectID: launch.request.Project.ProjectID, WorkspaceID: launch.request.Project.WorkspaceID, WorkUnitID: launch.request.Work.Unit.ID, AttemptID: launch.request.Work.AttemptID, }, }, nil } type daemonLifecycleInvocation struct { request agenttask.DispatchRequest root string started agenttask.StartedConfinement locator agenttask.LocatorRecord } func (invocation *daemonLifecycleInvocation) Locators() []agenttask.LocatorRecord { return []agenttask.LocatorRecord{invocation.locator} } func (invocation *daemonLifecycleInvocation) Wait( context.Context, ) (agenttask.Submission, error) { if err := daemonLifecycleWait(invocation.started); err != nil { return agenttask.Submission{}, err } target := "" for _, candidate := range invocation.request.Work.Unit.DeclaredWriteSet { if !strings.HasPrefix(candidate, "agent-task/") { target = candidate break } } if target == "" { return agenttask.Submission{}, errors.New("daemon fake target is missing") } if err := os.WriteFile( filepath.Join(invocation.root, filepath.FromSlash(target)), []byte("integrated "+string(invocation.request.Work.Unit.ID)+"\n"), 0o600, ); err != nil { return agenttask.Submission{}, err } return agenttask.Submission{ ProjectID: invocation.request.Project.ProjectID, WorkUnitID: invocation.request.Work.Unit.ID, AttemptID: invocation.request.Work.AttemptID, ArtifactID: agenttask.ArtifactID(daemonLifecycleDigest( "artifact", string(invocation.request.Work.Unit.ID), string(invocation.request.Work.AttemptID), )), Ready: true, Locators: invocation.Locators(), }, nil } func (invocation *daemonLifecycleInvocation) Cancel(context.Context) error { return invocation.started.Abort() } func daemonLifecycleWait(started agenttask.StartedConfinement) error { if stdin := started.Stdin(); stdin != nil { _ = stdin.Close() } if stdout := started.Stdout(); stdout != nil { _, _ = io.Copy(io.Discard, stdout) _ = stdout.Close() } if stderr := started.Stderr(); stderr != nil { _, _ = io.Copy(io.Discard, stderr) _ = stderr.Close() } return started.Child().Wait() } func daemonLifecycleDigest(parts ...string) string { hash := sha256.New() for _, part := range parts { _, _ = fmt.Fprintf(hash, "%d:", len(part)) _, _ = hash.Write([]byte(part)) } return "sha256:" + hex.EncodeToString(hash.Sum(nil)) } type daemonLifecycleReviewer struct { mu sync.Mutex count int } func (reviewer *daemonLifecycleReviewer) ExecuteReview( _ context.Context, _ agenttask.ReviewRequest, root, _, reviewRelative string, ) error { reviewer.mu.Lock() reviewer.count++ reviewer.mu.Unlock() file, err := os.OpenFile( filepath.Join(root, filepath.FromSlash(reviewRelative)), os.O_APPEND|os.O_WRONLY, 0, ) if err != nil { return err } defer file.Close() _, err = io.WriteString(file, "\n## Code Review Result\n\nOverall Verdict: PASS\n") return err } func (reviewer *daemonLifecycleReviewer) Count() int { reviewer.mu.Lock() defer reviewer.mu.Unlock() return reviewer.count } type daemonLifecycleValidator struct { mu sync.Mutex count int } func (validator *daemonLifecycleValidator) Validate( _ context.Context, request agentworkspace.ValidationRequest, ) error { validator.mu.Lock() validator.count++ validator.mu.Unlock() if request.ValidationRoot == request.CanonicalRoot { return errors.New("daemon validator did not receive an isolated candidate") } content, err := os.ReadFile(filepath.Join(request.ValidationRoot, "rejected.txt")) if err != nil { return err } if string(content) != "base rejected\n" { return errors.New("daemon fake validator rejected candidate") } return nil } func (validator *daemonLifecycleValidator) Count() int { validator.mu.Lock() defer validator.mu.Unlock() return validator.count } type partialFailBackend struct { mu sync.Mutex failKind clientprocess.ClientKind failErr error cleanupErr error processes map[clientprocess.ClientKind]*fakeOwnedProcess } func newPartialFailBackend(failKind clientprocess.ClientKind, failErr error, cleanupErr error) *partialFailBackend { return &partialFailBackend{ failKind: failKind, failErr: failErr, cleanupErr: cleanupErr, processes: make(map[clientprocess.ClientKind]*fakeOwnedProcess), } } func (b *partialFailBackend) Start(ctx context.Context, kind clientprocess.ClientKind, spec agentconfig.ClientProcessSpec) (clientprocess.OwnedProcess, error) { b.mu.Lock() defer b.mu.Unlock() if kind == b.failKind { return nil, b.failErr } p := &fakeOwnedProcess{ identity: clientprocess.ProcessIdentity{PID: 1234, StartToken: "token-1234"}, done: make(chan struct{}), } b.processes[kind] = p return p, nil } func (b *partialFailBackend) Inspect(ctx context.Context, identity clientprocess.ProcessIdentity) (clientprocess.IdentityObservation, error) { return clientprocess.IdentityObservation{State: clientprocess.IdentityLive}, nil } func (b *partialFailBackend) Signal(ctx context.Context, identity clientprocess.ProcessIdentity, sig os.Signal) error { b.mu.Lock() defer b.mu.Unlock() for _, p := range b.processes { if p.identity == identity { p.Abort() } } return b.cleanupErr } func (b *partialFailBackend) Kill(ctx context.Context, identity clientprocess.ProcessIdentity) error { b.mu.Lock() defer b.mu.Unlock() for _, p := range b.processes { if p.identity == identity { p.Abort() } } return b.cleanupErr } func (b *partialFailBackend) Focus(ctx context.Context, kind clientprocess.ClientKind, spec agentconfig.ClientProcessSpec) error { return nil } func TestDaemonPartialClientLaunchFailureRollsBack(t *testing.T) { stateRoot := t.TempDir() _ = os.Chmod(stateRoot, 0700) overlayRoot := filepath.Join(stateRoot, "overlays") logRoot := filepath.Join(stateRoot, "logs") globalConfig := `version: "1" defaults: default_profile: p1 isolation: default_mode: overlay retention: completed_days: 14 ` localConfig := fmt.Sprintf(`version: "1" device: state_root: %s overlay_root: %s log_root: %s clients: flutter: executable: /bin/echo working_directory: /tmp launch_on_start: true unity: executable: /bin/echo working_directory: /tmp launch_on_start: true `, stateRoot, overlayRoot, logRoot) snapshot, err := agentconfig.LoadRuntimeConfigBytes([]byte(globalConfig), []byte(localConfig)) if err != nil { t.Fatalf("LoadRuntimeConfigBytes error = %v", err) } launchErr := errors.New("unity process launch failed") cleanupErr := errors.New("flutter process cleanup failed") backend := newPartialFailBackend(clientprocess.ClientUnity, launchErr, cleanupErr) daemon, err := NewDaemonModule( context.Background(), snapshot, WithProcessBackend(backend), WithTaskCatalog(bootstrapTestCatalog(snapshot)), ) if err != nil { t.Fatalf("NewDaemonModule() error = %v", err) } ctx := context.Background() runErr := daemon.Run(ctx) if runErr == nil { t.Fatal("expected daemon.Run to return error on partial launch failure, got nil") } if !errors.Is(runErr, launchErr) { t.Fatalf("daemon.Run error = %v, want errors.Is launchErr", runErr) } if !errors.Is(runErr, cleanupErr) { t.Fatalf("daemon.Run error = %v, want errors.Is cleanupErr", runErr) } backend.mu.Lock() flutterProc := backend.processes[clientprocess.ClientFlutter] backend.mu.Unlock() if flutterProc == nil { t.Fatal("expected flutter process to have been launched initially") } select { case <-flutterProc.done: // Succeeded: flutter process was aborted/closed during rollback default: t.Fatal("expected flutter process to be aborted/reaped on partial startup rollback") } status := daemon.Status() if !status.Stopped { t.Fatal("expected host status to record stopped/failed state") } socketPath := daemon.ControlServer.Path() if _, err := os.Lstat(socketPath); !os.IsNotExist(err) { t.Errorf("expected local control socket to be removed after rollback, got %v", err) } }