package clientprocess import ( "bufio" "context" "errors" "fmt" "os" "os/signal" "path/filepath" "sort" "strings" "sync" "syscall" "testing" "time" "iop/packages/go/agentconfig" "iop/packages/go/agentstate" ) func TestHelperProcess(t *testing.T) { separator := -1 for index, argument := range os.Args { if argument == "--" { separator = index break } } if separator < 0 || len(os.Args) <= separator+2 { return } mode := os.Args[separator+1] tracePath := os.Args[separator+2] appendHelperTrace(t, tracePath, fmt.Sprintf("%s:%d", mode, os.Getpid())) if mode == "focus" { return } if mode == "crash-trigger" || mode == "crash-trigger-once" { if len(os.Args) <= separator+3 { t.Fatal("crash helper trigger path is missing") } triggerPath := os.Args[separator+3] for { if _, err := os.Stat(triggerPath); err == nil { break } else if !errors.Is(err, os.ErrNotExist) { t.Fatalf("inspect crash trigger: %v", err) } time.Sleep(5 * time.Millisecond) } if mode == "crash-trigger" { os.Exit(2) } if len(os.Args) <= separator+4 { t.Fatal("crash helper consumed path is missing") } consumedPath := os.Args[separator+4] file, err := os.OpenFile( consumedPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600, ) if err == nil { _ = file.Close() os.Exit(2) } if !errors.Is(err, os.ErrExist) { t.Fatalf("record consumed crash trigger: %v", err) } } signals := make(chan os.Signal, 1) signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT) defer signal.Stop(signals) <-signals } func TestManagerOwnsSingletonAndReapsClient(t *testing.T) { manager, daemonContext, tracePath := newFixtureManager( t, map[ClientKind]fixturePolicy{ ClientFlutter: {}, }, ) const callers = 24 results := make(chan Result, callers) errorsChannel := make(chan error, callers) var wait sync.WaitGroup for index := 0; index < callers; index++ { wait.Add(1) go func(index int) { defer wait.Done() result, err := manager.Start( context.Background(), ClientFlutter, fmt.Sprintf("start-%d", index), ) results <- result errorsChannel <- err }(index) } wait.Wait() close(results) close(errorsChannel) for err := range errorsChannel { if err != nil { t.Fatalf("Start: %v", err) } } var identity ProcessIdentity for result := range results { if result.Record.Identity == nil { t.Fatalf("start result has no identity: %+v", result) } if identity.PID == 0 { identity = *result.Record.Identity continue } if *result.Record.Identity != identity { t.Fatalf( "duplicate start identity = %+v, want %+v", *result.Record.Identity, identity, ) } } waitForTraceCount(t, tracePath[ClientFlutter], 1) if _, err := manager.SetConnected( context.Background(), ClientFlutter, true, ); err != nil { t.Fatalf("SetConnected: %v", err) } if _, err := manager.SetConnected( context.Background(), ClientFlutter, false, ); err != nil { t.Fatalf("SetDisconnected: %v", err) } if _, err := manager.SetConnected( context.Background(), ClientFlutter, true, ); err != nil { t.Fatalf("SetReconnected: %v", err) } stopped, err := manager.Stop( context.Background(), ClientFlutter, "stop-one", ) if err != nil { t.Fatalf("Stop: %v", err) } if stopped.Record.State != StateStopped || stopped.Record.Identity != nil || stopped.Record.LastIdentity == nil || *stopped.Record.LastIdentity != identity { t.Fatalf("stopped record = %+v", stopped.Record) } if _, err := manager.Stop( context.Background(), ClientFlutter, "stop-two", ); err != nil { t.Fatalf("idempotent Stop: %v", err) } if _, err := currentProcessIdentity(identity.PID); !errors.Is(err, os.ErrProcessDone) { t.Fatalf("child identity remains live: %v", err) } select { case <-daemonContext.Done(): t.Fatal("client lifecycle cancelled the daemon context") default: } } func TestManagerRejectsLaunchAfterClose(t *testing.T) { manager, _, traces := newFixtureManager( t, map[ClientKind]fixturePolicy{ ClientFlutter: {}, }, ) if err := manager.Close(context.Background()); err != nil { t.Fatalf("Close: %v", err) } if _, err := manager.Start( context.Background(), ClientFlutter, "closed-start", ); !errors.Is(err, ErrManagerClosed) { t.Fatalf("Start error = %v, want ErrManagerClosed", err) } if _, err := manager.StartOrFocusFlutter( context.Background(), "closed-detail", ); !errors.Is(err, ErrManagerClosed) { t.Fatalf("StartOrFocusFlutter error = %v, want ErrManagerClosed", err) } if lines := readTraceLines(t, traces[ClientFlutter]); len(lines) != 0 { t.Fatalf("closed manager launched clients: %v", lines) } } func TestDuplicateLaunchConvergesAfterManagerRestart(t *testing.T) { root := t.TempDir() statePath := filepath.Join(root, "state.json") tracePath := filepath.Join(root, "flutter.trace") spec := helperSpec(t, root, tracePath, filepath.Join(root, "focus.trace")) store, err := agentstate.NewStore(statePath) if err != nil { t.Fatalf("NewStore: %v", err) } firstContext, firstCancel := context.WithCancel(context.Background()) defer firstCancel() first, err := NewManager( firstContext, map[string]agentconfig.ClientProcessSpec{ string(ClientFlutter): spec, }, store, ) if err != nil { t.Fatalf("NewManager first: %v", err) } started, err := first.Start(context.Background(), ClientFlutter, "first") if err != nil { t.Fatalf("first Start: %v", err) } waitForTraceCount(t, tracePath, 1) secondContext, secondCancel := context.WithCancel(context.Background()) defer secondCancel() second, err := NewManager( secondContext, map[string]agentconfig.ClientProcessSpec{ string(ClientFlutter): spec, }, store, ) if err != nil { t.Fatalf("NewManager second: %v", err) } if err := second.Reconcile(context.Background()); err != nil { t.Fatalf("second Reconcile: %v", err) } replayed, err := second.Start( context.Background(), ClientFlutter, "second", ) if err != nil { t.Fatalf("second Start: %v", err) } if replayed.Record.Identity == nil || started.Record.Identity == nil || *replayed.Record.Identity != *started.Record.Identity { t.Fatalf( "restart identity changed: first=%+v second=%+v", started.Record.Identity, replayed.Record.Identity, ) } waitForTraceCount(t, tracePath, 1) if _, err := second.Stop( context.Background(), ClientFlutter, "cleanup", ); err != nil { t.Fatalf("second Stop: %v", err) } firstCancel() first.waiters.Wait() secondCancel() second.waiters.Wait() } func TestDaemonSurvivesCrashAndBoundedRestart(t *testing.T) { manager, daemonContext, tracePath := newFixtureManager( t, map[ClientKind]fixturePolicy{ ClientFlutter: { restart: true, limit: 1, backoff: 10, crashMode: "crash-trigger", }, }, ) _, err := manager.Start( context.Background(), ClientFlutter, "initial", ) if err != nil { t.Fatalf("Start: %v", err) } if _, err := manager.SetConnected( context.Background(), ClientFlutter, true, ); err != nil { t.Fatalf("SetConnected: %v", err) } if err := os.WriteFile( tracePath[ClientKind("flutter-crash")], []byte("crash"), 0o600, ); err != nil { t.Fatalf("write crash trigger: %v", err) } waitForTraceCount(t, tracePath[ClientFlutter], 2) waitForCondition(t, func() bool { record, statusErr := manager.Status(ClientFlutter) return statusErr == nil && record.State == StateCrashed && record.Identity == nil && record.RestartAttempts == 1 }) time.Sleep(50 * time.Millisecond) waitForTraceCount(t, tracePath[ClientFlutter], 2) select { case <-daemonContext.Done(): t.Fatal("client crash cancelled the daemon context") default: } } func TestS15ClientLifecycleTrace(t *testing.T) { manager, daemonContext, traces := newFixtureManager( t, map[ClientKind]fixturePolicy{ ClientFlutter: {}, ClientUnity: { restart: true, limit: 1, backoff: 10, crashMode: "crash-trigger-once", }, }, ) flutter, err := manager.Start( context.Background(), ClientFlutter, "flutter-start", ) if err != nil { t.Fatalf("start Flutter: %v", err) } unity, err := manager.Start( context.Background(), ClientUnity, "unity-start", ) if err != nil { t.Fatalf("start Unity: %v", err) } duplicate, err := manager.Start( context.Background(), ClientFlutter, "flutter-duplicate", ) if err != nil { t.Fatalf("duplicate Flutter: %v", err) } if *duplicate.Record.Identity != *flutter.Record.Identity { t.Fatalf("Flutter duplicate identity changed") } if _, err := manager.SetConnected( context.Background(), ClientFlutter, true, ); err != nil { t.Fatalf("connect Flutter: %v", err) } if _, err := manager.SetConnected( context.Background(), ClientFlutter, false, ); err != nil { t.Fatalf("disconnect Flutter: %v", err) } if _, err := manager.SetConnected( context.Background(), ClientFlutter, true, ); err != nil { t.Fatalf("reconnect Flutter: %v", err) } detail, err := manager.StartOrFocusFlutter( context.Background(), "unity-detail", ) if err != nil { t.Fatalf("Unity detail route: %v", err) } if detail.Action != "focus" || detail.Record.FocusCount != 1 || *detail.Record.Identity != *flutter.Record.Identity { t.Fatalf("detail result = %+v", detail) } waitForTraceCount(t, traces[ClientFlutter], 1) waitForTraceCount(t, traces[ClientUnity], 1) waitForTraceCount(t, traces[ClientKind("flutter-focus")], 1) if err := os.WriteFile( traces[ClientKind("unity-crash")], []byte("crash"), 0o600, ); err != nil { t.Fatalf("write Unity crash trigger: %v", err) } var restartedUnity Record waitForCondition(t, func() bool { record, statusErr := manager.Status(ClientUnity) if statusErr != nil || record.Identity == nil || record.Identity.PID == unity.Record.Identity.PID { return false } restartedUnity = record return true }) waitForTraceCount(t, traces[ClientUnity], 2) t.Logf( "S15 trace flutter=%d/%s duplicate=%d unity=%d/%s restart=%d/%s focus=%d daemon=live", flutter.Record.Identity.PID, flutter.Record.Identity.StartToken, duplicate.Record.Identity.PID, unity.Record.Identity.PID, unity.Record.Identity.StartToken, restartedUnity.Identity.PID, restartedUnity.Identity.StartToken, detail.Record.FocusCount, ) select { case <-daemonContext.Done(): t.Fatal("S15 client lifecycle stopped daemon ownership") default: } } type fixturePolicy struct { restart bool limit int backoff int crashMode string } func newFixtureManager( t *testing.T, policies map[ClientKind]fixturePolicy, ) (*Manager, context.Context, map[ClientKind]string) { t.Helper() root := t.TempDir() specs := make(map[string]agentconfig.ClientProcessSpec, len(policies)) traces := make(map[ClientKind]string, len(policies)+1) for kind, policy := range policies { tracePath := filepath.Join(root, string(kind)+".trace") focusPath := filepath.Join(root, string(kind)+".focus.trace") spec := helperSpec(t, root, tracePath, focusPath) spec.RestartOnCrash = policy.restart spec.RestartLimit = policy.limit spec.RestartBackoffMillis = policy.backoff if policy.crashMode != "" { triggerPath := filepath.Join(root, string(kind)+".crash") consumedPath := filepath.Join(root, string(kind)+".crash-consumed") spec.Args = []string{ "-test.run=^TestHelperProcess$", "--", policy.crashMode, tracePath, triggerPath, } if policy.crashMode == "crash-trigger-once" { spec.Args = append(spec.Args, consumedPath) } traces[ClientKind(string(kind)+"-crash")] = triggerPath } if kind == ClientUnity { spec.FocusArgs = nil } specs[string(kind)] = spec traces[kind] = tracePath if kind == ClientFlutter { traces[ClientKind("flutter-focus")] = focusPath } } store, err := agentstate.NewStore(filepath.Join(root, "state.json")) if err != nil { t.Fatalf("NewStore: %v", err) } ctx, cancel := context.WithCancel(context.Background()) manager, err := NewManager( ctx, specs, store, WithStopTimeout(time.Second), WithPollInterval(10*time.Millisecond), ) if err != nil { cancel() t.Fatalf("NewManager: %v", err) } t.Cleanup(func() { _ = manager.Close(context.Background()) cancel() }) return manager, ctx, traces } func helperSpec( t *testing.T, workingDirectory string, tracePath string, focusTracePath string, ) agentconfig.ClientProcessSpec { t.Helper() executable, err := os.Executable() if err != nil { t.Fatalf("Executable: %v", err) } return agentconfig.ClientProcessSpec{ Executable: executable, Args: []string{"-test.run=^TestHelperProcess$", "--", "run", tracePath}, WorkingDirectory: workingDirectory, FocusArgs: []string{ "-test.run=^TestHelperProcess$", "--", "focus", focusTracePath, }, } } func appendHelperTrace(t *testing.T, path, line string) { t.Helper() file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600) if err != nil { t.Fatalf("open helper trace: %v", err) } if _, err := fmt.Fprintln(file, line); err != nil { _ = file.Close() t.Fatalf("write helper trace: %v", err) } if err := file.Close(); err != nil { t.Fatalf("close helper trace: %v", err) } } func waitForTraceCount(t *testing.T, path string, count int) { t.Helper() deadline := time.Now().Add(5 * time.Second) for time.Now().Before(deadline) { if len(readTraceLines(t, path)) == count { return } time.Sleep(10 * time.Millisecond) } t.Fatalf( "trace %s has %d lines, want %d: %v", path, len(readTraceLines(t, path)), count, readTraceLines(t, path), ) } func readTraceLines(t *testing.T, path string) []string { t.Helper() file, err := os.Open(path) if errors.Is(err, os.ErrNotExist) { return nil } if err != nil { t.Fatalf("open trace: %v", err) } defer file.Close() var lines []string scanner := bufio.NewScanner(file) for scanner.Scan() { lines = append(lines, strings.TrimSpace(scanner.Text())) } if err := scanner.Err(); err != nil { t.Fatalf("scan trace: %v", err) } sort.Strings(lines) return lines } func waitForCondition(t *testing.T, condition func() bool) { t.Helper() deadline := time.Now().Add(5 * time.Second) for time.Now().Before(deadline) { if condition() { return } time.Sleep(10 * time.Millisecond) } t.Fatal("condition was not satisfied before deadline") } func TestManagerReconcilesRetainedClientAfterConfigRemoval(t *testing.T) { root := t.TempDir() statePath := filepath.Join(root, "state.json") tracePath := filepath.Join(root, "flutter.trace") spec := helperSpec(t, root, tracePath, filepath.Join(root, "focus.trace")) store, err := agentstate.NewStore(statePath) if err != nil { t.Fatalf("NewStore: %v", err) } ctx1, cancel1 := context.WithCancel(context.Background()) defer cancel1() m1, err := NewManager( ctx1, map[string]agentconfig.ClientProcessSpec{string(ClientFlutter): spec}, store, ) if err != nil { t.Fatalf("NewManager 1: %v", err) } res, err := m1.Start(context.Background(), ClientFlutter, "cmd-1") if err != nil { t.Fatalf("Start m1: %v", err) } pid := res.Record.Identity.PID waitForTraceCount(t, tracePath, 1) ctx2, cancel2 := context.WithCancel(context.Background()) defer cancel2() m2, err := NewManager( ctx2, map[string]agentconfig.ClientProcessSpec{}, store, WithStopTimeout(time.Second), WithPollInterval(10*time.Millisecond), ) if err != nil { t.Fatalf("NewManager 2: %v", err) } if err := m2.Reconcile(context.Background()); err != nil { t.Fatalf("Reconcile m2: %v", err) } status, err := m2.Status(ClientFlutter) if err != nil { t.Fatalf("Status m2: %v", err) } if status.Identity == nil || status.Identity.PID != pid { t.Fatalf("m2 status identity = %+v, want PID %d", status.Identity, pid) } if _, err := m2.Start( context.Background(), ClientFlutter, "cmd-2", ); !errors.Is(err, ErrClientNotConfigured) { t.Fatalf("m2 Start err = %v, want ErrClientNotConfigured", err) } if _, err := m2.Stop(context.Background(), ClientFlutter, "stop-unconfig"); err != nil { t.Fatalf("m2 Stop err = %v", err) } statusAfter, err := m2.Status(ClientFlutter) if err != nil { t.Fatalf("Status after stop: %v", err) } if statusAfter.State != StateStopped { t.Fatalf("statusAfter state = %s, want stopped", statusAfter.State) } } func TestStartConfiguredLaunchesAfterDaemonRestart(t *testing.T) { backend := &sharedLifecycleBackend{} store, err := agentstate.NewStore(filepath.Join(t.TempDir(), "state.json")) if err != nil { t.Fatalf("NewStore: %v", err) } spec := fakeSpec() spec.LaunchOnStart = true specs := map[string]agentconfig.ClientProcessSpec{string(ClientFlutter): spec} firstCtx, firstCancel := context.WithCancel(context.Background()) first, err := NewManager( firstCtx, specs, store, WithProcessBackend(backend), WithStopTimeout(time.Second), WithPollInterval(10*time.Millisecond), ) if err != nil { firstCancel() t.Fatalf("NewManager first: %v", err) } if err := first.StartConfigured(context.Background()); err != nil { t.Fatalf("first StartConfigured: %v", err) } if backend.startCount() != 1 { t.Fatalf("first generation starts = %d, want 1", backend.startCount()) } firstStatus, err := first.Status(ClientFlutter) if err != nil || firstStatus.Identity == nil { t.Fatalf("first generation not launched: %+v err=%v", firstStatus, err) } firstIdentity := *firstStatus.Identity if err := first.Close(context.Background()); err != nil { t.Fatalf("first Close: %v", err) } firstCancel() secondCtx, secondCancel := context.WithCancel(context.Background()) defer secondCancel() second, err := NewManager( secondCtx, specs, store, WithProcessBackend(backend), WithStopTimeout(time.Second), WithPollInterval(10*time.Millisecond), ) if err != nil { t.Fatalf("NewManager second: %v", err) } defer second.Close(context.Background()) if err := second.StartConfigured(context.Background()); err != nil { t.Fatalf("second StartConfigured: %v", err) } if backend.startCount() != 2 { t.Fatalf("second generation starts = %d, want 2 (relaunch after restart)", backend.startCount()) } secondStatus, err := second.Status(ClientFlutter) if err != nil || secondStatus.Identity == nil { t.Fatalf("second generation not launched: %+v err=%v", secondStatus, err) } if *secondStatus.Identity == firstIdentity { t.Fatalf("second generation reused restarted identity %+v", firstIdentity) } if secondStatus.State != StateStarting { t.Fatalf("second generation state = %s, want starting", secondStatus.State) } } func TestCloseStopsCurrentGenerationAfterPriorLifecycleReceipts(t *testing.T) { store, err := agentstate.NewStore(filepath.Join(t.TempDir(), "state.json")) if err != nil { t.Fatalf("NewStore: %v", err) } // Seed durable state as if a prior buggy generation persisted fixed // daemon-start/daemon-stop caller receipts. The current generation must // launch and stop its own process without replaying them. priorIdentity := ProcessIdentity{PID: 999, StartToken: "gen-prior"} seed := initialRecord(ClientFlutter) seed.LastIdentity = &priorIdentity seed.Commands = map[string]CommandReceipt{ "daemon-start": { CommandID: "daemon-start", Action: "start", Status: CommandReceiptCompleted, Result: CommandResultSnapshot{State: StateStarting, Action: "start"}, }, "daemon-stop": { CommandID: "daemon-stop", Action: "stop", Status: CommandReceiptCompleted, Result: CommandResultSnapshot{State: StateStopped, Action: "stop"}, }, } seed.UpdatedAt = time.Now().UTC() durable, _ := newDurableStore(store) if _, err := durable.save(context.Background(), seed, ""); err != nil { t.Fatalf("seed record: %v", err) } backend := &sharedLifecycleBackend{} spec := fakeSpec() spec.LaunchOnStart = true ctx, cancel := context.WithCancel(context.Background()) defer cancel() manager, err := NewManager( ctx, map[string]agentconfig.ClientProcessSpec{string(ClientFlutter): spec}, store, WithProcessBackend(backend), WithStopTimeout(time.Second), WithPollInterval(10*time.Millisecond), ) if err != nil { t.Fatalf("NewManager: %v", err) } defer manager.Close(context.Background()) if err := manager.StartConfigured(context.Background()); err != nil { t.Fatalf("StartConfigured: %v", err) } if backend.startCount() != 1 { t.Fatalf("starts = %d, want 1 despite prior daemon-start receipt", backend.startCount()) } status, err := manager.Status(ClientFlutter) if err != nil || status.Identity == nil { t.Fatalf("current generation not launched: %+v err=%v", status, err) } currentIdentity := *status.Identity if currentIdentity == priorIdentity { t.Fatalf("current generation reused prior identity %+v", priorIdentity) } if err := manager.Close(context.Background()); err != nil { t.Fatalf("Close: %v", err) } if !backend.wasStopped(currentIdentity) { t.Fatalf("close did not stop the current identity %+v", currentIdentity) } final, err := manager.Status(ClientFlutter) if err != nil { t.Fatalf("Status: %v", err) } if final.State != StateStopped || final.Identity != nil { t.Fatalf("final state = %+v, want stopped without identity", final) } } // sharedLifecycleBackend is a deterministic in-memory backend that can be // shared across daemon generations. It assigns a unique identity per launch and // tracks which identities were stopped. type sharedLifecycleBackend struct { mu sync.Mutex starts int stopped []ProcessIdentity processes map[ProcessIdentity]*fakeOwnedProcess } func (b *sharedLifecycleBackend) Start( context.Context, ClientKind, agentconfig.ClientProcessSpec, ) (OwnedProcess, error) { b.mu.Lock() defer b.mu.Unlock() b.starts++ id := ProcessIdentity{ PID: 1000 + b.starts, StartToken: fmt.Sprintf("gen-%d", b.starts), } process := &fakeOwnedProcess{identity: id, done: make(chan struct{})} if b.processes == nil { b.processes = make(map[ProcessIdentity]*fakeOwnedProcess) } b.processes[id] = process return process, nil } func (b *sharedLifecycleBackend) Inspect( _ context.Context, id ProcessIdentity, ) (IdentityObservation, error) { b.mu.Lock() defer b.mu.Unlock() process, ok := b.processes[id] if !ok { return IdentityObservation{State: IdentityExited}, nil } select { case <-process.done: return IdentityObservation{State: IdentityExited}, nil default: return IdentityObservation{State: IdentityLive, Identity: id}, nil } } func (b *sharedLifecycleBackend) Signal( _ context.Context, id ProcessIdentity, _ os.Signal, ) error { b.mu.Lock() process, ok := b.processes[id] if ok { b.stopped = append(b.stopped, id) } b.mu.Unlock() if ok { _ = process.Abort() } return nil } func (b *sharedLifecycleBackend) Kill(ctx context.Context, id ProcessIdentity) error { return b.Signal(ctx, id, os.Kill) } func (b *sharedLifecycleBackend) Focus( context.Context, ClientKind, agentconfig.ClientProcessSpec, ) error { return nil } func (b *sharedLifecycleBackend) startCount() int { b.mu.Lock() defer b.mu.Unlock() return b.starts } func (b *sharedLifecycleBackend) wasStopped(id ProcessIdentity) bool { b.mu.Lock() defer b.mu.Unlock() for _, stopped := range b.stopped { if stopped == id { return true } } return false } func (b *sharedLifecycleBackend) wasAborted(id ProcessIdentity) bool { b.mu.Lock() defer b.mu.Unlock() process, ok := b.processes[id] return ok && process.aborted } func TestConcurrentCloseAbortsInFlightLaunch(t *testing.T) { barrier := make(chan struct{}) backend := &barrierProcessBackend{barrier: barrier} store, err := agentstate.NewStore(filepath.Join(t.TempDir(), "state.json")) if err != nil { t.Fatalf("NewStore: %v", err) } m, err := NewManager( context.Background(), map[string]agentconfig.ClientProcessSpec{ string(ClientFlutter): fakeSpec(), }, store, WithProcessBackend(backend), ) if err != nil { t.Fatalf("NewManager: %v", err) } var startErr error var wg sync.WaitGroup wg.Add(1) go func() { defer wg.Done() _, startErr = m.Start(context.Background(), ClientFlutter, "in-flight-start") }() <-barrier closeDone := make(chan error, 1) go func() { closeDone <- m.Close(context.Background()) }() select { case err := <-closeDone: t.Fatalf("Close returned before the admitted launch finished: %v", err) case <-time.After(20 * time.Millisecond): } close(barrier) if closeErr := <-closeDone; closeErr != nil { t.Fatalf("Close err = %v", closeErr) } wg.Wait() if !errors.Is(startErr, ErrManagerClosed) { t.Fatalf("startErr = %v, want ErrManagerClosed", startErr) } if backend.createdProcess != nil && !backend.createdProcess.aborted { t.Fatalf("in-flight process was not aborted") } } func TestConcurrentCloseCancelsInFlightFocus(t *testing.T) { assertConcurrentCloseCancelsMutation(t, func(m *Manager) (Result, error) { return m.Focus(context.Background(), ClientFlutter, "focus-1") }) } func TestConcurrentCloseCancelsInFlightDetail(t *testing.T) { assertConcurrentCloseCancelsMutation(t, func(m *Manager) (Result, error) { return m.StartOrFocusFlutter(context.Background(), "detail-1") }) } func assertConcurrentCloseCancelsMutation( t *testing.T, invoke func(*Manager) (Result, error), ) { t.Helper() backend := &focusBarrierBackend{focusEntered: make(chan struct{}, 1)} store, err := agentstate.NewStore(filepath.Join(t.TempDir(), "state.json")) if err != nil { t.Fatalf("NewStore: %v", err) } m, err := NewManager( context.Background(), map[string]agentconfig.ClientProcessSpec{string(ClientFlutter): fakeSpec()}, store, WithProcessBackend(backend), WithStopTimeout(time.Second), WithPollInterval(10*time.Millisecond), ) if err != nil { t.Fatalf("NewManager: %v", err) } if _, err := m.Start(context.Background(), ClientFlutter, "start-1"); err != nil { t.Fatalf("Start: %v", err) } if _, err := m.SetConnected(context.Background(), ClientFlutter, true); err != nil { t.Fatalf("SetConnected: %v", err) } status, err := m.Status(ClientFlutter) if err != nil || status.Identity == nil { t.Fatalf("client not live: %+v err=%v", status, err) } identity := *status.Identity var mutationErr error var wg sync.WaitGroup wg.Add(1) go func() { defer wg.Done() _, mutationErr = invoke(m) }() <-backend.focusEntered if err := m.Close(context.Background()); err != nil { t.Fatalf("Close: %v", err) } wg.Wait() if !errors.Is(mutationErr, ErrManagerClosed) { t.Fatalf("mutation error = %v, want ErrManagerClosed", mutationErr) } if !backend.focusWasCancelled() { t.Fatalf("manager cancellation did not reach the blocking mutation") } final, err := m.Status(ClientFlutter) if err != nil { t.Fatalf("Status: %v", err) } if final.FocusCount != 0 { t.Fatalf("focus committed after close: FocusCount=%d", final.FocusCount) } if final.State != StateStopped || final.Identity != nil { t.Fatalf("current client not reaped: %+v", final) } if !backend.wasStopped(identity) { t.Fatalf("current identity %+v not reaped", identity) } } func TestConcurrentCloseFencesConnectionMutation(t *testing.T) { base, err := agentstate.NewStore(filepath.Join(t.TempDir(), "state.json")) if err != nil { t.Fatalf("NewStore: %v", err) } store := &connectionBarrierStore{ Store: base, armKey: clientRecordKey(ClientFlutter), entered: make(chan struct{}, 1), } backend := &sharedLifecycleBackend{} m, err := NewManager( context.Background(), map[string]agentconfig.ClientProcessSpec{string(ClientFlutter): fakeSpec()}, store, WithProcessBackend(backend), WithStopTimeout(time.Second), WithPollInterval(10*time.Millisecond), ) if err != nil { t.Fatalf("NewManager: %v", err) } if _, err := m.Start(context.Background(), ClientFlutter, "start-1"); err != nil { t.Fatalf("Start: %v", err) } if _, err := m.SetConnected(context.Background(), ClientFlutter, true); err != nil { t.Fatalf("SetConnected true: %v", err) } status, err := m.Status(ClientFlutter) if err != nil || status.Identity == nil { t.Fatalf("client not live: %+v err=%v", status, err) } identity := *status.Identity // Block the disconnect's durable projection commit after admission. store.arm() var connectErr error var wg sync.WaitGroup wg.Add(1) go func() { defer wg.Done() _, connectErr = m.SetConnected(context.Background(), ClientFlutter, false) }() <-store.entered if err := m.Close(context.Background()); err != nil { t.Fatalf("Close: %v", err) } wg.Wait() if connectErr == nil { t.Fatalf("connection mutation committed across close") } final, err := m.Status(ClientFlutter) if err != nil { t.Fatalf("Status: %v", err) } if final.State != StateStopped || final.Identity != nil { t.Fatalf("current client not reaped: %+v", final) } if !backend.wasStopped(identity) { t.Fatalf("current identity %+v not reaped", identity) } } // focusBarrierBackend blocks its Focus call after admission until the mutation // context is cancelled, so a concurrent Close must cancel and join the mutation. type focusBarrierBackend struct { sharedLifecycleBackend focusEntered chan struct{} focusMu sync.Mutex cancelled bool } func (b *focusBarrierBackend) Focus( ctx context.Context, _ ClientKind, _ agentconfig.ClientProcessSpec, ) error { select { case b.focusEntered <- struct{}{}: default: } <-ctx.Done() b.focusMu.Lock() b.cancelled = true b.focusMu.Unlock() return ctx.Err() } func (b *focusBarrierBackend) focusWasCancelled() bool { b.focusMu.Lock() defer b.focusMu.Unlock() return b.cancelled } // connectionBarrierStore blocks exactly one CompareAndSwap on the armed key // after admission until the write context is cancelled. type connectionBarrierStore struct { *agentstate.Store armKey string entered chan struct{} mu sync.Mutex armed bool } func (s *connectionBarrierStore) arm() { s.mu.Lock() s.armed = true s.mu.Unlock() } func (s *connectionBarrierStore) CompareAndSwapIntegrationRecord( ctx context.Context, key string, revision string, payload []byte, ) (string, error) { s.mu.Lock() block := s.armed && key == s.armKey if block { s.armed = false } s.mu.Unlock() if block { select { case s.entered <- struct{}{}: default: } <-ctx.Done() return "", ctx.Err() } return s.Store.CompareAndSwapIntegrationRecord(ctx, key, revision, payload) } type barrierProcessBackend struct { fakeProcessBackend barrier chan struct{} createdProcess *fakeOwnedProcess } func (b *barrierProcessBackend) Start( ctx context.Context, kind ClientKind, spec agentconfig.ClientProcessSpec, ) (OwnedProcess, error) { p := &fakeOwnedProcess{ identity: ProcessIdentity{PID: 456, StartToken: "fake-barrier"}, done: make(chan struct{}), } b.createdProcess = p b.barrier <- struct{}{} <-b.barrier return p, nil } func TestAdoptedCleanupRequiresObservedExit(t *testing.T) { backend := &adoptableBackend{live: true} store, err := agentstate.NewStore(filepath.Join(t.TempDir(), "state.json")) if err != nil { t.Fatalf("NewStore: %v", err) } durable, _ := newDurableStore(store) identity := ProcessIdentity{PID: 888, StartToken: "adopt-token"} rec := initialRecord(ClientFlutter) rec.State = StateConnected rec.Connected = true rec.Identity = &identity rec.UpdatedAt = time.Now().UTC() if _, err := durable.save(context.Background(), rec, ""); err != nil { t.Fatalf("seed record: %v", err) } m, err := NewManager( context.Background(), map[string]agentconfig.ClientProcessSpec{ string(ClientFlutter): fakeSpec(), }, store, WithProcessBackend(backend), WithPollInterval(10*time.Millisecond), WithStopTimeout(200*time.Millisecond), ) if err != nil { t.Fatalf("NewManager: %v", err) } if err := m.Reconcile(context.Background()); err != nil { t.Fatalf("Reconcile: %v", err) } stopDone := make(chan error, 1) go func() { _, err := m.Stop(context.Background(), ClientFlutter, "stop-adopted") stopDone <- err }() select { case err := <-stopDone: if err != nil { t.Fatalf("Stop error = %v", err) } case <-time.After(3 * time.Second): t.Fatal("Stop timed out waiting for adopted exit observation") } status, err := m.Status(ClientFlutter) if err != nil { t.Fatalf("Status: %v", err) } if status.State != StateStopped { t.Fatalf("adopted status state = %s, want stopped", status.State) } } type adoptableBackend struct { fakeProcessBackend mu sync.Mutex live bool } func (b *adoptableBackend) Inspect( ctx context.Context, id ProcessIdentity, ) (IdentityObservation, error) { b.mu.Lock() defer b.mu.Unlock() if b.live { return IdentityObservation{State: IdentityLive, Identity: id}, nil } return IdentityObservation{State: IdentityExited}, nil } func (b *adoptableBackend) Signal( ctx context.Context, id ProcessIdentity, sig os.Signal, ) error { return nil } func (b *adoptableBackend) Kill( ctx context.Context, id ProcessIdentity, ) error { b.mu.Lock() defer b.mu.Unlock() b.live = false return nil } func TestCommandReceiptCompletionSaveFailureStaysPending(t *testing.T) { injected := errors.New("injected completion receipt save failure") tests := []struct { name string setup func(t *testing.T, manager *Manager, store *nthCASFailureStore) call func(*Manager) (Result, error) }{ { name: "start", setup: func(t *testing.T, _ *Manager, store *nthCASFailureStore) { t.Helper() store.failOnNextCompletion() }, call: func(manager *Manager) (Result, error) { return manager.Start(context.Background(), ClientFlutter, "completion-start") }, }, { name: "already running start", setup: func(t *testing.T, manager *Manager, store *nthCASFailureStore) { t.Helper() startLiveFixture(t, manager) store.failOnNextCompletion() }, call: func(manager *Manager) (Result, error) { return manager.Start(context.Background(), ClientFlutter, "completion-start-live") }, }, { name: "already stopped", setup: func(t *testing.T, _ *Manager, store *nthCASFailureStore) { t.Helper() store.failOnNextCompletion() }, call: func(manager *Manager) (Result, error) { return manager.Stop(context.Background(), ClientFlutter, "completion-stop") }, }, { name: "focus", setup: func(t *testing.T, manager *Manager, store *nthCASFailureStore) { t.Helper() startLiveFixture(t, manager) store.failOnNextCompletion() }, call: func(manager *Manager) (Result, error) { return manager.Focus(context.Background(), ClientFlutter, "completion-focus") }, }, { name: "detail focus", setup: func(t *testing.T, manager *Manager, store *nthCASFailureStore) { t.Helper() startLiveFixture(t, manager) store.failOnNextCompletion() }, call: func(manager *Manager) (Result, error) { return manager.StartOrFocusFlutter(context.Background(), "completion-detail") }, }, { name: "detail start", setup: func(t *testing.T, _ *Manager, store *nthCASFailureStore) { t.Helper() store.failOnNextCompletion() }, call: func(manager *Manager) (Result, error) { return manager.StartOrFocusFlutter(context.Background(), "completion-detail-start") }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { base, err := agentstate.NewStore(filepath.Join(t.TempDir(), "state.json")) if err != nil { t.Fatalf("NewStore: %v", err) } store := &nthCASFailureStore{Store: base, failure: injected} backend := &sharedLifecycleBackend{} manager, err := NewManager( context.Background(), map[string]agentconfig.ClientProcessSpec{string(ClientFlutter): fakeSpec()}, store, WithProcessBackend(backend), WithStopTimeout(time.Second), WithPollInterval(10*time.Millisecond), ) if err != nil { t.Fatalf("NewManager: %v", err) } tt.setup(t, manager, store) if _, err := tt.call(manager); !errors.Is(err, injected) { t.Fatalf("completion action error = %v, want injected failure", err) } if tt.name == "start" || tt.name == "detail start" { status, err := manager.Status(ClientFlutter) if err != nil { t.Fatalf("Status: %v", err) } if status.Identity != nil || backend.startCount() != 1 { t.Fatalf("failed start status = %+v starts=%d", status, backend.startCount()) } identity := ProcessIdentity{PID: 1001, StartToken: "gen-1"} if !backend.wasAborted(identity) { t.Fatalf("failed start process %+v was not aborted", identity) } } if _, err := tt.call(manager); !errors.Is(err, ErrCommandPending) { t.Fatalf("same-manager retry error = %v, want ErrCommandPending", err) } restarted, err := NewManager( context.Background(), map[string]agentconfig.ClientProcessSpec{string(ClientFlutter): fakeSpec()}, base, WithProcessBackend(backend), ) if err != nil { t.Fatalf("NewManager restart: %v", err) } if _, err := tt.call(restarted); !errors.Is(err, ErrCommandPending) { t.Fatalf("restart retry error = %v, want ErrCommandPending", err) } restarted.cancel() _ = manager.Close(context.Background()) }) } } func startLiveFixture(t *testing.T, manager *Manager) { t.Helper() if _, err := manager.Start(context.Background(), ClientFlutter, ""); err != nil { t.Fatalf("fixture Start: %v", err) } if _, err := manager.SetConnected(context.Background(), ClientFlutter, true); err != nil { t.Fatalf("fixture SetConnected: %v", err) } } // nthCASFailureStore injects one deterministic CAS failure after a pending // command receipt has been durably saved, so tests exercise only the // completion-save rollback boundary. type nthCASFailureStore struct { *agentstate.Store mu sync.Mutex calls int failAt int failure error } func (s *nthCASFailureStore) failOnNextCompletion() { s.mu.Lock() defer s.mu.Unlock() s.failAt = s.calls + 2 } func (s *nthCASFailureStore) CompareAndSwapIntegrationRecord( ctx context.Context, key string, revision string, payload []byte, ) (string, error) { s.mu.Lock() s.calls++ fail := s.failAt != 0 && s.calls == s.failAt s.mu.Unlock() if fail { return "", s.failure } return s.Store.CompareAndSwapIntegrationRecord(ctx, key, revision, payload) } func TestClosePreservesAmbiguousIdentityBeforeAdoption(t *testing.T) { assertClosePreservesAmbiguousIdentity(t, false) } func TestClosePreservesAmbiguousAdoptedIdentity(t *testing.T) { assertClosePreservesAmbiguousIdentity(t, true) } func assertClosePreservesAmbiguousIdentity(t *testing.T, adopt bool) { t.Helper() base, err := agentstate.NewStore(filepath.Join(t.TempDir(), "state.json")) if err != nil { t.Fatalf("NewStore: %v", err) } identity := ProcessIdentity{PID: 733, StartToken: "ambiguous-token"} durable, err := newDurableStore(base) if err != nil { t.Fatalf("newDurableStore: %v", err) } record := initialRecord(ClientFlutter) record.State = StateConnected record.Connected = true record.Identity = &identity record.UpdatedAt = time.Now().UTC() if _, err := durable.save(context.Background(), record, ""); err != nil { t.Fatalf("seed record: %v", err) } backend := &mutableInspectionBackend{state: IdentityAmbiguous} manager, err := NewManager( context.Background(), map[string]agentconfig.ClientProcessSpec{string(ClientFlutter): fakeSpec()}, base, WithProcessBackend(backend), WithPollInterval(10*time.Millisecond), ) if err != nil { t.Fatalf("NewManager: %v", err) } if adopt { backend.setState(IdentityLive) if err := manager.Reconcile(context.Background()); err != nil { t.Fatalf("Reconcile: %v", err) } backend.setState(IdentityAmbiguous) } closeContext, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() if err := manager.Close(closeContext); !errors.Is(err, ErrIdentityAmbiguous) { t.Fatalf("Close error = %v, want ErrIdentityAmbiguous", err) } if closeContext.Err() != nil { t.Fatalf("Close exceeded its context: %v", closeContext.Err()) } status, err := manager.Status(ClientFlutter) if err != nil { t.Fatalf("Status: %v", err) } if status.Identity == nil || *status.Identity != identity || status.State != StateConnected || status.Blocker == "" { t.Fatalf("ambiguous close status = %+v", status) } manager.waiters.Wait() } type mutableInspectionBackend struct { fakeProcessBackend mu sync.Mutex state IdentityState } func (b *mutableInspectionBackend) setState(state IdentityState) { b.mu.Lock() b.state = state b.mu.Unlock() } func (b *mutableInspectionBackend) Inspect( context.Context, ProcessIdentity, ) (IdentityObservation, error) { b.mu.Lock() defer b.mu.Unlock() return IdentityObservation{State: b.state}, nil }