package clientprocess import ( "context" "errors" "fmt" "os" "sort" "sync" "syscall" "time" "iop/packages/go/agentconfig" ) const ( defaultStopTimeout = 3 * time.Second defaultPollInterval = 100 * time.Millisecond ) type Option func(*Manager) func WithProcessBackend(backend ProcessBackend) Option { return func(manager *Manager) { if backend != nil { manager.backend = backend } } } func WithStopTimeout(timeout time.Duration) Option { return func(manager *Manager) { if timeout > 0 { manager.stopTimeout = timeout } } } func WithPollInterval(interval time.Duration) Option { return func(manager *Manager) { if interval > 0 { manager.pollInterval = interval } } } type processSlot struct { mu sync.Mutex configured bool spec agentconfig.ClientProcessSpec record Record durableRecord Record revision string process OwnedProcess waitDone chan struct{} startingDone chan struct{} stopping bool } type Manager struct { ctx context.Context cancel context.CancelFunc store *durableStore backend ProcessBackend stopTimeout time.Duration pollInterval time.Duration slots map[ClientKind]*processSlot now func() time.Time closeOnce sync.Once closedMu sync.RWMutex closed bool waiters sync.WaitGroup mutationCtx context.Context mutationCancel context.CancelFunc mutationMu sync.Mutex mutationClosed bool mutations sync.WaitGroup } func NewManager( ctx context.Context, specs map[string]agentconfig.ClientProcessSpec, store StateStore, options ...Option, ) (*Manager, error) { if ctx == nil { return nil, fmt.Errorf("clientprocess: manager context is required") } durable, err := newDurableStore(store) if err != nil { return nil, err } for rawKind := range specs { kind := ClientKind(rawKind) if !validKind(kind) { return nil, fmt.Errorf("%w: %q", ErrUnknownClient, rawKind) } } managerContext, cancel := context.WithCancel(ctx) mutationContext, mutationCancel := context.WithCancel(managerContext) manager := &Manager{ ctx: managerContext, cancel: cancel, mutationCtx: mutationContext, mutationCancel: mutationCancel, store: durable, backend: osProcessBackend{}, stopTimeout: defaultStopTimeout, pollInterval: defaultPollInterval, slots: make(map[ClientKind]*processSlot, len(canonicalClientKinds)), now: func() time.Time { return time.Now().UTC() }, } for _, option := range options { option(manager) } for _, kind := range canonicalClientKinds { spec, configured := specs[string(kind)] record, revision, _, err := durable.load(ctx, kind) if err != nil { cancel() return nil, err } manager.slots[kind] = &processSlot{ configured: configured, spec: cloneSpec(spec), record: record, durableRecord: cloneRecord(record), revision: revision, } } return manager, nil } func (m *Manager) Reconcile(ctx context.Context) error { if err := m.ensureOpen(); err != nil { return err } for _, kind := range m.kinds() { slot := m.slots[kind] slot.mu.Lock() err := m.reconcileLocked(ctx, kind, slot) slot.mu.Unlock() if err != nil { return err } } return nil } func (m *Manager) StartConfigured(ctx context.Context) error { if err := m.Reconcile(ctx); err != nil { return err } for _, kind := range m.kinds() { slot := m.slots[kind] if !slot.spec.LaunchOnStart { continue } if _, err := m.startInternal(ctx, kind); err != nil { return err } } return nil } func (m *Manager) Start( ctx context.Context, kind ClientKind, commandID string, ) (Result, error) { slot, err := m.slot(kind) if err != nil { return Result{}, err } if err := m.ensureOpen(); err != nil { return Result{}, err } mutationCtx, finishMutation, err := m.beginMutation(ctx) if err != nil { return Result{}, err } defer finishMutation() slot.mu.Lock() for { if err := m.ensureOpen(); err != nil { slot.mu.Unlock() return Result{}, err } if !slot.configured { slot.mu.Unlock() return Result{}, ErrClientNotConfigured } if res, handled, receiptErr := m.checkReceiptLocked(slot, commandID, "start"); handled { slot.mu.Unlock() return res, receiptErr } if err := m.admitReceiptLocked(slot, commandID); err != nil { slot.mu.Unlock() return Result{}, err } if slot.startingDone != nil { startingDone := slot.startingDone slot.mu.Unlock() select { case <-startingDone: slot.mu.Lock() continue case <-ctx.Done(): return Result{}, ctx.Err() case <-m.ctx.Done(): return Result{}, ErrManagerClosed } } live, err := m.ensureLiveStateLocked(mutationCtx, kind, slot) if err != nil { slot.mu.Unlock() return Result{}, err } if live { slot.record.LastCommandID = commandID slot.record.LastAction = "start" slot.record.LastResult = "already-running" m.setReceiptLocked(slot, commandID, "start", CommandReceiptPending, Result{}) if err := m.persistLocked(mutationCtx, slot); err != nil { slot.mu.Unlock() return Result{}, err } res := Result{ Record: cloneRecord(slot.record), Action: "start", } if err := m.completeReceiptLocked( mutationCtx, slot, commandID, "start", res, ); err != nil { slot.mu.Unlock() return Result{}, err } slot.mu.Unlock() return res, nil } slot.record.RestartAttempts = 0 return m.startLocked(mutationCtx, kind, slot, commandID, "start", false) } } func (m *Manager) Stop( ctx context.Context, kind ClientKind, commandID string, ) (Result, error) { if err := m.ensureOpen(); err != nil { return Result{}, err } mutationCtx, finishMutation, err := m.beginMutation(ctx) if err != nil { return Result{}, err } defer finishMutation() return m.stop(mutationCtx, kind, commandID) } // stop is the shared stop path. The public caller admits a close-fenced // mutation before entering it; Close invokes it directly after cancellation so // it can still reap the exact current lifecycle generation. func (m *Manager) stop( ctx context.Context, kind ClientKind, commandID string, ) (Result, error) { slot, err := m.slot(kind) if err != nil { return Result{}, err } slot.mu.Lock() if res, handled, receiptErr := m.checkReceiptLocked(slot, commandID, "stop"); handled { slot.mu.Unlock() return res, receiptErr } if err := m.admitReceiptLocked(slot, commandID); err != nil { slot.mu.Unlock() return Result{}, err } live, liveErr := m.ensureLiveStateLocked(ctx, kind, slot) if liveErr != nil && !errors.Is(liveErr, ErrIdentityAmbiguous) { slot.mu.Unlock() return Result{}, liveErr } if errors.Is(liveErr, ErrIdentityAmbiguous) { slot.mu.Unlock() return Result{}, liveErr } if !live { slot.record.State = StateStopped slot.record.Connected = false slot.record.LastCommandID = commandID slot.record.LastAction = "stop" slot.record.LastResult = "already-stopped" slot.record.Blocker = "" m.setReceiptLocked(slot, commandID, "stop", CommandReceiptPending, Result{}) if err := m.persistLocked(ctx, slot); err != nil { slot.mu.Unlock() return Result{}, err } res := Result{Record: cloneRecord(slot.record), Action: "stop"} if err := m.completeReceiptLocked( ctx, slot, commandID, "stop", res, ); err != nil { slot.mu.Unlock() return Result{}, err } slot.mu.Unlock() return res, nil } identity := *slot.record.Identity waitDone := slot.waitDone slot.stopping = true slot.record.LastCommandID = commandID slot.record.LastAction = "stop" slot.record.LastResult = "stopping" m.setReceiptLocked(slot, commandID, "stop", CommandReceiptPending, Result{}) if err := m.persistLocked(ctx, slot); err != nil { slot.stopping = false slot.mu.Unlock() return Result{}, err } slot.mu.Unlock() signalErr := m.backend.Signal(ctx, identity, syscall.SIGTERM) if signalErr != nil && !errors.Is(signalErr, os.ErrProcessDone) { return Result{}, signalErr } if !waitForProcess(ctx, waitDone, m.stopTimeout) { if err := m.backend.Kill(ctx, identity); err != nil && !errors.Is(err, os.ErrProcessDone) { return Result{}, err } if !waitForProcess(ctx, waitDone, m.stopTimeout) { return Result{}, fmt.Errorf("clientprocess: client did not exit after kill") } } slot.mu.Lock() defer slot.mu.Unlock() slot.stopping = false slot.record.State = StateStopped slot.record.Connected = false slot.record.Identity = nil slot.record.LastIdentity = &identity slot.record.LastResult = "stopped" slot.record.Blocker = "" res := Result{ Record: cloneRecord(slot.record), Changed: true, Action: "stop", } if err := m.completeReceiptLocked( ctx, slot, commandID, "stop", res, ); err != nil { return Result{}, err } return res, nil } func (m *Manager) Focus( ctx context.Context, kind ClientKind, commandID string, ) (Result, error) { slot, err := m.slot(kind) if err != nil { return Result{}, err } if kind != ClientFlutter { return Result{}, ErrFocusUnsupported } mutationCtx, finishMutation, err := m.beginMutation(ctx) if err != nil { return Result{}, err } defer finishMutation() slot.mu.Lock() if err := m.ensureOpen(); err != nil { slot.mu.Unlock() return Result{}, err } if !slot.configured { slot.mu.Unlock() return Result{}, ErrClientNotConfigured } if res, handled, receiptErr := m.checkReceiptLocked(slot, commandID, "focus"); handled { slot.mu.Unlock() return res, receiptErr } if err := m.admitReceiptLocked(slot, commandID); err != nil { slot.mu.Unlock() return Result{}, err } live, err := m.ensureLiveStateLocked(mutationCtx, kind, slot) if err != nil { slot.mu.Unlock() return Result{}, err } if !live { slot.mu.Unlock() return Result{}, ErrClientNotRunning } m.setReceiptLocked(slot, commandID, "focus", CommandReceiptPending, Result{}) if err := m.persistLocked(mutationCtx, slot); err != nil { slot.mu.Unlock() return Result{}, err } spec := slot.spec slot.mu.Unlock() focusErr := m.backend.Focus(mutationCtx, kind, spec) slot.mu.Lock() defer slot.mu.Unlock() if err := m.ensureOpen(); err != nil { return Result{}, err } if focusErr != nil { return Result{}, focusErr } slot.record.FocusCount++ slot.record.LastCommandID = commandID slot.record.LastAction = "focus" slot.record.LastResult = "focused" res := Result{ Record: cloneRecord(slot.record), Changed: true, Action: "focus", } if err := m.completeReceiptLocked( mutationCtx, slot, commandID, "focus", res, ); err != nil { return Result{}, err } return res, nil } func (m *Manager) StartOrFocusFlutter( ctx context.Context, commandID string, ) (Result, error) { slot, err := m.slot(ClientFlutter) if err != nil { return Result{}, err } if err := m.ensureOpen(); err != nil { return Result{}, err } mutationCtx, finishMutation, err := m.beginMutation(ctx) if err != nil { return Result{}, err } defer finishMutation() slot.mu.Lock() if err := m.ensureOpen(); err != nil { slot.mu.Unlock() return Result{}, err } if !slot.configured { slot.mu.Unlock() return Result{}, ErrClientNotConfigured } if res, handled, receiptErr := m.checkReceiptLocked(slot, commandID, "detail"); handled { slot.mu.Unlock() return res, receiptErr } if err := m.admitReceiptLocked(slot, commandID); err != nil { slot.mu.Unlock() return Result{}, err } live, err := m.ensureLiveStateLocked(mutationCtx, ClientFlutter, slot) if err != nil { slot.mu.Unlock() return Result{}, err } if !live { slot.record.RestartAttempts = 0 return m.startLocked( mutationCtx, ClientFlutter, slot, commandID, "detail", false, ) } m.setReceiptLocked(slot, commandID, "detail", CommandReceiptPending, Result{}) if err := m.persistLocked(mutationCtx, slot); err != nil { slot.mu.Unlock() return Result{}, err } spec := slot.spec slot.mu.Unlock() focusErr := m.backend.Focus(mutationCtx, ClientFlutter, spec) slot.mu.Lock() defer slot.mu.Unlock() if err := m.ensureOpen(); err != nil { return Result{}, err } if focusErr != nil { return Result{}, focusErr } slot.record.FocusCount++ slot.record.LastCommandID = commandID slot.record.LastAction = "detail" slot.record.LastResult = "focused" res := Result{ Record: cloneRecord(slot.record), Changed: true, Action: "focus", } if err := m.completeReceiptLocked( mutationCtx, slot, commandID, "detail", res, ); err != nil { return Result{}, err } return res, nil } func (m *Manager) SetConnected( ctx context.Context, kind ClientKind, connected bool, ) (Record, error) { slot, err := m.slot(kind) if err != nil { return Record{}, err } if err := m.ensureOpen(); err != nil { return Record{}, err } mutationCtx, finishMutation, err := m.beginMutation(ctx) if err != nil { return Record{}, err } defer finishMutation() slot.mu.Lock() defer slot.mu.Unlock() if err := m.ensureOpen(); err != nil { return Record{}, err } live, err := m.ensureLiveStateLocked(mutationCtx, kind, slot) if err != nil { return Record{}, err } if !live { return Record{}, ErrClientNotRunning } slot.record.Connected = connected if connected { slot.record.State = StateConnected slot.record.LastResult = "connected" } else { slot.record.State = StateStarting slot.record.LastResult = "disconnected" } if err := m.persistLocked(mutationCtx, slot); err != nil { return Record{}, err } return cloneRecord(slot.record), nil } func (m *Manager) Status(kind ClientKind) (Record, error) { slot, err := m.slot(kind) if err != nil { return Record{}, err } slot.mu.Lock() defer slot.mu.Unlock() return cloneRecord(slot.record), nil } func (m *Manager) Close(ctx context.Context) error { var closeErr error m.closeOnce.Do(func() { m.closedMu.Lock() m.closed = true m.closedMu.Unlock() // Atomically stop admitting new mutations, cancel every admitted // blocking mutation, and wait for them to finish before reaping the // current client generation. m.mutationMu.Lock() m.mutationClosed = true m.mutationMu.Unlock() m.mutationCancel() m.mutations.Wait() for _, kind := range m.kinds() { if _, err := m.stopInternal(ctx, kind); err != nil && !errors.Is(err, ErrClientNotRunning) && !errors.Is(err, ErrClientNotConfigured) { closeErr = errors.Join(closeErr, err) } } m.cancel() m.waiters.Wait() }) return closeErr } // startInternal launches a configured client for an automatic daemon lifecycle // action. It reuses the external start path but supplies no caller command_id, // so it never writes or replays a durable caller receipt and every daemon // generation launches its current process. func (m *Manager) startInternal(ctx context.Context, kind ClientKind) (Result, error) { return m.Start(ctx, kind, "") } // stopInternal stops the current client generation for an automatic daemon // lifecycle action without participating in caller command_id receipt replay. func (m *Manager) stopInternal(ctx context.Context, kind ClientKind) (Result, error) { return m.stop(ctx, kind, "") } // beginMutation admits one manager-owned blocking mutation. It fails closed // once the manager begins closing, returns a context cancelled by either the // caller or manager closure, and registers the mutation so Close waits for it // to finish before reaping the client. func (m *Manager) beginMutation( caller context.Context, ) (context.Context, func(), error) { m.mutationMu.Lock() if m.mutationClosed || m.isClosed() { m.mutationMu.Unlock() return nil, nil, ErrManagerClosed } m.mutations.Add(1) m.mutationMu.Unlock() ctx, cancel := context.WithCancel(m.mutationCtx) stop := context.AfterFunc(caller, cancel) var once sync.Once finish := func() { once.Do(func() { stop() cancel() m.mutations.Done() }) } return ctx, finish, nil } func (m *Manager) startLocked( ctx context.Context, kind ClientKind, slot *processSlot, commandID string, receiptAction string, automatic bool, ) (Result, error) { if err := m.ensureOpen(); err != nil { slot.mu.Unlock() return Result{}, err } if automatic { slot.record.RestartAttempts++ } startingDone := make(chan struct{}) slot.startingDone = startingDone cleanupStartingDone := func() { if slot.startingDone == startingDone { close(startingDone) slot.startingDone = nil } } slot.record.State = StateStarting slot.record.Connected = false slot.record.Identity = nil slot.record.LastCommandID = commandID slot.record.LastAction = "start" slot.record.LastResult = "starting" slot.record.Blocker = "" m.setReceiptLocked(slot, commandID, receiptAction, CommandReceiptPending, Result{}) if err := m.persistLocked(ctx, slot); err != nil { cleanupStartingDone() slot.mu.Unlock() return Result{}, err } spec := slot.spec slot.mu.Unlock() // The backend start context owns the child after a successful launch (the OS // backend uses exec.CommandContext), so it must remain tied to the manager // lifecycle rather than the per-command mutation scope. Close fences the // in-flight admission and aborts a process returned after closure. process, startErr := m.backend.Start(m.ctx, kind, spec) slot.mu.Lock() cleanupStartingDone() defer slot.mu.Unlock() if err := m.ensureOpen(); err != nil { if process != nil { _ = process.Abort() } slot.record.State = StateCrashed slot.record.LastResult = "start-failed" _ = m.persistLocked(ctx, slot) return Result{}, err } if startErr != nil { slot.record.State = StateCrashed slot.record.LastResult = "start-failed" _ = m.persistLocked(ctx, slot) return Result{}, startErr } identity := process.Identity() if identity.PID <= 0 || identity.StartToken == "" { _ = process.Abort() slot.record.State = StateCrashed slot.record.LastResult = "start-failed" _ = m.persistLocked(ctx, slot) return Result{}, ErrIdentityAmbiguous } slot.process = process slot.waitDone = make(chan struct{}) slot.record.Identity = &identity slot.record.LastIdentity = &identity slot.record.LastResult = "started" res := Result{ Record: cloneRecord(slot.record), Changed: true, Action: "start", } if err := m.completeReceiptLocked( ctx, slot, commandID, receiptAction, res, ); err != nil { _ = process.Abort() slot.process = nil close(slot.waitDone) slot.waitDone = nil return Result{}, err } m.startOwnedWaiter(kind, slot, process, identity, slot.waitDone) return res, nil } func (m *Manager) reconcileLocked( ctx context.Context, kind ClientKind, slot *processSlot, ) error { if slot.record.Identity == nil { switch slot.record.State { case StateStopped: return nil case StateCrashed: return m.maybeScheduleRestartLocked(kind, slot) default: slot.record.Blocker = "process identity is missing for an in-flight client" if err := m.persistLocked(ctx, slot); err != nil { return err } return ErrIdentityAmbiguous } } observation, err := m.backend.Inspect(ctx, *slot.record.Identity) if err != nil || observation.State == IdentityAmbiguous { slot.record.Blocker = "process identity could not be verified" if persistErr := m.persistLocked(ctx, slot); persistErr != nil { return persistErr } return ErrIdentityAmbiguous } switch observation.State { case IdentityLive: slot.record.Blocker = "" if slot.waitDone == nil { slot.waitDone = make(chan struct{}) m.startAdoptedWatcher(kind, slot, *slot.record.Identity, slot.waitDone) } return m.persistLocked(ctx, slot) case IdentityExited, IdentityStale: identity := *slot.record.Identity slot.record.LastIdentity = &identity slot.record.Identity = nil slot.record.Connected = false if slot.record.State != StateStopped { slot.record.State = StateCrashed slot.record.LastResult = string(observation.State) } slot.record.Blocker = "" if err := m.persistLocked(ctx, slot); err != nil { return err } return m.maybeScheduleRestartLocked(kind, slot) default: return ErrIdentityAmbiguous } } func (m *Manager) ensureLiveStateLocked( ctx context.Context, kind ClientKind, slot *processSlot, ) (bool, error) { if slot.record.Identity == nil { if slot.record.State == StateStarting { return false, ErrIdentityAmbiguous } return false, nil } observation, err := m.backend.Inspect(ctx, *slot.record.Identity) if err != nil || observation.State == IdentityAmbiguous { slot.record.Blocker = "process identity could not be verified" if persistErr := m.persistLocked(ctx, slot); persistErr != nil { return false, persistErr } return false, ErrIdentityAmbiguous } if observation.State == IdentityLive { return true, nil } identity := *slot.record.Identity slot.record.LastIdentity = &identity slot.record.Identity = nil slot.record.Connected = false if slot.record.State != StateStopped { slot.record.State = StateCrashed } slot.record.LastResult = string(observation.State) slot.record.Blocker = "" if err := m.persistLocked(ctx, slot); err != nil { return false, err } return false, nil } func (m *Manager) startOwnedWaiter( kind ClientKind, slot *processSlot, process OwnedProcess, identity ProcessIdentity, done chan struct{}, ) { m.waiters.Add(1) go func() { defer m.waiters.Done() _ = process.Wait() m.finishExit(kind, slot, identity, done) }() } func (m *Manager) startAdoptedWatcher( kind ClientKind, slot *processSlot, identity ProcessIdentity, done chan struct{}, ) { m.waiters.Add(1) go func() { defer m.waiters.Done() ticker := time.NewTicker(m.pollInterval) defer ticker.Stop() for { select { case <-m.ctx.Done(): // Manager cancellation is not process-exit evidence. Retain the // adopted identity for a later daemon to reconcile instead of // fabricating a stopped or crashed transition. return case <-ticker.C: observation, err := m.backend.Inspect(m.ctx, identity) if err == nil && observation.State == IdentityLive { continue } if err != nil || observation.State == IdentityAmbiguous { continue } m.finishExit(kind, slot, identity, done) return } } }() } func (m *Manager) finishExit( kind ClientKind, slot *processSlot, identity ProcessIdentity, done chan struct{}, ) { slot.mu.Lock() defer slot.mu.Unlock() defer close(done) if slot.record.Identity == nil || *slot.record.Identity != identity { return } slot.record.LastIdentity = &identity slot.record.Identity = nil slot.record.Connected = false slot.process = nil slot.waitDone = nil if slot.stopping { slot.record.State = StateStopped slot.record.LastResult = "stopped" } else { slot.record.State = StateCrashed slot.record.LastResult = "crashed" } if err := m.persistLocked(context.Background(), slot); err != nil { slot.record.Blocker = "client exit could not be persisted" return } if !slot.stopping { _ = m.maybeScheduleRestartLocked(kind, slot) } } func (m *Manager) maybeScheduleRestartLocked( kind ClientKind, slot *processSlot, ) error { if !slot.configured || !slot.spec.RestartOnCrash || slot.record.RestartAttempts >= slot.spec.RestartLimit || m.isClosed() { return nil } delay := time.Duration(slot.spec.RestartBackoffMillis) * time.Millisecond m.waiters.Add(1) go func() { defer m.waiters.Done() timer := time.NewTimer(delay) defer timer.Stop() select { case <-m.ctx.Done(): return case <-timer.C: } mutationCtx, finishMutation, err := m.beginMutation(context.Background()) if err != nil { return } defer finishMutation() slot.mu.Lock() if !slot.configured || slot.record.State != StateCrashed || slot.record.Identity != nil || slot.record.RestartAttempts >= slot.spec.RestartLimit || m.isClosed() { slot.mu.Unlock() return } _, _ = m.startLocked( mutationCtx, kind, slot, "", "start", true, ) }() return nil } // checkReceiptLocked replays a durable caller receipt for an exact command_id // and expected action. A completed receipt restores the immutable original // result from its snapshot rather than current slot state; a pending receipt // fails closed; an action mismatch or corrupt projection is a typed error. func (m *Manager) checkReceiptLocked( slot *processSlot, commandID string, expectedAction string, ) (Result, bool, error) { if commandID == "" || slot.record.Commands == nil { return Result{}, false, nil } receipt, ok := slot.record.Commands[commandID] if !ok { return Result{}, false, nil } if receipt.Action != expectedAction { return Result{}, true, ErrCommandActionMismatch } switch receipt.Status { case CommandReceiptCompleted: if !validCompletedReceipt(receipt) { return Result{}, true, ErrCommandReceiptCorrupt } return receipt.Result.Restore(slot.record.Kind), true, nil case CommandReceiptPending: if !validPendingReceipt(receipt) { return Result{}, true, ErrCommandReceiptCorrupt } return Result{}, true, ErrCommandPending default: return Result{}, true, ErrCommandReceiptCorrupt } } // admitReceiptLocked fails closed when a new command_id would exceed the // ledger-aligned receipt capacity, instead of evicting an arbitrary recoverable // command. An existing command_id or an internal lifecycle action (empty id) is // always admitted. func (m *Manager) admitReceiptLocked(slot *processSlot, commandID string) error { if commandID == "" || slot.record.Commands == nil { return nil } if _, exists := slot.record.Commands[commandID]; exists { return nil } if len(slot.record.Commands) >= maxCommandReceipts { return ErrReceiptCapacityReached } return nil } func (m *Manager) setReceiptLocked( slot *processSlot, commandID string, action string, status CommandReceiptStatus, res Result, ) { if commandID == "" { return } if slot.record.Commands == nil { slot.record.Commands = make(map[string]CommandReceipt) } receipt := CommandReceipt{ CommandID: commandID, Action: action, Status: status, } if status == CommandReceiptCompleted { receipt.Result = CommandResultSnapshot{ State: slot.record.State, Connected: slot.record.Connected, Changed: res.Changed, Action: res.Action, } } slot.record.Commands[commandID] = receipt } // completeReceiptLocked persists a completed caller receipt atomically with // respect to the manager's in-memory projection. Callers must first persist // the pending receipt. If the completed projection cannot be saved, restoring // the most recent exact durable projection (which still carries that pending // receipt) prevents an aborted or otherwise non-durable side effect from // being replayed as accepted success. func (m *Manager) completeReceiptLocked( ctx context.Context, slot *processSlot, commandID string, action string, res Result, ) error { pendingRecord := cloneRecord(slot.durableRecord) pendingRevision := slot.revision m.setReceiptLocked(slot, commandID, action, CommandReceiptCompleted, res) if err := m.persistLocked(ctx, slot); err != nil { slot.record = pendingRecord slot.revision = pendingRevision return err } return nil } func (m *Manager) persistLocked( ctx context.Context, slot *processSlot, ) error { slot.record.UpdatedAt = m.now() revision, err := m.store.save(ctx, slot.record, slot.revision) if err != nil { return err } slot.revision = revision slot.durableRecord = cloneRecord(slot.record) return nil } func (m *Manager) slot(kind ClientKind) (*processSlot, error) { if !validKind(kind) { return nil, ErrUnknownClient } slot, ok := m.slots[kind] if !ok { return nil, ErrClientNotConfigured } return slot, nil } func (m *Manager) kinds() []ClientKind { kinds := make([]ClientKind, 0, len(m.slots)) for kind := range m.slots { kinds = append(kinds, kind) } sort.Slice(kinds, func(i, j int) bool { return kinds[i] < kinds[j] }) return kinds } func (m *Manager) isClosed() bool { m.closedMu.RLock() defer m.closedMu.RUnlock() return m.closed } func (m *Manager) ensureOpen() error { if m.isClosed() || m.ctx.Err() != nil { return ErrManagerClosed } return nil } func waitForProcess( ctx context.Context, done <-chan struct{}, timeout time.Duration, ) bool { if done == nil { return true } timer := time.NewTimer(timeout) defer timer.Stop() select { case <-done: return true case <-ctx.Done(): return false case <-timer.C: return false } } func cloneSpec(spec agentconfig.ClientProcessSpec) agentconfig.ClientProcessSpec { out := spec out.Args = append([]string(nil), spec.Args...) out.FocusArgs = append([]string(nil), spec.FocusArgs...) return out }