iop/apps/agent/internal/clientprocess/manager.go

package clientprocess

import (
	"context"
	"errors"
	"fmt"
	"os"
	"sort"
	"sync"
	"syscall"
	"time"

	"iop/packages/go/agentconfig"
)

const (
	defaultStopTimeout  = 3 * time.Second
	defaultPollInterval = 100 * time.Millisecond
)

type Option func(*Manager)

func WithProcessBackend(backend ProcessBackend) Option {
	return func(manager *Manager) {
		if backend != nil {
			manager.backend = backend
		}
	}
}

func WithStopTimeout(timeout time.Duration) Option {
	return func(manager *Manager) {
		if timeout > 0 {
			manager.stopTimeout = timeout
		}
	}
}

func WithPollInterval(interval time.Duration) Option {
	return func(manager *Manager) {
		if interval > 0 {
			manager.pollInterval = interval
		}
	}
}

type processSlot struct {
	mu            sync.Mutex
	configured    bool
	spec          agentconfig.ClientProcessSpec
	record        Record
	durableRecord Record
	revision      string
	process       OwnedProcess
	waitDone      chan struct{}
	startingDone  chan struct{}
	stopping      bool
}

type Manager struct {
	ctx          context.Context
	cancel       context.CancelFunc
	store        *durableStore
	backend      ProcessBackend
	stopTimeout  time.Duration
	pollInterval time.Duration
	slots        map[ClientKind]*processSlot
	now          func() time.Time

	closeOnce sync.Once
	closedMu  sync.RWMutex
	closed    bool
	waiters   sync.WaitGroup

	mutationCtx    context.Context
	mutationCancel context.CancelFunc
	mutationMu     sync.Mutex
	mutationClosed bool
	mutations      sync.WaitGroup
}

func NewManager(
	ctx context.Context,
	specs map[string]agentconfig.ClientProcessSpec,
	store StateStore,
	options ...Option,
) (*Manager, error) {
	if ctx == nil {
		return nil, fmt.Errorf("clientprocess: manager context is required")
	}
	durable, err := newDurableStore(store)
	if err != nil {
		return nil, err
	}
	for rawKind := range specs {
		kind := ClientKind(rawKind)
		if !validKind(kind) {
			return nil, fmt.Errorf("%w: %q", ErrUnknownClient, rawKind)
		}
	}
	managerContext, cancel := context.WithCancel(ctx)
	mutationContext, mutationCancel := context.WithCancel(managerContext)
	manager := &Manager{
		ctx:            managerContext,
		cancel:         cancel,
		mutationCtx:    mutationContext,
		mutationCancel: mutationCancel,
		store:          durable,
		backend:        osProcessBackend{},
		stopTimeout:    defaultStopTimeout,
		pollInterval:   defaultPollInterval,
		slots:          make(map[ClientKind]*processSlot, len(canonicalClientKinds)),
		now:            func() time.Time { return time.Now().UTC() },
	}
	for _, option := range options {
		option(manager)
	}
	for _, kind := range canonicalClientKinds {
		spec, configured := specs[string(kind)]
		record, revision, _, err := durable.load(ctx, kind)
		if err != nil {
			cancel()
			return nil, err
		}
		manager.slots[kind] = &processSlot{
			configured:    configured,
			spec:          cloneSpec(spec),
			record:        record,
			durableRecord: cloneRecord(record),
			revision:      revision,
		}
	}
	return manager, nil
}

func (m *Manager) Reconcile(ctx context.Context) error {
	if err := m.ensureOpen(); err != nil {
		return err
	}
	for _, kind := range m.kinds() {
		slot := m.slots[kind]
		slot.mu.Lock()
		err := m.reconcileLocked(ctx, kind, slot)
		slot.mu.Unlock()
		if err != nil {
			return err
		}
	}
	return nil
}

func (m *Manager) StartConfigured(ctx context.Context) error {
	if err := m.Reconcile(ctx); err != nil {
		return err
	}
	for _, kind := range m.kinds() {
		slot := m.slots[kind]
		if !slot.spec.LaunchOnStart {
			continue
		}
		if _, err := m.startInternal(ctx, kind); err != nil {
			return err
		}
	}
	return nil
}

func (m *Manager) Start(
	ctx context.Context,
	kind ClientKind,
	commandID string,
) (Result, error) {
	slot, err := m.slot(kind)
	if err != nil {
		return Result{}, err
	}
	if err := m.ensureOpen(); err != nil {
		return Result{}, err
	}
	mutationCtx, finishMutation, err := m.beginMutation(ctx)
	if err != nil {
		return Result{}, err
	}
	defer finishMutation()
	slot.mu.Lock()
	for {
		if err := m.ensureOpen(); err != nil {
			slot.mu.Unlock()
			return Result{}, err
		}
		if !slot.configured {
			slot.mu.Unlock()
			return Result{}, ErrClientNotConfigured
		}
		if res, handled, receiptErr := m.checkReceiptLocked(slot, commandID, "start"); handled {
			slot.mu.Unlock()
			return res, receiptErr
		}
		if err := m.admitReceiptLocked(slot, commandID); err != nil {
			slot.mu.Unlock()
			return Result{}, err
		}
		if slot.startingDone != nil {
			startingDone := slot.startingDone
			slot.mu.Unlock()
			select {
			case <-startingDone:
				slot.mu.Lock()
				continue
			case <-ctx.Done():
				return Result{}, ctx.Err()
			case <-m.ctx.Done():
				return Result{}, ErrManagerClosed
			}
		}
		live, err := m.ensureLiveStateLocked(mutationCtx, kind, slot)
		if err != nil {
			slot.mu.Unlock()
			return Result{}, err
		}
		if live {
			slot.record.LastCommandID = commandID
			slot.record.LastAction = "start"
			slot.record.LastResult = "already-running"
			m.setReceiptLocked(slot, commandID, "start", CommandReceiptPending, Result{})
			if err := m.persistLocked(mutationCtx, slot); err != nil {
				slot.mu.Unlock()
				return Result{}, err
			}
			res := Result{
				Record: cloneRecord(slot.record),
				Action: "start",
			}
			if err := m.completeReceiptLocked(
				mutationCtx,
				slot,
				commandID,
				"start",
				res,
			); err != nil {
				slot.mu.Unlock()
				return Result{}, err
			}
			slot.mu.Unlock()
			return res, nil
		}
		slot.record.RestartAttempts = 0
		return m.startLocked(mutationCtx, kind, slot, commandID, "start", false)
	}
}

func (m *Manager) Stop(
	ctx context.Context,
	kind ClientKind,
	commandID string,
) (Result, error) {
	if err := m.ensureOpen(); err != nil {
		return Result{}, err
	}
	mutationCtx, finishMutation, err := m.beginMutation(ctx)
	if err != nil {
		return Result{}, err
	}
	defer finishMutation()
	return m.stop(mutationCtx, kind, commandID)
}

// stop is the shared stop path. The public caller admits a close-fenced
// mutation before entering it; Close invokes it directly after cancellation so
// it can still reap the exact current lifecycle generation.
func (m *Manager) stop(
	ctx context.Context,
	kind ClientKind,
	commandID string,
) (Result, error) {
	slot, err := m.slot(kind)
	if err != nil {
		return Result{}, err
	}
	slot.mu.Lock()
	if res, handled, receiptErr := m.checkReceiptLocked(slot, commandID, "stop"); handled {
		slot.mu.Unlock()
		return res, receiptErr
	}
	if err := m.admitReceiptLocked(slot, commandID); err != nil {
		slot.mu.Unlock()
		return Result{}, err
	}
	live, liveErr := m.ensureLiveStateLocked(ctx, kind, slot)
	if liveErr != nil && !errors.Is(liveErr, ErrIdentityAmbiguous) {
		slot.mu.Unlock()
		return Result{}, liveErr
	}
	if errors.Is(liveErr, ErrIdentityAmbiguous) {
		slot.mu.Unlock()
		return Result{}, liveErr
	}
	if !live {
		slot.record.State = StateStopped
		slot.record.Connected = false
		slot.record.LastCommandID = commandID
		slot.record.LastAction = "stop"
		slot.record.LastResult = "already-stopped"
		slot.record.Blocker = ""
		m.setReceiptLocked(slot, commandID, "stop", CommandReceiptPending, Result{})
		if err := m.persistLocked(ctx, slot); err != nil {
			slot.mu.Unlock()
			return Result{}, err
		}
		res := Result{Record: cloneRecord(slot.record), Action: "stop"}
		if err := m.completeReceiptLocked(
			ctx,
			slot,
			commandID,
			"stop",
			res,
		); err != nil {
			slot.mu.Unlock()
			return Result{}, err
		}
		slot.mu.Unlock()
		return res, nil
	}
	identity := *slot.record.Identity
	waitDone := slot.waitDone
	slot.stopping = true
	slot.record.LastCommandID = commandID
	slot.record.LastAction = "stop"
	slot.record.LastResult = "stopping"
	m.setReceiptLocked(slot, commandID, "stop", CommandReceiptPending, Result{})
	if err := m.persistLocked(ctx, slot); err != nil {
		slot.stopping = false
		slot.mu.Unlock()
		return Result{}, err
	}
	slot.mu.Unlock()

	signalErr := m.backend.Signal(ctx, identity, syscall.SIGTERM)
	if signalErr != nil && !errors.Is(signalErr, os.ErrProcessDone) {
		return Result{}, signalErr
	}
	if !waitForProcess(ctx, waitDone, m.stopTimeout) {
		if err := m.backend.Kill(ctx, identity); err != nil &&
			!errors.Is(err, os.ErrProcessDone) {
			return Result{}, err
		}
		if !waitForProcess(ctx, waitDone, m.stopTimeout) {
			return Result{}, fmt.Errorf("clientprocess: client did not exit after kill")
		}
	}

	slot.mu.Lock()
	defer slot.mu.Unlock()
	slot.stopping = false
	slot.record.State = StateStopped
	slot.record.Connected = false
	slot.record.Identity = nil
	slot.record.LastIdentity = &identity
	slot.record.LastResult = "stopped"
	slot.record.Blocker = ""
	res := Result{
		Record:  cloneRecord(slot.record),
		Changed: true,
		Action:  "stop",
	}
	if err := m.completeReceiptLocked(
		ctx,
		slot,
		commandID,
		"stop",
		res,
	); err != nil {
		return Result{}, err
	}
	return res, nil
}

func (m *Manager) Focus(
	ctx context.Context,
	kind ClientKind,
	commandID string,
) (Result, error) {
	slot, err := m.slot(kind)
	if err != nil {
		return Result{}, err
	}
	if kind != ClientFlutter {
		return Result{}, ErrFocusUnsupported
	}
	mutationCtx, finishMutation, err := m.beginMutation(ctx)
	if err != nil {
		return Result{}, err
	}
	defer finishMutation()
	slot.mu.Lock()
	if err := m.ensureOpen(); err != nil {
		slot.mu.Unlock()
		return Result{}, err
	}
	if !slot.configured {
		slot.mu.Unlock()
		return Result{}, ErrClientNotConfigured
	}
	if res, handled, receiptErr := m.checkReceiptLocked(slot, commandID, "focus"); handled {
		slot.mu.Unlock()
		return res, receiptErr
	}
	if err := m.admitReceiptLocked(slot, commandID); err != nil {
		slot.mu.Unlock()
		return Result{}, err
	}
	live, err := m.ensureLiveStateLocked(mutationCtx, kind, slot)
	if err != nil {
		slot.mu.Unlock()
		return Result{}, err
	}
	if !live {
		slot.mu.Unlock()
		return Result{}, ErrClientNotRunning
	}
	m.setReceiptLocked(slot, commandID, "focus", CommandReceiptPending, Result{})
	if err := m.persistLocked(mutationCtx, slot); err != nil {
		slot.mu.Unlock()
		return Result{}, err
	}
	spec := slot.spec
	slot.mu.Unlock()

	focusErr := m.backend.Focus(mutationCtx, kind, spec)

	slot.mu.Lock()
	defer slot.mu.Unlock()
	if err := m.ensureOpen(); err != nil {
		return Result{}, err
	}
	if focusErr != nil {
		return Result{}, focusErr
	}
	slot.record.FocusCount++
	slot.record.LastCommandID = commandID
	slot.record.LastAction = "focus"
	slot.record.LastResult = "focused"
	res := Result{
		Record:  cloneRecord(slot.record),
		Changed: true,
		Action:  "focus",
	}
	if err := m.completeReceiptLocked(
		mutationCtx,
		slot,
		commandID,
		"focus",
		res,
	); err != nil {
		return Result{}, err
	}
	return res, nil
}

func (m *Manager) StartOrFocusFlutter(
	ctx context.Context,
	commandID string,
) (Result, error) {
	slot, err := m.slot(ClientFlutter)
	if err != nil {
		return Result{}, err
	}
	if err := m.ensureOpen(); err != nil {
		return Result{}, err
	}
	mutationCtx, finishMutation, err := m.beginMutation(ctx)
	if err != nil {
		return Result{}, err
	}
	defer finishMutation()
	slot.mu.Lock()
	if err := m.ensureOpen(); err != nil {
		slot.mu.Unlock()
		return Result{}, err
	}
	if !slot.configured {
		slot.mu.Unlock()
		return Result{}, ErrClientNotConfigured
	}
	if res, handled, receiptErr := m.checkReceiptLocked(slot, commandID, "detail"); handled {
		slot.mu.Unlock()
		return res, receiptErr
	}
	if err := m.admitReceiptLocked(slot, commandID); err != nil {
		slot.mu.Unlock()
		return Result{}, err
	}
	live, err := m.ensureLiveStateLocked(mutationCtx, ClientFlutter, slot)
	if err != nil {
		slot.mu.Unlock()
		return Result{}, err
	}
	if !live {
		slot.record.RestartAttempts = 0
		return m.startLocked(
			mutationCtx,
			ClientFlutter,
			slot,
			commandID,
			"detail",
			false,
		)
	}
	m.setReceiptLocked(slot, commandID, "detail", CommandReceiptPending, Result{})
	if err := m.persistLocked(mutationCtx, slot); err != nil {
		slot.mu.Unlock()
		return Result{}, err
	}
	spec := slot.spec
	slot.mu.Unlock()

	focusErr := m.backend.Focus(mutationCtx, ClientFlutter, spec)

	slot.mu.Lock()
	defer slot.mu.Unlock()
	if err := m.ensureOpen(); err != nil {
		return Result{}, err
	}
	if focusErr != nil {
		return Result{}, focusErr
	}
	slot.record.FocusCount++
	slot.record.LastCommandID = commandID
	slot.record.LastAction = "detail"
	slot.record.LastResult = "focused"
	res := Result{
		Record:  cloneRecord(slot.record),
		Changed: true,
		Action:  "focus",
	}
	if err := m.completeReceiptLocked(
		mutationCtx,
		slot,
		commandID,
		"detail",
		res,
	); err != nil {
		return Result{}, err
	}
	return res, nil
}

func (m *Manager) SetConnected(
	ctx context.Context,
	kind ClientKind,
	connected bool,
) (Record, error) {
	slot, err := m.slot(kind)
	if err != nil {
		return Record{}, err
	}
	if err := m.ensureOpen(); err != nil {
		return Record{}, err
	}
	mutationCtx, finishMutation, err := m.beginMutation(ctx)
	if err != nil {
		return Record{}, err
	}
	defer finishMutation()
	slot.mu.Lock()
	defer slot.mu.Unlock()
	if err := m.ensureOpen(); err != nil {
		return Record{}, err
	}
	live, err := m.ensureLiveStateLocked(mutationCtx, kind, slot)
	if err != nil {
		return Record{}, err
	}
	if !live {
		return Record{}, ErrClientNotRunning
	}
	slot.record.Connected = connected
	if connected {
		slot.record.State = StateConnected
		slot.record.LastResult = "connected"
	} else {
		slot.record.State = StateStarting
		slot.record.LastResult = "disconnected"
	}
	if err := m.persistLocked(mutationCtx, slot); err != nil {
		return Record{}, err
	}
	return cloneRecord(slot.record), nil
}

func (m *Manager) Status(kind ClientKind) (Record, error) {
	slot, err := m.slot(kind)
	if err != nil {
		return Record{}, err
	}
	slot.mu.Lock()
	defer slot.mu.Unlock()
	return cloneRecord(slot.record), nil
}

func (m *Manager) Close(ctx context.Context) error {
	var closeErr error
	m.closeOnce.Do(func() {
		m.closedMu.Lock()
		m.closed = true
		m.closedMu.Unlock()

		// Atomically stop admitting new mutations, cancel every admitted
		// blocking mutation, and wait for them to finish before reaping the
		// current client generation.
		m.mutationMu.Lock()
		m.mutationClosed = true
		m.mutationMu.Unlock()
		m.mutationCancel()
		m.mutations.Wait()

		for _, kind := range m.kinds() {
			if _, err := m.stopInternal(ctx, kind); err != nil &&
				!errors.Is(err, ErrClientNotRunning) &&
				!errors.Is(err, ErrClientNotConfigured) {
				closeErr = errors.Join(closeErr, err)
			}
		}
		m.cancel()
		m.waiters.Wait()
	})
	return closeErr
}

// startInternal launches a configured client for an automatic daemon lifecycle
// action. It reuses the external start path but supplies no caller command_id,
// so it never writes or replays a durable caller receipt and every daemon
// generation launches its current process.
func (m *Manager) startInternal(ctx context.Context, kind ClientKind) (Result, error) {
	return m.Start(ctx, kind, "")
}

// stopInternal stops the current client generation for an automatic daemon
// lifecycle action without participating in caller command_id receipt replay.
func (m *Manager) stopInternal(ctx context.Context, kind ClientKind) (Result, error) {
	return m.stop(ctx, kind, "")
}

// beginMutation admits one manager-owned blocking mutation. It fails closed
// once the manager begins closing, returns a context cancelled by either the
// caller or manager closure, and registers the mutation so Close waits for it
// to finish before reaping the client.
func (m *Manager) beginMutation(
	caller context.Context,
) (context.Context, func(), error) {
	m.mutationMu.Lock()
	if m.mutationClosed || m.isClosed() {
		m.mutationMu.Unlock()
		return nil, nil, ErrManagerClosed
	}
	m.mutations.Add(1)
	m.mutationMu.Unlock()

	ctx, cancel := context.WithCancel(m.mutationCtx)
	stop := context.AfterFunc(caller, cancel)
	var once sync.Once
	finish := func() {
		once.Do(func() {
			stop()
			cancel()
			m.mutations.Done()
		})
	}
	return ctx, finish, nil
}

func (m *Manager) startLocked(
	ctx context.Context,
	kind ClientKind,
	slot *processSlot,
	commandID string,
	receiptAction string,
	automatic bool,
) (Result, error) {
	if err := m.ensureOpen(); err != nil {
		slot.mu.Unlock()
		return Result{}, err
	}
	if automatic {
		slot.record.RestartAttempts++
	}
	startingDone := make(chan struct{})
	slot.startingDone = startingDone
	cleanupStartingDone := func() {
		if slot.startingDone == startingDone {
			close(startingDone)
			slot.startingDone = nil
		}
	}
	slot.record.State = StateStarting
	slot.record.Connected = false
	slot.record.Identity = nil
	slot.record.LastCommandID = commandID
	slot.record.LastAction = "start"
	slot.record.LastResult = "starting"
	slot.record.Blocker = ""
	m.setReceiptLocked(slot, commandID, receiptAction, CommandReceiptPending, Result{})
	if err := m.persistLocked(ctx, slot); err != nil {
		cleanupStartingDone()
		slot.mu.Unlock()
		return Result{}, err
	}

	spec := slot.spec
	slot.mu.Unlock()

	// The backend start context owns the child after a successful launch (the OS
	// backend uses exec.CommandContext), so it must remain tied to the manager
	// lifecycle rather than the per-command mutation scope. Close fences the
	// in-flight admission and aborts a process returned after closure.
	process, startErr := m.backend.Start(m.ctx, kind, spec)

	slot.mu.Lock()
	cleanupStartingDone()
	defer slot.mu.Unlock()

	if err := m.ensureOpen(); err != nil {
		if process != nil {
			_ = process.Abort()
		}
		slot.record.State = StateCrashed
		slot.record.LastResult = "start-failed"
		_ = m.persistLocked(ctx, slot)
		return Result{}, err
	}
	if startErr != nil {
		slot.record.State = StateCrashed
		slot.record.LastResult = "start-failed"
		_ = m.persistLocked(ctx, slot)
		return Result{}, startErr
	}
	identity := process.Identity()
	if identity.PID <= 0 || identity.StartToken == "" {
		_ = process.Abort()
		slot.record.State = StateCrashed
		slot.record.LastResult = "start-failed"
		_ = m.persistLocked(ctx, slot)
		return Result{}, ErrIdentityAmbiguous
	}
	slot.process = process
	slot.waitDone = make(chan struct{})
	slot.record.Identity = &identity
	slot.record.LastIdentity = &identity
	slot.record.LastResult = "started"
	res := Result{
		Record:  cloneRecord(slot.record),
		Changed: true,
		Action:  "start",
	}
	if err := m.completeReceiptLocked(
		ctx,
		slot,
		commandID,
		receiptAction,
		res,
	); err != nil {
		_ = process.Abort()
		slot.process = nil
		close(slot.waitDone)
		slot.waitDone = nil
		return Result{}, err
	}
	m.startOwnedWaiter(kind, slot, process, identity, slot.waitDone)
	return res, nil
}

func (m *Manager) reconcileLocked(
	ctx context.Context,
	kind ClientKind,
	slot *processSlot,
) error {
	if slot.record.Identity == nil {
		switch slot.record.State {
		case StateStopped:
			return nil
		case StateCrashed:
			return m.maybeScheduleRestartLocked(kind, slot)
		default:
			slot.record.Blocker = "process identity is missing for an in-flight client"
			if err := m.persistLocked(ctx, slot); err != nil {
				return err
			}
			return ErrIdentityAmbiguous
		}
	}
	observation, err := m.backend.Inspect(ctx, *slot.record.Identity)
	if err != nil || observation.State == IdentityAmbiguous {
		slot.record.Blocker = "process identity could not be verified"
		if persistErr := m.persistLocked(ctx, slot); persistErr != nil {
			return persistErr
		}
		return ErrIdentityAmbiguous
	}
	switch observation.State {
	case IdentityLive:
		slot.record.Blocker = ""
		if slot.waitDone == nil {
			slot.waitDone = make(chan struct{})
			m.startAdoptedWatcher(kind, slot, *slot.record.Identity, slot.waitDone)
		}
		return m.persistLocked(ctx, slot)
	case IdentityExited, IdentityStale:
		identity := *slot.record.Identity
		slot.record.LastIdentity = &identity
		slot.record.Identity = nil
		slot.record.Connected = false
		if slot.record.State != StateStopped {
			slot.record.State = StateCrashed
			slot.record.LastResult = string(observation.State)
		}
		slot.record.Blocker = ""
		if err := m.persistLocked(ctx, slot); err != nil {
			return err
		}
		return m.maybeScheduleRestartLocked(kind, slot)
	default:
		return ErrIdentityAmbiguous
	}
}

func (m *Manager) ensureLiveStateLocked(
	ctx context.Context,
	kind ClientKind,
	slot *processSlot,
) (bool, error) {
	if slot.record.Identity == nil {
		if slot.record.State == StateStarting {
			return false, ErrIdentityAmbiguous
		}
		return false, nil
	}
	observation, err := m.backend.Inspect(ctx, *slot.record.Identity)
	if err != nil || observation.State == IdentityAmbiguous {
		slot.record.Blocker = "process identity could not be verified"
		if persistErr := m.persistLocked(ctx, slot); persistErr != nil {
			return false, persistErr
		}
		return false, ErrIdentityAmbiguous
	}
	if observation.State == IdentityLive {
		return true, nil
	}
	identity := *slot.record.Identity
	slot.record.LastIdentity = &identity
	slot.record.Identity = nil
	slot.record.Connected = false
	if slot.record.State != StateStopped {
		slot.record.State = StateCrashed
	}
	slot.record.LastResult = string(observation.State)
	slot.record.Blocker = ""
	if err := m.persistLocked(ctx, slot); err != nil {
		return false, err
	}
	return false, nil
}

func (m *Manager) startOwnedWaiter(
	kind ClientKind,
	slot *processSlot,
	process OwnedProcess,
	identity ProcessIdentity,
	done chan struct{},
) {
	m.waiters.Add(1)
	go func() {
		defer m.waiters.Done()
		_ = process.Wait()
		m.finishExit(kind, slot, identity, done)
	}()
}

func (m *Manager) startAdoptedWatcher(
	kind ClientKind,
	slot *processSlot,
	identity ProcessIdentity,
	done chan struct{},
) {
	m.waiters.Add(1)
	go func() {
		defer m.waiters.Done()
		ticker := time.NewTicker(m.pollInterval)
		defer ticker.Stop()
		for {
			select {
			case <-m.ctx.Done():
				// Manager cancellation is not process-exit evidence. Retain the
				// adopted identity for a later daemon to reconcile instead of
				// fabricating a stopped or crashed transition.
				return
			case <-ticker.C:
				observation, err := m.backend.Inspect(m.ctx, identity)
				if err == nil && observation.State == IdentityLive {
					continue
				}
				if err != nil || observation.State == IdentityAmbiguous {
					continue
				}
				m.finishExit(kind, slot, identity, done)
				return
			}
		}
	}()
}

func (m *Manager) finishExit(
	kind ClientKind,
	slot *processSlot,
	identity ProcessIdentity,
	done chan struct{},
) {
	slot.mu.Lock()
	defer slot.mu.Unlock()
	defer close(done)
	if slot.record.Identity == nil ||
		*slot.record.Identity != identity {
		return
	}
	slot.record.LastIdentity = &identity
	slot.record.Identity = nil
	slot.record.Connected = false
	slot.process = nil
	slot.waitDone = nil
	if slot.stopping {
		slot.record.State = StateStopped
		slot.record.LastResult = "stopped"
	} else {
		slot.record.State = StateCrashed
		slot.record.LastResult = "crashed"
	}
	if err := m.persistLocked(context.Background(), slot); err != nil {
		slot.record.Blocker = "client exit could not be persisted"
		return
	}
	if !slot.stopping {
		_ = m.maybeScheduleRestartLocked(kind, slot)
	}
}

func (m *Manager) maybeScheduleRestartLocked(
	kind ClientKind,
	slot *processSlot,
) error {
	if !slot.configured ||
		!slot.spec.RestartOnCrash ||
		slot.record.RestartAttempts >= slot.spec.RestartLimit ||
		m.isClosed() {
		return nil
	}
	delay := time.Duration(slot.spec.RestartBackoffMillis) * time.Millisecond
	m.waiters.Add(1)
	go func() {
		defer m.waiters.Done()
		timer := time.NewTimer(delay)
		defer timer.Stop()
		select {
		case <-m.ctx.Done():
			return
		case <-timer.C:
		}
		mutationCtx, finishMutation, err := m.beginMutation(context.Background())
		if err != nil {
			return
		}
		defer finishMutation()
		slot.mu.Lock()
		if !slot.configured ||
			slot.record.State != StateCrashed ||
			slot.record.Identity != nil ||
			slot.record.RestartAttempts >= slot.spec.RestartLimit ||
			m.isClosed() {
			slot.mu.Unlock()
			return
		}
		_, _ = m.startLocked(
			mutationCtx,
			kind,
			slot,
			"",
			"start",
			true,
		)
	}()
	return nil
}

// checkReceiptLocked replays a durable caller receipt for an exact command_id
// and expected action. A completed receipt restores the immutable original
// result from its snapshot rather than current slot state; a pending receipt
// fails closed; an action mismatch or corrupt projection is a typed error.
func (m *Manager) checkReceiptLocked(
	slot *processSlot,
	commandID string,
	expectedAction string,
) (Result, bool, error) {
	if commandID == "" || slot.record.Commands == nil {
		return Result{}, false, nil
	}
	receipt, ok := slot.record.Commands[commandID]
	if !ok {
		return Result{}, false, nil
	}
	if receipt.Action != expectedAction {
		return Result{}, true, ErrCommandActionMismatch
	}
	switch receipt.Status {
	case CommandReceiptCompleted:
		if !validCompletedReceipt(receipt) {
			return Result{}, true, ErrCommandReceiptCorrupt
		}
		return receipt.Result.Restore(slot.record.Kind), true, nil
	case CommandReceiptPending:
		if !validPendingReceipt(receipt) {
			return Result{}, true, ErrCommandReceiptCorrupt
		}
		return Result{}, true, ErrCommandPending
	default:
		return Result{}, true, ErrCommandReceiptCorrupt
	}
}

// admitReceiptLocked fails closed when a new command_id would exceed the
// ledger-aligned receipt capacity, instead of evicting an arbitrary recoverable
// command. An existing command_id or an internal lifecycle action (empty id) is
// always admitted.
func (m *Manager) admitReceiptLocked(slot *processSlot, commandID string) error {
	if commandID == "" || slot.record.Commands == nil {
		return nil
	}
	if _, exists := slot.record.Commands[commandID]; exists {
		return nil
	}
	if len(slot.record.Commands) >= maxCommandReceipts {
		return ErrReceiptCapacityReached
	}
	return nil
}

func (m *Manager) setReceiptLocked(
	slot *processSlot,
	commandID string,
	action string,
	status CommandReceiptStatus,
	res Result,
) {
	if commandID == "" {
		return
	}
	if slot.record.Commands == nil {
		slot.record.Commands = make(map[string]CommandReceipt)
	}
	receipt := CommandReceipt{
		CommandID: commandID,
		Action:    action,
		Status:    status,
	}
	if status == CommandReceiptCompleted {
		receipt.Result = CommandResultSnapshot{
			State:     slot.record.State,
			Connected: slot.record.Connected,
			Changed:   res.Changed,
			Action:    res.Action,
		}
	}
	slot.record.Commands[commandID] = receipt
}

// completeReceiptLocked persists a completed caller receipt atomically with
// respect to the manager's in-memory projection. Callers must first persist
// the pending receipt. If the completed projection cannot be saved, restoring
// the most recent exact durable projection (which still carries that pending
// receipt) prevents an aborted or otherwise non-durable side effect from
// being replayed as accepted success.
func (m *Manager) completeReceiptLocked(
	ctx context.Context,
	slot *processSlot,
	commandID string,
	action string,
	res Result,
) error {
	pendingRecord := cloneRecord(slot.durableRecord)
	pendingRevision := slot.revision
	m.setReceiptLocked(slot, commandID, action, CommandReceiptCompleted, res)
	if err := m.persistLocked(ctx, slot); err != nil {
		slot.record = pendingRecord
		slot.revision = pendingRevision
		return err
	}
	return nil
}

func (m *Manager) persistLocked(
	ctx context.Context,
	slot *processSlot,
) error {
	slot.record.UpdatedAt = m.now()
	revision, err := m.store.save(ctx, slot.record, slot.revision)
	if err != nil {
		return err
	}
	slot.revision = revision
	slot.durableRecord = cloneRecord(slot.record)
	return nil
}

func (m *Manager) slot(kind ClientKind) (*processSlot, error) {
	if !validKind(kind) {
		return nil, ErrUnknownClient
	}
	slot, ok := m.slots[kind]
	if !ok {
		return nil, ErrClientNotConfigured
	}
	return slot, nil
}

func (m *Manager) kinds() []ClientKind {
	kinds := make([]ClientKind, 0, len(m.slots))
	for kind := range m.slots {
		kinds = append(kinds, kind)
	}
	sort.Slice(kinds, func(i, j int) bool { return kinds[i] < kinds[j] })
	return kinds
}

func (m *Manager) isClosed() bool {
	m.closedMu.RLock()
	defer m.closedMu.RUnlock()
	return m.closed
}

func (m *Manager) ensureOpen() error {
	if m.isClosed() || m.ctx.Err() != nil {
		return ErrManagerClosed
	}
	return nil
}

func waitForProcess(
	ctx context.Context,
	done <-chan struct{},
	timeout time.Duration,
) bool {
	if done == nil {
		return true
	}
	timer := time.NewTimer(timeout)
	defer timer.Stop()
	select {
	case <-done:
		return true
	case <-ctx.Done():
		return false
	case <-timer.C:
		return false
	}
}

func cloneSpec(spec agentconfig.ClientProcessSpec) agentconfig.ClientProcessSpec {
	out := spec
	out.Args = append([]string(nil), spec.Args...)
	out.FocusArgs = append([]string(nil), spec.FocusArgs...)
	return out
}