iop/apps/agent/internal/clientprocess/store_test.go

package clientprocess

import (
	"context"
	"errors"
	"os"
	"path/filepath"
	"strconv"
	"sync"
	"testing"
	"time"

	"iop/packages/go/agentconfig"
	"iop/packages/go/agentstate"
	"iop/packages/go/agenttask"
)

func TestStoreRoundTripAndConflict(t *testing.T) {
	base, err := agentstate.NewStore(filepath.Join(t.TempDir(), "state.json"))
	if err != nil {
		t.Fatalf("NewStore: %v", err)
	}
	store, err := newDurableStore(base)
	if err != nil {
		t.Fatalf("newDurableStore: %v", err)
	}
	record := initialRecord(ClientFlutter)
	record.UpdatedAt = time.Now().UTC()
	revision, err := store.save(context.Background(), record, "")
	if err != nil {
		t.Fatalf("save: %v", err)
	}
	loaded, loadedRevision, found, err := store.load(
		context.Background(),
		ClientFlutter,
	)
	if err != nil || !found {
		t.Fatalf("load found=%t err=%v", found, err)
	}
	if loadedRevision != revision ||
		loaded.Kind != ClientFlutter ||
		loaded.State != StateStopped {
		t.Fatalf("loaded record=%+v revision=%q", loaded, loadedRevision)
	}
	if _, err := store.save(
		context.Background(),
		record,
		"",
	); !errors.Is(err, ErrStateConflict) {
		t.Fatalf("stale save error = %v, want ErrStateConflict", err)
	}
}

func TestRecordRejectsContradictoryLifecycleProjection(t *testing.T) {
	identity := ProcessIdentity{PID: 100, StartToken: "token-1"}
	validTime := time.Now().UTC()

	tests := []struct {
		name    string
		record  Record
		wantErr bool
	}{
		{
			name: "valid stopped",
			record: Record{
				SchemaVersion: RecordSchemaVersion,
				Kind:          ClientFlutter,
				State:         StateStopped,
				UpdatedAt:     validTime,
			},
			wantErr: false,
		},
		{
			name: "stopped with identity",
			record: Record{
				SchemaVersion: RecordSchemaVersion,
				Kind:          ClientFlutter,
				State:         StateStopped,
				Identity:      &identity,
				UpdatedAt:     validTime,
			},
			wantErr: true,
		},
		{
			name: "stopped with connected",
			record: Record{
				SchemaVersion: RecordSchemaVersion,
				Kind:          ClientFlutter,
				State:         StateStopped,
				Connected:     true,
				UpdatedAt:     validTime,
			},
			wantErr: true,
		},
		{
			name: "crashed with identity",
			record: Record{
				SchemaVersion: RecordSchemaVersion,
				Kind:          ClientFlutter,
				State:         StateCrashed,
				Identity:      &identity,
				UpdatedAt:     validTime,
			},
			wantErr: true,
		},
		{
			name: "crashed with connected",
			record: Record{
				SchemaVersion: RecordSchemaVersion,
				Kind:          ClientFlutter,
				State:         StateCrashed,
				Connected:     true,
				UpdatedAt:     validTime,
			},
			wantErr: true,
		},
		{
			name: "connected with connected false",
			record: Record{
				SchemaVersion: RecordSchemaVersion,
				Kind:          ClientFlutter,
				State:         StateConnected,
				Identity:      &identity,
				Connected:     false,
				UpdatedAt:     validTime,
			},
			wantErr: true,
		},
		{
			name: "connected with nil identity",
			record: Record{
				SchemaVersion: RecordSchemaVersion,
				Kind:          ClientFlutter,
				State:         StateConnected,
				Identity:      nil,
				Connected:     true,
				UpdatedAt:     validTime,
			},
			wantErr: true,
		},
		{
			name: "starting with connected true",
			record: Record{
				SchemaVersion: RecordSchemaVersion,
				Kind:          ClientFlutter,
				State:         StateStarting,
				Identity:      &identity,
				Connected:     true,
				UpdatedAt:     validTime,
			},
			wantErr: true,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			err := validateRecord(tt.record, tt.record.Kind)
			if (err != nil) != tt.wantErr {
				t.Errorf("validateRecord() error = %v, wantErr %v", err, tt.wantErr)
			}
		})
	}
}

func TestRecordRejectsInvalidCommandReceiptProjection(t *testing.T) {
	identity := ProcessIdentity{PID: 100, StartToken: "token-1"}
	base := func(commands map[string]CommandReceipt) Record {
		return Record{
			SchemaVersion: RecordSchemaVersion,
			Kind:          ClientFlutter,
			State:         StateConnected,
			Connected:     true,
			Identity:      &identity,
			Commands:      commands,
			UpdatedAt:     time.Now().UTC(),
		}
	}
	tests := []struct {
		name    string
		record  Record
		wantErr bool
	}{
		{
			name: "valid start receipt while starting",
			record: base(map[string]CommandReceipt{
				"cmd-1": {
					CommandID: "cmd-1",
					Action:    "start",
					Status:    CommandReceiptCompleted,
					Result: CommandResultSnapshot{
						State: StateStarting, Action: "start", Changed: true,
					},
				},
			}),
			wantErr: false,
		},
		{
			name: "valid start receipt while connected",
			record: base(map[string]CommandReceipt{
				"cmd-1": {
					CommandID: "cmd-1", Action: "start", Status: CommandReceiptCompleted,
					Result: CommandResultSnapshot{State: StateConnected, Connected: true, Action: "start"},
				},
			}),
			wantErr: false,
		},
		{
			name: "valid stopped receipt",
			record: base(map[string]CommandReceipt{
				"cmd-1": {
					CommandID: "cmd-1", Action: "stop", Status: CommandReceiptCompleted,
					Result: CommandResultSnapshot{State: StateStopped, Action: "stop", Changed: true},
				},
			}),
			wantErr: false,
		},
		{
			name: "valid focus receipt",
			record: base(map[string]CommandReceipt{
				"cmd-1": {
					CommandID: "cmd-1", Action: "focus", Status: CommandReceiptCompleted,
					Result: CommandResultSnapshot{State: StateConnected, Connected: true, Action: "focus", Changed: true},
				},
			}),
			wantErr: false,
		},
		{
			name: "valid detail start receipt",
			record: base(map[string]CommandReceipt{
				"cmd-1": {
					CommandID: "cmd-1", Action: "detail", Status: CommandReceiptCompleted,
					Result: CommandResultSnapshot{State: StateStarting, Action: "start", Changed: true},
				},
			}),
			wantErr: false,
		},
		{
			name: "valid detail focus receipt",
			record: base(map[string]CommandReceipt{
				"cmd-1": {
					CommandID: "cmd-1", Action: "detail", Status: CommandReceiptCompleted,
					Result: CommandResultSnapshot{State: StateConnected, Connected: true, Action: "focus", Changed: true},
				},
			}),
			wantErr: false,
		},
		{
			name: "valid pending receipt without result",
			record: base(map[string]CommandReceipt{
				"cmd-1": {
					CommandID: "cmd-1",
					Action:    "detail",
					Status:    CommandReceiptPending,
				},
			}),
			wantErr: false,
		},
		{
			name: "key mismatch",
			record: base(map[string]CommandReceipt{
				"cmd-1": {
					CommandID: "other",
					Action:    "start",
					Status:    CommandReceiptCompleted,
					Result:    CommandResultSnapshot{State: StateStarting, Action: "start"},
				},
			}),
			wantErr: true,
		},
		{
			name: "invalid command action",
			record: base(map[string]CommandReceipt{
				"cmd-1": {
					CommandID: "cmd-1",
					Action:    "resize",
					Status:    CommandReceiptCompleted,
					Result:    CommandResultSnapshot{State: StateConnected, Action: "focus"},
				},
			}),
			wantErr: true,
		},
		{
			name: "invalid status",
			record: base(map[string]CommandReceipt{
				"cmd-1": {
					CommandID: "cmd-1",
					Action:    "focus",
					Status:    CommandReceiptStatus("archived"),
				},
			}),
			wantErr: true,
		},
		{
			name: "completed with invalid result action",
			record: base(map[string]CommandReceipt{
				"cmd-1": {
					CommandID: "cmd-1",
					Action:    "detail",
					Status:    CommandReceiptCompleted,
					Result:    CommandResultSnapshot{State: StateConnected, Action: "detail"},
				},
			}),
			wantErr: true,
		},
		{
			name: "completed with invalid result state",
			record: base(map[string]CommandReceipt{
				"cmd-1": {
					CommandID: "cmd-1",
					Action:    "focus",
					Status:    CommandReceiptCompleted,
					Result:    CommandResultSnapshot{State: LifecycleState("frozen"), Action: "focus"},
				},
			}),
			wantErr: true,
		},
		{
			name: "pending with completed projection",
			record: base(map[string]CommandReceipt{
				"cmd-1": {
					CommandID: "cmd-1",
					Action:    "focus",
					Status:    CommandReceiptPending,
					Result: CommandResultSnapshot{
						State: StateConnected, Connected: true, Action: "focus",
					},
				},
			}),
			wantErr: true,
		},
		{
			name: "completed start with stopped result",
			record: base(map[string]CommandReceipt{
				"cmd-1": {
					CommandID: "cmd-1",
					Action:    "start",
					Status:    CommandReceiptCompleted,
					Result: CommandResultSnapshot{
						State: StateStopped, Action: "start",
					},
				},
			}),
			wantErr: true,
		},
		{
			name: "completed stop with connected result",
			record: base(map[string]CommandReceipt{
				"cmd-1": {
					CommandID: "cmd-1",
					Action:    "stop",
					Status:    CommandReceiptCompleted,
					Result:    CommandResultSnapshot{State: StateConnected, Connected: true, Action: "stop"},
				},
			}),
			wantErr: true,
		},
		{
			name: "completed focus without changed result",
			record: base(map[string]CommandReceipt{
				"cmd-1": {
					CommandID: "cmd-1", Action: "focus", Status: CommandReceiptCompleted,
					Result: CommandResultSnapshot{State: StateConnected, Connected: true, Action: "focus"},
				},
			}),
			wantErr: true,
		},
		{
			name: "completed focus with crashed result",
			record: base(map[string]CommandReceipt{
				"cmd-1": {
					CommandID: "cmd-1", Action: "focus", Status: CommandReceiptCompleted,
					Result: CommandResultSnapshot{State: StateCrashed, Action: "focus", Changed: true},
				},
			}),
			wantErr: true,
		},
		{
			name: "completed detail start with connected result",
			record: base(map[string]CommandReceipt{
				"cmd-1": {
					CommandID: "cmd-1", Action: "detail", Status: CommandReceiptCompleted,
					Result: CommandResultSnapshot{State: StateConnected, Connected: true, Action: "start", Changed: true},
				},
			}),
			wantErr: true,
		},
		{
			name: "completed detail focus without changed result",
			record: base(map[string]CommandReceipt{
				"cmd-1": {
					CommandID: "cmd-1", Action: "detail", Status: CommandReceiptCompleted,
					Result: CommandResultSnapshot{State: StateConnected, Connected: true, Action: "focus"},
				},
			}),
			wantErr: true,
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			err := validateRecord(tt.record, tt.record.Kind)
			if (err != nil) != tt.wantErr {
				t.Errorf("validateRecord() error = %v, wantErr %v", err, tt.wantErr)
			}
		})
	}
}

func TestCommandReceiptCapacityMatchesLedger(t *testing.T) {
	// The manager receipt bound must retain a replayable receipt for every
	// command the authoritative 4096-entry local-control ledger may recover.
	if maxCommandReceipts != 4096 {
		t.Fatalf("maxCommandReceipts = %d, want 4096 to match the local-control ledger", maxCommandReceipts)
	}
	build := func(count int) Record {
		commands := make(map[string]CommandReceipt, count)
		for i := 0; i < count; i++ {
			id := "cmd-" + strconv.Itoa(i)
			commands[id] = CommandReceipt{
				CommandID: id,
				Action:    "focus",
				Status:    CommandReceiptPending,
			}
		}
		return Record{
			SchemaVersion: RecordSchemaVersion,
			Kind:          ClientFlutter,
			State:         StateStopped,
			Commands:      commands,
			UpdatedAt:     time.Now().UTC(),
		}
	}
	if err := validateRecord(build(maxCommandReceipts), ClientFlutter); err != nil {
		t.Fatalf("record at ledger capacity rejected: %v", err)
	}
	if err := validateRecord(build(maxCommandReceipts+1), ClientFlutter); err == nil {
		t.Fatalf("record above ledger capacity was accepted")
	}
}

func TestReconcileBlocksAmbiguousIdentityWithoutLaunch(t *testing.T) {
	base, err := agentstate.NewStore(filepath.Join(t.TempDir(), "state.json"))
	if err != nil {
		t.Fatalf("NewStore: %v", err)
	}
	durable, _ := newDurableStore(base)
	identity := ProcessIdentity{PID: 71, StartToken: "start-a"}
	record := initialRecord(ClientFlutter)
	record.State = StateConnected
	record.Connected = true
	record.Identity = &identity
	record.UpdatedAt = time.Now().UTC()
	if _, err := durable.save(context.Background(), record, ""); err != nil {
		t.Fatalf("seed record: %v", err)
	}
	backend := &fakeProcessBackend{
		observation: IdentityObservation{State: IdentityAmbiguous},
		inspectErr:  ErrIdentityAmbiguous,
	}
	manager, err := NewManager(
		context.Background(),
		map[string]agentconfig.ClientProcessSpec{
			string(ClientFlutter): fakeSpec(),
		},
		base,
		WithProcessBackend(backend),
	)
	if err != nil {
		t.Fatalf("NewManager: %v", err)
	}
	if err := manager.Reconcile(
		context.Background(),
	); !errors.Is(err, ErrIdentityAmbiguous) {
		t.Fatalf("Reconcile error = %v, want ErrIdentityAmbiguous", err)
	}
	if backend.startCount() != 0 {
		t.Fatalf("ambiguous recovery starts = %d, want 0", backend.startCount())
	}
	status, err := manager.Status(ClientFlutter)
	if err != nil {
		t.Fatalf("Status: %v", err)
	}
	if status.Identity == nil || status.Blocker == "" {
		t.Fatalf("ambiguous status = %+v", status)
	}
	manager.cancel()
}

func TestReconcileClassifiesExitedAndStaleIdentity(t *testing.T) {
	for _, state := range []IdentityState{IdentityExited, IdentityStale} {
		t.Run(string(state), func(t *testing.T) {
			base, err := agentstate.NewStore(filepath.Join(t.TempDir(), "state.json"))
			if err != nil {
				t.Fatalf("NewStore: %v", err)
			}
			durable, _ := newDurableStore(base)
			identity := ProcessIdentity{PID: 72, StartToken: "start-old"}
			record := initialRecord(ClientUnity)
			record.State = StateConnected
			record.Connected = true
			record.Identity = &identity
			record.UpdatedAt = time.Now().UTC()
			if _, err := durable.save(context.Background(), record, ""); err != nil {
				t.Fatalf("seed record: %v", err)
			}
			backend := &fakeProcessBackend{
				observation: IdentityObservation{
					State: state,
					Identity: ProcessIdentity{
						PID:        identity.PID,
						StartToken: "start-new",
					},
				},
			}
			manager, err := NewManager(
				context.Background(),
				map[string]agentconfig.ClientProcessSpec{
					string(ClientUnity): fakeSpec(),
				},
				base,
				WithProcessBackend(backend),
			)
			if err != nil {
				t.Fatalf("NewManager: %v", err)
			}
			if err := manager.Reconcile(context.Background()); err != nil {
				t.Fatalf("Reconcile: %v", err)
			}
			status, err := manager.Status(ClientUnity)
			if err != nil {
				t.Fatalf("Status: %v", err)
			}
			if status.State != StateCrashed ||
				status.Identity != nil ||
				status.LastIdentity == nil ||
				*status.LastIdentity != identity {
				t.Fatalf("reconciled status = %+v", status)
			}
			if backend.startCount() != 0 {
				t.Fatalf("reconcile starts = %d, want 0", backend.startCount())
			}
			manager.cancel()
		})
	}
}

func TestStoreCASConflictPreventsProcessStart(t *testing.T) {
	base, err := agentstate.NewStore(filepath.Join(t.TempDir(), "state.json"))
	if err != nil {
		t.Fatalf("NewStore: %v", err)
	}
	store := &conflictStateStore{StateStore: base, conflict: true}
	backend := &fakeProcessBackend{
		observation: IdentityObservation{State: IdentityExited},
	}
	manager, err := NewManager(
		context.Background(),
		map[string]agentconfig.ClientProcessSpec{
			string(ClientFlutter): fakeSpec(),
		},
		store,
		WithProcessBackend(backend),
	)
	if err != nil {
		t.Fatalf("NewManager: %v", err)
	}
	if _, err := manager.Start(
		context.Background(),
		ClientFlutter,
		"conflict",
	); !errors.Is(err, ErrStateConflict) {
		t.Fatalf("Start error = %v, want ErrStateConflict", err)
	}
	if backend.startCount() != 0 {
		t.Fatalf("CAS conflict starts = %d, want 0", backend.startCount())
	}
	manager.cancel()
}

type conflictStateStore struct {
	StateStore
	mu       sync.Mutex
	conflict bool
}

func (s *conflictStateStore) CompareAndSwapIntegrationRecord(
	ctx context.Context,
	key string,
	revision string,
	payload []byte,
) (string, error) {
	s.mu.Lock()
	if s.conflict {
		s.conflict = false
		s.mu.Unlock()
		return "", agenttask.ErrRevisionConflict
	}
	s.mu.Unlock()
	return s.StateStore.CompareAndSwapIntegrationRecord(
		ctx,
		key,
		revision,
		payload,
	)
}

type fakeProcessBackend struct {
	mu          sync.Mutex
	observation IdentityObservation
	inspectErr  error
	starts      int
}

func (b *fakeProcessBackend) Start(
	context.Context,
	ClientKind,
	agentconfig.ClientProcessSpec,
) (OwnedProcess, error) {
	b.mu.Lock()
	defer b.mu.Unlock()
	b.starts++
	return &fakeOwnedProcess{
		identity: ProcessIdentity{PID: 100 + b.starts, StartToken: "fake"},
		done:     make(chan struct{}),
	}, nil
}

func (b *fakeProcessBackend) Inspect(
	context.Context,
	ProcessIdentity,
) (IdentityObservation, error) {
	return b.observation, b.inspectErr
}

func (*fakeProcessBackend) Signal(
	context.Context,
	ProcessIdentity,
	os.Signal,
) error {
	return nil
}

func (*fakeProcessBackend) Kill(context.Context, ProcessIdentity) error {
	return nil
}

func (*fakeProcessBackend) Focus(
	context.Context,
	ClientKind,
	agentconfig.ClientProcessSpec,
) error {
	return nil
}

func (b *fakeProcessBackend) startCount() int {
	b.mu.Lock()
	defer b.mu.Unlock()
	return b.starts
}

type fakeOwnedProcess struct {
	identity ProcessIdentity
	done     chan struct{}
	abort    sync.Once
	aborted  bool
}

func (p *fakeOwnedProcess) Identity() ProcessIdentity {
	return p.identity
}

func (p *fakeOwnedProcess) Wait() error {
	<-p.done
	return nil
}

func (p *fakeOwnedProcess) Abort() error {
	p.abort.Do(func() {
		p.aborted = true
		close(p.done)
	})
	return nil
}

func fakeSpec() agentconfig.ClientProcessSpec {
	return agentconfig.ClientProcessSpec{
		Executable:       "/bin/false",
		WorkingDirectory: "/tmp",
	}
}