1565 lines
40 KiB
Go
1565 lines
40 KiB
Go
package clientprocess
|
|
|
|
import (
|
|
"bufio"
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"os/signal"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"syscall"
|
|
"testing"
|
|
"time"
|
|
|
|
"iop/packages/go/agentconfig"
|
|
"iop/packages/go/agentstate"
|
|
)
|
|
|
|
func TestHelperProcess(t *testing.T) {
|
|
separator := -1
|
|
for index, argument := range os.Args {
|
|
if argument == "--" {
|
|
separator = index
|
|
break
|
|
}
|
|
}
|
|
if separator < 0 || len(os.Args) <= separator+2 {
|
|
return
|
|
}
|
|
mode := os.Args[separator+1]
|
|
tracePath := os.Args[separator+2]
|
|
appendHelperTrace(t, tracePath, fmt.Sprintf("%s:%d", mode, os.Getpid()))
|
|
if mode == "focus" {
|
|
return
|
|
}
|
|
if mode == "crash-trigger" || mode == "crash-trigger-once" {
|
|
if len(os.Args) <= separator+3 {
|
|
t.Fatal("crash helper trigger path is missing")
|
|
}
|
|
triggerPath := os.Args[separator+3]
|
|
for {
|
|
if _, err := os.Stat(triggerPath); err == nil {
|
|
break
|
|
} else if !errors.Is(err, os.ErrNotExist) {
|
|
t.Fatalf("inspect crash trigger: %v", err)
|
|
}
|
|
time.Sleep(5 * time.Millisecond)
|
|
}
|
|
if mode == "crash-trigger" {
|
|
os.Exit(2)
|
|
}
|
|
if len(os.Args) <= separator+4 {
|
|
t.Fatal("crash helper consumed path is missing")
|
|
}
|
|
consumedPath := os.Args[separator+4]
|
|
file, err := os.OpenFile(
|
|
consumedPath,
|
|
os.O_CREATE|os.O_EXCL|os.O_WRONLY,
|
|
0o600,
|
|
)
|
|
if err == nil {
|
|
_ = file.Close()
|
|
os.Exit(2)
|
|
}
|
|
if !errors.Is(err, os.ErrExist) {
|
|
t.Fatalf("record consumed crash trigger: %v", err)
|
|
}
|
|
}
|
|
signals := make(chan os.Signal, 1)
|
|
signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT)
|
|
defer signal.Stop(signals)
|
|
<-signals
|
|
}
|
|
|
|
func TestManagerOwnsSingletonAndReapsClient(t *testing.T) {
|
|
manager, daemonContext, tracePath := newFixtureManager(
|
|
t,
|
|
map[ClientKind]fixturePolicy{
|
|
ClientFlutter: {},
|
|
},
|
|
)
|
|
const callers = 24
|
|
results := make(chan Result, callers)
|
|
errorsChannel := make(chan error, callers)
|
|
var wait sync.WaitGroup
|
|
for index := 0; index < callers; index++ {
|
|
wait.Add(1)
|
|
go func(index int) {
|
|
defer wait.Done()
|
|
result, err := manager.Start(
|
|
context.Background(),
|
|
ClientFlutter,
|
|
fmt.Sprintf("start-%d", index),
|
|
)
|
|
results <- result
|
|
errorsChannel <- err
|
|
}(index)
|
|
}
|
|
wait.Wait()
|
|
close(results)
|
|
close(errorsChannel)
|
|
for err := range errorsChannel {
|
|
if err != nil {
|
|
t.Fatalf("Start: %v", err)
|
|
}
|
|
}
|
|
var identity ProcessIdentity
|
|
for result := range results {
|
|
if result.Record.Identity == nil {
|
|
t.Fatalf("start result has no identity: %+v", result)
|
|
}
|
|
if identity.PID == 0 {
|
|
identity = *result.Record.Identity
|
|
continue
|
|
}
|
|
if *result.Record.Identity != identity {
|
|
t.Fatalf(
|
|
"duplicate start identity = %+v, want %+v",
|
|
*result.Record.Identity,
|
|
identity,
|
|
)
|
|
}
|
|
}
|
|
waitForTraceCount(t, tracePath[ClientFlutter], 1)
|
|
if _, err := manager.SetConnected(
|
|
context.Background(),
|
|
ClientFlutter,
|
|
true,
|
|
); err != nil {
|
|
t.Fatalf("SetConnected: %v", err)
|
|
}
|
|
if _, err := manager.SetConnected(
|
|
context.Background(),
|
|
ClientFlutter,
|
|
false,
|
|
); err != nil {
|
|
t.Fatalf("SetDisconnected: %v", err)
|
|
}
|
|
if _, err := manager.SetConnected(
|
|
context.Background(),
|
|
ClientFlutter,
|
|
true,
|
|
); err != nil {
|
|
t.Fatalf("SetReconnected: %v", err)
|
|
}
|
|
|
|
stopped, err := manager.Stop(
|
|
context.Background(),
|
|
ClientFlutter,
|
|
"stop-one",
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("Stop: %v", err)
|
|
}
|
|
if stopped.Record.State != StateStopped ||
|
|
stopped.Record.Identity != nil ||
|
|
stopped.Record.LastIdentity == nil ||
|
|
*stopped.Record.LastIdentity != identity {
|
|
t.Fatalf("stopped record = %+v", stopped.Record)
|
|
}
|
|
if _, err := manager.Stop(
|
|
context.Background(),
|
|
ClientFlutter,
|
|
"stop-two",
|
|
); err != nil {
|
|
t.Fatalf("idempotent Stop: %v", err)
|
|
}
|
|
if _, err := currentProcessIdentity(identity.PID); !errors.Is(err, os.ErrProcessDone) {
|
|
t.Fatalf("child identity remains live: %v", err)
|
|
}
|
|
select {
|
|
case <-daemonContext.Done():
|
|
t.Fatal("client lifecycle cancelled the daemon context")
|
|
default:
|
|
}
|
|
}
|
|
|
|
func TestManagerRejectsLaunchAfterClose(t *testing.T) {
|
|
manager, _, traces := newFixtureManager(
|
|
t,
|
|
map[ClientKind]fixturePolicy{
|
|
ClientFlutter: {},
|
|
},
|
|
)
|
|
if err := manager.Close(context.Background()); err != nil {
|
|
t.Fatalf("Close: %v", err)
|
|
}
|
|
if _, err := manager.Start(
|
|
context.Background(),
|
|
ClientFlutter,
|
|
"closed-start",
|
|
); !errors.Is(err, ErrManagerClosed) {
|
|
t.Fatalf("Start error = %v, want ErrManagerClosed", err)
|
|
}
|
|
if _, err := manager.StartOrFocusFlutter(
|
|
context.Background(),
|
|
"closed-detail",
|
|
); !errors.Is(err, ErrManagerClosed) {
|
|
t.Fatalf("StartOrFocusFlutter error = %v, want ErrManagerClosed", err)
|
|
}
|
|
if lines := readTraceLines(t, traces[ClientFlutter]); len(lines) != 0 {
|
|
t.Fatalf("closed manager launched clients: %v", lines)
|
|
}
|
|
}
|
|
|
|
func TestDuplicateLaunchConvergesAfterManagerRestart(t *testing.T) {
|
|
root := t.TempDir()
|
|
statePath := filepath.Join(root, "state.json")
|
|
tracePath := filepath.Join(root, "flutter.trace")
|
|
spec := helperSpec(t, root, tracePath, filepath.Join(root, "focus.trace"))
|
|
store, err := agentstate.NewStore(statePath)
|
|
if err != nil {
|
|
t.Fatalf("NewStore: %v", err)
|
|
}
|
|
firstContext, firstCancel := context.WithCancel(context.Background())
|
|
defer firstCancel()
|
|
first, err := NewManager(
|
|
firstContext,
|
|
map[string]agentconfig.ClientProcessSpec{
|
|
string(ClientFlutter): spec,
|
|
},
|
|
store,
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("NewManager first: %v", err)
|
|
}
|
|
started, err := first.Start(context.Background(), ClientFlutter, "first")
|
|
if err != nil {
|
|
t.Fatalf("first Start: %v", err)
|
|
}
|
|
waitForTraceCount(t, tracePath, 1)
|
|
|
|
secondContext, secondCancel := context.WithCancel(context.Background())
|
|
defer secondCancel()
|
|
second, err := NewManager(
|
|
secondContext,
|
|
map[string]agentconfig.ClientProcessSpec{
|
|
string(ClientFlutter): spec,
|
|
},
|
|
store,
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("NewManager second: %v", err)
|
|
}
|
|
if err := second.Reconcile(context.Background()); err != nil {
|
|
t.Fatalf("second Reconcile: %v", err)
|
|
}
|
|
replayed, err := second.Start(
|
|
context.Background(),
|
|
ClientFlutter,
|
|
"second",
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("second Start: %v", err)
|
|
}
|
|
if replayed.Record.Identity == nil ||
|
|
started.Record.Identity == nil ||
|
|
*replayed.Record.Identity != *started.Record.Identity {
|
|
t.Fatalf(
|
|
"restart identity changed: first=%+v second=%+v",
|
|
started.Record.Identity,
|
|
replayed.Record.Identity,
|
|
)
|
|
}
|
|
waitForTraceCount(t, tracePath, 1)
|
|
if _, err := second.Stop(
|
|
context.Background(),
|
|
ClientFlutter,
|
|
"cleanup",
|
|
); err != nil {
|
|
t.Fatalf("second Stop: %v", err)
|
|
}
|
|
firstCancel()
|
|
first.waiters.Wait()
|
|
secondCancel()
|
|
second.waiters.Wait()
|
|
}
|
|
|
|
func TestDaemonSurvivesCrashAndBoundedRestart(t *testing.T) {
|
|
manager, daemonContext, tracePath := newFixtureManager(
|
|
t,
|
|
map[ClientKind]fixturePolicy{
|
|
ClientFlutter: {
|
|
restart: true,
|
|
limit: 1,
|
|
backoff: 10,
|
|
crashMode: "crash-trigger",
|
|
},
|
|
},
|
|
)
|
|
_, err := manager.Start(
|
|
context.Background(),
|
|
ClientFlutter,
|
|
"initial",
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("Start: %v", err)
|
|
}
|
|
if _, err := manager.SetConnected(
|
|
context.Background(),
|
|
ClientFlutter,
|
|
true,
|
|
); err != nil {
|
|
t.Fatalf("SetConnected: %v", err)
|
|
}
|
|
if err := os.WriteFile(
|
|
tracePath[ClientKind("flutter-crash")],
|
|
[]byte("crash"),
|
|
0o600,
|
|
); err != nil {
|
|
t.Fatalf("write crash trigger: %v", err)
|
|
}
|
|
waitForTraceCount(t, tracePath[ClientFlutter], 2)
|
|
waitForCondition(t, func() bool {
|
|
record, statusErr := manager.Status(ClientFlutter)
|
|
return statusErr == nil &&
|
|
record.State == StateCrashed &&
|
|
record.Identity == nil &&
|
|
record.RestartAttempts == 1
|
|
})
|
|
time.Sleep(50 * time.Millisecond)
|
|
waitForTraceCount(t, tracePath[ClientFlutter], 2)
|
|
select {
|
|
case <-daemonContext.Done():
|
|
t.Fatal("client crash cancelled the daemon context")
|
|
default:
|
|
}
|
|
}
|
|
|
|
func TestS15ClientLifecycleTrace(t *testing.T) {
|
|
manager, daemonContext, traces := newFixtureManager(
|
|
t,
|
|
map[ClientKind]fixturePolicy{
|
|
ClientFlutter: {},
|
|
ClientUnity: {
|
|
restart: true,
|
|
limit: 1,
|
|
backoff: 10,
|
|
crashMode: "crash-trigger-once",
|
|
},
|
|
},
|
|
)
|
|
flutter, err := manager.Start(
|
|
context.Background(),
|
|
ClientFlutter,
|
|
"flutter-start",
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("start Flutter: %v", err)
|
|
}
|
|
unity, err := manager.Start(
|
|
context.Background(),
|
|
ClientUnity,
|
|
"unity-start",
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("start Unity: %v", err)
|
|
}
|
|
duplicate, err := manager.Start(
|
|
context.Background(),
|
|
ClientFlutter,
|
|
"flutter-duplicate",
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("duplicate Flutter: %v", err)
|
|
}
|
|
if *duplicate.Record.Identity != *flutter.Record.Identity {
|
|
t.Fatalf("Flutter duplicate identity changed")
|
|
}
|
|
if _, err := manager.SetConnected(
|
|
context.Background(),
|
|
ClientFlutter,
|
|
true,
|
|
); err != nil {
|
|
t.Fatalf("connect Flutter: %v", err)
|
|
}
|
|
if _, err := manager.SetConnected(
|
|
context.Background(),
|
|
ClientFlutter,
|
|
false,
|
|
); err != nil {
|
|
t.Fatalf("disconnect Flutter: %v", err)
|
|
}
|
|
if _, err := manager.SetConnected(
|
|
context.Background(),
|
|
ClientFlutter,
|
|
true,
|
|
); err != nil {
|
|
t.Fatalf("reconnect Flutter: %v", err)
|
|
}
|
|
detail, err := manager.StartOrFocusFlutter(
|
|
context.Background(),
|
|
"unity-detail",
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("Unity detail route: %v", err)
|
|
}
|
|
if detail.Action != "focus" ||
|
|
detail.Record.FocusCount != 1 ||
|
|
*detail.Record.Identity != *flutter.Record.Identity {
|
|
t.Fatalf("detail result = %+v", detail)
|
|
}
|
|
waitForTraceCount(t, traces[ClientFlutter], 1)
|
|
waitForTraceCount(t, traces[ClientUnity], 1)
|
|
waitForTraceCount(t, traces[ClientKind("flutter-focus")], 1)
|
|
|
|
if err := os.WriteFile(
|
|
traces[ClientKind("unity-crash")],
|
|
[]byte("crash"),
|
|
0o600,
|
|
); err != nil {
|
|
t.Fatalf("write Unity crash trigger: %v", err)
|
|
}
|
|
var restartedUnity Record
|
|
waitForCondition(t, func() bool {
|
|
record, statusErr := manager.Status(ClientUnity)
|
|
if statusErr != nil ||
|
|
record.Identity == nil ||
|
|
record.Identity.PID == unity.Record.Identity.PID {
|
|
return false
|
|
}
|
|
restartedUnity = record
|
|
return true
|
|
})
|
|
waitForTraceCount(t, traces[ClientUnity], 2)
|
|
|
|
t.Logf(
|
|
"S15 trace flutter=%d/%s duplicate=%d unity=%d/%s restart=%d/%s focus=%d daemon=live",
|
|
flutter.Record.Identity.PID,
|
|
flutter.Record.Identity.StartToken,
|
|
duplicate.Record.Identity.PID,
|
|
unity.Record.Identity.PID,
|
|
unity.Record.Identity.StartToken,
|
|
restartedUnity.Identity.PID,
|
|
restartedUnity.Identity.StartToken,
|
|
detail.Record.FocusCount,
|
|
)
|
|
select {
|
|
case <-daemonContext.Done():
|
|
t.Fatal("S15 client lifecycle stopped daemon ownership")
|
|
default:
|
|
}
|
|
}
|
|
|
|
type fixturePolicy struct {
|
|
restart bool
|
|
limit int
|
|
backoff int
|
|
crashMode string
|
|
}
|
|
|
|
func newFixtureManager(
|
|
t *testing.T,
|
|
policies map[ClientKind]fixturePolicy,
|
|
) (*Manager, context.Context, map[ClientKind]string) {
|
|
t.Helper()
|
|
root := t.TempDir()
|
|
specs := make(map[string]agentconfig.ClientProcessSpec, len(policies))
|
|
traces := make(map[ClientKind]string, len(policies)+1)
|
|
for kind, policy := range policies {
|
|
tracePath := filepath.Join(root, string(kind)+".trace")
|
|
focusPath := filepath.Join(root, string(kind)+".focus.trace")
|
|
spec := helperSpec(t, root, tracePath, focusPath)
|
|
spec.RestartOnCrash = policy.restart
|
|
spec.RestartLimit = policy.limit
|
|
spec.RestartBackoffMillis = policy.backoff
|
|
if policy.crashMode != "" {
|
|
triggerPath := filepath.Join(root, string(kind)+".crash")
|
|
consumedPath := filepath.Join(root, string(kind)+".crash-consumed")
|
|
spec.Args = []string{
|
|
"-test.run=^TestHelperProcess$",
|
|
"--",
|
|
policy.crashMode,
|
|
tracePath,
|
|
triggerPath,
|
|
}
|
|
if policy.crashMode == "crash-trigger-once" {
|
|
spec.Args = append(spec.Args, consumedPath)
|
|
}
|
|
traces[ClientKind(string(kind)+"-crash")] = triggerPath
|
|
}
|
|
if kind == ClientUnity {
|
|
spec.FocusArgs = nil
|
|
}
|
|
specs[string(kind)] = spec
|
|
traces[kind] = tracePath
|
|
if kind == ClientFlutter {
|
|
traces[ClientKind("flutter-focus")] = focusPath
|
|
}
|
|
}
|
|
store, err := agentstate.NewStore(filepath.Join(root, "state.json"))
|
|
if err != nil {
|
|
t.Fatalf("NewStore: %v", err)
|
|
}
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
manager, err := NewManager(
|
|
ctx,
|
|
specs,
|
|
store,
|
|
WithStopTimeout(time.Second),
|
|
WithPollInterval(10*time.Millisecond),
|
|
)
|
|
if err != nil {
|
|
cancel()
|
|
t.Fatalf("NewManager: %v", err)
|
|
}
|
|
t.Cleanup(func() {
|
|
_ = manager.Close(context.Background())
|
|
cancel()
|
|
})
|
|
return manager, ctx, traces
|
|
}
|
|
|
|
func helperSpec(
|
|
t *testing.T,
|
|
workingDirectory string,
|
|
tracePath string,
|
|
focusTracePath string,
|
|
) agentconfig.ClientProcessSpec {
|
|
t.Helper()
|
|
executable, err := os.Executable()
|
|
if err != nil {
|
|
t.Fatalf("Executable: %v", err)
|
|
}
|
|
return agentconfig.ClientProcessSpec{
|
|
Executable: executable,
|
|
Args: []string{"-test.run=^TestHelperProcess$", "--", "run", tracePath},
|
|
WorkingDirectory: workingDirectory,
|
|
FocusArgs: []string{
|
|
"-test.run=^TestHelperProcess$",
|
|
"--",
|
|
"focus",
|
|
focusTracePath,
|
|
},
|
|
}
|
|
}
|
|
|
|
func appendHelperTrace(t *testing.T, path, line string) {
|
|
t.Helper()
|
|
file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600)
|
|
if err != nil {
|
|
t.Fatalf("open helper trace: %v", err)
|
|
}
|
|
if _, err := fmt.Fprintln(file, line); err != nil {
|
|
_ = file.Close()
|
|
t.Fatalf("write helper trace: %v", err)
|
|
}
|
|
if err := file.Close(); err != nil {
|
|
t.Fatalf("close helper trace: %v", err)
|
|
}
|
|
}
|
|
|
|
func waitForTraceCount(t *testing.T, path string, count int) {
|
|
t.Helper()
|
|
deadline := time.Now().Add(5 * time.Second)
|
|
for time.Now().Before(deadline) {
|
|
if len(readTraceLines(t, path)) == count {
|
|
return
|
|
}
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
t.Fatalf(
|
|
"trace %s has %d lines, want %d: %v",
|
|
path,
|
|
len(readTraceLines(t, path)),
|
|
count,
|
|
readTraceLines(t, path),
|
|
)
|
|
}
|
|
|
|
func readTraceLines(t *testing.T, path string) []string {
|
|
t.Helper()
|
|
file, err := os.Open(path)
|
|
if errors.Is(err, os.ErrNotExist) {
|
|
return nil
|
|
}
|
|
if err != nil {
|
|
t.Fatalf("open trace: %v", err)
|
|
}
|
|
defer file.Close()
|
|
var lines []string
|
|
scanner := bufio.NewScanner(file)
|
|
for scanner.Scan() {
|
|
lines = append(lines, strings.TrimSpace(scanner.Text()))
|
|
}
|
|
if err := scanner.Err(); err != nil {
|
|
t.Fatalf("scan trace: %v", err)
|
|
}
|
|
sort.Strings(lines)
|
|
return lines
|
|
}
|
|
|
|
func waitForCondition(t *testing.T, condition func() bool) {
|
|
t.Helper()
|
|
deadline := time.Now().Add(5 * time.Second)
|
|
for time.Now().Before(deadline) {
|
|
if condition() {
|
|
return
|
|
}
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
t.Fatal("condition was not satisfied before deadline")
|
|
}
|
|
|
|
func TestManagerReconcilesRetainedClientAfterConfigRemoval(t *testing.T) {
|
|
root := t.TempDir()
|
|
statePath := filepath.Join(root, "state.json")
|
|
tracePath := filepath.Join(root, "flutter.trace")
|
|
spec := helperSpec(t, root, tracePath, filepath.Join(root, "focus.trace"))
|
|
store, err := agentstate.NewStore(statePath)
|
|
if err != nil {
|
|
t.Fatalf("NewStore: %v", err)
|
|
}
|
|
ctx1, cancel1 := context.WithCancel(context.Background())
|
|
defer cancel1()
|
|
m1, err := NewManager(
|
|
ctx1,
|
|
map[string]agentconfig.ClientProcessSpec{string(ClientFlutter): spec},
|
|
store,
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("NewManager 1: %v", err)
|
|
}
|
|
res, err := m1.Start(context.Background(), ClientFlutter, "cmd-1")
|
|
if err != nil {
|
|
t.Fatalf("Start m1: %v", err)
|
|
}
|
|
pid := res.Record.Identity.PID
|
|
waitForTraceCount(t, tracePath, 1)
|
|
|
|
ctx2, cancel2 := context.WithCancel(context.Background())
|
|
defer cancel2()
|
|
m2, err := NewManager(
|
|
ctx2,
|
|
map[string]agentconfig.ClientProcessSpec{},
|
|
store,
|
|
WithStopTimeout(time.Second),
|
|
WithPollInterval(10*time.Millisecond),
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("NewManager 2: %v", err)
|
|
}
|
|
if err := m2.Reconcile(context.Background()); err != nil {
|
|
t.Fatalf("Reconcile m2: %v", err)
|
|
}
|
|
status, err := m2.Status(ClientFlutter)
|
|
if err != nil {
|
|
t.Fatalf("Status m2: %v", err)
|
|
}
|
|
if status.Identity == nil || status.Identity.PID != pid {
|
|
t.Fatalf("m2 status identity = %+v, want PID %d", status.Identity, pid)
|
|
}
|
|
if _, err := m2.Start(
|
|
context.Background(),
|
|
ClientFlutter,
|
|
"cmd-2",
|
|
); !errors.Is(err, ErrClientNotConfigured) {
|
|
t.Fatalf("m2 Start err = %v, want ErrClientNotConfigured", err)
|
|
}
|
|
if _, err := m2.Stop(context.Background(), ClientFlutter, "stop-unconfig"); err != nil {
|
|
t.Fatalf("m2 Stop err = %v", err)
|
|
}
|
|
statusAfter, err := m2.Status(ClientFlutter)
|
|
if err != nil {
|
|
t.Fatalf("Status after stop: %v", err)
|
|
}
|
|
if statusAfter.State != StateStopped {
|
|
t.Fatalf("statusAfter state = %s, want stopped", statusAfter.State)
|
|
}
|
|
}
|
|
|
|
func TestStartConfiguredLaunchesAfterDaemonRestart(t *testing.T) {
|
|
backend := &sharedLifecycleBackend{}
|
|
store, err := agentstate.NewStore(filepath.Join(t.TempDir(), "state.json"))
|
|
if err != nil {
|
|
t.Fatalf("NewStore: %v", err)
|
|
}
|
|
spec := fakeSpec()
|
|
spec.LaunchOnStart = true
|
|
specs := map[string]agentconfig.ClientProcessSpec{string(ClientFlutter): spec}
|
|
|
|
firstCtx, firstCancel := context.WithCancel(context.Background())
|
|
first, err := NewManager(
|
|
firstCtx,
|
|
specs,
|
|
store,
|
|
WithProcessBackend(backend),
|
|
WithStopTimeout(time.Second),
|
|
WithPollInterval(10*time.Millisecond),
|
|
)
|
|
if err != nil {
|
|
firstCancel()
|
|
t.Fatalf("NewManager first: %v", err)
|
|
}
|
|
if err := first.StartConfigured(context.Background()); err != nil {
|
|
t.Fatalf("first StartConfigured: %v", err)
|
|
}
|
|
if backend.startCount() != 1 {
|
|
t.Fatalf("first generation starts = %d, want 1", backend.startCount())
|
|
}
|
|
firstStatus, err := first.Status(ClientFlutter)
|
|
if err != nil || firstStatus.Identity == nil {
|
|
t.Fatalf("first generation not launched: %+v err=%v", firstStatus, err)
|
|
}
|
|
firstIdentity := *firstStatus.Identity
|
|
if err := first.Close(context.Background()); err != nil {
|
|
t.Fatalf("first Close: %v", err)
|
|
}
|
|
firstCancel()
|
|
|
|
secondCtx, secondCancel := context.WithCancel(context.Background())
|
|
defer secondCancel()
|
|
second, err := NewManager(
|
|
secondCtx,
|
|
specs,
|
|
store,
|
|
WithProcessBackend(backend),
|
|
WithStopTimeout(time.Second),
|
|
WithPollInterval(10*time.Millisecond),
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("NewManager second: %v", err)
|
|
}
|
|
defer second.Close(context.Background())
|
|
if err := second.StartConfigured(context.Background()); err != nil {
|
|
t.Fatalf("second StartConfigured: %v", err)
|
|
}
|
|
if backend.startCount() != 2 {
|
|
t.Fatalf("second generation starts = %d, want 2 (relaunch after restart)", backend.startCount())
|
|
}
|
|
secondStatus, err := second.Status(ClientFlutter)
|
|
if err != nil || secondStatus.Identity == nil {
|
|
t.Fatalf("second generation not launched: %+v err=%v", secondStatus, err)
|
|
}
|
|
if *secondStatus.Identity == firstIdentity {
|
|
t.Fatalf("second generation reused restarted identity %+v", firstIdentity)
|
|
}
|
|
if secondStatus.State != StateStarting {
|
|
t.Fatalf("second generation state = %s, want starting", secondStatus.State)
|
|
}
|
|
}
|
|
|
|
func TestCloseStopsCurrentGenerationAfterPriorLifecycleReceipts(t *testing.T) {
|
|
store, err := agentstate.NewStore(filepath.Join(t.TempDir(), "state.json"))
|
|
if err != nil {
|
|
t.Fatalf("NewStore: %v", err)
|
|
}
|
|
// Seed durable state as if a prior buggy generation persisted fixed
|
|
// daemon-start/daemon-stop caller receipts. The current generation must
|
|
// launch and stop its own process without replaying them.
|
|
priorIdentity := ProcessIdentity{PID: 999, StartToken: "gen-prior"}
|
|
seed := initialRecord(ClientFlutter)
|
|
seed.LastIdentity = &priorIdentity
|
|
seed.Commands = map[string]CommandReceipt{
|
|
"daemon-start": {
|
|
CommandID: "daemon-start",
|
|
Action: "start",
|
|
Status: CommandReceiptCompleted,
|
|
Result: CommandResultSnapshot{State: StateStarting, Action: "start"},
|
|
},
|
|
"daemon-stop": {
|
|
CommandID: "daemon-stop",
|
|
Action: "stop",
|
|
Status: CommandReceiptCompleted,
|
|
Result: CommandResultSnapshot{State: StateStopped, Action: "stop"},
|
|
},
|
|
}
|
|
seed.UpdatedAt = time.Now().UTC()
|
|
durable, _ := newDurableStore(store)
|
|
if _, err := durable.save(context.Background(), seed, ""); err != nil {
|
|
t.Fatalf("seed record: %v", err)
|
|
}
|
|
|
|
backend := &sharedLifecycleBackend{}
|
|
spec := fakeSpec()
|
|
spec.LaunchOnStart = true
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
manager, err := NewManager(
|
|
ctx,
|
|
map[string]agentconfig.ClientProcessSpec{string(ClientFlutter): spec},
|
|
store,
|
|
WithProcessBackend(backend),
|
|
WithStopTimeout(time.Second),
|
|
WithPollInterval(10*time.Millisecond),
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("NewManager: %v", err)
|
|
}
|
|
defer manager.Close(context.Background())
|
|
|
|
if err := manager.StartConfigured(context.Background()); err != nil {
|
|
t.Fatalf("StartConfigured: %v", err)
|
|
}
|
|
if backend.startCount() != 1 {
|
|
t.Fatalf("starts = %d, want 1 despite prior daemon-start receipt", backend.startCount())
|
|
}
|
|
status, err := manager.Status(ClientFlutter)
|
|
if err != nil || status.Identity == nil {
|
|
t.Fatalf("current generation not launched: %+v err=%v", status, err)
|
|
}
|
|
currentIdentity := *status.Identity
|
|
if currentIdentity == priorIdentity {
|
|
t.Fatalf("current generation reused prior identity %+v", priorIdentity)
|
|
}
|
|
|
|
if err := manager.Close(context.Background()); err != nil {
|
|
t.Fatalf("Close: %v", err)
|
|
}
|
|
if !backend.wasStopped(currentIdentity) {
|
|
t.Fatalf("close did not stop the current identity %+v", currentIdentity)
|
|
}
|
|
final, err := manager.Status(ClientFlutter)
|
|
if err != nil {
|
|
t.Fatalf("Status: %v", err)
|
|
}
|
|
if final.State != StateStopped || final.Identity != nil {
|
|
t.Fatalf("final state = %+v, want stopped without identity", final)
|
|
}
|
|
}
|
|
|
|
// sharedLifecycleBackend is a deterministic in-memory backend that can be
|
|
// shared across daemon generations. It assigns a unique identity per launch and
|
|
// tracks which identities were stopped.
|
|
type sharedLifecycleBackend struct {
|
|
mu sync.Mutex
|
|
starts int
|
|
stopped []ProcessIdentity
|
|
processes map[ProcessIdentity]*fakeOwnedProcess
|
|
}
|
|
|
|
func (b *sharedLifecycleBackend) Start(
|
|
context.Context,
|
|
ClientKind,
|
|
agentconfig.ClientProcessSpec,
|
|
) (OwnedProcess, error) {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
b.starts++
|
|
id := ProcessIdentity{
|
|
PID: 1000 + b.starts,
|
|
StartToken: fmt.Sprintf("gen-%d", b.starts),
|
|
}
|
|
process := &fakeOwnedProcess{identity: id, done: make(chan struct{})}
|
|
if b.processes == nil {
|
|
b.processes = make(map[ProcessIdentity]*fakeOwnedProcess)
|
|
}
|
|
b.processes[id] = process
|
|
return process, nil
|
|
}
|
|
|
|
func (b *sharedLifecycleBackend) Inspect(
|
|
_ context.Context,
|
|
id ProcessIdentity,
|
|
) (IdentityObservation, error) {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
process, ok := b.processes[id]
|
|
if !ok {
|
|
return IdentityObservation{State: IdentityExited}, nil
|
|
}
|
|
select {
|
|
case <-process.done:
|
|
return IdentityObservation{State: IdentityExited}, nil
|
|
default:
|
|
return IdentityObservation{State: IdentityLive, Identity: id}, nil
|
|
}
|
|
}
|
|
|
|
func (b *sharedLifecycleBackend) Signal(
|
|
_ context.Context,
|
|
id ProcessIdentity,
|
|
_ os.Signal,
|
|
) error {
|
|
b.mu.Lock()
|
|
process, ok := b.processes[id]
|
|
if ok {
|
|
b.stopped = append(b.stopped, id)
|
|
}
|
|
b.mu.Unlock()
|
|
if ok {
|
|
_ = process.Abort()
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (b *sharedLifecycleBackend) Kill(ctx context.Context, id ProcessIdentity) error {
|
|
return b.Signal(ctx, id, os.Kill)
|
|
}
|
|
|
|
func (b *sharedLifecycleBackend) Focus(
|
|
context.Context,
|
|
ClientKind,
|
|
agentconfig.ClientProcessSpec,
|
|
) error {
|
|
return nil
|
|
}
|
|
|
|
func (b *sharedLifecycleBackend) startCount() int {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
return b.starts
|
|
}
|
|
|
|
func (b *sharedLifecycleBackend) wasStopped(id ProcessIdentity) bool {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
for _, stopped := range b.stopped {
|
|
if stopped == id {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func (b *sharedLifecycleBackend) wasAborted(id ProcessIdentity) bool {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
process, ok := b.processes[id]
|
|
return ok && process.aborted
|
|
}
|
|
|
|
func TestConcurrentCloseAbortsInFlightLaunch(t *testing.T) {
|
|
barrier := make(chan struct{})
|
|
backend := &barrierProcessBackend{barrier: barrier}
|
|
store, err := agentstate.NewStore(filepath.Join(t.TempDir(), "state.json"))
|
|
if err != nil {
|
|
t.Fatalf("NewStore: %v", err)
|
|
}
|
|
m, err := NewManager(
|
|
context.Background(),
|
|
map[string]agentconfig.ClientProcessSpec{
|
|
string(ClientFlutter): fakeSpec(),
|
|
},
|
|
store,
|
|
WithProcessBackend(backend),
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("NewManager: %v", err)
|
|
}
|
|
|
|
var startErr error
|
|
var wg sync.WaitGroup
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
_, startErr = m.Start(context.Background(), ClientFlutter, "in-flight-start")
|
|
}()
|
|
|
|
<-barrier
|
|
closeDone := make(chan error, 1)
|
|
go func() {
|
|
closeDone <- m.Close(context.Background())
|
|
}()
|
|
select {
|
|
case err := <-closeDone:
|
|
t.Fatalf("Close returned before the admitted launch finished: %v", err)
|
|
case <-time.After(20 * time.Millisecond):
|
|
}
|
|
close(barrier)
|
|
if closeErr := <-closeDone; closeErr != nil {
|
|
t.Fatalf("Close err = %v", closeErr)
|
|
}
|
|
wg.Wait()
|
|
|
|
if !errors.Is(startErr, ErrManagerClosed) {
|
|
t.Fatalf("startErr = %v, want ErrManagerClosed", startErr)
|
|
}
|
|
if backend.createdProcess != nil && !backend.createdProcess.aborted {
|
|
t.Fatalf("in-flight process was not aborted")
|
|
}
|
|
}
|
|
|
|
func TestConcurrentCloseCancelsInFlightFocus(t *testing.T) {
|
|
assertConcurrentCloseCancelsMutation(t, func(m *Manager) (Result, error) {
|
|
return m.Focus(context.Background(), ClientFlutter, "focus-1")
|
|
})
|
|
}
|
|
|
|
func TestConcurrentCloseCancelsInFlightDetail(t *testing.T) {
|
|
assertConcurrentCloseCancelsMutation(t, func(m *Manager) (Result, error) {
|
|
return m.StartOrFocusFlutter(context.Background(), "detail-1")
|
|
})
|
|
}
|
|
|
|
func assertConcurrentCloseCancelsMutation(
|
|
t *testing.T,
|
|
invoke func(*Manager) (Result, error),
|
|
) {
|
|
t.Helper()
|
|
backend := &focusBarrierBackend{focusEntered: make(chan struct{}, 1)}
|
|
store, err := agentstate.NewStore(filepath.Join(t.TempDir(), "state.json"))
|
|
if err != nil {
|
|
t.Fatalf("NewStore: %v", err)
|
|
}
|
|
m, err := NewManager(
|
|
context.Background(),
|
|
map[string]agentconfig.ClientProcessSpec{string(ClientFlutter): fakeSpec()},
|
|
store,
|
|
WithProcessBackend(backend),
|
|
WithStopTimeout(time.Second),
|
|
WithPollInterval(10*time.Millisecond),
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("NewManager: %v", err)
|
|
}
|
|
if _, err := m.Start(context.Background(), ClientFlutter, "start-1"); err != nil {
|
|
t.Fatalf("Start: %v", err)
|
|
}
|
|
if _, err := m.SetConnected(context.Background(), ClientFlutter, true); err != nil {
|
|
t.Fatalf("SetConnected: %v", err)
|
|
}
|
|
status, err := m.Status(ClientFlutter)
|
|
if err != nil || status.Identity == nil {
|
|
t.Fatalf("client not live: %+v err=%v", status, err)
|
|
}
|
|
identity := *status.Identity
|
|
|
|
var mutationErr error
|
|
var wg sync.WaitGroup
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
_, mutationErr = invoke(m)
|
|
}()
|
|
|
|
<-backend.focusEntered
|
|
if err := m.Close(context.Background()); err != nil {
|
|
t.Fatalf("Close: %v", err)
|
|
}
|
|
wg.Wait()
|
|
|
|
if !errors.Is(mutationErr, ErrManagerClosed) {
|
|
t.Fatalf("mutation error = %v, want ErrManagerClosed", mutationErr)
|
|
}
|
|
if !backend.focusWasCancelled() {
|
|
t.Fatalf("manager cancellation did not reach the blocking mutation")
|
|
}
|
|
final, err := m.Status(ClientFlutter)
|
|
if err != nil {
|
|
t.Fatalf("Status: %v", err)
|
|
}
|
|
if final.FocusCount != 0 {
|
|
t.Fatalf("focus committed after close: FocusCount=%d", final.FocusCount)
|
|
}
|
|
if final.State != StateStopped || final.Identity != nil {
|
|
t.Fatalf("current client not reaped: %+v", final)
|
|
}
|
|
if !backend.wasStopped(identity) {
|
|
t.Fatalf("current identity %+v not reaped", identity)
|
|
}
|
|
}
|
|
|
|
func TestConcurrentCloseFencesConnectionMutation(t *testing.T) {
|
|
base, err := agentstate.NewStore(filepath.Join(t.TempDir(), "state.json"))
|
|
if err != nil {
|
|
t.Fatalf("NewStore: %v", err)
|
|
}
|
|
store := &connectionBarrierStore{
|
|
Store: base,
|
|
armKey: clientRecordKey(ClientFlutter),
|
|
entered: make(chan struct{}, 1),
|
|
}
|
|
backend := &sharedLifecycleBackend{}
|
|
m, err := NewManager(
|
|
context.Background(),
|
|
map[string]agentconfig.ClientProcessSpec{string(ClientFlutter): fakeSpec()},
|
|
store,
|
|
WithProcessBackend(backend),
|
|
WithStopTimeout(time.Second),
|
|
WithPollInterval(10*time.Millisecond),
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("NewManager: %v", err)
|
|
}
|
|
if _, err := m.Start(context.Background(), ClientFlutter, "start-1"); err != nil {
|
|
t.Fatalf("Start: %v", err)
|
|
}
|
|
if _, err := m.SetConnected(context.Background(), ClientFlutter, true); err != nil {
|
|
t.Fatalf("SetConnected true: %v", err)
|
|
}
|
|
status, err := m.Status(ClientFlutter)
|
|
if err != nil || status.Identity == nil {
|
|
t.Fatalf("client not live: %+v err=%v", status, err)
|
|
}
|
|
identity := *status.Identity
|
|
|
|
// Block the disconnect's durable projection commit after admission.
|
|
store.arm()
|
|
var connectErr error
|
|
var wg sync.WaitGroup
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
_, connectErr = m.SetConnected(context.Background(), ClientFlutter, false)
|
|
}()
|
|
|
|
<-store.entered
|
|
if err := m.Close(context.Background()); err != nil {
|
|
t.Fatalf("Close: %v", err)
|
|
}
|
|
wg.Wait()
|
|
|
|
if connectErr == nil {
|
|
t.Fatalf("connection mutation committed across close")
|
|
}
|
|
final, err := m.Status(ClientFlutter)
|
|
if err != nil {
|
|
t.Fatalf("Status: %v", err)
|
|
}
|
|
if final.State != StateStopped || final.Identity != nil {
|
|
t.Fatalf("current client not reaped: %+v", final)
|
|
}
|
|
if !backend.wasStopped(identity) {
|
|
t.Fatalf("current identity %+v not reaped", identity)
|
|
}
|
|
}
|
|
|
|
// focusBarrierBackend blocks its Focus call after admission until the mutation
|
|
// context is cancelled, so a concurrent Close must cancel and join the mutation.
|
|
type focusBarrierBackend struct {
|
|
sharedLifecycleBackend
|
|
focusEntered chan struct{}
|
|
focusMu sync.Mutex
|
|
cancelled bool
|
|
}
|
|
|
|
func (b *focusBarrierBackend) Focus(
|
|
ctx context.Context,
|
|
_ ClientKind,
|
|
_ agentconfig.ClientProcessSpec,
|
|
) error {
|
|
select {
|
|
case b.focusEntered <- struct{}{}:
|
|
default:
|
|
}
|
|
<-ctx.Done()
|
|
b.focusMu.Lock()
|
|
b.cancelled = true
|
|
b.focusMu.Unlock()
|
|
return ctx.Err()
|
|
}
|
|
|
|
func (b *focusBarrierBackend) focusWasCancelled() bool {
|
|
b.focusMu.Lock()
|
|
defer b.focusMu.Unlock()
|
|
return b.cancelled
|
|
}
|
|
|
|
// connectionBarrierStore blocks exactly one CompareAndSwap on the armed key
|
|
// after admission until the write context is cancelled.
|
|
type connectionBarrierStore struct {
|
|
*agentstate.Store
|
|
armKey string
|
|
entered chan struct{}
|
|
mu sync.Mutex
|
|
armed bool
|
|
}
|
|
|
|
func (s *connectionBarrierStore) arm() {
|
|
s.mu.Lock()
|
|
s.armed = true
|
|
s.mu.Unlock()
|
|
}
|
|
|
|
func (s *connectionBarrierStore) CompareAndSwapIntegrationRecord(
|
|
ctx context.Context,
|
|
key string,
|
|
revision string,
|
|
payload []byte,
|
|
) (string, error) {
|
|
s.mu.Lock()
|
|
block := s.armed && key == s.armKey
|
|
if block {
|
|
s.armed = false
|
|
}
|
|
s.mu.Unlock()
|
|
if block {
|
|
select {
|
|
case s.entered <- struct{}{}:
|
|
default:
|
|
}
|
|
<-ctx.Done()
|
|
return "", ctx.Err()
|
|
}
|
|
return s.Store.CompareAndSwapIntegrationRecord(ctx, key, revision, payload)
|
|
}
|
|
|
|
type barrierProcessBackend struct {
|
|
fakeProcessBackend
|
|
barrier chan struct{}
|
|
createdProcess *fakeOwnedProcess
|
|
}
|
|
|
|
func (b *barrierProcessBackend) Start(
|
|
ctx context.Context,
|
|
kind ClientKind,
|
|
spec agentconfig.ClientProcessSpec,
|
|
) (OwnedProcess, error) {
|
|
p := &fakeOwnedProcess{
|
|
identity: ProcessIdentity{PID: 456, StartToken: "fake-barrier"},
|
|
done: make(chan struct{}),
|
|
}
|
|
b.createdProcess = p
|
|
b.barrier <- struct{}{}
|
|
<-b.barrier
|
|
return p, nil
|
|
}
|
|
|
|
func TestAdoptedCleanupRequiresObservedExit(t *testing.T) {
|
|
backend := &adoptableBackend{live: true}
|
|
store, err := agentstate.NewStore(filepath.Join(t.TempDir(), "state.json"))
|
|
if err != nil {
|
|
t.Fatalf("NewStore: %v", err)
|
|
}
|
|
durable, _ := newDurableStore(store)
|
|
identity := ProcessIdentity{PID: 888, StartToken: "adopt-token"}
|
|
rec := initialRecord(ClientFlutter)
|
|
rec.State = StateConnected
|
|
rec.Connected = true
|
|
rec.Identity = &identity
|
|
rec.UpdatedAt = time.Now().UTC()
|
|
if _, err := durable.save(context.Background(), rec, ""); err != nil {
|
|
t.Fatalf("seed record: %v", err)
|
|
}
|
|
|
|
m, err := NewManager(
|
|
context.Background(),
|
|
map[string]agentconfig.ClientProcessSpec{
|
|
string(ClientFlutter): fakeSpec(),
|
|
},
|
|
store,
|
|
WithProcessBackend(backend),
|
|
WithPollInterval(10*time.Millisecond),
|
|
WithStopTimeout(200*time.Millisecond),
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("NewManager: %v", err)
|
|
}
|
|
if err := m.Reconcile(context.Background()); err != nil {
|
|
t.Fatalf("Reconcile: %v", err)
|
|
}
|
|
|
|
stopDone := make(chan error, 1)
|
|
go func() {
|
|
_, err := m.Stop(context.Background(), ClientFlutter, "stop-adopted")
|
|
stopDone <- err
|
|
}()
|
|
|
|
select {
|
|
case err := <-stopDone:
|
|
if err != nil {
|
|
t.Fatalf("Stop error = %v", err)
|
|
}
|
|
case <-time.After(3 * time.Second):
|
|
t.Fatal("Stop timed out waiting for adopted exit observation")
|
|
}
|
|
|
|
status, err := m.Status(ClientFlutter)
|
|
if err != nil {
|
|
t.Fatalf("Status: %v", err)
|
|
}
|
|
if status.State != StateStopped {
|
|
t.Fatalf("adopted status state = %s, want stopped", status.State)
|
|
}
|
|
}
|
|
|
|
type adoptableBackend struct {
|
|
fakeProcessBackend
|
|
mu sync.Mutex
|
|
live bool
|
|
}
|
|
|
|
func (b *adoptableBackend) Inspect(
|
|
ctx context.Context,
|
|
id ProcessIdentity,
|
|
) (IdentityObservation, error) {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
if b.live {
|
|
return IdentityObservation{State: IdentityLive, Identity: id}, nil
|
|
}
|
|
return IdentityObservation{State: IdentityExited}, nil
|
|
}
|
|
|
|
func (b *adoptableBackend) Signal(
|
|
ctx context.Context,
|
|
id ProcessIdentity,
|
|
sig os.Signal,
|
|
) error {
|
|
return nil
|
|
}
|
|
|
|
func (b *adoptableBackend) Kill(
|
|
ctx context.Context,
|
|
id ProcessIdentity,
|
|
) error {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
b.live = false
|
|
return nil
|
|
}
|
|
|
|
func TestCommandReceiptCompletionSaveFailureStaysPending(t *testing.T) {
|
|
injected := errors.New("injected completion receipt save failure")
|
|
tests := []struct {
|
|
name string
|
|
setup func(t *testing.T, manager *Manager, store *nthCASFailureStore)
|
|
call func(*Manager) (Result, error)
|
|
}{
|
|
{
|
|
name: "start",
|
|
setup: func(t *testing.T, _ *Manager, store *nthCASFailureStore) {
|
|
t.Helper()
|
|
store.failOnNextCompletion()
|
|
},
|
|
call: func(manager *Manager) (Result, error) {
|
|
return manager.Start(context.Background(), ClientFlutter, "completion-start")
|
|
},
|
|
},
|
|
{
|
|
name: "already running start",
|
|
setup: func(t *testing.T, manager *Manager, store *nthCASFailureStore) {
|
|
t.Helper()
|
|
startLiveFixture(t, manager)
|
|
store.failOnNextCompletion()
|
|
},
|
|
call: func(manager *Manager) (Result, error) {
|
|
return manager.Start(context.Background(), ClientFlutter, "completion-start-live")
|
|
},
|
|
},
|
|
{
|
|
name: "already stopped",
|
|
setup: func(t *testing.T, _ *Manager, store *nthCASFailureStore) {
|
|
t.Helper()
|
|
store.failOnNextCompletion()
|
|
},
|
|
call: func(manager *Manager) (Result, error) {
|
|
return manager.Stop(context.Background(), ClientFlutter, "completion-stop")
|
|
},
|
|
},
|
|
{
|
|
name: "focus",
|
|
setup: func(t *testing.T, manager *Manager, store *nthCASFailureStore) {
|
|
t.Helper()
|
|
startLiveFixture(t, manager)
|
|
store.failOnNextCompletion()
|
|
},
|
|
call: func(manager *Manager) (Result, error) {
|
|
return manager.Focus(context.Background(), ClientFlutter, "completion-focus")
|
|
},
|
|
},
|
|
{
|
|
name: "detail focus",
|
|
setup: func(t *testing.T, manager *Manager, store *nthCASFailureStore) {
|
|
t.Helper()
|
|
startLiveFixture(t, manager)
|
|
store.failOnNextCompletion()
|
|
},
|
|
call: func(manager *Manager) (Result, error) {
|
|
return manager.StartOrFocusFlutter(context.Background(), "completion-detail")
|
|
},
|
|
},
|
|
{
|
|
name: "detail start",
|
|
setup: func(t *testing.T, _ *Manager, store *nthCASFailureStore) {
|
|
t.Helper()
|
|
store.failOnNextCompletion()
|
|
},
|
|
call: func(manager *Manager) (Result, error) {
|
|
return manager.StartOrFocusFlutter(context.Background(), "completion-detail-start")
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
base, err := agentstate.NewStore(filepath.Join(t.TempDir(), "state.json"))
|
|
if err != nil {
|
|
t.Fatalf("NewStore: %v", err)
|
|
}
|
|
store := &nthCASFailureStore{Store: base, failure: injected}
|
|
backend := &sharedLifecycleBackend{}
|
|
manager, err := NewManager(
|
|
context.Background(),
|
|
map[string]agentconfig.ClientProcessSpec{string(ClientFlutter): fakeSpec()},
|
|
store,
|
|
WithProcessBackend(backend),
|
|
WithStopTimeout(time.Second),
|
|
WithPollInterval(10*time.Millisecond),
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("NewManager: %v", err)
|
|
}
|
|
tt.setup(t, manager, store)
|
|
|
|
if _, err := tt.call(manager); !errors.Is(err, injected) {
|
|
t.Fatalf("completion action error = %v, want injected failure", err)
|
|
}
|
|
if tt.name == "start" || tt.name == "detail start" {
|
|
status, err := manager.Status(ClientFlutter)
|
|
if err != nil {
|
|
t.Fatalf("Status: %v", err)
|
|
}
|
|
if status.Identity != nil || backend.startCount() != 1 {
|
|
t.Fatalf("failed start status = %+v starts=%d", status, backend.startCount())
|
|
}
|
|
identity := ProcessIdentity{PID: 1001, StartToken: "gen-1"}
|
|
if !backend.wasAborted(identity) {
|
|
t.Fatalf("failed start process %+v was not aborted", identity)
|
|
}
|
|
}
|
|
|
|
if _, err := tt.call(manager); !errors.Is(err, ErrCommandPending) {
|
|
t.Fatalf("same-manager retry error = %v, want ErrCommandPending", err)
|
|
}
|
|
restarted, err := NewManager(
|
|
context.Background(),
|
|
map[string]agentconfig.ClientProcessSpec{string(ClientFlutter): fakeSpec()},
|
|
base,
|
|
WithProcessBackend(backend),
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("NewManager restart: %v", err)
|
|
}
|
|
if _, err := tt.call(restarted); !errors.Is(err, ErrCommandPending) {
|
|
t.Fatalf("restart retry error = %v, want ErrCommandPending", err)
|
|
}
|
|
restarted.cancel()
|
|
_ = manager.Close(context.Background())
|
|
})
|
|
}
|
|
}
|
|
|
|
func startLiveFixture(t *testing.T, manager *Manager) {
|
|
t.Helper()
|
|
if _, err := manager.Start(context.Background(), ClientFlutter, ""); err != nil {
|
|
t.Fatalf("fixture Start: %v", err)
|
|
}
|
|
if _, err := manager.SetConnected(context.Background(), ClientFlutter, true); err != nil {
|
|
t.Fatalf("fixture SetConnected: %v", err)
|
|
}
|
|
}
|
|
|
|
// nthCASFailureStore injects one deterministic CAS failure after a pending
|
|
// command receipt has been durably saved, so tests exercise only the
|
|
// completion-save rollback boundary.
|
|
type nthCASFailureStore struct {
|
|
*agentstate.Store
|
|
mu sync.Mutex
|
|
calls int
|
|
failAt int
|
|
failure error
|
|
}
|
|
|
|
func (s *nthCASFailureStore) failOnNextCompletion() {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
s.failAt = s.calls + 2
|
|
}
|
|
|
|
func (s *nthCASFailureStore) CompareAndSwapIntegrationRecord(
|
|
ctx context.Context,
|
|
key string,
|
|
revision string,
|
|
payload []byte,
|
|
) (string, error) {
|
|
s.mu.Lock()
|
|
s.calls++
|
|
fail := s.failAt != 0 && s.calls == s.failAt
|
|
s.mu.Unlock()
|
|
if fail {
|
|
return "", s.failure
|
|
}
|
|
return s.Store.CompareAndSwapIntegrationRecord(ctx, key, revision, payload)
|
|
}
|
|
|
|
func TestClosePreservesAmbiguousIdentityBeforeAdoption(t *testing.T) {
|
|
assertClosePreservesAmbiguousIdentity(t, false)
|
|
}
|
|
|
|
func TestClosePreservesAmbiguousAdoptedIdentity(t *testing.T) {
|
|
assertClosePreservesAmbiguousIdentity(t, true)
|
|
}
|
|
|
|
func assertClosePreservesAmbiguousIdentity(t *testing.T, adopt bool) {
|
|
t.Helper()
|
|
base, err := agentstate.NewStore(filepath.Join(t.TempDir(), "state.json"))
|
|
if err != nil {
|
|
t.Fatalf("NewStore: %v", err)
|
|
}
|
|
identity := ProcessIdentity{PID: 733, StartToken: "ambiguous-token"}
|
|
durable, err := newDurableStore(base)
|
|
if err != nil {
|
|
t.Fatalf("newDurableStore: %v", err)
|
|
}
|
|
record := initialRecord(ClientFlutter)
|
|
record.State = StateConnected
|
|
record.Connected = true
|
|
record.Identity = &identity
|
|
record.UpdatedAt = time.Now().UTC()
|
|
if _, err := durable.save(context.Background(), record, ""); err != nil {
|
|
t.Fatalf("seed record: %v", err)
|
|
}
|
|
|
|
backend := &mutableInspectionBackend{state: IdentityAmbiguous}
|
|
manager, err := NewManager(
|
|
context.Background(),
|
|
map[string]agentconfig.ClientProcessSpec{string(ClientFlutter): fakeSpec()},
|
|
base,
|
|
WithProcessBackend(backend),
|
|
WithPollInterval(10*time.Millisecond),
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("NewManager: %v", err)
|
|
}
|
|
if adopt {
|
|
backend.setState(IdentityLive)
|
|
if err := manager.Reconcile(context.Background()); err != nil {
|
|
t.Fatalf("Reconcile: %v", err)
|
|
}
|
|
backend.setState(IdentityAmbiguous)
|
|
}
|
|
|
|
closeContext, cancel := context.WithTimeout(context.Background(), time.Second)
|
|
defer cancel()
|
|
if err := manager.Close(closeContext); !errors.Is(err, ErrIdentityAmbiguous) {
|
|
t.Fatalf("Close error = %v, want ErrIdentityAmbiguous", err)
|
|
}
|
|
if closeContext.Err() != nil {
|
|
t.Fatalf("Close exceeded its context: %v", closeContext.Err())
|
|
}
|
|
status, err := manager.Status(ClientFlutter)
|
|
if err != nil {
|
|
t.Fatalf("Status: %v", err)
|
|
}
|
|
if status.Identity == nil || *status.Identity != identity || status.State != StateConnected || status.Blocker == "" {
|
|
t.Fatalf("ambiguous close status = %+v", status)
|
|
}
|
|
manager.waiters.Wait()
|
|
}
|
|
|
|
type mutableInspectionBackend struct {
|
|
fakeProcessBackend
|
|
mu sync.Mutex
|
|
state IdentityState
|
|
}
|
|
|
|
func (b *mutableInspectionBackend) setState(state IdentityState) {
|
|
b.mu.Lock()
|
|
b.state = state
|
|
b.mu.Unlock()
|
|
}
|
|
|
|
func (b *mutableInspectionBackend) Inspect(
|
|
context.Context,
|
|
ProcessIdentity,
|
|
) (IdentityObservation, error) {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
return IdentityObservation{State: b.state}, nil
|
|
}
|