package bootstrap import ( "bytes" "context" "encoding/json" "fmt" "net" "net/http" "os" "path/filepath" "reflect" "strings" "sync" "testing" "time" toki "git.toki-labs.com/toki/proto-socket/go" "google.golang.org/protobuf/proto" "iop/apps/edge/internal/configrefresh" edgenode "iop/apps/edge/internal/node" edgeservice "iop/apps/edge/internal/service" "iop/packages/go/config" iop "iop/proto/gen/iop" ) // TestRefreshAdminDryRunAndApplyReachRunningRuntime verifies S06: a running // Edge process can receive dry-run and apply refresh requests via the admin API // and returns mode-specific results. func TestRefreshAdminDryRunAndApplyReachRunningRuntime(t *testing.T) { dir := t.TempDir() serverAddr := freeTCPAddr(t) // Write base and candidate YAMLs with explicit immutable field values so // that config.LoadEdge produces configs with identical defaults on both sides. // Only capacity differs between base and candidate. baseYAML := fmt.Sprintf(` server: listen: %q bootstrap: listen: "0.0.0.0:18080" artifact_dir: "artifacts" logging: level: "error" refresh: enabled: true listen: "127.0.0.1:0" openai: listen: "0.0.0.0:18081" a2a: listen: "0.0.0.0:8081" metrics: port: 0 nodes: - id: "node-1" alias: "n1" token: "tok-1" adapters: cli: enabled: true providers: - id: "prov-a" type: "ollama" category: "local_inference" adapter: "cli" models: ["llama3.1"] capacity: 2 max_queue: 4 queue_timeout_ms: 5000 `, serverAddr) candidateYAML := fmt.Sprintf(` server: listen: %q bootstrap: listen: "0.0.0.0:18080" artifact_dir: "artifacts" logging: level: "error" refresh: enabled: true listen: "127.0.0.1:0" openai: listen: "0.0.0.0:18081" a2a: listen: "0.0.0.0:8081" metrics: port: 0 nodes: - id: "node-1" alias: "n1" token: "tok-1" adapters: cli: enabled: true providers: - id: "prov-a" type: "ollama" category: "local_inference" adapter: "cli" models: ["llama3.1"] capacity: 8 max_queue: 4 queue_timeout_ms: 5000 `, serverAddr) basePath := filepath.Join(dir, "base.yaml") if err := os.WriteFile(basePath, []byte(baseYAML), 0o600); err != nil { t.Fatalf("write base: %v", err) } candidatePath := filepath.Join(dir, "candidate.yaml") if err := os.WriteFile(candidatePath, []byte(candidateYAML), 0o600); err != nil { t.Fatalf("write candidate: %v", err) } currentCfg, err := config.LoadEdge(basePath) if err != nil { t.Fatalf("load base config: %v", err) } // Apply serve-equivalent normalization so the current config matches // what iop-edge serve would hold after loadRuntimeConfig. currentCfg.Logging.Path = serveNormalizedLogPath() currentCfg.Bootstrap.ArtifactDir = serveNormalizedArtifactDir(currentCfg.Bootstrap.ArtifactDir) // Override fields that don't need to match between runtime and candidate. currentCfg.Refresh.Enabled = true currentCfg.Refresh.Listen = "127.0.0.1:0" currentCfg.Logging.Level = "error" rt, err := NewRuntime(currentCfg) if err != nil { t.Fatalf("NewRuntime: %v", err) } ctx := context.Background() if err := rt.Start(ctx); err != nil { t.Fatalf("Start: %v", err) } defer func() { if err := rt.Stop(); err != nil { t.Fatalf("Stop: %v", err) } }() adminAddr := rt.RefreshAdmin.Addr() if adminAddr == "" { t.Fatal("refresh admin addr is empty after Start") } adminURL := "http://" + adminAddr + "/refresh" client := &http.Client{Timeout: 5 * time.Second} postRefresh := func(t *testing.T, mode configrefresh.Mode) configrefresh.Result { t.Helper() body, _ := json.Marshal(configrefresh.Request{ Mode: mode, ConfigPath: candidatePath, RequestID: string(mode) + "-req", }) resp, err := client.Post(adminURL, "application/json", bytes.NewReader(body)) if err != nil { t.Fatalf("POST /refresh mode=%s: %v", mode, err) } defer resp.Body.Close() var result configrefresh.Result if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { t.Fatalf("decode response: %v", err) } return result } // dry-run: process is reached, result has mode=dry_run. dryResult := postRefresh(t, configrefresh.ModeDryRun) if dryResult.Mode != configrefresh.ModeDryRun { t.Errorf("dry-run: expected mode=%q, got %q", configrefresh.ModeDryRun, dryResult.Mode) } if dryResult.Status != configrefresh.StatusApplied { t.Errorf("dry-run: expected status=%q (capacity change), got %q", configrefresh.StatusApplied, dryResult.Status) } // apply: process is reached, result has mode=apply and status=applied. applyResult := postRefresh(t, configrefresh.ModeApply) if applyResult.Mode != configrefresh.ModeApply { t.Errorf("apply: expected mode=%q, got %q", configrefresh.ModeApply, applyResult.Mode) } if applyResult.Status != configrefresh.StatusApplied { t.Errorf("apply: expected status=%q, got %q", configrefresh.StatusApplied, applyResult.Status) } // verify dry-run and apply have distinct observable results: the apply // committed the capacity change, so the next dry-run shows no changes. dryResult2 := postRefresh(t, configrefresh.ModeDryRun) if len(dryResult2.Changes) != 0 { t.Errorf("after apply, expected no diff changes on second dry-run, got %+v", dryResult2.Changes) } } func TestRefreshApplyConcurrentRuntimeReaders(t *testing.T) { dir := t.TempDir() serverAddr := freeTCPAddr(t) openAIAddr := freeTCPAddr(t) writeConfig := func(t *testing.T, name string, capacity int, displayName string) string { t.Helper() path := filepath.Join(dir, name) yaml := fmt.Sprintf(` server: listen: %q bootstrap: listen: "0.0.0.0:18080" artifact_dir: "artifacts" logging: level: "error" refresh: enabled: false listen: "127.0.0.1:19093" openai: enabled: true listen: %q adapter: "openai_compat" target: "" a2a: listen: "0.0.0.0:8081" metrics: port: 0 models: - id: "qwen3.6:35b" display_name: %q providers: prov-a: "served-qwen" nodes: - id: "node-1" alias: "n1" token: "tok-1" adapters: openai_compat_instances: - name: "vllm-gpu" enabled: true provider: "vllm" endpoint: "http://127.0.0.1:8000/v1" providers: - id: "prov-a" type: "vllm" category: "api" adapter: "vllm-gpu" models: ["served-qwen"] health: "available" capacity: %d max_queue: 4 queue_timeout_ms: 5000 `, serverAddr, openAIAddr, displayName, capacity) if err := os.WriteFile(path, []byte(yaml), 0o600); err != nil { t.Fatalf("write %s: %v", name, err) } return path } basePath := writeConfig(t, "base.yaml", 2, "Qwen Base") candidatePath := writeConfig(t, "candidate.yaml", 8, "Qwen Candidate") currentCfg, err := config.LoadEdge(basePath) if err != nil { t.Fatalf("load base config: %v", err) } // Apply serve-equivalent normalization. currentCfg.Logging.Path = serveNormalizedLogPath() currentCfg.Bootstrap.ArtifactDir = serveNormalizedArtifactDir(currentCfg.Bootstrap.ArtifactDir) rt, err := NewRuntime(currentCfg) if err != nil { t.Fatalf("NewRuntime: %v", err) } if err := rt.Start(context.Background()); err != nil { t.Fatalf("Start: %v", err) } defer func() { if err := rt.Stop(); err != nil { t.Fatalf("Stop: %v", err) } }() client := &http.Client{Timeout: 5 * time.Second} start := make(chan struct{}) var wg sync.WaitGroup errCh := make(chan error, 3) wg.Add(1) go func() { defer wg.Done() <-start for i := 0; i < 50; i++ { result, err := rt.RefreshConfig(context.Background(), configrefresh.Request{ Mode: configrefresh.ModeApply, ConfigPath: candidatePath, RequestID: fmt.Sprintf("apply-%d", i), }) if err != nil { errCh <- err return } if result.Status != configrefresh.StatusApplied { errCh <- fmt.Errorf("refresh status=%s summary=%s changes=%+v", result.Status, result.Summary, result.Changes) return } } }() wg.Add(1) go func() { defer wg.Done() <-start for i := 0; i < 100; i++ { _ = rt.Service.ListNodeSnapshots() } }() wg.Add(1) go func() { defer wg.Done() <-start url := "http://" + openAIAddr + "/v1/models" for i := 0; i < 100; i++ { resp, err := client.Get(url) if err != nil { errCh <- err return } resp.Body.Close() if resp.StatusCode != http.StatusOK { errCh <- fmt.Errorf("GET /v1/models status=%d", resp.StatusCode) return } } }() close(start) wg.Wait() close(errCh) for err := range errCh { if err != nil { t.Fatalf("concurrent refresh reader failed: %v", err) } } } // TestRefreshRejectedPreservesModels verifies SDD S10: an invalid candidate // apply is rejected without mutating the running runtime config or the // /v1/models surface. Uses stable model ID comparison to avoid timestamp flakiness. func TestRefreshRejectedPreservesModels(t *testing.T) { dir := t.TempDir() serverAddr := freeTCPAddr(t) openAIAddr := freeTCPAddr(t) basePath := filepath.Join(dir, "base.yaml") if err := os.WriteFile(basePath, []byte(providerPoolConfigYAML(serverAddr, openAIAddr, validAdapterBlock)), 0o600); err != nil { t.Fatalf("write base: %v", err) } candidatePath := filepath.Join(dir, "invalid.yaml") if err := os.WriteFile(candidatePath, []byte(providerPoolConfigYAML(serverAddr, openAIAddr, invalidAdapterBlock)), 0o600); err != nil { t.Fatalf("write candidate: %v", err) } rt, err := NewRuntime(loadServeNormalizedConfig(t, basePath)) if err != nil { t.Fatalf("NewRuntime: %v", err) } if err := rt.Start(context.Background()); err != nil { t.Fatalf("Start: %v", err) } defer func() { if err := rt.Stop(); err != nil { t.Fatalf("Stop: %v", err) } }() client := &http.Client{Timeout: 5 * time.Second} beforeIDs := modelIDs(t, client, openAIAddr) if len(beforeIDs) == 0 { t.Fatal("expected at least one model before refresh") } cfgBefore := rt.Cfg result, err := rt.RefreshConfig(context.Background(), configrefresh.Request{ Mode: configrefresh.ModeApply, ConfigPath: candidatePath, RequestID: "reject-1", }) if err != nil { t.Fatalf("RefreshConfig: %v", err) } if result.Status != configrefresh.StatusRejected { t.Fatalf("expected status=rejected for invalid candidate, got %q (summary=%s)", result.Status, result.Summary) } if rt.Cfg != cfgBefore { t.Fatal("runtime config snapshot mutated after rejected apply") } afterIDs := modelIDs(t, client, openAIAddr) if !reflect.DeepEqual(afterIDs, beforeIDs) { t.Fatalf("/v1/models changed after rejected apply:\nbefore=%v\nafter=%v", beforeIDs, afterIDs) } } // TestRefreshApplyDoesNotMutateOnRestartRequired verifies that a candidate with // a restart_required change leaves the running runtime snapshot and /v1/models // untouched. Uses stable model ID comparison. func TestRefreshApplyDoesNotMutateOnRestartRequired(t *testing.T) { dir := t.TempDir() serverAddr := freeTCPAddr(t) openAIAddr := freeTCPAddr(t) basePath := filepath.Join(dir, "base.yaml") if err := os.WriteFile(basePath, []byte(providerPoolConfigYAML(serverAddr, openAIAddr, validAdapterBlock)), 0o600); err != nil { t.Fatalf("write base: %v", err) } // Candidate changes server.listen, which is classified restart_required. candidateYAML := providerPoolConfigYAML(freeTCPAddr(t), openAIAddr, validAdapterBlock) candidatePath := filepath.Join(dir, "restart.yaml") if err := os.WriteFile(candidatePath, []byte(candidateYAML), 0o600); err != nil { t.Fatalf("write candidate: %v", err) } rt, err := NewRuntime(loadServeNormalizedConfig(t, basePath)) if err != nil { t.Fatalf("NewRuntime: %v", err) } if err := rt.Start(context.Background()); err != nil { t.Fatalf("Start: %v", err) } defer func() { if err := rt.Stop(); err != nil { t.Fatalf("Stop: %v", err) } }() client := &http.Client{Timeout: 5 * time.Second} beforeIDs := modelIDs(t, client, openAIAddr) if len(beforeIDs) == 0 { t.Fatal("expected at least one model before refresh") } cfgBefore := rt.Cfg result, err := rt.RefreshConfig(context.Background(), configrefresh.Request{ Mode: configrefresh.ModeApply, ConfigPath: candidatePath, RequestID: "restart-1", }) if err != nil { t.Fatalf("RefreshConfig: %v", err) } if result.Status != configrefresh.StatusRestartRequired { t.Fatalf("expected status=restart_required, got %q (summary=%s)", result.Status, result.Summary) } if len(result.RestartRequiredPaths) == 0 { t.Fatal("expected restart_required_paths to be reported") } if rt.Cfg != cfgBefore { t.Fatal("runtime config snapshot mutated after restart_required apply") } afterIDs := modelIDs(t, client, openAIAddr) if !reflect.DeepEqual(afterIDs, beforeIDs) { t.Fatalf("/v1/models changed after restart_required apply:\nbefore=%v\nafter=%v", beforeIDs, afterIDs) } } // TestRefreshAPIReportIncludesChangedItems verifies SDD S11: the refresh API // response exposes the changed provider/model items so an operator can confirm // what a refresh touched. func TestRefreshAPIReportIncludesChangedItems(t *testing.T) { dir := t.TempDir() serverAddr := freeTCPAddr(t) openAIAddr := freeTCPAddr(t) basePath := filepath.Join(dir, "base.yaml") if err := os.WriteFile(basePath, []byte(providerPoolConfigYAML(serverAddr, openAIAddr, validAdapterBlock)), 0o600); err != nil { t.Fatalf("write base: %v", err) } // Candidate raises provider capacity (applied) and renames the model display // name (applied), so both a provider and a model item change. candidateYAML := strings.Replace( strings.Replace(providerPoolConfigYAML(serverAddr, openAIAddr, validAdapterBlock), "capacity: 2", "capacity: 8", 1), `display_name: "Qwen"`, `display_name: "Qwen Refreshed"`, 1) candidatePath := filepath.Join(dir, "candidate.yaml") if err := os.WriteFile(candidatePath, []byte(candidateYAML), 0o600); err != nil { t.Fatalf("write candidate: %v", err) } rt, err := NewRuntime(loadServeNormalizedConfig(t, basePath)) if err != nil { t.Fatalf("NewRuntime: %v", err) } if err := rt.Start(context.Background()); err != nil { t.Fatalf("Start: %v", err) } defer func() { if err := rt.Stop(); err != nil { t.Fatalf("Stop: %v", err) } }() adminURL := "http://" + rt.RefreshAdmin.Addr() + "/refresh" client := &http.Client{Timeout: 5 * time.Second} body, _ := json.Marshal(configrefresh.Request{ Mode: configrefresh.ModeDryRun, ConfigPath: candidatePath, RequestID: "report-1", }) resp, err := client.Post(adminURL, "application/json", bytes.NewReader(body)) if err != nil { t.Fatalf("POST /refresh: %v", err) } defer resp.Body.Close() var result configrefresh.Result if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { t.Fatalf("decode response: %v", err) } if result.Status != configrefresh.StatusApplied { t.Fatalf("expected status=applied, got %q (summary=%s)", result.Status, result.Summary) } if !containsString(result.ChangedProviders, "prov-a") { t.Errorf("expected changed_providers to include prov-a, got %v", result.ChangedProviders) } if !containsString(result.ChangedModels, "qwen3.6:35b") { t.Errorf("expected changed_models to include qwen3.6:35b, got %v", result.ChangedModels) } } // TestRefreshProviderPoolPolicy verifies that NewRuntime initial assembly and // RefreshConfig apply both propagate the root provider-pool policy into the // queue manager. The initial SetRuntimeConfig call in NewRuntime must use the // same atomic handoff as the apply path, so the root max_queue and timeout // are observable on the queue immediately after NewRuntime returns. func TestRefreshProviderPoolPolicy(t *testing.T) { dir := t.TempDir() serverAddr := freeTCPAddr(t) cfgYAML := fmt.Sprintf(` server: listen: %q bootstrap: listen: "0.0.0.0:18080" artifact_dir: "artifacts" logging: level: "error" refresh: enabled: false listen: "127.0.0.1:0" metrics: port: 0 models: - id: "qwen3.6:35b" display_name: "Qwen" providers: prov-a: "served-qwen" nodes: - id: "node-rp" alias: "n-rp" token: "tok-rp" adapters: openai_compat_instances: - name: "vllm-gpu" enabled: true provider: "vllm" endpoint: "http://127.0.0.1:8000/v1" providers: - id: "prov-a" type: "vllm" category: "api" adapter: "vllm-gpu" models: ["served-qwen"] health: "available" capacity: 2 max_queue: 4 queue_timeout_ms: 5000 `, serverAddr) cfgPath := filepath.Join(dir, "base.yaml") if err := os.WriteFile(cfgPath, []byte(cfgYAML), 0o600); err != nil { t.Fatalf("write config: %v", err) } cfg, err := config.LoadEdge(cfgPath) if err != nil { t.Fatalf("load config: %v", err) } cfg.Logging.Path = serveNormalizedLogPath() cfg.Bootstrap.ArtifactDir = serveNormalizedArtifactDir(cfg.Bootstrap.ArtifactDir) rt, err := NewRuntime(cfg) if err != nil { t.Fatalf("NewRuntime: %v", err) } // Initial handoff: the root policy must be propagated to the queue. initialPolicy := rt.Service.ProviderPoolPolicy() if initialPolicy.MaxQueue != 4 { t.Errorf("expected initial maxQueue=4, got %d", initialPolicy.MaxQueue) } if initialPolicy.QueueTimeoutSet != true { t.Errorf("expected initial queueTimeoutSet=true") } if initialPolicy.QueueTimeout != 5*time.Second { t.Errorf("expected initial queueTimeout=5s, got %v", initialPolicy.QueueTimeout) } // Apply a new policy via RefreshConfig and verify it replaces the root. candidatePath := filepath.Join(dir, "candidate.yaml") candidateYAML := strings.Replace(cfgYAML, "capacity: 2", "capacity: 4", 1) candidateYAML = strings.Replace(candidateYAML, "max_queue: 4", "max_queue: 2", 1) candidateYAML = strings.Replace(candidateYAML, "queue_timeout_ms: 5000", "queue_timeout_ms: 2000", 1) if err := os.WriteFile(candidatePath, []byte(candidateYAML), 0o600); err != nil { t.Fatalf("write candidate: %v", err) } result, err := rt.RefreshConfig(context.Background(), configrefresh.Request{ Mode: configrefresh.ModeApply, ConfigPath: candidatePath, RequestID: "rp-apply-1", }) if err != nil { t.Fatalf("RefreshConfig: %v", err) } if result.Status != configrefresh.StatusApplied { t.Fatalf("expected status=applied, got %q (summary=%s)", result.Status, result.Summary) } afterPolicy := rt.Service.ProviderPoolPolicy() if afterPolicy.MaxQueue != 2 { t.Errorf("expected maxQueue=2 after apply, got %d", afterPolicy.MaxQueue) } if afterPolicy.QueueTimeoutSet != true { t.Errorf("expected queueTimeoutSet=true after apply") } if afterPolicy.QueueTimeout != 2*time.Second { t.Errorf("expected queueTimeout=2s after apply, got %v", afterPolicy.QueueTimeout) } } // TestRefreshRuntimeSnapshotPreservesLeaseAndAppliesPolicyAtomically verifies // that a single Runtime.RefreshConfig apply updates provider attributes // (capacity) and the root provider-pool queue policy from the same config // snapshot. The observable contract is: after a refresh that changes both // capacity and policy, both reflect the new values in a single apply cycle, // existing leases survive the shrink, and the queued waiter remains blocked // until occupancy falls below the refreshed capacity. func TestRefreshRuntimeSnapshotPreservesLeaseAndAppliesPolicyAtomically(t *testing.T) { dir := t.TempDir() serverAddr := freeTCPAddr(t) cfgYAML := fmt.Sprintf(` server: listen: %q bootstrap: listen: "0.0.0.0:18080" artifact_dir: "artifacts" logging: level: "error" refresh: enabled: false listen: "127.0.0.1:0" metrics: port: 0 provider_pool: max_queue: 8 queue_timeout_ms: 5000 models: - id: "group-atomic" providers: prov-atomic: "served-atomic" nodes: - id: "node-atomic" alias: "n-atm" token: "tok-atm" adapters: openai_compat_instances: - name: "vllm-gpu" enabled: true provider: "vllm" endpoint: "http://127.0.0.1:0/v1" providers: - id: "prov-atomic" type: "vllm" category: "api" adapter: "vllm-gpu" models: ["served-atomic"] health: "available" capacity: 2 `, serverAddr) cfgPath := filepath.Join(dir, "base.yaml") if err := os.WriteFile(cfgPath, []byte(cfgYAML), 0o600); err != nil { t.Fatalf("write config: %v", err) } cfg, err := config.LoadEdge(cfgPath) if err != nil { t.Fatalf("load config: %v", err) } cfg.Logging.Path = serveNormalizedLogPath() cfg.Bootstrap.ArtifactDir = serveNormalizedArtifactDir(cfg.Bootstrap.ArtifactDir) rt, err := NewRuntime(cfg) if err != nil { t.Fatalf("NewRuntime: %v", err) } initialPolicy := rt.Service.ProviderPoolPolicy() if initialPolicy.MaxQueue != 8 { t.Fatalf("expected initial maxQueue=8, got %d", initialPolicy.MaxQueue) } if initialPolicy.QueueTimeout != 5*time.Second { t.Fatalf("expected initial queueTimeout=5s, got %v", initialPolicy.QueueTimeout) } edgeConn, nodeConn := net.Pipe() t.Cleanup(func() { edgeConn.Close() nodeConn.Close() }) parserMap := toki.ParserMap{ toki.TypeNameOf(&iop.RunRequest{}): func(b []byte) (proto.Message, error) { m := &iop.RunRequest{} return m, proto.Unmarshal(b, m) }, toki.TypeNameOf(&iop.NodeConfigRefreshRequest{}): func(b []byte) (proto.Message, error) { m := &iop.NodeConfigRefreshRequest{} return m, proto.Unmarshal(b, m) }, toki.TypeNameOf(&iop.NodeConfigRefreshResponse{}): func(b []byte) (proto.Message, error) { m := &iop.NodeConfigRefreshResponse{} return m, proto.Unmarshal(b, m) }, } edgeClient := toki.NewTcpClient(edgeConn, 0, 0, parserMap) nodeClient := toki.NewTcpClient(nodeConn, 0, 0, parserMap) toki.AddListenerTyped[*iop.RunRequest](&nodeClient.Communicator, func(*iop.RunRequest) {}) toki.AddRequestListenerTyped[*iop.NodeConfigRefreshRequest, *iop.NodeConfigRefreshResponse]( &nodeClient.Communicator, func(req *iop.NodeConfigRefreshRequest) (*iop.NodeConfigRefreshResponse, error) { return &iop.NodeConfigRefreshResponse{ RequestId: req.GetRequestId(), Status: iop.NodeConfigRefreshStatus_NODE_CONFIG_REFRESH_STATUS_APPLIED, }, nil }, ) rt.Registry.Register(&edgenode.NodeEntry{ NodeID: "node-atomic", LifecycleState: edgenode.LifecycleConnected, Client: edgeClient, }) submit := func(runID string) edgeservice.RunResult { t.Helper() run, submitErr := rt.Service.SubmitRun(t.Context(), edgeservice.SubmitRunRequest{ RunID: runID, ModelGroupKey: "group-atomic", ProviderPool: true, Background: true, }) if submitErr != nil { t.Fatalf("SubmitRun %q: %v", runID, submitErr) } return run } held1 := submit("atomic-held-1") held2 := submit("atomic-held-2") type submitResult struct { run edgeservice.RunResult err error } waiterCh := make(chan submitResult, 1) go func() { run, submitErr := rt.Service.SubmitRun(t.Context(), edgeservice.SubmitRunRequest{ RunID: "atomic-waiter", ModelGroupKey: "group-atomic", ProviderPool: true, Background: true, }) waiterCh <- submitResult{run: run, err: submitErr} }() providerSnapshot := func() *iop.ProviderSnapshot { t.Helper() for _, node := range rt.Service.ListNodeSnapshots() { if node.NodeID != "node-atomic" { continue } for _, provider := range node.ProviderSnapshots { if provider.GetId() == "prov-atomic" { return provider } } } t.Fatal("provider prov-atomic snapshot not found") return nil } queueDeadline := time.Now().Add(2 * time.Second) for time.Now().Before(queueDeadline) { if providerSnapshot().GetQueued() == 1 { break } time.Sleep(time.Millisecond) } beforeRefresh := providerSnapshot() if beforeRefresh.GetInFlight() != 2 || beforeRefresh.GetQueued() != 1 { t.Fatalf("expected pre-refresh inFlight=2 queued=1, got inFlight=%d queued=%d", beforeRefresh.GetInFlight(), beforeRefresh.GetQueued()) } // Apply a candidate that shrinks capacity AND changes policy atomically. candidatePath := filepath.Join(dir, "candidate.yaml") candidateYAML := strings.Replace(cfgYAML, "capacity: 2", "capacity: 1", 1) candidateYAML = strings.Replace(candidateYAML, "max_queue: 8", "max_queue: 4", 1) candidateYAML = strings.Replace(candidateYAML, "queue_timeout_ms: 5000", "queue_timeout_ms: 1000", 1) if err := os.WriteFile(candidatePath, []byte(candidateYAML), 0o600); err != nil { t.Fatalf("write candidate: %v", err) } result, err := rt.RefreshConfig(context.Background(), configrefresh.Request{ Mode: configrefresh.ModeApply, ConfigPath: candidatePath, RequestID: "atomic-apply-1", }) if err != nil { t.Fatalf("RefreshConfig: %v", err) } if result.Status != configrefresh.StatusApplied { t.Fatalf("expected status=applied, got %q", result.Status) } // Verify the policy snapshot reflects the new config. afterPolicy := rt.Service.ProviderPoolPolicy() if afterPolicy.MaxQueue != 4 { t.Errorf("expected maxQueue=4 after atomic apply, got %d", afterPolicy.MaxQueue) } if afterPolicy.QueueTimeout != 1*time.Second { t.Errorf("expected queueTimeout=1s after atomic apply, got %v", afterPolicy.QueueTimeout) } afterRefresh := providerSnapshot() if afterRefresh.GetCapacity() != 1 || afterRefresh.GetInFlight() != 2 || afterRefresh.GetQueued() != 1 { t.Fatalf( "expected atomic snapshot capacity=1 inFlight=2 queued=1, got capacity=%d inFlight=%d queued=%d", afterRefresh.GetCapacity(), afterRefresh.GetInFlight(), afterRefresh.GetQueued(), ) } // Verify the node store reflects the new capacity. store := rt.Service.NodeStore() records := store.All() if len(records) != 1 { t.Fatalf("expected 1 node record, got %d", len(records)) } rec := records[0] found := false for _, p := range rec.Providers { if p.ID == "prov-atomic" { if p.Capacity != 1 { t.Errorf("expected capacity=1 after atomic apply, got %d", p.Capacity) } found = true } } if !found { t.Fatal("provider prov-atomic not found in node record") } // The capacity shrink preserves both leases and blocks the waiter until // occupancy falls below the new capacity. rt.Service.HandleRunLifecycleEvent(&iop.RunEvent{RunId: held1.Dispatch().RunID, Type: "complete"}) held1.Close() afterFirstRelease := providerSnapshot() if afterFirstRelease.GetInFlight() != 1 || afterFirstRelease.GetQueued() != 1 { t.Fatalf( "expected waiter blocked at shrunken capacity after one release, got inFlight=%d queued=%d", afterFirstRelease.GetInFlight(), afterFirstRelease.GetQueued(), ) } rt.Service.HandleRunLifecycleEvent(&iop.RunEvent{RunId: held2.Dispatch().RunID, Type: "complete"}) held2.Close() var waiter edgeservice.RunResult select { case got := <-waiterCh: if got.err != nil { t.Fatalf("queued SubmitRun after second release: %v", got.err) } waiter = got.run case <-time.After(2 * time.Second): t.Fatal("queued waiter was not dispatched after occupancy fell below refreshed capacity") } rt.Service.HandleRunLifecycleEvent(&iop.RunEvent{RunId: waiter.Dispatch().RunID, Type: "complete"}) waiter.Close() finalSnapshot := providerSnapshot() if finalSnapshot.GetInFlight() != 0 || finalSnapshot.GetQueued() != 0 { t.Errorf("expected final drain, got inFlight=%d queued=%d", finalSnapshot.GetInFlight(), finalSnapshot.GetQueued()) } finalPolicy := rt.Service.ProviderPoolPolicy() if finalPolicy.MaxQueue != 4 || finalPolicy.QueueTimeout != 1*time.Second { t.Errorf("final snapshot inconsistent: maxQueue=%d queueTimeout=%v", finalPolicy.MaxQueue, finalPolicy.QueueTimeout) } }