package bootstrap import ( "bytes" "context" "encoding/json" "fmt" "net" "net/http" "os" "path/filepath" "reflect" "sort" "strings" "sync" "sync/atomic" "testing" "time" toki "git.toki-labs.com/toki/proto-socket/go" "google.golang.org/protobuf/proto" iop "iop/proto/gen/iop" "iop/apps/edge/internal/configrefresh" edgenode "iop/apps/edge/internal/node" edgeservice "iop/apps/edge/internal/service" "iop/packages/go/config" ) func newTestConfig() *config.EdgeConfig { return &config.EdgeConfig{ Server: config.EdgeServerConf{Listen: "127.0.0.1:0"}, Logging: config.LoggingConf{Level: "error"}, Metrics: config.MetricsConf{Port: 0}, Nodes: []config.NodeDefinition{ {ID: "node-1", Alias: "alias-1", Token: "tok-1"}, }, } } func TestNewRuntimeSeedsNodeStore(t *testing.T) { cfg := newTestConfig() rt, err := NewRuntime(cfg) if err != nil { t.Fatalf("NewRuntime: %v", err) } if rt.NodeStore == nil { t.Fatal("NodeStore is nil") } rec, ok := rt.NodeStore.FindByToken("tok-1") if !ok { t.Fatal("expected node record seeded by token tok-1") } if rec.ID != "node-1" || rec.Alias != "alias-1" { t.Errorf("unexpected record: id=%q alias=%q", rec.ID, rec.Alias) } if rt.Registry == nil || rt.EventBus == nil || rt.Service == nil || rt.Server == nil || rt.Logger == nil { t.Error("runtime dependencies must be wired by NewRuntime") } } func TestRuntimeWiresTransportHandlers(t *testing.T) { cfg := newTestConfig() rt, err := NewRuntime(cfg) if err != nil { t.Fatalf("NewRuntime: %v", err) } if rt.Server.HasRunEventHandler() || rt.Server.HasNodeEventHandler() { t.Fatal("handlers must not be wired before wireHandlers") } rt.wireHandlers() if !rt.Server.HasRunEventHandler() { t.Error("run event handler not wired") } if !rt.Server.HasNodeEventHandler() { t.Error("node event handler not wired") } } func TestNewRuntimeWiresInputManager(t *testing.T) { cfg := newTestConfig() rt, err := NewRuntime(cfg) if err != nil { t.Fatalf("NewRuntime: %v", err) } if rt.Input == nil { t.Fatal("Input manager is nil") } if rt.Input.OpenAI == nil { t.Fatal("Input.OpenAI is nil") } if rt.Input.A2A == nil { t.Fatal("Input.A2A is nil") } } func TestRuntimeKeepsListenersAfterStartupContextCancel(t *testing.T) { edgeAddr := freeTCPAddr(t) openAIAddr := freeTCPAddr(t) cfg := newTestConfig() cfg.Server.Listen = edgeAddr cfg.OpenAI = config.EdgeOpenAIConf{ Enabled: true, Listen: openAIAddr, } rt, err := NewRuntime(cfg) if err != nil { t.Fatalf("NewRuntime: %v", err) } startupCtx, cancelStartup := context.WithCancel(context.Background()) if err := rt.Start(startupCtx); err != nil { t.Fatalf("Start: %v", err) } defer func() { if err := rt.Stop(); err != nil { t.Fatalf("Stop: %v", err) } }() cancelStartup() conn, err := net.DialTimeout("tcp", edgeAddr, time.Second) if err != nil { t.Fatalf("edge listener closed with startup context: %v", err) } _ = conn.Close() client := &http.Client{Timeout: time.Second} resp, err := client.Get("http://" + openAIAddr + "/healthz") if err != nil { t.Fatalf("openai listener closed with startup context: %v", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { t.Fatalf("openai health status = %d, want %d", resp.StatusCode, http.StatusOK) } } func TestNewRuntimeWritesLogsToConfiguredPath(t *testing.T) { dir := t.TempDir() logPath := filepath.Join(dir, "logs", "edge.log") cfg := newTestConfig() cfg.Logging.Level = "info" cfg.Logging.Path = logPath rt, err := NewRuntime(cfg) if err != nil { t.Fatalf("NewRuntime: %v", err) } rt.Logger.Info("runtime-log-entry") _ = rt.Logger.Sync() info, err := os.Stat(logPath) if err != nil { t.Fatalf("stat log path: %v", err) } if info.Size() == 0 { t.Fatalf("log file empty: %s", logPath) } contents, err := os.ReadFile(logPath) if err != nil { t.Fatalf("read log file: %v", err) } if !strings.Contains(string(contents), "runtime-log-entry") { t.Fatalf("expected runtime-log-entry in log file; contents=%q", string(contents)) } } func TestNewRuntimeRejectsDuplicateToken(t *testing.T) { cfg := &config.EdgeConfig{ Server: config.EdgeServerConf{Listen: "127.0.0.1:0"}, Logging: config.LoggingConf{Level: "error"}, Nodes: []config.NodeDefinition{ {ID: "node-1", Alias: "a1", Token: "dup"}, {ID: "node-2", Alias: "a2", Token: "dup"}, }, } if _, err := NewRuntime(cfg); err == nil { t.Fatal("expected error on duplicate token seed") } } // TestRefreshAdminDryRunAndApplyReachRunningRuntime verifies S06: a running // Edge process can receive dry-run and apply refresh requests via the admin API // and returns mode-specific results. func TestRefreshAdminDryRunAndApplyReachRunningRuntime(t *testing.T) { dir := t.TempDir() serverAddr := freeTCPAddr(t) // Write base and candidate YAMLs with explicit immutable field values so // that config.LoadEdge produces configs with identical defaults on both sides. // Only capacity differs between base and candidate. baseYAML := fmt.Sprintf(` server: listen: %q bootstrap: listen: "0.0.0.0:18080" artifact_dir: "artifacts" logging: level: "error" refresh: enabled: true listen: "127.0.0.1:0" openai: listen: "0.0.0.0:18081" a2a: listen: "0.0.0.0:8081" metrics: port: 0 nodes: - id: "node-1" alias: "n1" token: "tok-1" adapters: cli: enabled: true providers: - id: "prov-a" type: "ollama" category: "local_inference" models: ["llama3.1"] capacity: 2 max_queue: 4 queue_timeout_ms: 5000 `, serverAddr) candidateYAML := fmt.Sprintf(` server: listen: %q bootstrap: listen: "0.0.0.0:18080" artifact_dir: "artifacts" logging: level: "error" refresh: enabled: true listen: "127.0.0.1:0" openai: listen: "0.0.0.0:18081" a2a: listen: "0.0.0.0:8081" metrics: port: 0 nodes: - id: "node-1" alias: "n1" token: "tok-1" adapters: cli: enabled: true providers: - id: "prov-a" type: "ollama" category: "local_inference" models: ["llama3.1"] capacity: 8 max_queue: 4 queue_timeout_ms: 5000 `, serverAddr) basePath := filepath.Join(dir, "base.yaml") if err := os.WriteFile(basePath, []byte(baseYAML), 0o600); err != nil { t.Fatalf("write base: %v", err) } candidatePath := filepath.Join(dir, "candidate.yaml") if err := os.WriteFile(candidatePath, []byte(candidateYAML), 0o600); err != nil { t.Fatalf("write candidate: %v", err) } currentCfg, err := config.LoadEdge(basePath) if err != nil { t.Fatalf("load base config: %v", err) } // Apply serve-equivalent normalization so the current config matches // what iop-edge serve would hold after loadRuntimeConfig. currentCfg.Logging.Path = serveNormalizedLogPath() currentCfg.Bootstrap.ArtifactDir = serveNormalizedArtifactDir(currentCfg.Bootstrap.ArtifactDir) // Override fields that don't need to match between runtime and candidate. currentCfg.Refresh.Enabled = true currentCfg.Refresh.Listen = "127.0.0.1:0" currentCfg.Logging.Level = "error" rt, err := NewRuntime(currentCfg) if err != nil { t.Fatalf("NewRuntime: %v", err) } ctx := context.Background() if err := rt.Start(ctx); err != nil { t.Fatalf("Start: %v", err) } defer func() { if err := rt.Stop(); err != nil { t.Fatalf("Stop: %v", err) } }() adminAddr := rt.RefreshAdmin.Addr() if adminAddr == "" { t.Fatal("refresh admin addr is empty after Start") } adminURL := "http://" + adminAddr + "/refresh" client := &http.Client{Timeout: 5 * time.Second} postRefresh := func(t *testing.T, mode configrefresh.Mode) configrefresh.Result { t.Helper() body, _ := json.Marshal(configrefresh.Request{ Mode: mode, ConfigPath: candidatePath, RequestID: string(mode) + "-req", }) resp, err := client.Post(adminURL, "application/json", bytes.NewReader(body)) if err != nil { t.Fatalf("POST /refresh mode=%s: %v", mode, err) } defer resp.Body.Close() var result configrefresh.Result if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { t.Fatalf("decode response: %v", err) } return result } // dry-run: process is reached, result has mode=dry_run. dryResult := postRefresh(t, configrefresh.ModeDryRun) if dryResult.Mode != configrefresh.ModeDryRun { t.Errorf("dry-run: expected mode=%q, got %q", configrefresh.ModeDryRun, dryResult.Mode) } if dryResult.Status != configrefresh.StatusApplied { t.Errorf("dry-run: expected status=%q (capacity change), got %q", configrefresh.StatusApplied, dryResult.Status) } // apply: process is reached, result has mode=apply and status=applied. applyResult := postRefresh(t, configrefresh.ModeApply) if applyResult.Mode != configrefresh.ModeApply { t.Errorf("apply: expected mode=%q, got %q", configrefresh.ModeApply, applyResult.Mode) } if applyResult.Status != configrefresh.StatusApplied { t.Errorf("apply: expected status=%q, got %q", configrefresh.StatusApplied, applyResult.Status) } // verify dry-run and apply have distinct observable results: the apply // committed the capacity change, so the next dry-run shows no changes. dryResult2 := postRefresh(t, configrefresh.ModeDryRun) if len(dryResult2.Changes) != 0 { t.Errorf("after apply, expected no diff changes on second dry-run, got %+v", dryResult2.Changes) } } func serveNormalizedLogPath() string { if exe, err := os.Executable(); err == nil { return filepath.Join(filepath.Dir(exe), "logs", "edge.log") } if wd, err := os.Getwd(); err == nil { return filepath.Join(wd, "logs", "edge.log") } return "logs/edge.log" } func serveNormalizedArtifactDir(dir string) string { if dir == "" { dir = "artifacts" } if filepath.IsAbs(dir) { return dir } bd := binaryDirForTest() return filepath.Join(bd, dir) } func binaryDirForTest() string { if exe, err := os.Executable(); err == nil { return filepath.Dir(exe) } if wd, err := os.Getwd(); err == nil { return wd } return "." } func TestRefreshApplyConcurrentRuntimeReaders(t *testing.T) { dir := t.TempDir() serverAddr := freeTCPAddr(t) openAIAddr := freeTCPAddr(t) writeConfig := func(t *testing.T, name string, capacity int, displayName string) string { t.Helper() path := filepath.Join(dir, name) yaml := fmt.Sprintf(` server: listen: %q bootstrap: listen: "0.0.0.0:18080" artifact_dir: "artifacts" logging: level: "error" refresh: enabled: false listen: "127.0.0.1:19093" openai: enabled: true listen: %q adapter: "openai_compat" target: "" a2a: listen: "0.0.0.0:8081" metrics: port: 0 models: - id: "qwen3.6:35b" display_name: %q providers: prov-a: "served-qwen" nodes: - id: "node-1" alias: "n1" token: "tok-1" adapters: openai_compat_instances: - name: "vllm-gpu" enabled: true provider: "vllm" endpoint: "http://127.0.0.1:8000/v1" providers: - id: "prov-a" type: "vllm" category: "api" adapter: "vllm-gpu" models: ["served-qwen"] health: "available" capacity: %d max_queue: 4 queue_timeout_ms: 5000 `, serverAddr, openAIAddr, displayName, capacity) if err := os.WriteFile(path, []byte(yaml), 0o600); err != nil { t.Fatalf("write %s: %v", name, err) } return path } basePath := writeConfig(t, "base.yaml", 2, "Qwen Base") candidatePath := writeConfig(t, "candidate.yaml", 8, "Qwen Candidate") currentCfg, err := config.LoadEdge(basePath) if err != nil { t.Fatalf("load base config: %v", err) } // Apply serve-equivalent normalization. currentCfg.Logging.Path = serveNormalizedLogPath() currentCfg.Bootstrap.ArtifactDir = serveNormalizedArtifactDir(currentCfg.Bootstrap.ArtifactDir) rt, err := NewRuntime(currentCfg) if err != nil { t.Fatalf("NewRuntime: %v", err) } if err := rt.Start(context.Background()); err != nil { t.Fatalf("Start: %v", err) } defer func() { if err := rt.Stop(); err != nil { t.Fatalf("Stop: %v", err) } }() client := &http.Client{Timeout: 5 * time.Second} start := make(chan struct{}) var wg sync.WaitGroup errCh := make(chan error, 3) wg.Add(1) go func() { defer wg.Done() <-start for i := 0; i < 50; i++ { result, err := rt.RefreshConfig(context.Background(), configrefresh.Request{ Mode: configrefresh.ModeApply, ConfigPath: candidatePath, RequestID: fmt.Sprintf("apply-%d", i), }) if err != nil { errCh <- err return } if result.Status != configrefresh.StatusApplied { errCh <- fmt.Errorf("refresh status=%s summary=%s changes=%+v", result.Status, result.Summary, result.Changes) return } } }() wg.Add(1) go func() { defer wg.Done() <-start for i := 0; i < 100; i++ { _ = rt.Service.ListNodeSnapshots() } }() wg.Add(1) go func() { defer wg.Done() <-start url := "http://" + openAIAddr + "/v1/models" for i := 0; i < 100; i++ { resp, err := client.Get(url) if err != nil { errCh <- err return } resp.Body.Close() if resp.StatusCode != http.StatusOK { errCh <- fmt.Errorf("GET /v1/models status=%d", resp.StatusCode) return } } }() close(start) wg.Wait() close(errCh) for err := range errCh { if err != nil { t.Fatalf("concurrent refresh reader failed: %v", err) } } } // loadServeNormalizedConfig loads an Edge config and applies the same // normalization iop-edge serve performs, so refresh classification compares // effective values. func loadServeNormalizedConfig(t *testing.T, path string) *config.EdgeConfig { t.Helper() cfg, err := config.LoadEdge(path) if err != nil { t.Fatalf("load config %s: %v", path, err) } cfg.Logging.Path = serveNormalizedLogPath() cfg.Bootstrap.ArtifactDir = serveNormalizedArtifactDir(cfg.Bootstrap.ArtifactDir) return cfg } // providerPoolConfigYAML builds an Edge config with one provider-pool model and // one node provider. The extra parameter lets callers inject a candidate // variation (e.g. a different capacity or an invalid adapter block). func providerPoolConfigYAML(serverAddr, openAIAddr, providerBlock string) string { return fmt.Sprintf(` server: listen: %q bootstrap: listen: "0.0.0.0:18080" artifact_dir: "artifacts" logging: level: "error" refresh: enabled: true listen: "127.0.0.1:0" openai: enabled: true listen: %q adapter: "openai_compat" target: "" a2a: listen: "0.0.0.0:8081" metrics: port: 0 models: - id: "qwen3.6:35b" display_name: "Qwen" providers: prov-a: "served-qwen" nodes: - id: "node-1" alias: "n1" token: "tok-1" adapters: %s providers: - id: "prov-a" type: "vllm" category: "api" adapter: "vllm-gpu" models: ["served-qwen"] health: "available" capacity: 2 max_queue: 4 queue_timeout_ms: 5000 `, serverAddr, openAIAddr, providerBlock) } const validAdapterBlock = ` openai_compat_instances: - name: "vllm-gpu" enabled: true provider: "vllm" endpoint: "http://127.0.0.1:8000/v1"` // invalidAdapterBlock enables an openai_compat instance but leaves endpoint // empty, which ValidateEdgeConfig rejects. const invalidAdapterBlock = ` openai_compat_instances: - name: "vllm-gpu" enabled: true provider: "vllm" endpoint: ""` // modelsResponse represents the JSON response from /v1/models. type modelsResponse struct { Object string `json:"object"` Data []modelItem `json:"data"` } // modelItem represents a single model entry in the /v1/models response. type modelItem struct { ID string `json:"id"` Object string `json:"object"` Created int64 `json:"created"` OwnedBy string `json:"owned_by"` } // modelIDs returns the sorted list of model IDs from /v1/models. // This avoids flaky timestamp comparisons on the `created` field. func modelIDs(t *testing.T, client *http.Client, openAIAddr string) []string { t.Helper() resp, err := client.Get("http://" + openAIAddr + "/v1/models") if err != nil { t.Fatalf("GET /v1/models: %v", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { t.Fatalf("GET /v1/models status=%d", resp.StatusCode) } var body modelsResponse if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { t.Fatalf("decode /v1/models body: %v", err) } ids := make([]string, 0, len(body.Data)) for _, m := range body.Data { ids = append(ids, m.ID) } sort.Strings(ids) return ids } // TestRefreshRejectedPreservesModels verifies SDD S10: an invalid candidate // apply is rejected without mutating the running runtime config or the // /v1/models surface. Uses stable model ID comparison to avoid timestamp flakiness. func TestRefreshRejectedPreservesModels(t *testing.T) { dir := t.TempDir() serverAddr := freeTCPAddr(t) openAIAddr := freeTCPAddr(t) basePath := filepath.Join(dir, "base.yaml") if err := os.WriteFile(basePath, []byte(providerPoolConfigYAML(serverAddr, openAIAddr, validAdapterBlock)), 0o600); err != nil { t.Fatalf("write base: %v", err) } candidatePath := filepath.Join(dir, "invalid.yaml") if err := os.WriteFile(candidatePath, []byte(providerPoolConfigYAML(serverAddr, openAIAddr, invalidAdapterBlock)), 0o600); err != nil { t.Fatalf("write candidate: %v", err) } rt, err := NewRuntime(loadServeNormalizedConfig(t, basePath)) if err != nil { t.Fatalf("NewRuntime: %v", err) } if err := rt.Start(context.Background()); err != nil { t.Fatalf("Start: %v", err) } defer func() { if err := rt.Stop(); err != nil { t.Fatalf("Stop: %v", err) } }() client := &http.Client{Timeout: 5 * time.Second} beforeIDs := modelIDs(t, client, openAIAddr) if len(beforeIDs) == 0 { t.Fatal("expected at least one model before refresh") } cfgBefore := rt.Cfg result, err := rt.RefreshConfig(context.Background(), configrefresh.Request{ Mode: configrefresh.ModeApply, ConfigPath: candidatePath, RequestID: "reject-1", }) if err != nil { t.Fatalf("RefreshConfig: %v", err) } if result.Status != configrefresh.StatusRejected { t.Fatalf("expected status=rejected for invalid candidate, got %q (summary=%s)", result.Status, result.Summary) } if rt.Cfg != cfgBefore { t.Fatal("runtime config snapshot mutated after rejected apply") } afterIDs := modelIDs(t, client, openAIAddr) if !reflect.DeepEqual(afterIDs, beforeIDs) { t.Fatalf("/v1/models changed after rejected apply:\nbefore=%v\nafter=%v", beforeIDs, afterIDs) } } // TestRefreshApplyDoesNotMutateOnRestartRequired verifies that a candidate with // a restart_required change leaves the running runtime snapshot and /v1/models // untouched. Uses stable model ID comparison. func TestRefreshApplyDoesNotMutateOnRestartRequired(t *testing.T) { dir := t.TempDir() serverAddr := freeTCPAddr(t) openAIAddr := freeTCPAddr(t) basePath := filepath.Join(dir, "base.yaml") if err := os.WriteFile(basePath, []byte(providerPoolConfigYAML(serverAddr, openAIAddr, validAdapterBlock)), 0o600); err != nil { t.Fatalf("write base: %v", err) } // Candidate changes server.listen, which is classified restart_required. candidateYAML := providerPoolConfigYAML(freeTCPAddr(t), openAIAddr, validAdapterBlock) candidatePath := filepath.Join(dir, "restart.yaml") if err := os.WriteFile(candidatePath, []byte(candidateYAML), 0o600); err != nil { t.Fatalf("write candidate: %v", err) } rt, err := NewRuntime(loadServeNormalizedConfig(t, basePath)) if err != nil { t.Fatalf("NewRuntime: %v", err) } if err := rt.Start(context.Background()); err != nil { t.Fatalf("Start: %v", err) } defer func() { if err := rt.Stop(); err != nil { t.Fatalf("Stop: %v", err) } }() client := &http.Client{Timeout: 5 * time.Second} beforeIDs := modelIDs(t, client, openAIAddr) if len(beforeIDs) == 0 { t.Fatal("expected at least one model before refresh") } cfgBefore := rt.Cfg result, err := rt.RefreshConfig(context.Background(), configrefresh.Request{ Mode: configrefresh.ModeApply, ConfigPath: candidatePath, RequestID: "restart-1", }) if err != nil { t.Fatalf("RefreshConfig: %v", err) } if result.Status != configrefresh.StatusRestartRequired { t.Fatalf("expected status=restart_required, got %q (summary=%s)", result.Status, result.Summary) } if len(result.RestartRequiredPaths) == 0 { t.Fatal("expected restart_required_paths to be reported") } if rt.Cfg != cfgBefore { t.Fatal("runtime config snapshot mutated after restart_required apply") } afterIDs := modelIDs(t, client, openAIAddr) if !reflect.DeepEqual(afterIDs, beforeIDs) { t.Fatalf("/v1/models changed after restart_required apply:\nbefore=%v\nafter=%v", beforeIDs, afterIDs) } } // TestRefreshAPIReportIncludesChangedItems verifies SDD S11: the refresh API // response exposes the changed provider/model items so an operator can confirm // what a refresh touched. func TestRefreshAPIReportIncludesChangedItems(t *testing.T) { dir := t.TempDir() serverAddr := freeTCPAddr(t) openAIAddr := freeTCPAddr(t) basePath := filepath.Join(dir, "base.yaml") if err := os.WriteFile(basePath, []byte(providerPoolConfigYAML(serverAddr, openAIAddr, validAdapterBlock)), 0o600); err != nil { t.Fatalf("write base: %v", err) } // Candidate raises provider capacity (applied) and renames the model display // name (applied), so both a provider and a model item change. candidateYAML := strings.Replace( strings.Replace(providerPoolConfigYAML(serverAddr, openAIAddr, validAdapterBlock), "capacity: 2", "capacity: 8", 1), `display_name: "Qwen"`, `display_name: "Qwen Refreshed"`, 1) candidatePath := filepath.Join(dir, "candidate.yaml") if err := os.WriteFile(candidatePath, []byte(candidateYAML), 0o600); err != nil { t.Fatalf("write candidate: %v", err) } rt, err := NewRuntime(loadServeNormalizedConfig(t, basePath)) if err != nil { t.Fatalf("NewRuntime: %v", err) } if err := rt.Start(context.Background()); err != nil { t.Fatalf("Start: %v", err) } defer func() { if err := rt.Stop(); err != nil { t.Fatalf("Stop: %v", err) } }() adminURL := "http://" + rt.RefreshAdmin.Addr() + "/refresh" client := &http.Client{Timeout: 5 * time.Second} body, _ := json.Marshal(configrefresh.Request{ Mode: configrefresh.ModeDryRun, ConfigPath: candidatePath, RequestID: "report-1", }) resp, err := client.Post(adminURL, "application/json", bytes.NewReader(body)) if err != nil { t.Fatalf("POST /refresh: %v", err) } defer resp.Body.Close() var result configrefresh.Result if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { t.Fatalf("decode response: %v", err) } if result.Status != configrefresh.StatusApplied { t.Fatalf("expected status=applied, got %q (summary=%s)", result.Status, result.Summary) } if !containsString(result.ChangedProviders, "prov-a") { t.Errorf("expected changed_providers to include prov-a, got %v", result.ChangedProviders) } if !containsString(result.ChangedModels, "qwen3.6:35b") { t.Errorf("expected changed_models to include qwen3.6:35b, got %v", result.ChangedModels) } } func containsString(s []string, want string) bool { for _, v := range s { if v == want { return true } } return false } // TestRefreshConfigApplyIncludesNodeResults verifies SDD S12: when a refresh // apply succeeds, the result includes per-node refresh outcomes. Uses a // net.Pipe-based fake node that registers with the edge and responds applied. func TestRefreshConfigApplyIncludesNodeResults(t *testing.T) { dir := t.TempDir() serverAddr := freeTCPAddr(t) writeRefreshConfig := func(name string, capacity int) string { path := filepath.Join(dir, name) yaml := fmt.Sprintf(` server: listen: %q bootstrap: listen: "0.0.0.0:18090" artifact_dir: "artifacts" logging: level: "error" refresh: enabled: false listen: "127.0.0.1:0" openai: listen: "0.0.0.0:18091" a2a: listen: "0.0.0.0:8082" metrics: port: 0 nodes: - id: "node-refresh-1" alias: "refresh-node" token: "tok-refresh" adapters: cli: enabled: true openai_compat_instances: - name: "primary" enabled: true provider: "vllm" endpoint: "http://127.0.0.1:9000/v1" capacity: 5 max_queue: 4 queue_timeout_ms: 5000 runtime: concurrency: 7 workspace_root: "/tmp/iop-refresh-ws" providers: - id: "prov-r" type: "ollama" category: "local_inference" models: ["llama3"] capacity: %d max_queue: 4 queue_timeout_ms: 5000 `, serverAddr, capacity) if err := os.WriteFile(path, []byte(yaml), 0o600); err != nil { t.Fatalf("write %s: %v", name, err) } return path } basePath := writeRefreshConfig("base.yaml", 2) candidatePath := writeRefreshConfig("candidate.yaml", 8) rt, err := NewRuntime(loadServeNormalizedConfig(t, basePath)) if err != nil { t.Fatalf("NewRuntime: %v", err) } if err := rt.Start(context.Background()); err != nil { t.Fatalf("Start: %v", err) } defer func() { if err := rt.Stop(); err != nil { t.Fatalf("Stop: %v", err) } }() // Build a fake node parser that can decode NodeConfigRefreshRequest (edge push) // and RegisterResponse (registration). nodeParserMap := toki.ParserMap{ toki.TypeNameOf(&iop.RegisterResponse{}): func(b []byte) (proto.Message, error) { m := &iop.RegisterResponse{} return m, proto.Unmarshal(b, m) }, toki.TypeNameOf(&iop.NodeConfigRefreshRequest{}): func(b []byte) (proto.Message, error) { m := &iop.NodeConfigRefreshRequest{} return m, proto.Unmarshal(b, m) }, } host, portStr, _ := net.SplitHostPort(serverAddr) port := 0 fmt.Sscanf(portStr, "%d", &port) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() fakeNode, err := toki.DialTcp(ctx, host, port, 30, 10, nodeParserMap) if err != nil { t.Fatalf("dial edge: %v", err) } defer fakeNode.Close() // Fake node captures the refresh request and responds restart_required // (current Node behavior for any non-empty payload/change). var capturedRefreshReq *iop.NodeConfigRefreshRequest var capturedRefreshMu sync.Mutex toki.AddRequestListenerTyped[*iop.NodeConfigRefreshRequest, *iop.NodeConfigRefreshResponse]( &fakeNode.Communicator, func(req *iop.NodeConfigRefreshRequest) (*iop.NodeConfigRefreshResponse, error) { capturedRefreshMu.Lock() capturedRefreshReq = req capturedRefreshMu.Unlock() return &iop.NodeConfigRefreshResponse{ RequestId: req.GetRequestId(), Status: iop.NodeConfigRefreshStatus_NODE_CONFIG_REFRESH_STATUS_RESTART_REQUIRED, RestartRequiredPaths: req.GetChangedPaths(), }, nil }, ) // Register the fake node. regResp, err := toki.SendRequestTyped[*iop.RegisterRequest, *iop.RegisterResponse]( &fakeNode.Communicator, &iop.RegisterRequest{Token: "tok-refresh"}, 2*time.Second, ) if err != nil { t.Fatalf("register: %v", err) } if !regResp.GetAccepted() { t.Fatalf("expected accepted, got reason %q", regResp.GetReason()) } // Wait for the node to appear in the registry. deadline := time.Now().Add(2 * time.Second) for time.Now().Before(deadline) { if _, ok := rt.Registry.Get("node-refresh-1"); ok { break } time.Sleep(25 * time.Millisecond) } if _, ok := rt.Registry.Get("node-refresh-1"); !ok { t.Fatal("fake node not registered in edge registry") } result, err := rt.RefreshConfig(context.Background(), configrefresh.Request{ Mode: configrefresh.ModeApply, ConfigPath: candidatePath, RequestID: "node-result-test", }) if err != nil { t.Fatalf("RefreshConfig: %v", err) } if result.Status != configrefresh.StatusApplied { t.Fatalf("expected status=applied, got %q (summary=%s)", result.Status, result.Summary) } if len(result.NodeResults) == 0 { t.Fatal("expected node_results to be populated after apply") } found := false for _, nr := range result.NodeResults { if nr.NodeID == "node-refresh-1" { found = true if nr.Status != "restart_required" { t.Errorf("expected node status=restart_required (current behavior for non-empty payload), got %q (error=%q)", nr.Status, nr.Error) } } } if !found { t.Fatalf("node-refresh-1 not found in node_results: %+v", result.NodeResults) } // Verify request_id propagation and payload contents on the fake node side. capturedRefreshMu.Lock() captured := capturedRefreshReq capturedRefreshMu.Unlock() if captured == nil { t.Fatal("fake node never received a NodeConfigRefreshRequest") } if captured.GetRequestId() != "node-result-test" { t.Errorf("request_id: got %q, want %q", captured.GetRequestId(), "node-result-test") } // Payload must carry the node-specific config edge builds from the NodeStore // record. Note: providers[].capacity is the refresh *trigger* (it drives // changed_paths), not a NodeConfigPayload field; the payload itself carries // the node's adapters + runtime, so we assert those concrete typed fields. payload := captured.GetConfig() if payload == nil { t.Fatal("expected non-nil NodeConfigPayload in refresh request") } // Runtime tuning is pushed verbatim from the node record. if got := payload.GetRuntime().GetConcurrency(); got != 7 { t.Errorf("runtime concurrency: got %d, want 7", got) } if got := payload.GetRuntime().GetWorkspaceRoot(); got != "/tmp/iop-refresh-ws" { t.Errorf("runtime workspace_root: got %q, want %q", got, "/tmp/iop-refresh-ws") } // The configured openai_compat adapter instance must be present with its // typed fields (provider/endpoint/capacity) intact, and the CLI adapter too. var oai *iop.OpenAICompatAdapterConfig hasCLI := false for _, a := range payload.GetAdapters() { switch a.GetType() { case "openai_compat": if a.GetName() == "primary" { oai = a.GetOpenaiCompat() } case "cli": hasCLI = true } } if oai == nil { t.Fatalf("expected openai_compat adapter %q in refresh payload, got %+v", "primary", payload.GetAdapters()) } if oai.GetProvider() != "vllm" { t.Errorf("openai_compat provider: got %q, want %q", oai.GetProvider(), "vllm") } if oai.GetEndpoint() != "http://127.0.0.1:9000/v1" { t.Errorf("openai_compat endpoint: got %q, want %q", oai.GetEndpoint(), "http://127.0.0.1:9000/v1") } if oai.GetCapacity() != 5 { t.Errorf("openai_compat capacity: got %d, want 5", oai.GetCapacity()) } if !hasCLI { t.Errorf("expected cli adapter in refresh payload, got %+v", payload.GetAdapters()) } } // TestRefreshConfigApplySkipsDisconnectedConfiguredNode verifies that a node // that is in the NodeStore but not in the live registry (post-disconnect state) // appears as "skipped" in node_results alongside connected nodes. func TestRefreshConfigApplySkipsDisconnectedConfiguredNode(t *testing.T) { dir := t.TempDir() serverAddr := freeTCPAddr(t) writeConfig := func(name string, capacity int) string { path := filepath.Join(dir, name) yaml := fmt.Sprintf(` server: listen: %q bootstrap: listen: "0.0.0.0:18092" artifact_dir: "artifacts" logging: level: "error" refresh: enabled: false listen: "127.0.0.1:0" openai: listen: "0.0.0.0:18093" a2a: listen: "0.0.0.0:8083" metrics: port: 0 nodes: - id: "node-connected" alias: "conn-node" token: "tok-conn" adapters: cli: enabled: true providers: - id: "prov-skip" type: "ollama" category: "local_inference" models: ["llama3"] capacity: %d max_queue: 4 queue_timeout_ms: 5000 - id: "node-disconnected" alias: "disc-node" token: "tok-disc" adapters: cli: enabled: true `, serverAddr, capacity) if err := os.WriteFile(path, []byte(yaml), 0o600); err != nil { t.Fatalf("write %s: %v", name, err) } return path } basePath := writeConfig("base.yaml", 2) candidatePath := writeConfig("candidate.yaml", 8) rt, err := NewRuntime(loadServeNormalizedConfig(t, basePath)) if err != nil { t.Fatalf("NewRuntime: %v", err) } if err := rt.Start(context.Background()); err != nil { t.Fatalf("Start: %v", err) } defer func() { if err := rt.Stop(); err != nil { t.Fatalf("Stop: %v", err) } }() nodeParserMap := toki.ParserMap{ toki.TypeNameOf(&iop.RegisterResponse{}): func(b []byte) (proto.Message, error) { m := &iop.RegisterResponse{} return m, proto.Unmarshal(b, m) }, toki.TypeNameOf(&iop.NodeConfigRefreshRequest{}): func(b []byte) (proto.Message, error) { m := &iop.NodeConfigRefreshRequest{} return m, proto.Unmarshal(b, m) }, } host, portStr, _ := net.SplitHostPort(serverAddr) port := 0 fmt.Sscanf(portStr, "%d", &port) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() // Only connect node-connected; node-disconnected never dials in. fakeNode, err := toki.DialTcp(ctx, host, port, 30, 10, nodeParserMap) if err != nil { t.Fatalf("dial edge: %v", err) } defer fakeNode.Close() toki.AddRequestListenerTyped[*iop.NodeConfigRefreshRequest, *iop.NodeConfigRefreshResponse]( &fakeNode.Communicator, func(req *iop.NodeConfigRefreshRequest) (*iop.NodeConfigRefreshResponse, error) { return &iop.NodeConfigRefreshResponse{ RequestId: req.GetRequestId(), Status: iop.NodeConfigRefreshStatus_NODE_CONFIG_REFRESH_STATUS_RESTART_REQUIRED, RestartRequiredPaths: req.GetChangedPaths(), }, nil }, ) regResp, err := toki.SendRequestTyped[*iop.RegisterRequest, *iop.RegisterResponse]( &fakeNode.Communicator, &iop.RegisterRequest{Token: "tok-conn"}, 2*time.Second, ) if err != nil { t.Fatalf("register: %v", err) } if !regResp.GetAccepted() { t.Fatalf("expected accepted, got reason %q", regResp.GetReason()) } deadline := time.Now().Add(2 * time.Second) for time.Now().Before(deadline) { if _, ok := rt.Registry.Get("node-connected"); ok { break } time.Sleep(25 * time.Millisecond) } if _, ok := rt.Registry.Get("node-connected"); !ok { t.Fatal("fake node not registered in edge registry") } result, err := rt.RefreshConfig(context.Background(), configrefresh.Request{ Mode: configrefresh.ModeApply, ConfigPath: candidatePath, RequestID: "skip-test", }) if err != nil { t.Fatalf("RefreshConfig: %v", err) } if result.Status != configrefresh.StatusApplied { t.Fatalf("expected status=applied, got %q (summary=%s)", result.Status, result.Summary) } if len(result.NodeResults) != 2 { t.Fatalf("expected 2 node_results (connected + disconnected), got %d: %+v", len(result.NodeResults), result.NodeResults) } statusByID := make(map[string]string) for _, nr := range result.NodeResults { statusByID[nr.NodeID] = nr.Status } if statusByID["node-connected"] != "restart_required" { t.Errorf("node-connected: expected restart_required, got %q", statusByID["node-connected"]) } if statusByID["node-disconnected"] != "skipped" { t.Errorf("node-disconnected: expected skipped, got %q", statusByID["node-disconnected"]) } } // TestRefreshConfigApplyNoChangeSkipsNodePush verifies that an identical config // apply (no-change) does NOT push a refresh request to connected nodes, // preventing spurious restart_required responses. func TestRefreshConfigApplyNoChangeSkipsNodePush(t *testing.T) { dir := t.TempDir() serverAddr := freeTCPAddr(t) writeConfig := func(name string) string { path := filepath.Join(dir, name) yaml := fmt.Sprintf(` server: listen: %q bootstrap: listen: "0.0.0.0:18094" artifact_dir: "artifacts" logging: level: "error" refresh: enabled: false listen: "127.0.0.1:0" openai: listen: "0.0.0.0:18095" a2a: listen: "0.0.0.0:8084" metrics: port: 0 nodes: - id: "node-noc-change" alias: "noc-change-node" token: "tok-noc-change" adapters: cli: enabled: true providers: - id: "prov-nc" type: "ollama" category: "local_inference" models: ["llama3"] capacity: 2 max_queue: 4 queue_timeout_ms: 5000 `, serverAddr) if err := os.WriteFile(path, []byte(yaml), 0o600); err != nil { t.Fatalf("write %s: %v", name, err) } return path } basePath := writeConfig("base.yaml") // candidate is identical to base (same file). candidatePath := basePath rt, err := NewRuntime(loadServeNormalizedConfig(t, basePath)) if err != nil { t.Fatalf("NewRuntime: %v", err) } if err := rt.Start(context.Background()); err != nil { t.Fatalf("Start: %v", err) } defer func() { if err := rt.Stop(); err != nil { t.Fatalf("Stop: %v", err) } }() // Build a fake node parser. nodeParserMap := toki.ParserMap{ toki.TypeNameOf(&iop.RegisterResponse{}): func(b []byte) (proto.Message, error) { m := &iop.RegisterResponse{} return m, proto.Unmarshal(b, m) }, toki.TypeNameOf(&iop.NodeConfigRefreshRequest{}): func(b []byte) (proto.Message, error) { m := &iop.NodeConfigRefreshRequest{} return m, proto.Unmarshal(b, m) }, } host, portStr, _ := net.SplitHostPort(serverAddr) port := 0 fmt.Sscanf(portStr, "%d", &port) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() fakeNode, err := toki.DialTcp(ctx, host, port, 30, 10, nodeParserMap) if err != nil { t.Fatalf("dial edge: %v", err) } defer fakeNode.Close() // Track whether NodeConfigRefreshRequest is received. var receivedRefresh atomic.Bool toki.AddRequestListenerTyped[*iop.NodeConfigRefreshRequest, *iop.NodeConfigRefreshResponse]( &fakeNode.Communicator, func(req *iop.NodeConfigRefreshRequest) (*iop.NodeConfigRefreshResponse, error) { receivedRefresh.Store(true) return &iop.NodeConfigRefreshResponse{ RequestId: req.GetRequestId(), Status: iop.NodeConfigRefreshStatus_NODE_CONFIG_REFRESH_STATUS_APPLIED, }, nil }, ) // Register the fake node. regResp, err := toki.SendRequestTyped[*iop.RegisterRequest, *iop.RegisterResponse]( &fakeNode.Communicator, &iop.RegisterRequest{Token: "tok-noc-change"}, 2*time.Second, ) if err != nil { t.Fatalf("register: %v", err) } if !regResp.GetAccepted() { t.Fatalf("expected accepted, got reason %q", regResp.GetReason()) } // Wait for the node to appear in the registry. deadline := time.Now().Add(2 * time.Second) for time.Now().Before(deadline) { if _, ok := rt.Registry.Get("node-noc-change"); ok { break } time.Sleep(25 * time.Millisecond) } if _, ok := rt.Registry.Get("node-noc-change"); !ok { t.Fatal("fake node not registered in edge registry") } // Apply identical config (no-change). result, err := rt.RefreshConfig(context.Background(), configrefresh.Request{ Mode: configrefresh.ModeApply, ConfigPath: candidatePath, RequestID: "no-change-1", }) if err != nil { t.Fatalf("RefreshConfig: %v", err) } if result.Status != configrefresh.StatusApplied { t.Fatalf("expected status=applied, got %q (summary=%s)", result.Status, result.Summary) } // No-change apply should produce no node_results (no push). if len(result.NodeResults) != 0 { t.Errorf("expected no node_results for no-change apply, got %d: %+v", len(result.NodeResults), result.NodeResults) } // The fake node must NOT have received a NodeConfigRefreshRequest. if receivedRefresh.Load() { t.Error("expected no NodeConfigRefreshRequest to be sent for no-change apply, but it was received") } } func freeTCPAddr(t *testing.T) string { t.Helper() ln, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatalf("listen free tcp addr: %v", err) } addr := ln.Addr().String() if err := ln.Close(); err != nil { t.Fatalf("close free tcp addr listener: %v", err) } return addr } // TestNewRuntimeWiresControlPlaneConnector verifies that NewRuntime always // wires a non-nil ControlPlane connector regardless of config. func TestNewRuntimeWiresControlPlaneConnector(t *testing.T) { cfg := newTestConfig() rt, err := NewRuntime(cfg) if err != nil { t.Fatalf("NewRuntime: %v", err) } if rt.ControlPlane == nil { t.Fatal("expected non-nil ControlPlane connector") } } // TestNewRuntimeWiresStatusProviderIntoConnector verifies that NewRuntime wires // the edge service as the Control Plane connector's status provider, so status // requests are answered from the Edge-owned node snapshot surface. func TestNewRuntimeWiresStatusProviderIntoConnector(t *testing.T) { cfg := newTestConfig() rt, err := NewRuntime(cfg) if err != nil { t.Fatalf("NewRuntime: %v", err) } if !rt.ControlPlane.StatusProviderConfigured() { t.Fatal("expected NewRuntime to wire a status provider into the connector") } } // TestNewRuntimeWiresNodeEventBusIntoConnector verifies that node lifecycle // events are relayed through the shared events.Bus boundary. func TestNewRuntimeWiresNodeEventBusIntoConnector(t *testing.T) { cfg := newTestConfig() rt, err := NewRuntime(cfg) if err != nil { t.Fatalf("NewRuntime: %v", err) } if !rt.ControlPlane.NodeEventBusConfigured() { t.Fatal("expected NewRuntime to wire the node event bus into the connector") } } // TestRuntimeStartsControlPlaneConnectorWhenEnabled verifies that Start does // not fail when a Control Plane connector is configured with a local fake // server accepting hello. Uses a local TCP fake endpoint. func TestRuntimeStartsControlPlaneConnectorWhenEnabled(t *testing.T) { // Start a minimal fake Control Plane server. ln, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatalf("listen fake cp: %v", err) } cpAddr := ln.Addr().String() go func() { // Accept and immediately close; connector will fail hello and retry, // but Start() itself must succeed (connector errors are async). conn, err := ln.Accept() if err != nil { return } _ = conn.Close() _ = ln.Close() }() cfg := newTestConfig() cfg.ControlPlane = config.EdgeControlPlaneConf{ Enabled: true, WireAddr: cpAddr, ReconnectIntervalSec: 60, } rt, err := NewRuntime(cfg) if err != nil { t.Fatalf("NewRuntime: %v", err) } if !rt.ControlPlane.IsEnabled() { t.Fatal("expected connector IsEnabled()==true") } ctx := context.Background() if err := rt.Start(ctx); err != nil { t.Fatalf("Start with enabled connector: %v", err) } if err := rt.Stop(); err != nil { t.Fatalf("Stop: %v", err) } } // TestRuntimeStopsControlPlaneConnector verifies that Stop() reaches the // ControlPlane connector. A disabled connector is used so no real network // activity occurs. func TestRuntimeStopsControlPlaneConnector(t *testing.T) { cfg := newTestConfig() // disabled connector; Start/Stop must be no-ops on the connector side. cfg.ControlPlane = config.EdgeControlPlaneConf{ Enabled: false, WireAddr: "", } rt, err := NewRuntime(cfg) if err != nil { t.Fatalf("NewRuntime: %v", err) } ctx := context.Background() if err := rt.Start(ctx); err != nil { t.Fatalf("Start: %v", err) } if err := rt.Stop(); err != nil { t.Fatalf("Stop: %v", err) } // After Stop, the connector must be in a stopped-or-no-op state. // For disabled connector, CurrentState is Disconnected (never started). // Calling Stop again must not panic. rt.ControlPlane.Stop() } // TestRefreshRejectedPreservesProviderDispatch verifies SDD S10: a rejected // refresh must preserve provider-pool dispatch state. It uses a net.Pipe-based // fake node to capture the RunRequest sent by SubmitRun(ProviderPool=true) // before and after a rejected config refresh. func TestRefreshRejectedPreservesProviderDispatch(t *testing.T) { dir := t.TempDir() serverAddr := freeTCPAddr(t) openAIAddr := freeTCPAddr(t) basePath := filepath.Join(dir, "base.yaml") if err := os.WriteFile(basePath, []byte(providerPoolConfigYAML(serverAddr, openAIAddr, validAdapterBlock)), 0o600); err != nil { t.Fatalf("write base: %v", err) } candidatePath := filepath.Join(dir, "invalid.yaml") if err := os.WriteFile(candidatePath, []byte(providerPoolConfigYAML(serverAddr, openAIAddr, invalidAdapterBlock)), 0o600); err != nil { t.Fatalf("write candidate: %v", err) } rt, err := NewRuntime(loadServeNormalizedConfig(t, basePath)) if err != nil { t.Fatalf("NewRuntime: %v", err) } if err := rt.Start(context.Background()); err != nil { t.Fatalf("Start: %v", err) } defer func() { if err := rt.Stop(); err != nil { t.Fatalf("Stop: %v", err) } }() // --- Dispatch capture setup using net.Pipe --- edgeConn, nodeConn := net.Pipe() defer edgeConn.Close() defer nodeConn.Close() parserMap := toki.ParserMap{ toki.TypeNameOf(&iop.RunRequest{}): func(b []byte) (proto.Message, error) { m := &iop.RunRequest{} return m, proto.Unmarshal(b, m) }, } edgeClient := toki.NewTcpClient(edgeConn, 0, 0, parserMap) nodeClient := toki.NewTcpClient(nodeConn, 0, 0, parserMap) var capturedReq *iop.RunRequest var capturedMu sync.Mutex toki.AddListenerTyped[*iop.RunRequest](&nodeClient.Communicator, func(req *iop.RunRequest) { capturedMu.Lock() capturedReq = req capturedMu.Unlock() }) // Add a provider-pool node record to the NodeStore so provider-pool // resolution finds a dispatchable provider. The provider config must // match the adapter (vllm-gpu) and served target (served-qwen) expected // by the model catalog entry (qwen3.6:35b -> prov-a). rt.NodeStore.Add(&edgenode.NodeRecord{ ID: "node-pool-dispatch", Alias: "pool-alias", Runtime: config.RuntimeConf{Concurrency: 4}, Providers: []config.NodeProviderConf{ { ID: "prov-a", Type: "vllm", Category: "api", Adapter: "vllm-gpu", Models: []string{"served-qwen"}, Health: "available", Capacity: 2, }, }, }) // Register the fake node into the runtime registry. rt.Registry.Register(&edgenode.NodeEntry{ NodeID: "node-pool-dispatch", LifecycleState: edgenode.LifecycleConnected, Client: edgeClient, }) // Submit RunRequest via the runtime Service before refresh. dispatchAndCapture := func() (*iop.RunRequest, error) { capturedMu.Lock() capturedReq = nil capturedMu.Unlock() result, err := rt.Service.SubmitRun(context.Background(), edgeservice.SubmitRunRequest{ RunID: "dispatch-test-" + time.Now().Format("150405"), ModelGroupKey: "qwen3.6:35b", ProviderPool: true, Background: true, }) if err != nil { return nil, err } // Drain the result handle to release resources. if result != nil { result.Close() } // Wait for the fake node to receive the request. time.Sleep(100 * time.Millisecond) capturedMu.Lock() defer capturedMu.Unlock() return capturedReq, nil } // --- Before rejected refresh: dispatch must succeed --- beforeReq, err := dispatchAndCapture() if err != nil { t.Fatalf("dispatch before refresh: %v", err) } if beforeReq == nil { t.Fatal("expected dispatch before refresh to produce a RunRequest") } if beforeReq.GetAdapter() != "vllm-gpu" { t.Errorf("before refresh adapter: got %q, want %q", beforeReq.GetAdapter(), "vllm-gpu") } if beforeReq.GetTarget() != "served-qwen" { t.Errorf("before refresh target: got %q, want %q", beforeReq.GetTarget(), "served-qwen") } // --- Execute rejected refresh --- result, err := rt.RefreshConfig(context.Background(), configrefresh.Request{ Mode: configrefresh.ModeApply, ConfigPath: candidatePath, RequestID: "reject-dispatch-1", }) if err != nil { t.Fatalf("RefreshConfig: %v", err) } if result.Status != configrefresh.StatusRejected { t.Fatalf("expected status=rejected, got %q (summary=%s)", result.Status, result.Summary) } // --- After rejected refresh: dispatch must preserve state --- afterReq, err := dispatchAndCapture() if err != nil { t.Fatalf("dispatch after refresh: %v", err) } if afterReq == nil { t.Fatal("expected dispatch after rejected refresh to produce a RunRequest") } if afterReq.GetAdapter() != "vllm-gpu" { t.Errorf("after rejected refresh adapter: got %q, want %q", afterReq.GetAdapter(), "vllm-gpu") } if afterReq.GetTarget() != "served-qwen" { t.Errorf("after rejected refresh target: got %q, want %q", afterReq.GetTarget(), "served-qwen") } } // TestRefreshConfigNodeRuntimeConcurrencyAppliedAndWorkspaceRestartRequired // verifies the S15 boundary at the running-edge level: a node runtime.concurrency // change is applied live and pushed to the connected node (node result reported), // while a runtime.workspace_root change is classified restart_required and is NOT // pushed to nodes (no node results) nor committed to the runtime snapshot. func TestRefreshConfigNodeRuntimeConcurrencyAppliedAndWorkspaceRestartRequired(t *testing.T) { dir := t.TempDir() serverAddr := freeTCPAddr(t) writeCfg := func(name string, concurrency int, workspaceRoot string) string { path := filepath.Join(dir, name) yaml := fmt.Sprintf(` server: listen: %q bootstrap: listen: "0.0.0.0:18190" artifact_dir: "artifacts" logging: level: "error" refresh: enabled: false listen: "127.0.0.1:0" openai: listen: "0.0.0.0:18191" a2a: listen: "0.0.0.0:8182" metrics: port: 0 nodes: - id: "node-rt-1" alias: "rt-node" token: "tok-rt" adapters: cli: enabled: true runtime: concurrency: %d workspace_root: %q `, serverAddr, concurrency, workspaceRoot) if err := os.WriteFile(path, []byte(yaml), 0o600); err != nil { t.Fatalf("write %s: %v", name, err) } return path } basePath := writeCfg("base.yaml", 2, "/var/lib/iop/ws-a") concurrencyPath := writeCfg("concurrency.yaml", 4, "/var/lib/iop/ws-a") workspacePath := writeCfg("workspace.yaml", 2, "/var/lib/iop/ws-b") rt, err := NewRuntime(loadServeNormalizedConfig(t, basePath)) if err != nil { t.Fatalf("NewRuntime: %v", err) } if err := rt.Start(context.Background()); err != nil { t.Fatalf("Start: %v", err) } defer func() { if err := rt.Stop(); err != nil { t.Fatalf("Stop: %v", err) } }() nodeParserMap := toki.ParserMap{ toki.TypeNameOf(&iop.RegisterResponse{}): func(b []byte) (proto.Message, error) { m := &iop.RegisterResponse{} return m, proto.Unmarshal(b, m) }, toki.TypeNameOf(&iop.NodeConfigRefreshRequest{}): func(b []byte) (proto.Message, error) { m := &iop.NodeConfigRefreshRequest{} return m, proto.Unmarshal(b, m) }, } host, portStr, _ := net.SplitHostPort(serverAddr) port := 0 fmt.Sscanf(portStr, "%d", &port) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() fakeNode, err := toki.DialTcp(ctx, host, port, 30, 10, nodeParserMap) if err != nil { t.Fatalf("dial edge: %v", err) } defer fakeNode.Close() var capturedConcurrency int32 = -1 var capturedMu sync.Mutex toki.AddRequestListenerTyped[*iop.NodeConfigRefreshRequest, *iop.NodeConfigRefreshResponse]( &fakeNode.Communicator, func(req *iop.NodeConfigRefreshRequest) (*iop.NodeConfigRefreshResponse, error) { capturedMu.Lock() capturedConcurrency = req.GetConfig().GetRuntime().GetConcurrency() capturedMu.Unlock() return &iop.NodeConfigRefreshResponse{ RequestId: req.GetRequestId(), Status: iop.NodeConfigRefreshStatus_NODE_CONFIG_REFRESH_STATUS_APPLIED, }, nil }, ) regResp, err := toki.SendRequestTyped[*iop.RegisterRequest, *iop.RegisterResponse]( &fakeNode.Communicator, &iop.RegisterRequest{Token: "tok-rt"}, 2*time.Second, ) if err != nil { t.Fatalf("register: %v", err) } if !regResp.GetAccepted() { t.Fatalf("expected accepted, got reason %q", regResp.GetReason()) } deadline := time.Now().Add(2 * time.Second) for time.Now().Before(deadline) { if _, ok := rt.Registry.Get("node-rt-1"); ok { break } time.Sleep(25 * time.Millisecond) } if _, ok := rt.Registry.Get("node-rt-1"); !ok { t.Fatal("fake node not registered in edge registry") } // workspace_root change → restart_required, NOT pushed, snapshot unchanged. wsResult, err := rt.RefreshConfig(context.Background(), configrefresh.Request{ Mode: configrefresh.ModeApply, ConfigPath: workspacePath, RequestID: "rt-ws", }) if err != nil { t.Fatalf("RefreshConfig workspace: %v", err) } if wsResult.Status != configrefresh.StatusRestartRequired { t.Fatalf("expected workspace change status=restart_required, got %q", wsResult.Status) } if len(wsResult.NodeResults) != 0 { t.Errorf("workspace restart_required must not push to nodes, got node_results=%+v", wsResult.NodeResults) } capturedMu.Lock() if capturedConcurrency != -1 { t.Errorf("node must not receive a push for a restart_required workspace change, captured concurrency=%d", capturedConcurrency) } capturedMu.Unlock() // concurrency change → applied, pushed to node, node result reported. concResult, err := rt.RefreshConfig(context.Background(), configrefresh.Request{ Mode: configrefresh.ModeApply, ConfigPath: concurrencyPath, RequestID: "rt-conc", }) if err != nil { t.Fatalf("RefreshConfig concurrency: %v", err) } if concResult.Status != configrefresh.StatusApplied { t.Fatalf("expected concurrency change status=applied, got %q (summary=%s)", concResult.Status, concResult.Summary) } found := false for _, nr := range concResult.NodeResults { if nr.NodeID == "node-rt-1" { found = true if nr.Status != "applied" { t.Errorf("expected node status=applied, got %q (error=%q)", nr.Status, nr.Error) } } } if !found { t.Fatalf("node-rt-1 not found in node_results: %+v", concResult.NodeResults) } capturedMu.Lock() got := capturedConcurrency capturedMu.Unlock() if got != 4 { t.Errorf("pushed runtime concurrency: got %d, want 4", got) } }