package bootstrap import ( "bytes" "context" "encoding/json" "fmt" "net" "net/http" "os" "path/filepath" "reflect" "sort" "strings" "sync" "testing" "time" toki "git.toki-labs.com/toki/proto-socket/go" "google.golang.org/protobuf/proto" iop "iop/proto/gen/iop" "iop/apps/edge/internal/configrefresh" edgenode "iop/apps/edge/internal/node" edgeservice "iop/apps/edge/internal/service" "iop/packages/go/config" ) func newTestConfig() *config.EdgeConfig { return &config.EdgeConfig{ Server: config.EdgeServerConf{Listen: "127.0.0.1:0"}, Logging: config.LoggingConf{Level: "error"}, Metrics: config.MetricsConf{Port: 0}, Nodes: []config.NodeDefinition{ {ID: "node-1", Alias: "alias-1", Token: "tok-1"}, }, } } func TestNewRuntimeSeedsNodeStore(t *testing.T) { cfg := newTestConfig() rt, err := NewRuntime(cfg) if err != nil { t.Fatalf("NewRuntime: %v", err) } if rt.NodeStore == nil { t.Fatal("NodeStore is nil") } rec, ok := rt.NodeStore.FindByToken("tok-1") if !ok { t.Fatal("expected node record seeded by token tok-1") } if rec.ID != "node-1" || rec.Alias != "alias-1" { t.Errorf("unexpected record: id=%q alias=%q", rec.ID, rec.Alias) } if rt.Registry == nil || rt.EventBus == nil || rt.Service == nil || rt.Server == nil || rt.Logger == nil { t.Error("runtime dependencies must be wired by NewRuntime") } } func TestRuntimeWiresTransportHandlers(t *testing.T) { cfg := newTestConfig() rt, err := NewRuntime(cfg) if err != nil { t.Fatalf("NewRuntime: %v", err) } if rt.Server.HasRunEventHandler() || rt.Server.HasNodeEventHandler() { t.Fatal("handlers must not be wired before wireHandlers") } rt.wireHandlers() if !rt.Server.HasRunEventHandler() { t.Error("run event handler not wired") } if !rt.Server.HasNodeEventHandler() { t.Error("node event handler not wired") } } func TestNewRuntimeWiresInputManager(t *testing.T) { cfg := newTestConfig() rt, err := NewRuntime(cfg) if err != nil { t.Fatalf("NewRuntime: %v", err) } if rt.Input == nil { t.Fatal("Input manager is nil") } if rt.Input.OpenAI == nil { t.Fatal("Input.OpenAI is nil") } if rt.Input.A2A == nil { t.Fatal("Input.A2A is nil") } } func TestRuntimeKeepsListenersAfterStartupContextCancel(t *testing.T) { edgeAddr := freeTCPAddr(t) openAIAddr := freeTCPAddr(t) cfg := newTestConfig() cfg.Server.Listen = edgeAddr cfg.OpenAI = config.EdgeOpenAIConf{ Enabled: true, Listen: openAIAddr, } rt, err := NewRuntime(cfg) if err != nil { t.Fatalf("NewRuntime: %v", err) } startupCtx, cancelStartup := context.WithCancel(context.Background()) if err := rt.Start(startupCtx); err != nil { t.Fatalf("Start: %v", err) } defer func() { if err := rt.Stop(); err != nil { t.Fatalf("Stop: %v", err) } }() cancelStartup() conn, err := net.DialTimeout("tcp", edgeAddr, time.Second) if err != nil { t.Fatalf("edge listener closed with startup context: %v", err) } _ = conn.Close() client := &http.Client{Timeout: time.Second} resp, err := client.Get("http://" + openAIAddr + "/healthz") if err != nil { t.Fatalf("openai listener closed with startup context: %v", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { t.Fatalf("openai health status = %d, want %d", resp.StatusCode, http.StatusOK) } } func TestNewRuntimeWritesLogsToConfiguredPath(t *testing.T) { dir := t.TempDir() logPath := filepath.Join(dir, "logs", "edge.log") cfg := newTestConfig() cfg.Logging.Level = "info" cfg.Logging.Path = logPath rt, err := NewRuntime(cfg) if err != nil { t.Fatalf("NewRuntime: %v", err) } rt.Logger.Info("runtime-log-entry") _ = rt.Logger.Sync() info, err := os.Stat(logPath) if err != nil { t.Fatalf("stat log path: %v", err) } if info.Size() == 0 { t.Fatalf("log file empty: %s", logPath) } contents, err := os.ReadFile(logPath) if err != nil { t.Fatalf("read log file: %v", err) } if !strings.Contains(string(contents), "runtime-log-entry") { t.Fatalf("expected runtime-log-entry in log file; contents=%q", string(contents)) } } func TestNewRuntimeRejectsDuplicateToken(t *testing.T) { cfg := &config.EdgeConfig{ Server: config.EdgeServerConf{Listen: "127.0.0.1:0"}, Logging: config.LoggingConf{Level: "error"}, Nodes: []config.NodeDefinition{ {ID: "node-1", Alias: "a1", Token: "dup"}, {ID: "node-2", Alias: "a2", Token: "dup"}, }, } if _, err := NewRuntime(cfg); err == nil { t.Fatal("expected error on duplicate token seed") } } // TestRefreshAdminDryRunAndApplyReachRunningRuntime verifies S06: a running // Edge process can receive dry-run and apply refresh requests via the admin API // and returns mode-specific results. func TestRefreshAdminDryRunAndApplyReachRunningRuntime(t *testing.T) { dir := t.TempDir() serverAddr := freeTCPAddr(t) // Write base and candidate YAMLs with explicit immutable field values so // that config.LoadEdge produces configs with identical defaults on both sides. // Only capacity differs between base and candidate. baseYAML := fmt.Sprintf(` server: listen: %q bootstrap: listen: "0.0.0.0:18080" artifact_dir: "artifacts" logging: level: "error" refresh: enabled: true listen: "127.0.0.1:0" openai: listen: "0.0.0.0:18081" a2a: listen: "0.0.0.0:8081" metrics: port: 0 nodes: - id: "node-1" alias: "n1" token: "tok-1" adapters: cli: enabled: true providers: - id: "prov-a" type: "ollama" category: "local_inference" models: ["llama3.1"] capacity: 2 max_queue: 4 queue_timeout_ms: 5000 `, serverAddr) candidateYAML := fmt.Sprintf(` server: listen: %q bootstrap: listen: "0.0.0.0:18080" artifact_dir: "artifacts" logging: level: "error" refresh: enabled: true listen: "127.0.0.1:0" openai: listen: "0.0.0.0:18081" a2a: listen: "0.0.0.0:8081" metrics: port: 0 nodes: - id: "node-1" alias: "n1" token: "tok-1" adapters: cli: enabled: true providers: - id: "prov-a" type: "ollama" category: "local_inference" models: ["llama3.1"] capacity: 8 max_queue: 4 queue_timeout_ms: 5000 `, serverAddr) basePath := filepath.Join(dir, "base.yaml") if err := os.WriteFile(basePath, []byte(baseYAML), 0o600); err != nil { t.Fatalf("write base: %v", err) } candidatePath := filepath.Join(dir, "candidate.yaml") if err := os.WriteFile(candidatePath, []byte(candidateYAML), 0o600); err != nil { t.Fatalf("write candidate: %v", err) } currentCfg, err := config.LoadEdge(basePath) if err != nil { t.Fatalf("load base config: %v", err) } // Apply serve-equivalent normalization so the current config matches // what iop-edge serve would hold after loadRuntimeConfig. currentCfg.Logging.Path = serveNormalizedLogPath() currentCfg.Bootstrap.ArtifactDir = serveNormalizedArtifactDir(currentCfg.Bootstrap.ArtifactDir) // Override fields that don't need to match between runtime and candidate. currentCfg.Refresh.Enabled = true currentCfg.Refresh.Listen = "127.0.0.1:0" currentCfg.Logging.Level = "error" rt, err := NewRuntime(currentCfg) if err != nil { t.Fatalf("NewRuntime: %v", err) } ctx := context.Background() if err := rt.Start(ctx); err != nil { t.Fatalf("Start: %v", err) } defer func() { if err := rt.Stop(); err != nil { t.Fatalf("Stop: %v", err) } }() adminAddr := rt.RefreshAdmin.Addr() if adminAddr == "" { t.Fatal("refresh admin addr is empty after Start") } adminURL := "http://" + adminAddr + "/refresh" client := &http.Client{Timeout: 5 * time.Second} postRefresh := func(t *testing.T, mode configrefresh.Mode) configrefresh.Result { t.Helper() body, _ := json.Marshal(configrefresh.Request{ Mode: mode, ConfigPath: candidatePath, RequestID: string(mode) + "-req", }) resp, err := client.Post(adminURL, "application/json", bytes.NewReader(body)) if err != nil { t.Fatalf("POST /refresh mode=%s: %v", mode, err) } defer resp.Body.Close() var result configrefresh.Result if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { t.Fatalf("decode response: %v", err) } return result } // dry-run: process is reached, result has mode=dry_run. dryResult := postRefresh(t, configrefresh.ModeDryRun) if dryResult.Mode != configrefresh.ModeDryRun { t.Errorf("dry-run: expected mode=%q, got %q", configrefresh.ModeDryRun, dryResult.Mode) } if dryResult.Status != configrefresh.StatusApplied { t.Errorf("dry-run: expected status=%q (capacity change), got %q", configrefresh.StatusApplied, dryResult.Status) } // apply: process is reached, result has mode=apply and status=applied. applyResult := postRefresh(t, configrefresh.ModeApply) if applyResult.Mode != configrefresh.ModeApply { t.Errorf("apply: expected mode=%q, got %q", configrefresh.ModeApply, applyResult.Mode) } if applyResult.Status != configrefresh.StatusApplied { t.Errorf("apply: expected status=%q, got %q", configrefresh.StatusApplied, applyResult.Status) } // verify dry-run and apply have distinct observable results: the apply // committed the capacity change, so the next dry-run shows no changes. dryResult2 := postRefresh(t, configrefresh.ModeDryRun) if len(dryResult2.Changes) != 0 { t.Errorf("after apply, expected no diff changes on second dry-run, got %+v", dryResult2.Changes) } } func serveNormalizedLogPath() string { if exe, err := os.Executable(); err == nil { return filepath.Join(filepath.Dir(exe), "logs", "edge.log") } if wd, err := os.Getwd(); err == nil { return filepath.Join(wd, "logs", "edge.log") } return "logs/edge.log" } func serveNormalizedArtifactDir(dir string) string { if dir == "" { dir = "artifacts" } if filepath.IsAbs(dir) { return dir } bd := binaryDirForTest() return filepath.Join(bd, dir) } func binaryDirForTest() string { if exe, err := os.Executable(); err == nil { return filepath.Dir(exe) } if wd, err := os.Getwd(); err == nil { return wd } return "." } func TestRefreshApplyConcurrentRuntimeReaders(t *testing.T) { dir := t.TempDir() serverAddr := freeTCPAddr(t) openAIAddr := freeTCPAddr(t) writeConfig := func(t *testing.T, name string, capacity int, displayName string) string { t.Helper() path := filepath.Join(dir, name) yaml := fmt.Sprintf(` server: listen: %q bootstrap: listen: "0.0.0.0:18080" artifact_dir: "artifacts" logging: level: "error" refresh: enabled: false listen: "127.0.0.1:19093" openai: enabled: true listen: %q adapter: "openai_compat" target: "" a2a: listen: "0.0.0.0:8081" metrics: port: 0 models: - id: "qwen3.6:35b" display_name: %q providers: prov-a: "served-qwen" nodes: - id: "node-1" alias: "n1" token: "tok-1" adapters: openai_compat_instances: - name: "vllm-gpu" enabled: true provider: "vllm" endpoint: "http://127.0.0.1:8000/v1" providers: - id: "prov-a" type: "vllm" category: "api" adapter: "vllm-gpu" models: ["served-qwen"] health: "available" capacity: %d max_queue: 4 queue_timeout_ms: 5000 `, serverAddr, openAIAddr, displayName, capacity) if err := os.WriteFile(path, []byte(yaml), 0o600); err != nil { t.Fatalf("write %s: %v", name, err) } return path } basePath := writeConfig(t, "base.yaml", 2, "Qwen Base") candidatePath := writeConfig(t, "candidate.yaml", 8, "Qwen Candidate") currentCfg, err := config.LoadEdge(basePath) if err != nil { t.Fatalf("load base config: %v", err) } // Apply serve-equivalent normalization. currentCfg.Logging.Path = serveNormalizedLogPath() currentCfg.Bootstrap.ArtifactDir = serveNormalizedArtifactDir(currentCfg.Bootstrap.ArtifactDir) rt, err := NewRuntime(currentCfg) if err != nil { t.Fatalf("NewRuntime: %v", err) } if err := rt.Start(context.Background()); err != nil { t.Fatalf("Start: %v", err) } defer func() { if err := rt.Stop(); err != nil { t.Fatalf("Stop: %v", err) } }() client := &http.Client{Timeout: 5 * time.Second} start := make(chan struct{}) var wg sync.WaitGroup errCh := make(chan error, 3) wg.Add(1) go func() { defer wg.Done() <-start for i := 0; i < 50; i++ { result, err := rt.RefreshConfig(context.Background(), configrefresh.Request{ Mode: configrefresh.ModeApply, ConfigPath: candidatePath, RequestID: fmt.Sprintf("apply-%d", i), }) if err != nil { errCh <- err return } if result.Status != configrefresh.StatusApplied { errCh <- fmt.Errorf("refresh status=%s summary=%s changes=%+v", result.Status, result.Summary, result.Changes) return } } }() wg.Add(1) go func() { defer wg.Done() <-start for i := 0; i < 100; i++ { _ = rt.Service.ListNodeSnapshots() } }() wg.Add(1) go func() { defer wg.Done() <-start url := "http://" + openAIAddr + "/v1/models" for i := 0; i < 100; i++ { resp, err := client.Get(url) if err != nil { errCh <- err return } resp.Body.Close() if resp.StatusCode != http.StatusOK { errCh <- fmt.Errorf("GET /v1/models status=%d", resp.StatusCode) return } } }() close(start) wg.Wait() close(errCh) for err := range errCh { if err != nil { t.Fatalf("concurrent refresh reader failed: %v", err) } } } // loadServeNormalizedConfig loads an Edge config and applies the same // normalization iop-edge serve performs, so refresh classification compares // effective values. func loadServeNormalizedConfig(t *testing.T, path string) *config.EdgeConfig { t.Helper() cfg, err := config.LoadEdge(path) if err != nil { t.Fatalf("load config %s: %v", path, err) } cfg.Logging.Path = serveNormalizedLogPath() cfg.Bootstrap.ArtifactDir = serveNormalizedArtifactDir(cfg.Bootstrap.ArtifactDir) return cfg } // providerPoolConfigYAML builds an Edge config with one provider-pool model and // one node provider. The extra parameter lets callers inject a candidate // variation (e.g. a different capacity or an invalid adapter block). func providerPoolConfigYAML(serverAddr, openAIAddr, providerBlock string) string { return fmt.Sprintf(` server: listen: %q bootstrap: listen: "0.0.0.0:18080" artifact_dir: "artifacts" logging: level: "error" refresh: enabled: true listen: "127.0.0.1:0" openai: enabled: true listen: %q adapter: "openai_compat" target: "" a2a: listen: "0.0.0.0:8081" metrics: port: 0 models: - id: "qwen3.6:35b" display_name: "Qwen" providers: prov-a: "served-qwen" nodes: - id: "node-1" alias: "n1" token: "tok-1" adapters: %s providers: - id: "prov-a" type: "vllm" category: "api" adapter: "vllm-gpu" models: ["served-qwen"] health: "available" capacity: 2 max_queue: 4 queue_timeout_ms: 5000 `, serverAddr, openAIAddr, providerBlock) } const validAdapterBlock = ` openai_compat_instances: - name: "vllm-gpu" enabled: true provider: "vllm" endpoint: "http://127.0.0.1:8000/v1"` // invalidAdapterBlock enables an openai_compat instance but leaves endpoint // empty, which ValidateEdgeConfig rejects. const invalidAdapterBlock = ` openai_compat_instances: - name: "vllm-gpu" enabled: true provider: "vllm" endpoint: ""` // modelsResponse represents the JSON response from /v1/models. type modelsResponse struct { Object string `json:"object"` Data []modelItem `json:"data"` } // modelItem represents a single model entry in the /v1/models response. type modelItem struct { ID string `json:"id"` Object string `json:"object"` Created int64 `json:"created"` OwnedBy string `json:"owned_by"` } // modelIDs returns the sorted list of model IDs from /v1/models. // This avoids flaky timestamp comparisons on the `created` field. func modelIDs(t *testing.T, client *http.Client, openAIAddr string) []string { t.Helper() resp, err := client.Get("http://" + openAIAddr + "/v1/models") if err != nil { t.Fatalf("GET /v1/models: %v", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { t.Fatalf("GET /v1/models status=%d", resp.StatusCode) } var body modelsResponse if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { t.Fatalf("decode /v1/models body: %v", err) } ids := make([]string, 0, len(body.Data)) for _, m := range body.Data { ids = append(ids, m.ID) } sort.Strings(ids) return ids } // TestRefreshRejectedPreservesModels verifies SDD S10: an invalid candidate // apply is rejected without mutating the running runtime config or the // /v1/models surface. Uses stable model ID comparison to avoid timestamp flakiness. func TestRefreshRejectedPreservesModels(t *testing.T) { dir := t.TempDir() serverAddr := freeTCPAddr(t) openAIAddr := freeTCPAddr(t) basePath := filepath.Join(dir, "base.yaml") if err := os.WriteFile(basePath, []byte(providerPoolConfigYAML(serverAddr, openAIAddr, validAdapterBlock)), 0o600); err != nil { t.Fatalf("write base: %v", err) } candidatePath := filepath.Join(dir, "invalid.yaml") if err := os.WriteFile(candidatePath, []byte(providerPoolConfigYAML(serverAddr, openAIAddr, invalidAdapterBlock)), 0o600); err != nil { t.Fatalf("write candidate: %v", err) } rt, err := NewRuntime(loadServeNormalizedConfig(t, basePath)) if err != nil { t.Fatalf("NewRuntime: %v", err) } if err := rt.Start(context.Background()); err != nil { t.Fatalf("Start: %v", err) } defer func() { if err := rt.Stop(); err != nil { t.Fatalf("Stop: %v", err) } }() client := &http.Client{Timeout: 5 * time.Second} beforeIDs := modelIDs(t, client, openAIAddr) if len(beforeIDs) == 0 { t.Fatal("expected at least one model before refresh") } cfgBefore := rt.Cfg result, err := rt.RefreshConfig(context.Background(), configrefresh.Request{ Mode: configrefresh.ModeApply, ConfigPath: candidatePath, RequestID: "reject-1", }) if err != nil { t.Fatalf("RefreshConfig: %v", err) } if result.Status != configrefresh.StatusRejected { t.Fatalf("expected status=rejected for invalid candidate, got %q (summary=%s)", result.Status, result.Summary) } if rt.Cfg != cfgBefore { t.Fatal("runtime config snapshot mutated after rejected apply") } afterIDs := modelIDs(t, client, openAIAddr) if !reflect.DeepEqual(afterIDs, beforeIDs) { t.Fatalf("/v1/models changed after rejected apply:\nbefore=%v\nafter=%v", beforeIDs, afterIDs) } } // TestRefreshApplyDoesNotMutateOnRestartRequired verifies that a candidate with // a restart_required change leaves the running runtime snapshot and /v1/models // untouched. Uses stable model ID comparison. func TestRefreshApplyDoesNotMutateOnRestartRequired(t *testing.T) { dir := t.TempDir() serverAddr := freeTCPAddr(t) openAIAddr := freeTCPAddr(t) basePath := filepath.Join(dir, "base.yaml") if err := os.WriteFile(basePath, []byte(providerPoolConfigYAML(serverAddr, openAIAddr, validAdapterBlock)), 0o600); err != nil { t.Fatalf("write base: %v", err) } // Candidate changes server.listen, which is classified restart_required. candidateYAML := providerPoolConfigYAML(freeTCPAddr(t), openAIAddr, validAdapterBlock) candidatePath := filepath.Join(dir, "restart.yaml") if err := os.WriteFile(candidatePath, []byte(candidateYAML), 0o600); err != nil { t.Fatalf("write candidate: %v", err) } rt, err := NewRuntime(loadServeNormalizedConfig(t, basePath)) if err != nil { t.Fatalf("NewRuntime: %v", err) } if err := rt.Start(context.Background()); err != nil { t.Fatalf("Start: %v", err) } defer func() { if err := rt.Stop(); err != nil { t.Fatalf("Stop: %v", err) } }() client := &http.Client{Timeout: 5 * time.Second} beforeIDs := modelIDs(t, client, openAIAddr) if len(beforeIDs) == 0 { t.Fatal("expected at least one model before refresh") } cfgBefore := rt.Cfg result, err := rt.RefreshConfig(context.Background(), configrefresh.Request{ Mode: configrefresh.ModeApply, ConfigPath: candidatePath, RequestID: "restart-1", }) if err != nil { t.Fatalf("RefreshConfig: %v", err) } if result.Status != configrefresh.StatusRestartRequired { t.Fatalf("expected status=restart_required, got %q (summary=%s)", result.Status, result.Summary) } if len(result.RestartRequiredPaths) == 0 { t.Fatal("expected restart_required_paths to be reported") } if rt.Cfg != cfgBefore { t.Fatal("runtime config snapshot mutated after restart_required apply") } afterIDs := modelIDs(t, client, openAIAddr) if !reflect.DeepEqual(afterIDs, beforeIDs) { t.Fatalf("/v1/models changed after restart_required apply:\nbefore=%v\nafter=%v", beforeIDs, afterIDs) } } // TestRefreshAPIReportIncludesChangedItems verifies SDD S11: the refresh API // response exposes the changed provider/model items so an operator can confirm // what a refresh touched. func TestRefreshAPIReportIncludesChangedItems(t *testing.T) { dir := t.TempDir() serverAddr := freeTCPAddr(t) openAIAddr := freeTCPAddr(t) basePath := filepath.Join(dir, "base.yaml") if err := os.WriteFile(basePath, []byte(providerPoolConfigYAML(serverAddr, openAIAddr, validAdapterBlock)), 0o600); err != nil { t.Fatalf("write base: %v", err) } // Candidate raises provider capacity (applied) and renames the model display // name (applied), so both a provider and a model item change. candidateYAML := strings.Replace( strings.Replace(providerPoolConfigYAML(serverAddr, openAIAddr, validAdapterBlock), "capacity: 2", "capacity: 8", 1), `display_name: "Qwen"`, `display_name: "Qwen Refreshed"`, 1) candidatePath := filepath.Join(dir, "candidate.yaml") if err := os.WriteFile(candidatePath, []byte(candidateYAML), 0o600); err != nil { t.Fatalf("write candidate: %v", err) } rt, err := NewRuntime(loadServeNormalizedConfig(t, basePath)) if err != nil { t.Fatalf("NewRuntime: %v", err) } if err := rt.Start(context.Background()); err != nil { t.Fatalf("Start: %v", err) } defer func() { if err := rt.Stop(); err != nil { t.Fatalf("Stop: %v", err) } }() adminURL := "http://" + rt.RefreshAdmin.Addr() + "/refresh" client := &http.Client{Timeout: 5 * time.Second} body, _ := json.Marshal(configrefresh.Request{ Mode: configrefresh.ModeDryRun, ConfigPath: candidatePath, RequestID: "report-1", }) resp, err := client.Post(adminURL, "application/json", bytes.NewReader(body)) if err != nil { t.Fatalf("POST /refresh: %v", err) } defer resp.Body.Close() var result configrefresh.Result if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { t.Fatalf("decode response: %v", err) } if result.Status != configrefresh.StatusApplied { t.Fatalf("expected status=applied, got %q (summary=%s)", result.Status, result.Summary) } if !containsString(result.ChangedProviders, "prov-a") { t.Errorf("expected changed_providers to include prov-a, got %v", result.ChangedProviders) } if !containsString(result.ChangedModels, "qwen3.6:35b") { t.Errorf("expected changed_models to include qwen3.6:35b, got %v", result.ChangedModels) } } func containsString(s []string, want string) bool { for _, v := range s { if v == want { return true } } return false } func freeTCPAddr(t *testing.T) string { t.Helper() ln, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatalf("listen free tcp addr: %v", err) } addr := ln.Addr().String() if err := ln.Close(); err != nil { t.Fatalf("close free tcp addr listener: %v", err) } return addr } // TestNewRuntimeWiresControlPlaneConnector verifies that NewRuntime always // wires a non-nil ControlPlane connector regardless of config. func TestNewRuntimeWiresControlPlaneConnector(t *testing.T) { cfg := newTestConfig() rt, err := NewRuntime(cfg) if err != nil { t.Fatalf("NewRuntime: %v", err) } if rt.ControlPlane == nil { t.Fatal("expected non-nil ControlPlane connector") } } // TestNewRuntimeWiresStatusProviderIntoConnector verifies that NewRuntime wires // the edge service as the Control Plane connector's status provider, so status // requests are answered from the Edge-owned node snapshot surface. func TestNewRuntimeWiresStatusProviderIntoConnector(t *testing.T) { cfg := newTestConfig() rt, err := NewRuntime(cfg) if err != nil { t.Fatalf("NewRuntime: %v", err) } if !rt.ControlPlane.StatusProviderConfigured() { t.Fatal("expected NewRuntime to wire a status provider into the connector") } } // TestNewRuntimeWiresNodeEventBusIntoConnector verifies that node lifecycle // events are relayed through the shared events.Bus boundary. func TestNewRuntimeWiresNodeEventBusIntoConnector(t *testing.T) { cfg := newTestConfig() rt, err := NewRuntime(cfg) if err != nil { t.Fatalf("NewRuntime: %v", err) } if !rt.ControlPlane.NodeEventBusConfigured() { t.Fatal("expected NewRuntime to wire the node event bus into the connector") } } // TestRuntimeStartsControlPlaneConnectorWhenEnabled verifies that Start does // not fail when a Control Plane connector is configured with a local fake // server accepting hello. Uses a local TCP fake endpoint. func TestRuntimeStartsControlPlaneConnectorWhenEnabled(t *testing.T) { // Start a minimal fake Control Plane server. ln, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatalf("listen fake cp: %v", err) } cpAddr := ln.Addr().String() go func() { // Accept and immediately close; connector will fail hello and retry, // but Start() itself must succeed (connector errors are async). conn, err := ln.Accept() if err != nil { return } _ = conn.Close() _ = ln.Close() }() cfg := newTestConfig() cfg.ControlPlane = config.EdgeControlPlaneConf{ Enabled: true, WireAddr: cpAddr, ReconnectIntervalSec: 60, } rt, err := NewRuntime(cfg) if err != nil { t.Fatalf("NewRuntime: %v", err) } if !rt.ControlPlane.IsEnabled() { t.Fatal("expected connector IsEnabled()==true") } ctx := context.Background() if err := rt.Start(ctx); err != nil { t.Fatalf("Start with enabled connector: %v", err) } if err := rt.Stop(); err != nil { t.Fatalf("Stop: %v", err) } } // TestRuntimeStopsControlPlaneConnector verifies that Stop() reaches the // ControlPlane connector. A disabled connector is used so no real network // activity occurs. func TestRuntimeStopsControlPlaneConnector(t *testing.T) { cfg := newTestConfig() // disabled connector; Start/Stop must be no-ops on the connector side. cfg.ControlPlane = config.EdgeControlPlaneConf{ Enabled: false, WireAddr: "", } rt, err := NewRuntime(cfg) if err != nil { t.Fatalf("NewRuntime: %v", err) } ctx := context.Background() if err := rt.Start(ctx); err != nil { t.Fatalf("Start: %v", err) } if err := rt.Stop(); err != nil { t.Fatalf("Stop: %v", err) } // After Stop, the connector must be in a stopped-or-no-op state. // For disabled connector, CurrentState is Disconnected (never started). // Calling Stop again must not panic. rt.ControlPlane.Stop() } // TestRefreshRejectedPreservesProviderDispatch verifies SDD S10: a rejected // refresh must preserve provider-pool dispatch state. It uses a net.Pipe-based // fake node to capture the RunRequest sent by SubmitRun(ProviderPool=true) // before and after a rejected config refresh. func TestRefreshRejectedPreservesProviderDispatch(t *testing.T) { dir := t.TempDir() serverAddr := freeTCPAddr(t) openAIAddr := freeTCPAddr(t) basePath := filepath.Join(dir, "base.yaml") if err := os.WriteFile(basePath, []byte(providerPoolConfigYAML(serverAddr, openAIAddr, validAdapterBlock)), 0o600); err != nil { t.Fatalf("write base: %v", err) } candidatePath := filepath.Join(dir, "invalid.yaml") if err := os.WriteFile(candidatePath, []byte(providerPoolConfigYAML(serverAddr, openAIAddr, invalidAdapterBlock)), 0o600); err != nil { t.Fatalf("write candidate: %v", err) } rt, err := NewRuntime(loadServeNormalizedConfig(t, basePath)) if err != nil { t.Fatalf("NewRuntime: %v", err) } if err := rt.Start(context.Background()); err != nil { t.Fatalf("Start: %v", err) } defer func() { if err := rt.Stop(); err != nil { t.Fatalf("Stop: %v", err) } }() // --- Dispatch capture setup using net.Pipe --- edgeConn, nodeConn := net.Pipe() defer edgeConn.Close() defer nodeConn.Close() parserMap := toki.ParserMap{ toki.TypeNameOf(&iop.RunRequest{}): func(b []byte) (proto.Message, error) { m := &iop.RunRequest{} return m, proto.Unmarshal(b, m) }, } edgeClient := toki.NewTcpClient(edgeConn, 0, 0, parserMap) nodeClient := toki.NewTcpClient(nodeConn, 0, 0, parserMap) var capturedReq *iop.RunRequest var capturedMu sync.Mutex toki.AddListenerTyped[*iop.RunRequest](&nodeClient.Communicator, func(req *iop.RunRequest) { capturedMu.Lock() capturedReq = req capturedMu.Unlock() }) // Add a provider-pool node record to the NodeStore so provider-pool // resolution finds a dispatchable provider. The provider config must // match the adapter (vllm-gpu) and served target (served-qwen) expected // by the model catalog entry (qwen3.6:35b -> prov-a). rt.NodeStore.Add(&edgenode.NodeRecord{ ID: "node-pool-dispatch", Alias: "pool-alias", Runtime: config.RuntimeConf{Concurrency: 4}, Providers: []config.NodeProviderConf{ { ID: "prov-a", Type: "vllm", Category: "api", Adapter: "vllm-gpu", Models: []string{"served-qwen"}, Health: "available", Capacity: 2, }, }, }) // Register the fake node into the runtime registry. rt.Registry.Register(&edgenode.NodeEntry{ NodeID: "node-pool-dispatch", LifecycleState: edgenode.LifecycleConnected, Client: edgeClient, }) // Submit RunRequest via the runtime Service before refresh. dispatchAndCapture := func() (*iop.RunRequest, error) { capturedMu.Lock() capturedReq = nil capturedMu.Unlock() result, err := rt.Service.SubmitRun(context.Background(), edgeservice.SubmitRunRequest{ RunID: "dispatch-test-" + time.Now().Format("150405"), ModelGroupKey: "qwen3.6:35b", ProviderPool: true, Background: true, }) if err != nil { return nil, err } // Drain the result handle to release resources. if result != nil { result.Close() } // Wait for the fake node to receive the request. time.Sleep(100 * time.Millisecond) capturedMu.Lock() defer capturedMu.Unlock() return capturedReq, nil } // --- Before rejected refresh: dispatch must succeed --- beforeReq, err := dispatchAndCapture() if err != nil { t.Fatalf("dispatch before refresh: %v", err) } if beforeReq == nil { t.Fatal("expected dispatch before refresh to produce a RunRequest") } if beforeReq.GetAdapter() != "vllm-gpu" { t.Errorf("before refresh adapter: got %q, want %q", beforeReq.GetAdapter(), "vllm-gpu") } if beforeReq.GetTarget() != "served-qwen" { t.Errorf("before refresh target: got %q, want %q", beforeReq.GetTarget(), "served-qwen") } // --- Execute rejected refresh --- result, err := rt.RefreshConfig(context.Background(), configrefresh.Request{ Mode: configrefresh.ModeApply, ConfigPath: candidatePath, RequestID: "reject-dispatch-1", }) if err != nil { t.Fatalf("RefreshConfig: %v", err) } if result.Status != configrefresh.StatusRejected { t.Fatalf("expected status=rejected, got %q (summary=%s)", result.Status, result.Summary) } // --- After rejected refresh: dispatch must preserve state --- afterReq, err := dispatchAndCapture() if err != nil { t.Fatalf("dispatch after refresh: %v", err) } if afterReq == nil { t.Fatal("expected dispatch after rejected refresh to produce a RunRequest") } if afterReq.GetAdapter() != "vllm-gpu" { t.Errorf("after rejected refresh adapter: got %q, want %q", afterReq.GetAdapter(), "vllm-gpu") } if afterReq.GetTarget() != "served-qwen" { t.Errorf("after rejected refresh target: got %q, want %q", afterReq.GetTarget(), "served-qwen") } }