package bootstrap import ( "bytes" "context" "encoding/json" "fmt" "net/http" "os" "path/filepath" "reflect" "strings" "sync" "testing" "time" "iop/apps/edge/internal/configrefresh" "iop/packages/go/config" ) // TestRefreshAdminDryRunAndApplyReachRunningRuntime verifies S06: a running // Edge process can receive dry-run and apply refresh requests via the admin API // and returns mode-specific results. func TestRefreshAdminDryRunAndApplyReachRunningRuntime(t *testing.T) { dir := t.TempDir() serverAddr := freeTCPAddr(t) // Write base and candidate YAMLs with explicit immutable field values so // that config.LoadEdge produces configs with identical defaults on both sides. // Only capacity differs between base and candidate. baseYAML := fmt.Sprintf(` server: listen: %q bootstrap: listen: "0.0.0.0:18080" artifact_dir: "artifacts" logging: level: "error" refresh: enabled: true listen: "127.0.0.1:0" openai: listen: "0.0.0.0:18081" a2a: listen: "0.0.0.0:8081" metrics: port: 0 nodes: - id: "node-1" alias: "n1" token: "tok-1" adapters: cli: enabled: true providers: - id: "prov-a" type: "ollama" category: "local_inference" adapter: "cli" models: ["llama3.1"] capacity: 2 max_queue: 4 queue_timeout_ms: 5000 `, serverAddr) candidateYAML := fmt.Sprintf(` server: listen: %q bootstrap: listen: "0.0.0.0:18080" artifact_dir: "artifacts" logging: level: "error" refresh: enabled: true listen: "127.0.0.1:0" openai: listen: "0.0.0.0:18081" a2a: listen: "0.0.0.0:8081" metrics: port: 0 nodes: - id: "node-1" alias: "n1" token: "tok-1" adapters: cli: enabled: true providers: - id: "prov-a" type: "ollama" category: "local_inference" adapter: "cli" models: ["llama3.1"] capacity: 8 max_queue: 4 queue_timeout_ms: 5000 `, serverAddr) basePath := filepath.Join(dir, "base.yaml") if err := os.WriteFile(basePath, []byte(baseYAML), 0o600); err != nil { t.Fatalf("write base: %v", err) } candidatePath := filepath.Join(dir, "candidate.yaml") if err := os.WriteFile(candidatePath, []byte(candidateYAML), 0o600); err != nil { t.Fatalf("write candidate: %v", err) } currentCfg, err := config.LoadEdge(basePath) if err != nil { t.Fatalf("load base config: %v", err) } // Apply serve-equivalent normalization so the current config matches // what iop-edge serve would hold after loadRuntimeConfig. currentCfg.Logging.Path = serveNormalizedLogPath() currentCfg.Bootstrap.ArtifactDir = serveNormalizedArtifactDir(currentCfg.Bootstrap.ArtifactDir) // Override fields that don't need to match between runtime and candidate. currentCfg.Refresh.Enabled = true currentCfg.Refresh.Listen = "127.0.0.1:0" currentCfg.Logging.Level = "error" rt, err := NewRuntime(currentCfg) if err != nil { t.Fatalf("NewRuntime: %v", err) } ctx := context.Background() if err := rt.Start(ctx); err != nil { t.Fatalf("Start: %v", err) } defer func() { if err := rt.Stop(); err != nil { t.Fatalf("Stop: %v", err) } }() adminAddr := rt.RefreshAdmin.Addr() if adminAddr == "" { t.Fatal("refresh admin addr is empty after Start") } adminURL := "http://" + adminAddr + "/refresh" client := &http.Client{Timeout: 5 * time.Second} postRefresh := func(t *testing.T, mode configrefresh.Mode) configrefresh.Result { t.Helper() body, _ := json.Marshal(configrefresh.Request{ Mode: mode, ConfigPath: candidatePath, RequestID: string(mode) + "-req", }) resp, err := client.Post(adminURL, "application/json", bytes.NewReader(body)) if err != nil { t.Fatalf("POST /refresh mode=%s: %v", mode, err) } defer resp.Body.Close() var result configrefresh.Result if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { t.Fatalf("decode response: %v", err) } return result } // dry-run: process is reached, result has mode=dry_run. dryResult := postRefresh(t, configrefresh.ModeDryRun) if dryResult.Mode != configrefresh.ModeDryRun { t.Errorf("dry-run: expected mode=%q, got %q", configrefresh.ModeDryRun, dryResult.Mode) } if dryResult.Status != configrefresh.StatusApplied { t.Errorf("dry-run: expected status=%q (capacity change), got %q", configrefresh.StatusApplied, dryResult.Status) } // apply: process is reached, result has mode=apply and status=applied. applyResult := postRefresh(t, configrefresh.ModeApply) if applyResult.Mode != configrefresh.ModeApply { t.Errorf("apply: expected mode=%q, got %q", configrefresh.ModeApply, applyResult.Mode) } if applyResult.Status != configrefresh.StatusApplied { t.Errorf("apply: expected status=%q, got %q", configrefresh.StatusApplied, applyResult.Status) } // verify dry-run and apply have distinct observable results: the apply // committed the capacity change, so the next dry-run shows no changes. dryResult2 := postRefresh(t, configrefresh.ModeDryRun) if len(dryResult2.Changes) != 0 { t.Errorf("after apply, expected no diff changes on second dry-run, got %+v", dryResult2.Changes) } } func TestRefreshApplyConcurrentRuntimeReaders(t *testing.T) { dir := t.TempDir() serverAddr := freeTCPAddr(t) openAIAddr := freeTCPAddr(t) writeConfig := func(t *testing.T, name string, capacity int, displayName string) string { t.Helper() path := filepath.Join(dir, name) yaml := fmt.Sprintf(` server: listen: %q bootstrap: listen: "0.0.0.0:18080" artifact_dir: "artifacts" logging: level: "error" refresh: enabled: false listen: "127.0.0.1:19093" openai: enabled: true listen: %q adapter: "openai_compat" target: "" a2a: listen: "0.0.0.0:8081" metrics: port: 0 models: - id: "qwen3.6:35b" display_name: %q providers: prov-a: "served-qwen" nodes: - id: "node-1" alias: "n1" token: "tok-1" adapters: openai_compat_instances: - name: "vllm-gpu" enabled: true provider: "vllm" endpoint: "http://127.0.0.1:8000/v1" providers: - id: "prov-a" type: "vllm" category: "api" adapter: "vllm-gpu" models: ["served-qwen"] health: "available" capacity: %d max_queue: 4 queue_timeout_ms: 5000 `, serverAddr, openAIAddr, displayName, capacity) if err := os.WriteFile(path, []byte(yaml), 0o600); err != nil { t.Fatalf("write %s: %v", name, err) } return path } basePath := writeConfig(t, "base.yaml", 2, "Qwen Base") candidatePath := writeConfig(t, "candidate.yaml", 8, "Qwen Candidate") currentCfg, err := config.LoadEdge(basePath) if err != nil { t.Fatalf("load base config: %v", err) } // Apply serve-equivalent normalization. currentCfg.Logging.Path = serveNormalizedLogPath() currentCfg.Bootstrap.ArtifactDir = serveNormalizedArtifactDir(currentCfg.Bootstrap.ArtifactDir) rt, err := NewRuntime(currentCfg) if err != nil { t.Fatalf("NewRuntime: %v", err) } if err := rt.Start(context.Background()); err != nil { t.Fatalf("Start: %v", err) } defer func() { if err := rt.Stop(); err != nil { t.Fatalf("Stop: %v", err) } }() client := &http.Client{Timeout: 5 * time.Second} start := make(chan struct{}) var wg sync.WaitGroup errCh := make(chan error, 3) wg.Add(1) go func() { defer wg.Done() <-start for i := 0; i < 50; i++ { result, err := rt.RefreshConfig(context.Background(), configrefresh.Request{ Mode: configrefresh.ModeApply, ConfigPath: candidatePath, RequestID: fmt.Sprintf("apply-%d", i), }) if err != nil { errCh <- err return } if result.Status != configrefresh.StatusApplied { errCh <- fmt.Errorf("refresh status=%s summary=%s changes=%+v", result.Status, result.Summary, result.Changes) return } } }() wg.Add(1) go func() { defer wg.Done() <-start for i := 0; i < 100; i++ { _ = rt.Service.ListNodeSnapshots() } }() wg.Add(1) go func() { defer wg.Done() <-start url := "http://" + openAIAddr + "/v1/models" for i := 0; i < 100; i++ { resp, err := client.Get(url) if err != nil { errCh <- err return } resp.Body.Close() if resp.StatusCode != http.StatusOK { errCh <- fmt.Errorf("GET /v1/models status=%d", resp.StatusCode) return } } }() close(start) wg.Wait() close(errCh) for err := range errCh { if err != nil { t.Fatalf("concurrent refresh reader failed: %v", err) } } } // TestRefreshRejectedPreservesModels verifies SDD S10: an invalid candidate // apply is rejected without mutating the running runtime config or the // /v1/models surface. Uses stable model ID comparison to avoid timestamp flakiness. func TestRefreshRejectedPreservesModels(t *testing.T) { dir := t.TempDir() serverAddr := freeTCPAddr(t) openAIAddr := freeTCPAddr(t) basePath := filepath.Join(dir, "base.yaml") if err := os.WriteFile(basePath, []byte(providerPoolConfigYAML(serverAddr, openAIAddr, validAdapterBlock)), 0o600); err != nil { t.Fatalf("write base: %v", err) } candidatePath := filepath.Join(dir, "invalid.yaml") if err := os.WriteFile(candidatePath, []byte(providerPoolConfigYAML(serverAddr, openAIAddr, invalidAdapterBlock)), 0o600); err != nil { t.Fatalf("write candidate: %v", err) } rt, err := NewRuntime(loadServeNormalizedConfig(t, basePath)) if err != nil { t.Fatalf("NewRuntime: %v", err) } if err := rt.Start(context.Background()); err != nil { t.Fatalf("Start: %v", err) } defer func() { if err := rt.Stop(); err != nil { t.Fatalf("Stop: %v", err) } }() client := &http.Client{Timeout: 5 * time.Second} beforeIDs := modelIDs(t, client, openAIAddr) if len(beforeIDs) == 0 { t.Fatal("expected at least one model before refresh") } cfgBefore := rt.Cfg result, err := rt.RefreshConfig(context.Background(), configrefresh.Request{ Mode: configrefresh.ModeApply, ConfigPath: candidatePath, RequestID: "reject-1", }) if err != nil { t.Fatalf("RefreshConfig: %v", err) } if result.Status != configrefresh.StatusRejected { t.Fatalf("expected status=rejected for invalid candidate, got %q (summary=%s)", result.Status, result.Summary) } if rt.Cfg != cfgBefore { t.Fatal("runtime config snapshot mutated after rejected apply") } afterIDs := modelIDs(t, client, openAIAddr) if !reflect.DeepEqual(afterIDs, beforeIDs) { t.Fatalf("/v1/models changed after rejected apply:\nbefore=%v\nafter=%v", beforeIDs, afterIDs) } } // TestRefreshApplyDoesNotMutateOnRestartRequired verifies that a candidate with // a restart_required change leaves the running runtime snapshot and /v1/models // untouched. Uses stable model ID comparison. func TestRefreshApplyDoesNotMutateOnRestartRequired(t *testing.T) { dir := t.TempDir() serverAddr := freeTCPAddr(t) openAIAddr := freeTCPAddr(t) basePath := filepath.Join(dir, "base.yaml") if err := os.WriteFile(basePath, []byte(providerPoolConfigYAML(serverAddr, openAIAddr, validAdapterBlock)), 0o600); err != nil { t.Fatalf("write base: %v", err) } // Candidate changes server.listen, which is classified restart_required. candidateYAML := providerPoolConfigYAML(freeTCPAddr(t), openAIAddr, validAdapterBlock) candidatePath := filepath.Join(dir, "restart.yaml") if err := os.WriteFile(candidatePath, []byte(candidateYAML), 0o600); err != nil { t.Fatalf("write candidate: %v", err) } rt, err := NewRuntime(loadServeNormalizedConfig(t, basePath)) if err != nil { t.Fatalf("NewRuntime: %v", err) } if err := rt.Start(context.Background()); err != nil { t.Fatalf("Start: %v", err) } defer func() { if err := rt.Stop(); err != nil { t.Fatalf("Stop: %v", err) } }() client := &http.Client{Timeout: 5 * time.Second} beforeIDs := modelIDs(t, client, openAIAddr) if len(beforeIDs) == 0 { t.Fatal("expected at least one model before refresh") } cfgBefore := rt.Cfg result, err := rt.RefreshConfig(context.Background(), configrefresh.Request{ Mode: configrefresh.ModeApply, ConfigPath: candidatePath, RequestID: "restart-1", }) if err != nil { t.Fatalf("RefreshConfig: %v", err) } if result.Status != configrefresh.StatusRestartRequired { t.Fatalf("expected status=restart_required, got %q (summary=%s)", result.Status, result.Summary) } if len(result.RestartRequiredPaths) == 0 { t.Fatal("expected restart_required_paths to be reported") } if rt.Cfg != cfgBefore { t.Fatal("runtime config snapshot mutated after restart_required apply") } afterIDs := modelIDs(t, client, openAIAddr) if !reflect.DeepEqual(afterIDs, beforeIDs) { t.Fatalf("/v1/models changed after restart_required apply:\nbefore=%v\nafter=%v", beforeIDs, afterIDs) } } // TestRefreshAPIReportIncludesChangedItems verifies SDD S11: the refresh API // response exposes the changed provider/model items so an operator can confirm // what a refresh touched. func TestRefreshAPIReportIncludesChangedItems(t *testing.T) { dir := t.TempDir() serverAddr := freeTCPAddr(t) openAIAddr := freeTCPAddr(t) basePath := filepath.Join(dir, "base.yaml") if err := os.WriteFile(basePath, []byte(providerPoolConfigYAML(serverAddr, openAIAddr, validAdapterBlock)), 0o600); err != nil { t.Fatalf("write base: %v", err) } // Candidate raises provider capacity (applied) and renames the model display // name (applied), so both a provider and a model item change. candidateYAML := strings.Replace( strings.Replace(providerPoolConfigYAML(serverAddr, openAIAddr, validAdapterBlock), "capacity: 2", "capacity: 8", 1), `display_name: "Qwen"`, `display_name: "Qwen Refreshed"`, 1) candidatePath := filepath.Join(dir, "candidate.yaml") if err := os.WriteFile(candidatePath, []byte(candidateYAML), 0o600); err != nil { t.Fatalf("write candidate: %v", err) } rt, err := NewRuntime(loadServeNormalizedConfig(t, basePath)) if err != nil { t.Fatalf("NewRuntime: %v", err) } if err := rt.Start(context.Background()); err != nil { t.Fatalf("Start: %v", err) } defer func() { if err := rt.Stop(); err != nil { t.Fatalf("Stop: %v", err) } }() adminURL := "http://" + rt.RefreshAdmin.Addr() + "/refresh" client := &http.Client{Timeout: 5 * time.Second} body, _ := json.Marshal(configrefresh.Request{ Mode: configrefresh.ModeDryRun, ConfigPath: candidatePath, RequestID: "report-1", }) resp, err := client.Post(adminURL, "application/json", bytes.NewReader(body)) if err != nil { t.Fatalf("POST /refresh: %v", err) } defer resp.Body.Close() var result configrefresh.Result if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { t.Fatalf("decode response: %v", err) } if result.Status != configrefresh.StatusApplied { t.Fatalf("expected status=applied, got %q (summary=%s)", result.Status, result.Summary) } if !containsString(result.ChangedProviders, "prov-a") { t.Errorf("expected changed_providers to include prov-a, got %v", result.ChangedProviders) } if !containsString(result.ChangedModels, "qwen3.6:35b") { t.Errorf("expected changed_models to include qwen3.6:35b, got %v", result.ChangedModels) } }