iop/apps/edge/internal/bootstrap/runtime_test.go

package bootstrap

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"reflect"
	"sort"
	"strings"
	"sync"
	"testing"
	"time"

	toki "git.toki-labs.com/toki/proto-socket/go"
	"google.golang.org/protobuf/proto"
	iop "iop/proto/gen/iop"

	"iop/apps/edge/internal/configrefresh"
	edgenode "iop/apps/edge/internal/node"
	edgeservice "iop/apps/edge/internal/service"
	"iop/packages/go/config"
)

func newTestConfig() *config.EdgeConfig {
	return &config.EdgeConfig{
		Server:  config.EdgeServerConf{Listen: "127.0.0.1:0"},
		Logging: config.LoggingConf{Level: "error"},
		Metrics: config.MetricsConf{Port: 0},
		Nodes: []config.NodeDefinition{
			{ID: "node-1", Alias: "alias-1", Token: "tok-1"},
		},
	}
}

func TestNewRuntimeSeedsNodeStore(t *testing.T) {
	cfg := newTestConfig()
	rt, err := NewRuntime(cfg)
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	if rt.NodeStore == nil {
		t.Fatal("NodeStore is nil")
	}
	rec, ok := rt.NodeStore.FindByToken("tok-1")
	if !ok {
		t.Fatal("expected node record seeded by token tok-1")
	}
	if rec.ID != "node-1" || rec.Alias != "alias-1" {
		t.Errorf("unexpected record: id=%q alias=%q", rec.ID, rec.Alias)
	}
	if rt.Registry == nil || rt.EventBus == nil || rt.Service == nil || rt.Server == nil || rt.Logger == nil {
		t.Error("runtime dependencies must be wired by NewRuntime")
	}
}

func TestRuntimeWiresTransportHandlers(t *testing.T) {
	cfg := newTestConfig()
	rt, err := NewRuntime(cfg)
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	if rt.Server.HasRunEventHandler() || rt.Server.HasNodeEventHandler() {
		t.Fatal("handlers must not be wired before wireHandlers")
	}
	rt.wireHandlers()
	if !rt.Server.HasRunEventHandler() {
		t.Error("run event handler not wired")
	}
	if !rt.Server.HasNodeEventHandler() {
		t.Error("node event handler not wired")
	}
}

func TestNewRuntimeWiresInputManager(t *testing.T) {
	cfg := newTestConfig()
	rt, err := NewRuntime(cfg)
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	if rt.Input == nil {
		t.Fatal("Input manager is nil")
	}
	if rt.Input.OpenAI == nil {
		t.Fatal("Input.OpenAI is nil")
	}
	if rt.Input.A2A == nil {
		t.Fatal("Input.A2A is nil")
	}
}

func TestRuntimeKeepsListenersAfterStartupContextCancel(t *testing.T) {
	edgeAddr := freeTCPAddr(t)
	openAIAddr := freeTCPAddr(t)
	cfg := newTestConfig()
	cfg.Server.Listen = edgeAddr
	cfg.OpenAI = config.EdgeOpenAIConf{
		Enabled: true,
		Listen:  openAIAddr,
	}

	rt, err := NewRuntime(cfg)
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	startupCtx, cancelStartup := context.WithCancel(context.Background())
	if err := rt.Start(startupCtx); err != nil {
		t.Fatalf("Start: %v", err)
	}
	defer func() {
		if err := rt.Stop(); err != nil {
			t.Fatalf("Stop: %v", err)
		}
	}()

	cancelStartup()

	conn, err := net.DialTimeout("tcp", edgeAddr, time.Second)
	if err != nil {
		t.Fatalf("edge listener closed with startup context: %v", err)
	}
	_ = conn.Close()

	client := &http.Client{Timeout: time.Second}
	resp, err := client.Get("http://" + openAIAddr + "/healthz")
	if err != nil {
		t.Fatalf("openai listener closed with startup context: %v", err)
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("openai health status = %d, want %d", resp.StatusCode, http.StatusOK)
	}
}

func TestNewRuntimeWritesLogsToConfiguredPath(t *testing.T) {
	dir := t.TempDir()
	logPath := filepath.Join(dir, "logs", "edge.log")

	cfg := newTestConfig()
	cfg.Logging.Level = "info"
	cfg.Logging.Path = logPath
	rt, err := NewRuntime(cfg)
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	rt.Logger.Info("runtime-log-entry")
	_ = rt.Logger.Sync()

	info, err := os.Stat(logPath)
	if err != nil {
		t.Fatalf("stat log path: %v", err)
	}
	if info.Size() == 0 {
		t.Fatalf("log file empty: %s", logPath)
	}
	contents, err := os.ReadFile(logPath)
	if err != nil {
		t.Fatalf("read log file: %v", err)
	}
	if !strings.Contains(string(contents), "runtime-log-entry") {
		t.Fatalf("expected runtime-log-entry in log file; contents=%q", string(contents))
	}
}

func TestNewRuntimeRejectsDuplicateToken(t *testing.T) {
	cfg := &config.EdgeConfig{
		Server:  config.EdgeServerConf{Listen: "127.0.0.1:0"},
		Logging: config.LoggingConf{Level: "error"},
		Nodes: []config.NodeDefinition{
			{ID: "node-1", Alias: "a1", Token: "dup"},
			{ID: "node-2", Alias: "a2", Token: "dup"},
		},
	}
	if _, err := NewRuntime(cfg); err == nil {
		t.Fatal("expected error on duplicate token seed")
	}
}

// TestRefreshAdminDryRunAndApplyReachRunningRuntime verifies S06: a running
// Edge process can receive dry-run and apply refresh requests via the admin API
// and returns mode-specific results.
func TestRefreshAdminDryRunAndApplyReachRunningRuntime(t *testing.T) {
	dir := t.TempDir()

	serverAddr := freeTCPAddr(t)

	// Write base and candidate YAMLs with explicit immutable field values so
	// that config.LoadEdge produces configs with identical defaults on both sides.
	// Only capacity differs between base and candidate.
	baseYAML := fmt.Sprintf(`
server:
  listen: %q
bootstrap:
  listen: "0.0.0.0:18080"
  artifact_dir: "artifacts"
logging:
  level: "error"
refresh:
  enabled: true
  listen: "127.0.0.1:0"
openai:
  listen: "0.0.0.0:18081"
a2a:
  listen: "0.0.0.0:8081"
metrics:
  port: 0
nodes:
  - id: "node-1"
    alias: "n1"
    token: "tok-1"
    adapters:
      cli:
        enabled: true
    providers:
      - id: "prov-a"
        type: "ollama"
        category: "local_inference"
        models: ["llama3.1"]
        capacity: 2
        max_queue: 4
        queue_timeout_ms: 5000
`, serverAddr)

	candidateYAML := fmt.Sprintf(`
server:
  listen: %q
bootstrap:
  listen: "0.0.0.0:18080"
  artifact_dir: "artifacts"
logging:
  level: "error"
refresh:
  enabled: true
  listen: "127.0.0.1:0"
openai:
  listen: "0.0.0.0:18081"
a2a:
  listen: "0.0.0.0:8081"
metrics:
  port: 0
nodes:
  - id: "node-1"
    alias: "n1"
    token: "tok-1"
    adapters:
      cli:
        enabled: true
    providers:
      - id: "prov-a"
        type: "ollama"
        category: "local_inference"
        models: ["llama3.1"]
        capacity: 8
        max_queue: 4
        queue_timeout_ms: 5000
`, serverAddr)

	basePath := filepath.Join(dir, "base.yaml")
	if err := os.WriteFile(basePath, []byte(baseYAML), 0o600); err != nil {
		t.Fatalf("write base: %v", err)
	}
	candidatePath := filepath.Join(dir, "candidate.yaml")
	if err := os.WriteFile(candidatePath, []byte(candidateYAML), 0o600); err != nil {
		t.Fatalf("write candidate: %v", err)
	}

	currentCfg, err := config.LoadEdge(basePath)
	if err != nil {
		t.Fatalf("load base config: %v", err)
	}
	// Apply serve-equivalent normalization so the current config matches
	// what iop-edge serve would hold after loadRuntimeConfig.
	currentCfg.Logging.Path = serveNormalizedLogPath()
	currentCfg.Bootstrap.ArtifactDir = serveNormalizedArtifactDir(currentCfg.Bootstrap.ArtifactDir)
	// Override fields that don't need to match between runtime and candidate.
	currentCfg.Refresh.Enabled = true
	currentCfg.Refresh.Listen = "127.0.0.1:0"
	currentCfg.Logging.Level = "error"

	rt, err := NewRuntime(currentCfg)
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	ctx := context.Background()
	if err := rt.Start(ctx); err != nil {
		t.Fatalf("Start: %v", err)
	}
	defer func() {
		if err := rt.Stop(); err != nil {
			t.Fatalf("Stop: %v", err)
		}
	}()

	adminAddr := rt.RefreshAdmin.Addr()
	if adminAddr == "" {
		t.Fatal("refresh admin addr is empty after Start")
	}
	adminURL := "http://" + adminAddr + "/refresh"

	client := &http.Client{Timeout: 5 * time.Second}

	postRefresh := func(t *testing.T, mode configrefresh.Mode) configrefresh.Result {
		t.Helper()
		body, _ := json.Marshal(configrefresh.Request{
			Mode:       mode,
			ConfigPath: candidatePath,
			RequestID:  string(mode) + "-req",
		})
		resp, err := client.Post(adminURL, "application/json", bytes.NewReader(body))
		if err != nil {
			t.Fatalf("POST /refresh mode=%s: %v", mode, err)
		}
		defer resp.Body.Close()
		var result configrefresh.Result
		if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
			t.Fatalf("decode response: %v", err)
		}
		return result
	}

	// dry-run: process is reached, result has mode=dry_run.
	dryResult := postRefresh(t, configrefresh.ModeDryRun)
	if dryResult.Mode != configrefresh.ModeDryRun {
		t.Errorf("dry-run: expected mode=%q, got %q", configrefresh.ModeDryRun, dryResult.Mode)
	}
	if dryResult.Status != configrefresh.StatusApplied {
		t.Errorf("dry-run: expected status=%q (capacity change), got %q", configrefresh.StatusApplied, dryResult.Status)
	}

	// apply: process is reached, result has mode=apply and status=applied.
	applyResult := postRefresh(t, configrefresh.ModeApply)
	if applyResult.Mode != configrefresh.ModeApply {
		t.Errorf("apply: expected mode=%q, got %q", configrefresh.ModeApply, applyResult.Mode)
	}
	if applyResult.Status != configrefresh.StatusApplied {
		t.Errorf("apply: expected status=%q, got %q", configrefresh.StatusApplied, applyResult.Status)
	}

	// verify dry-run and apply have distinct observable results: the apply
	// committed the capacity change, so the next dry-run shows no changes.
	dryResult2 := postRefresh(t, configrefresh.ModeDryRun)
	if len(dryResult2.Changes) != 0 {
		t.Errorf("after apply, expected no diff changes on second dry-run, got %+v", dryResult2.Changes)
	}
}

func serveNormalizedLogPath() string {
	if exe, err := os.Executable(); err == nil {
		return filepath.Join(filepath.Dir(exe), "logs", "edge.log")
	}
	if wd, err := os.Getwd(); err == nil {
		return filepath.Join(wd, "logs", "edge.log")
	}
	return "logs/edge.log"
}

func serveNormalizedArtifactDir(dir string) string {
	if dir == "" {
		dir = "artifacts"
	}
	if filepath.IsAbs(dir) {
		return dir
	}
	bd := binaryDirForTest()
	return filepath.Join(bd, dir)
}

func binaryDirForTest() string {
	if exe, err := os.Executable(); err == nil {
		return filepath.Dir(exe)
	}
	if wd, err := os.Getwd(); err == nil {
		return wd
	}
	return "."
}

func TestRefreshApplyConcurrentRuntimeReaders(t *testing.T) {
	dir := t.TempDir()
	serverAddr := freeTCPAddr(t)
	openAIAddr := freeTCPAddr(t)

	writeConfig := func(t *testing.T, name string, capacity int, displayName string) string {
		t.Helper()
		path := filepath.Join(dir, name)
		yaml := fmt.Sprintf(`
server:
  listen: %q
bootstrap:
  listen: "0.0.0.0:18080"
  artifact_dir: "artifacts"
logging:
  level: "error"
refresh:
  enabled: false
  listen: "127.0.0.1:19093"
openai:
  enabled: true
  listen: %q
  adapter: "openai_compat"
  target: ""
a2a:
  listen: "0.0.0.0:8081"
metrics:
  port: 0
models:
  - id: "qwen3.6:35b"
    display_name: %q
    providers:
      prov-a: "served-qwen"
nodes:
  - id: "node-1"
    alias: "n1"
    token: "tok-1"
    adapters:
      openai_compat_instances:
        - name: "vllm-gpu"
          enabled: true
          provider: "vllm"
          endpoint: "http://127.0.0.1:8000/v1"
    providers:
      - id: "prov-a"
        type: "vllm"
        category: "api"
        adapter: "vllm-gpu"
        models: ["served-qwen"]
        health: "available"
        capacity: %d
        max_queue: 4
        queue_timeout_ms: 5000
`, serverAddr, openAIAddr, displayName, capacity)
		if err := os.WriteFile(path, []byte(yaml), 0o600); err != nil {
			t.Fatalf("write %s: %v", name, err)
		}
		return path
	}

	basePath := writeConfig(t, "base.yaml", 2, "Qwen Base")
	candidatePath := writeConfig(t, "candidate.yaml", 8, "Qwen Candidate")

	currentCfg, err := config.LoadEdge(basePath)
	if err != nil {
		t.Fatalf("load base config: %v", err)
	}
	// Apply serve-equivalent normalization.
	currentCfg.Logging.Path = serveNormalizedLogPath()
	currentCfg.Bootstrap.ArtifactDir = serveNormalizedArtifactDir(currentCfg.Bootstrap.ArtifactDir)
	rt, err := NewRuntime(currentCfg)
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	if err := rt.Start(context.Background()); err != nil {
		t.Fatalf("Start: %v", err)
	}
	defer func() {
		if err := rt.Stop(); err != nil {
			t.Fatalf("Stop: %v", err)
		}
	}()

	client := &http.Client{Timeout: 5 * time.Second}
	start := make(chan struct{})
	var wg sync.WaitGroup
	errCh := make(chan error, 3)

	wg.Add(1)
	go func() {
		defer wg.Done()
		<-start
		for i := 0; i < 50; i++ {
			result, err := rt.RefreshConfig(context.Background(), configrefresh.Request{
				Mode:       configrefresh.ModeApply,
				ConfigPath: candidatePath,
				RequestID:  fmt.Sprintf("apply-%d", i),
			})
			if err != nil {
				errCh <- err
				return
			}
			if result.Status != configrefresh.StatusApplied {
				errCh <- fmt.Errorf("refresh status=%s summary=%s changes=%+v", result.Status, result.Summary, result.Changes)
				return
			}
		}
	}()

	wg.Add(1)
	go func() {
		defer wg.Done()
		<-start
		for i := 0; i < 100; i++ {
			_ = rt.Service.ListNodeSnapshots()
		}
	}()

	wg.Add(1)
	go func() {
		defer wg.Done()
		<-start
		url := "http://" + openAIAddr + "/v1/models"
		for i := 0; i < 100; i++ {
			resp, err := client.Get(url)
			if err != nil {
				errCh <- err
				return
			}
			resp.Body.Close()
			if resp.StatusCode != http.StatusOK {
				errCh <- fmt.Errorf("GET /v1/models status=%d", resp.StatusCode)
				return
			}
		}
	}()

	close(start)
	wg.Wait()
	close(errCh)
	for err := range errCh {
		if err != nil {
			t.Fatalf("concurrent refresh reader failed: %v", err)
		}
	}
}

// loadServeNormalizedConfig loads an Edge config and applies the same
// normalization iop-edge serve performs, so refresh classification compares
// effective values.
func loadServeNormalizedConfig(t *testing.T, path string) *config.EdgeConfig {
	t.Helper()
	cfg, err := config.LoadEdge(path)
	if err != nil {
		t.Fatalf("load config %s: %v", path, err)
	}
	cfg.Logging.Path = serveNormalizedLogPath()
	cfg.Bootstrap.ArtifactDir = serveNormalizedArtifactDir(cfg.Bootstrap.ArtifactDir)
	return cfg
}

// providerPoolConfigYAML builds an Edge config with one provider-pool model and
// one node provider. The extra parameter lets callers inject a candidate
// variation (e.g. a different capacity or an invalid adapter block).
func providerPoolConfigYAML(serverAddr, openAIAddr, providerBlock string) string {
	return fmt.Sprintf(`
server:
  listen: %q
bootstrap:
  listen: "0.0.0.0:18080"
  artifact_dir: "artifacts"
logging:
  level: "error"
refresh:
  enabled: true
  listen: "127.0.0.1:0"
openai:
  enabled: true
  listen: %q
  adapter: "openai_compat"
  target: ""
a2a:
  listen: "0.0.0.0:8081"
metrics:
  port: 0
models:
  - id: "qwen3.6:35b"
    display_name: "Qwen"
    providers:
      prov-a: "served-qwen"
nodes:
  - id: "node-1"
    alias: "n1"
    token: "tok-1"
    adapters:
%s
    providers:
      - id: "prov-a"
        type: "vllm"
        category: "api"
        adapter: "vllm-gpu"
        models: ["served-qwen"]
        health: "available"
        capacity: 2
        max_queue: 4
        queue_timeout_ms: 5000
`, serverAddr, openAIAddr, providerBlock)
}

const validAdapterBlock = `      openai_compat_instances:
        - name: "vllm-gpu"
          enabled: true
          provider: "vllm"
          endpoint: "http://127.0.0.1:8000/v1"`

// invalidAdapterBlock enables an openai_compat instance but leaves endpoint
// empty, which ValidateEdgeConfig rejects.
const invalidAdapterBlock = `      openai_compat_instances:
        - name: "vllm-gpu"
          enabled: true
          provider: "vllm"
          endpoint: ""`

// modelsResponse represents the JSON response from /v1/models.
type modelsResponse struct {
	Object string      `json:"object"`
	Data   []modelItem `json:"data"`
}

// modelItem represents a single model entry in the /v1/models response.
type modelItem struct {
	ID      string `json:"id"`
	Object  string `json:"object"`
	Created int64  `json:"created"`
	OwnedBy string `json:"owned_by"`
}

// modelIDs returns the sorted list of model IDs from /v1/models.
// This avoids flaky timestamp comparisons on the `created` field.
func modelIDs(t *testing.T, client *http.Client, openAIAddr string) []string {
	t.Helper()
	resp, err := client.Get("http://" + openAIAddr + "/v1/models")
	if err != nil {
		t.Fatalf("GET /v1/models: %v", err)
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("GET /v1/models status=%d", resp.StatusCode)
	}
	var body modelsResponse
	if err := json.NewDecoder(resp.Body).Decode(&body); err != nil {
		t.Fatalf("decode /v1/models body: %v", err)
	}
	ids := make([]string, 0, len(body.Data))
	for _, m := range body.Data {
		ids = append(ids, m.ID)
	}
	sort.Strings(ids)
	return ids
}

// TestRefreshRejectedPreservesModels verifies SDD S10: an invalid candidate
// apply is rejected without mutating the running runtime config or the
// /v1/models surface. Uses stable model ID comparison to avoid timestamp flakiness.
func TestRefreshRejectedPreservesModels(t *testing.T) {
	dir := t.TempDir()
	serverAddr := freeTCPAddr(t)
	openAIAddr := freeTCPAddr(t)

	basePath := filepath.Join(dir, "base.yaml")
	if err := os.WriteFile(basePath, []byte(providerPoolConfigYAML(serverAddr, openAIAddr, validAdapterBlock)), 0o600); err != nil {
		t.Fatalf("write base: %v", err)
	}
	candidatePath := filepath.Join(dir, "invalid.yaml")
	if err := os.WriteFile(candidatePath, []byte(providerPoolConfigYAML(serverAddr, openAIAddr, invalidAdapterBlock)), 0o600); err != nil {
		t.Fatalf("write candidate: %v", err)
	}

	rt, err := NewRuntime(loadServeNormalizedConfig(t, basePath))
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	if err := rt.Start(context.Background()); err != nil {
		t.Fatalf("Start: %v", err)
	}
	defer func() {
		if err := rt.Stop(); err != nil {
			t.Fatalf("Stop: %v", err)
		}
	}()

	client := &http.Client{Timeout: 5 * time.Second}
	beforeIDs := modelIDs(t, client, openAIAddr)
	if len(beforeIDs) == 0 {
		t.Fatal("expected at least one model before refresh")
	}
	cfgBefore := rt.Cfg

	result, err := rt.RefreshConfig(context.Background(), configrefresh.Request{
		Mode:       configrefresh.ModeApply,
		ConfigPath: candidatePath,
		RequestID:  "reject-1",
	})
	if err != nil {
		t.Fatalf("RefreshConfig: %v", err)
	}
	if result.Status != configrefresh.StatusRejected {
		t.Fatalf("expected status=rejected for invalid candidate, got %q (summary=%s)", result.Status, result.Summary)
	}

	if rt.Cfg != cfgBefore {
		t.Fatal("runtime config snapshot mutated after rejected apply")
	}
	afterIDs := modelIDs(t, client, openAIAddr)
	if !reflect.DeepEqual(afterIDs, beforeIDs) {
		t.Fatalf("/v1/models changed after rejected apply:\nbefore=%v\nafter=%v", beforeIDs, afterIDs)
	}
}

// TestRefreshApplyDoesNotMutateOnRestartRequired verifies that a candidate with
// a restart_required change leaves the running runtime snapshot and /v1/models
// untouched. Uses stable model ID comparison.
func TestRefreshApplyDoesNotMutateOnRestartRequired(t *testing.T) {
	dir := t.TempDir()
	serverAddr := freeTCPAddr(t)
	openAIAddr := freeTCPAddr(t)

	basePath := filepath.Join(dir, "base.yaml")
	if err := os.WriteFile(basePath, []byte(providerPoolConfigYAML(serverAddr, openAIAddr, validAdapterBlock)), 0o600); err != nil {
		t.Fatalf("write base: %v", err)
	}
	// Candidate changes server.listen, which is classified restart_required.
	candidateYAML := providerPoolConfigYAML(freeTCPAddr(t), openAIAddr, validAdapterBlock)
	candidatePath := filepath.Join(dir, "restart.yaml")
	if err := os.WriteFile(candidatePath, []byte(candidateYAML), 0o600); err != nil {
		t.Fatalf("write candidate: %v", err)
	}

	rt, err := NewRuntime(loadServeNormalizedConfig(t, basePath))
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	if err := rt.Start(context.Background()); err != nil {
		t.Fatalf("Start: %v", err)
	}
	defer func() {
		if err := rt.Stop(); err != nil {
			t.Fatalf("Stop: %v", err)
		}
	}()

	client := &http.Client{Timeout: 5 * time.Second}
	beforeIDs := modelIDs(t, client, openAIAddr)
	if len(beforeIDs) == 0 {
		t.Fatal("expected at least one model before refresh")
	}
	cfgBefore := rt.Cfg

	result, err := rt.RefreshConfig(context.Background(), configrefresh.Request{
		Mode:       configrefresh.ModeApply,
		ConfigPath: candidatePath,
		RequestID:  "restart-1",
	})
	if err != nil {
		t.Fatalf("RefreshConfig: %v", err)
	}
	if result.Status != configrefresh.StatusRestartRequired {
		t.Fatalf("expected status=restart_required, got %q (summary=%s)", result.Status, result.Summary)
	}
	if len(result.RestartRequiredPaths) == 0 {
		t.Fatal("expected restart_required_paths to be reported")
	}
	if rt.Cfg != cfgBefore {
		t.Fatal("runtime config snapshot mutated after restart_required apply")
	}
	afterIDs := modelIDs(t, client, openAIAddr)
	if !reflect.DeepEqual(afterIDs, beforeIDs) {
		t.Fatalf("/v1/models changed after restart_required apply:\nbefore=%v\nafter=%v", beforeIDs, afterIDs)
	}
}

// TestRefreshAPIReportIncludesChangedItems verifies SDD S11: the refresh API
// response exposes the changed provider/model items so an operator can confirm
// what a refresh touched.
func TestRefreshAPIReportIncludesChangedItems(t *testing.T) {
	dir := t.TempDir()
	serverAddr := freeTCPAddr(t)
	openAIAddr := freeTCPAddr(t)

	basePath := filepath.Join(dir, "base.yaml")
	if err := os.WriteFile(basePath, []byte(providerPoolConfigYAML(serverAddr, openAIAddr, validAdapterBlock)), 0o600); err != nil {
		t.Fatalf("write base: %v", err)
	}
	// Candidate raises provider capacity (applied) and renames the model display
	// name (applied), so both a provider and a model item change.
	candidateYAML := strings.Replace(
		strings.Replace(providerPoolConfigYAML(serverAddr, openAIAddr, validAdapterBlock), "capacity: 2", "capacity: 8", 1),
		`display_name: "Qwen"`, `display_name: "Qwen Refreshed"`, 1)
	candidatePath := filepath.Join(dir, "candidate.yaml")
	if err := os.WriteFile(candidatePath, []byte(candidateYAML), 0o600); err != nil {
		t.Fatalf("write candidate: %v", err)
	}

	rt, err := NewRuntime(loadServeNormalizedConfig(t, basePath))
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	if err := rt.Start(context.Background()); err != nil {
		t.Fatalf("Start: %v", err)
	}
	defer func() {
		if err := rt.Stop(); err != nil {
			t.Fatalf("Stop: %v", err)
		}
	}()

	adminURL := "http://" + rt.RefreshAdmin.Addr() + "/refresh"
	client := &http.Client{Timeout: 5 * time.Second}
	body, _ := json.Marshal(configrefresh.Request{
		Mode:       configrefresh.ModeDryRun,
		ConfigPath: candidatePath,
		RequestID:  "report-1",
	})
	resp, err := client.Post(adminURL, "application/json", bytes.NewReader(body))
	if err != nil {
		t.Fatalf("POST /refresh: %v", err)
	}
	defer resp.Body.Close()
	var result configrefresh.Result
	if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
		t.Fatalf("decode response: %v", err)
	}

	if result.Status != configrefresh.StatusApplied {
		t.Fatalf("expected status=applied, got %q (summary=%s)", result.Status, result.Summary)
	}
	if !containsString(result.ChangedProviders, "prov-a") {
		t.Errorf("expected changed_providers to include prov-a, got %v", result.ChangedProviders)
	}
	if !containsString(result.ChangedModels, "qwen3.6:35b") {
		t.Errorf("expected changed_models to include qwen3.6:35b, got %v", result.ChangedModels)
	}
}

func containsString(s []string, want string) bool {
	for _, v := range s {
		if v == want {
			return true
		}
	}
	return false
}

func freeTCPAddr(t *testing.T) string {
	t.Helper()
	ln, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatalf("listen free tcp addr: %v", err)
	}
	addr := ln.Addr().String()
	if err := ln.Close(); err != nil {
		t.Fatalf("close free tcp addr listener: %v", err)
	}
	return addr
}

// TestNewRuntimeWiresControlPlaneConnector verifies that NewRuntime always
// wires a non-nil ControlPlane connector regardless of config.
func TestNewRuntimeWiresControlPlaneConnector(t *testing.T) {
	cfg := newTestConfig()
	rt, err := NewRuntime(cfg)
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	if rt.ControlPlane == nil {
		t.Fatal("expected non-nil ControlPlane connector")
	}
}

// TestNewRuntimeWiresStatusProviderIntoConnector verifies that NewRuntime wires
// the edge service as the Control Plane connector's status provider, so status
// requests are answered from the Edge-owned node snapshot surface.
func TestNewRuntimeWiresStatusProviderIntoConnector(t *testing.T) {
	cfg := newTestConfig()
	rt, err := NewRuntime(cfg)
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	if !rt.ControlPlane.StatusProviderConfigured() {
		t.Fatal("expected NewRuntime to wire a status provider into the connector")
	}
}

// TestNewRuntimeWiresNodeEventBusIntoConnector verifies that node lifecycle
// events are relayed through the shared events.Bus boundary.
func TestNewRuntimeWiresNodeEventBusIntoConnector(t *testing.T) {
	cfg := newTestConfig()
	rt, err := NewRuntime(cfg)
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	if !rt.ControlPlane.NodeEventBusConfigured() {
		t.Fatal("expected NewRuntime to wire the node event bus into the connector")
	}
}

// TestRuntimeStartsControlPlaneConnectorWhenEnabled verifies that Start does
// not fail when a Control Plane connector is configured with a local fake
// server accepting hello. Uses a local TCP fake endpoint.
func TestRuntimeStartsControlPlaneConnectorWhenEnabled(t *testing.T) {
	// Start a minimal fake Control Plane server.
	ln, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatalf("listen fake cp: %v", err)
	}
	cpAddr := ln.Addr().String()
	go func() {
		// Accept and immediately close; connector will fail hello and retry,
		// but Start() itself must succeed (connector errors are async).
		conn, err := ln.Accept()
		if err != nil {
			return
		}
		_ = conn.Close()
		_ = ln.Close()
	}()

	cfg := newTestConfig()
	cfg.ControlPlane = config.EdgeControlPlaneConf{
		Enabled:              true,
		WireAddr:             cpAddr,
		ReconnectIntervalSec: 60,
	}

	rt, err := NewRuntime(cfg)
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	if !rt.ControlPlane.IsEnabled() {
		t.Fatal("expected connector IsEnabled()==true")
	}

	ctx := context.Background()
	if err := rt.Start(ctx); err != nil {
		t.Fatalf("Start with enabled connector: %v", err)
	}
	if err := rt.Stop(); err != nil {
		t.Fatalf("Stop: %v", err)
	}
}

// TestRuntimeStopsControlPlaneConnector verifies that Stop() reaches the
// ControlPlane connector. A disabled connector is used so no real network
// activity occurs.
func TestRuntimeStopsControlPlaneConnector(t *testing.T) {
	cfg := newTestConfig()
	// disabled connector; Start/Stop must be no-ops on the connector side.
	cfg.ControlPlane = config.EdgeControlPlaneConf{
		Enabled:  false,
		WireAddr: "",
	}

	rt, err := NewRuntime(cfg)
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}

	ctx := context.Background()
	if err := rt.Start(ctx); err != nil {
		t.Fatalf("Start: %v", err)
	}
	if err := rt.Stop(); err != nil {
		t.Fatalf("Stop: %v", err)
	}
	// After Stop, the connector must be in a stopped-or-no-op state.
	// For disabled connector, CurrentState is Disconnected (never started).
	// Calling Stop again must not panic.
	rt.ControlPlane.Stop()
}

// TestRefreshRejectedPreservesProviderDispatch verifies SDD S10: a rejected
// refresh must preserve provider-pool dispatch state. It uses a net.Pipe-based
// fake node to capture the RunRequest sent by SubmitRun(ProviderPool=true)
// before and after a rejected config refresh.
func TestRefreshRejectedPreservesProviderDispatch(t *testing.T) {
	dir := t.TempDir()
	serverAddr := freeTCPAddr(t)
	openAIAddr := freeTCPAddr(t)

	basePath := filepath.Join(dir, "base.yaml")
	if err := os.WriteFile(basePath, []byte(providerPoolConfigYAML(serverAddr, openAIAddr, validAdapterBlock)), 0o600); err != nil {
		t.Fatalf("write base: %v", err)
	}
	candidatePath := filepath.Join(dir, "invalid.yaml")
	if err := os.WriteFile(candidatePath, []byte(providerPoolConfigYAML(serverAddr, openAIAddr, invalidAdapterBlock)), 0o600); err != nil {
		t.Fatalf("write candidate: %v", err)
	}

	rt, err := NewRuntime(loadServeNormalizedConfig(t, basePath))
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	if err := rt.Start(context.Background()); err != nil {
		t.Fatalf("Start: %v", err)
	}
	defer func() {
		if err := rt.Stop(); err != nil {
			t.Fatalf("Stop: %v", err)
		}
	}()

	// --- Dispatch capture setup using net.Pipe ---
	edgeConn, nodeConn := net.Pipe()
	defer edgeConn.Close()
	defer nodeConn.Close()

	parserMap := toki.ParserMap{
		toki.TypeNameOf(&iop.RunRequest{}): func(b []byte) (proto.Message, error) {
			m := &iop.RunRequest{}
			return m, proto.Unmarshal(b, m)
		},
	}

	edgeClient := toki.NewTcpClient(edgeConn, 0, 0, parserMap)
	nodeClient := toki.NewTcpClient(nodeConn, 0, 0, parserMap)

	var capturedReq *iop.RunRequest
	var capturedMu sync.Mutex
	toki.AddListenerTyped[*iop.RunRequest](&nodeClient.Communicator, func(req *iop.RunRequest) {
		capturedMu.Lock()
		capturedReq = req
		capturedMu.Unlock()
	})

	// Add a provider-pool node record to the NodeStore so provider-pool
	// resolution finds a dispatchable provider. The provider config must
	// match the adapter (vllm-gpu) and served target (served-qwen) expected
	// by the model catalog entry (qwen3.6:35b -> prov-a).
	rt.NodeStore.Add(&edgenode.NodeRecord{
		ID:      "node-pool-dispatch",
		Alias:   "pool-alias",
		Runtime: config.RuntimeConf{Concurrency: 4},
		Providers: []config.NodeProviderConf{
			{
				ID:       "prov-a",
				Type:     "vllm",
				Category: "api",
				Adapter:  "vllm-gpu",
				Models:   []string{"served-qwen"},
				Health:   "available",
				Capacity: 2,
			},
		},
	})

	// Register the fake node into the runtime registry.
	rt.Registry.Register(&edgenode.NodeEntry{
		NodeID:         "node-pool-dispatch",
		LifecycleState: edgenode.LifecycleConnected,
		Client:         edgeClient,
	})

	// Submit RunRequest via the runtime Service before refresh.
	dispatchAndCapture := func() (*iop.RunRequest, error) {
		capturedMu.Lock()
		capturedReq = nil
		capturedMu.Unlock()

		result, err := rt.Service.SubmitRun(context.Background(), edgeservice.SubmitRunRequest{
			RunID:         "dispatch-test-" + time.Now().Format("150405"),
			ModelGroupKey: "qwen3.6:35b",
			ProviderPool:  true,
			Background:    true,
		})
		if err != nil {
			return nil, err
		}
		// Drain the result handle to release resources.
		if result != nil {
			result.Close()
		}

		// Wait for the fake node to receive the request.
		time.Sleep(100 * time.Millisecond)

		capturedMu.Lock()
		defer capturedMu.Unlock()
		return capturedReq, nil
	}

	// --- Before rejected refresh: dispatch must succeed ---
	beforeReq, err := dispatchAndCapture()
	if err != nil {
		t.Fatalf("dispatch before refresh: %v", err)
	}
	if beforeReq == nil {
		t.Fatal("expected dispatch before refresh to produce a RunRequest")
	}
	if beforeReq.GetAdapter() != "vllm-gpu" {
		t.Errorf("before refresh adapter: got %q, want %q", beforeReq.GetAdapter(), "vllm-gpu")
	}
	if beforeReq.GetTarget() != "served-qwen" {
		t.Errorf("before refresh target: got %q, want %q", beforeReq.GetTarget(), "served-qwen")
	}

	// --- Execute rejected refresh ---
	result, err := rt.RefreshConfig(context.Background(), configrefresh.Request{
		Mode:       configrefresh.ModeApply,
		ConfigPath: candidatePath,
		RequestID:  "reject-dispatch-1",
	})
	if err != nil {
		t.Fatalf("RefreshConfig: %v", err)
	}
	if result.Status != configrefresh.StatusRejected {
		t.Fatalf("expected status=rejected, got %q (summary=%s)", result.Status, result.Summary)
	}

	// --- After rejected refresh: dispatch must preserve state ---
	afterReq, err := dispatchAndCapture()
	if err != nil {
		t.Fatalf("dispatch after refresh: %v", err)
	}
	if afterReq == nil {
		t.Fatal("expected dispatch after rejected refresh to produce a RunRequest")
	}
	if afterReq.GetAdapter() != "vllm-gpu" {
		t.Errorf("after rejected refresh adapter: got %q, want %q", afterReq.GetAdapter(), "vllm-gpu")
	}
	if afterReq.GetTarget() != "served-qwen" {
		t.Errorf("after rejected refresh target: got %q, want %q", afterReq.GetTarget(), "served-qwen")
	}
}