iop/apps/edge/internal/bootstrap/runtime_test.go

package bootstrap

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"reflect"
	"sort"
	"strings"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	toki "git.toki-labs.com/toki/proto-socket/go"
	"google.golang.org/protobuf/proto"
	iop "iop/proto/gen/iop"

	"iop/apps/edge/internal/configrefresh"
	edgenode "iop/apps/edge/internal/node"
	edgeservice "iop/apps/edge/internal/service"
	"iop/packages/go/config"
)

func newTestConfig() *config.EdgeConfig {
	return &config.EdgeConfig{
		Server:  config.EdgeServerConf{Listen: "127.0.0.1:0"},
		Logging: config.LoggingConf{Level: "error"},
		Metrics: config.MetricsConf{Port: 0},
		Nodes: []config.NodeDefinition{
			{ID: "node-1", Alias: "alias-1", Token: "tok-1"},
		},
	}
}

func TestNewRuntimeSeedsNodeStore(t *testing.T) {
	cfg := newTestConfig()
	rt, err := NewRuntime(cfg)
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	if rt.NodeStore == nil {
		t.Fatal("NodeStore is nil")
	}
	rec, ok := rt.NodeStore.FindByToken("tok-1")
	if !ok {
		t.Fatal("expected node record seeded by token tok-1")
	}
	if rec.ID != "node-1" || rec.Alias != "alias-1" {
		t.Errorf("unexpected record: id=%q alias=%q", rec.ID, rec.Alias)
	}
	if rt.Registry == nil || rt.EventBus == nil || rt.Service == nil || rt.Server == nil || rt.Logger == nil {
		t.Error("runtime dependencies must be wired by NewRuntime")
	}
}

func TestRuntimeWiresTransportHandlers(t *testing.T) {
	cfg := newTestConfig()
	rt, err := NewRuntime(cfg)
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	if rt.Server.HasRunEventHandler() || rt.Server.HasNodeEventHandler() {
		t.Fatal("handlers must not be wired before wireHandlers")
	}
	rt.wireHandlers()
	if !rt.Server.HasRunEventHandler() {
		t.Error("run event handler not wired")
	}
	if !rt.Server.HasNodeEventHandler() {
		t.Error("node event handler not wired")
	}
}

func TestNewRuntimeWiresInputManager(t *testing.T) {
	cfg := newTestConfig()
	rt, err := NewRuntime(cfg)
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	if rt.Input == nil {
		t.Fatal("Input manager is nil")
	}
	if rt.Input.OpenAI == nil {
		t.Fatal("Input.OpenAI is nil")
	}
	if rt.Input.A2A == nil {
		t.Fatal("Input.A2A is nil")
	}
}

func TestRuntimeKeepsListenersAfterStartupContextCancel(t *testing.T) {
	edgeAddr := freeTCPAddr(t)
	openAIAddr := freeTCPAddr(t)
	cfg := newTestConfig()
	cfg.Server.Listen = edgeAddr
	cfg.OpenAI = config.EdgeOpenAIConf{
		Enabled: true,
		Listen:  openAIAddr,
	}

	rt, err := NewRuntime(cfg)
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	startupCtx, cancelStartup := context.WithCancel(context.Background())
	if err := rt.Start(startupCtx); err != nil {
		t.Fatalf("Start: %v", err)
	}
	defer func() {
		if err := rt.Stop(); err != nil {
			t.Fatalf("Stop: %v", err)
		}
	}()

	cancelStartup()

	conn, err := net.DialTimeout("tcp", edgeAddr, time.Second)
	if err != nil {
		t.Fatalf("edge listener closed with startup context: %v", err)
	}
	_ = conn.Close()

	client := &http.Client{Timeout: time.Second}
	resp, err := client.Get("http://" + openAIAddr + "/healthz")
	if err != nil {
		t.Fatalf("openai listener closed with startup context: %v", err)
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("openai health status = %d, want %d", resp.StatusCode, http.StatusOK)
	}
}

func TestNewRuntimeWritesLogsToConfiguredPath(t *testing.T) {
	dir := t.TempDir()
	logPath := filepath.Join(dir, "logs", "edge.log")

	cfg := newTestConfig()
	cfg.Logging.Level = "info"
	cfg.Logging.Path = logPath
	rt, err := NewRuntime(cfg)
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	rt.Logger.Info("runtime-log-entry")
	_ = rt.Logger.Sync()

	info, err := os.Stat(logPath)
	if err != nil {
		t.Fatalf("stat log path: %v", err)
	}
	if info.Size() == 0 {
		t.Fatalf("log file empty: %s", logPath)
	}
	contents, err := os.ReadFile(logPath)
	if err != nil {
		t.Fatalf("read log file: %v", err)
	}
	if !strings.Contains(string(contents), "runtime-log-entry") {
		t.Fatalf("expected runtime-log-entry in log file; contents=%q", string(contents))
	}
}

func TestNewRuntimeRejectsDuplicateToken(t *testing.T) {
	cfg := &config.EdgeConfig{
		Server:  config.EdgeServerConf{Listen: "127.0.0.1:0"},
		Logging: config.LoggingConf{Level: "error"},
		Nodes: []config.NodeDefinition{
			{ID: "node-1", Alias: "a1", Token: "dup"},
			{ID: "node-2", Alias: "a2", Token: "dup"},
		},
	}
	if _, err := NewRuntime(cfg); err == nil {
		t.Fatal("expected error on duplicate token seed")
	}
}

// TestRefreshAdminDryRunAndApplyReachRunningRuntime verifies S06: a running
// Edge process can receive dry-run and apply refresh requests via the admin API
// and returns mode-specific results.
func TestRefreshAdminDryRunAndApplyReachRunningRuntime(t *testing.T) {
	dir := t.TempDir()

	serverAddr := freeTCPAddr(t)

	// Write base and candidate YAMLs with explicit immutable field values so
	// that config.LoadEdge produces configs with identical defaults on both sides.
	// Only capacity differs between base and candidate.
	baseYAML := fmt.Sprintf(`
server:
  listen: %q
bootstrap:
  listen: "0.0.0.0:18080"
  artifact_dir: "artifacts"
logging:
  level: "error"
refresh:
  enabled: true
  listen: "127.0.0.1:0"
openai:
  listen: "0.0.0.0:18081"
a2a:
  listen: "0.0.0.0:8081"
metrics:
  port: 0
nodes:
  - id: "node-1"
    alias: "n1"
    token: "tok-1"
    adapters:
      cli:
        enabled: true
    providers:
      - id: "prov-a"
        type: "ollama"
        category: "local_inference"
        models: ["llama3.1"]
        capacity: 2
        max_queue: 4
        queue_timeout_ms: 5000
`, serverAddr)

	candidateYAML := fmt.Sprintf(`
server:
  listen: %q
bootstrap:
  listen: "0.0.0.0:18080"
  artifact_dir: "artifacts"
logging:
  level: "error"
refresh:
  enabled: true
  listen: "127.0.0.1:0"
openai:
  listen: "0.0.0.0:18081"
a2a:
  listen: "0.0.0.0:8081"
metrics:
  port: 0
nodes:
  - id: "node-1"
    alias: "n1"
    token: "tok-1"
    adapters:
      cli:
        enabled: true
    providers:
      - id: "prov-a"
        type: "ollama"
        category: "local_inference"
        models: ["llama3.1"]
        capacity: 8
        max_queue: 4
        queue_timeout_ms: 5000
`, serverAddr)

	basePath := filepath.Join(dir, "base.yaml")
	if err := os.WriteFile(basePath, []byte(baseYAML), 0o600); err != nil {
		t.Fatalf("write base: %v", err)
	}
	candidatePath := filepath.Join(dir, "candidate.yaml")
	if err := os.WriteFile(candidatePath, []byte(candidateYAML), 0o600); err != nil {
		t.Fatalf("write candidate: %v", err)
	}

	currentCfg, err := config.LoadEdge(basePath)
	if err != nil {
		t.Fatalf("load base config: %v", err)
	}
	// Apply serve-equivalent normalization so the current config matches
	// what iop-edge serve would hold after loadRuntimeConfig.
	currentCfg.Logging.Path = serveNormalizedLogPath()
	currentCfg.Bootstrap.ArtifactDir = serveNormalizedArtifactDir(currentCfg.Bootstrap.ArtifactDir)
	// Override fields that don't need to match between runtime and candidate.
	currentCfg.Refresh.Enabled = true
	currentCfg.Refresh.Listen = "127.0.0.1:0"
	currentCfg.Logging.Level = "error"

	rt, err := NewRuntime(currentCfg)
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	ctx := context.Background()
	if err := rt.Start(ctx); err != nil {
		t.Fatalf("Start: %v", err)
	}
	defer func() {
		if err := rt.Stop(); err != nil {
			t.Fatalf("Stop: %v", err)
		}
	}()

	adminAddr := rt.RefreshAdmin.Addr()
	if adminAddr == "" {
		t.Fatal("refresh admin addr is empty after Start")
	}
	adminURL := "http://" + adminAddr + "/refresh"

	client := &http.Client{Timeout: 5 * time.Second}

	postRefresh := func(t *testing.T, mode configrefresh.Mode) configrefresh.Result {
		t.Helper()
		body, _ := json.Marshal(configrefresh.Request{
			Mode:       mode,
			ConfigPath: candidatePath,
			RequestID:  string(mode) + "-req",
		})
		resp, err := client.Post(adminURL, "application/json", bytes.NewReader(body))
		if err != nil {
			t.Fatalf("POST /refresh mode=%s: %v", mode, err)
		}
		defer resp.Body.Close()
		var result configrefresh.Result
		if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
			t.Fatalf("decode response: %v", err)
		}
		return result
	}

	// dry-run: process is reached, result has mode=dry_run.
	dryResult := postRefresh(t, configrefresh.ModeDryRun)
	if dryResult.Mode != configrefresh.ModeDryRun {
		t.Errorf("dry-run: expected mode=%q, got %q", configrefresh.ModeDryRun, dryResult.Mode)
	}
	if dryResult.Status != configrefresh.StatusApplied {
		t.Errorf("dry-run: expected status=%q (capacity change), got %q", configrefresh.StatusApplied, dryResult.Status)
	}

	// apply: process is reached, result has mode=apply and status=applied.
	applyResult := postRefresh(t, configrefresh.ModeApply)
	if applyResult.Mode != configrefresh.ModeApply {
		t.Errorf("apply: expected mode=%q, got %q", configrefresh.ModeApply, applyResult.Mode)
	}
	if applyResult.Status != configrefresh.StatusApplied {
		t.Errorf("apply: expected status=%q, got %q", configrefresh.StatusApplied, applyResult.Status)
	}

	// verify dry-run and apply have distinct observable results: the apply
	// committed the capacity change, so the next dry-run shows no changes.
	dryResult2 := postRefresh(t, configrefresh.ModeDryRun)
	if len(dryResult2.Changes) != 0 {
		t.Errorf("after apply, expected no diff changes on second dry-run, got %+v", dryResult2.Changes)
	}
}

func serveNormalizedLogPath() string {
	if exe, err := os.Executable(); err == nil {
		return filepath.Join(filepath.Dir(exe), "logs", "edge.log")
	}
	if wd, err := os.Getwd(); err == nil {
		return filepath.Join(wd, "logs", "edge.log")
	}
	return "logs/edge.log"
}

func serveNormalizedArtifactDir(dir string) string {
	if dir == "" {
		dir = "artifacts"
	}
	if filepath.IsAbs(dir) {
		return dir
	}
	bd := binaryDirForTest()
	return filepath.Join(bd, dir)
}

func binaryDirForTest() string {
	if exe, err := os.Executable(); err == nil {
		return filepath.Dir(exe)
	}
	if wd, err := os.Getwd(); err == nil {
		return wd
	}
	return "."
}

func TestRefreshApplyConcurrentRuntimeReaders(t *testing.T) {
	dir := t.TempDir()
	serverAddr := freeTCPAddr(t)
	openAIAddr := freeTCPAddr(t)

	writeConfig := func(t *testing.T, name string, capacity int, displayName string) string {
		t.Helper()
		path := filepath.Join(dir, name)
		yaml := fmt.Sprintf(`
server:
  listen: %q
bootstrap:
  listen: "0.0.0.0:18080"
  artifact_dir: "artifacts"
logging:
  level: "error"
refresh:
  enabled: false
  listen: "127.0.0.1:19093"
openai:
  enabled: true
  listen: %q
  adapter: "openai_compat"
  target: ""
a2a:
  listen: "0.0.0.0:8081"
metrics:
  port: 0
models:
  - id: "qwen3.6:35b"
    display_name: %q
    providers:
      prov-a: "served-qwen"
nodes:
  - id: "node-1"
    alias: "n1"
    token: "tok-1"
    adapters:
      openai_compat_instances:
        - name: "vllm-gpu"
          enabled: true
          provider: "vllm"
          endpoint: "http://127.0.0.1:8000/v1"
    providers:
      - id: "prov-a"
        type: "vllm"
        category: "api"
        adapter: "vllm-gpu"
        models: ["served-qwen"]
        health: "available"
        capacity: %d
        max_queue: 4
        queue_timeout_ms: 5000
`, serverAddr, openAIAddr, displayName, capacity)
		if err := os.WriteFile(path, []byte(yaml), 0o600); err != nil {
			t.Fatalf("write %s: %v", name, err)
		}
		return path
	}

	basePath := writeConfig(t, "base.yaml", 2, "Qwen Base")
	candidatePath := writeConfig(t, "candidate.yaml", 8, "Qwen Candidate")

	currentCfg, err := config.LoadEdge(basePath)
	if err != nil {
		t.Fatalf("load base config: %v", err)
	}
	// Apply serve-equivalent normalization.
	currentCfg.Logging.Path = serveNormalizedLogPath()
	currentCfg.Bootstrap.ArtifactDir = serveNormalizedArtifactDir(currentCfg.Bootstrap.ArtifactDir)
	rt, err := NewRuntime(currentCfg)
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	if err := rt.Start(context.Background()); err != nil {
		t.Fatalf("Start: %v", err)
	}
	defer func() {
		if err := rt.Stop(); err != nil {
			t.Fatalf("Stop: %v", err)
		}
	}()

	client := &http.Client{Timeout: 5 * time.Second}
	start := make(chan struct{})
	var wg sync.WaitGroup
	errCh := make(chan error, 3)

	wg.Add(1)
	go func() {
		defer wg.Done()
		<-start
		for i := 0; i < 50; i++ {
			result, err := rt.RefreshConfig(context.Background(), configrefresh.Request{
				Mode:       configrefresh.ModeApply,
				ConfigPath: candidatePath,
				RequestID:  fmt.Sprintf("apply-%d", i),
			})
			if err != nil {
				errCh <- err
				return
			}
			if result.Status != configrefresh.StatusApplied {
				errCh <- fmt.Errorf("refresh status=%s summary=%s changes=%+v", result.Status, result.Summary, result.Changes)
				return
			}
		}
	}()

	wg.Add(1)
	go func() {
		defer wg.Done()
		<-start
		for i := 0; i < 100; i++ {
			_ = rt.Service.ListNodeSnapshots()
		}
	}()

	wg.Add(1)
	go func() {
		defer wg.Done()
		<-start
		url := "http://" + openAIAddr + "/v1/models"
		for i := 0; i < 100; i++ {
			resp, err := client.Get(url)
			if err != nil {
				errCh <- err
				return
			}
			resp.Body.Close()
			if resp.StatusCode != http.StatusOK {
				errCh <- fmt.Errorf("GET /v1/models status=%d", resp.StatusCode)
				return
			}
		}
	}()

	close(start)
	wg.Wait()
	close(errCh)
	for err := range errCh {
		if err != nil {
			t.Fatalf("concurrent refresh reader failed: %v", err)
		}
	}
}

// loadServeNormalizedConfig loads an Edge config and applies the same
// normalization iop-edge serve performs, so refresh classification compares
// effective values.
func loadServeNormalizedConfig(t *testing.T, path string) *config.EdgeConfig {
	t.Helper()
	cfg, err := config.LoadEdge(path)
	if err != nil {
		t.Fatalf("load config %s: %v", path, err)
	}
	cfg.Logging.Path = serveNormalizedLogPath()
	cfg.Bootstrap.ArtifactDir = serveNormalizedArtifactDir(cfg.Bootstrap.ArtifactDir)
	return cfg
}

// providerPoolConfigYAML builds an Edge config with one provider-pool model and
// one node provider. The extra parameter lets callers inject a candidate
// variation (e.g. a different capacity or an invalid adapter block).
func providerPoolConfigYAML(serverAddr, openAIAddr, providerBlock string) string {
	return fmt.Sprintf(`
server:
  listen: %q
bootstrap:
  listen: "0.0.0.0:18080"
  artifact_dir: "artifacts"
logging:
  level: "error"
refresh:
  enabled: true
  listen: "127.0.0.1:0"
openai:
  enabled: true
  listen: %q
  adapter: "openai_compat"
  target: ""
a2a:
  listen: "0.0.0.0:8081"
metrics:
  port: 0
models:
  - id: "qwen3.6:35b"
    display_name: "Qwen"
    providers:
      prov-a: "served-qwen"
nodes:
  - id: "node-1"
    alias: "n1"
    token: "tok-1"
    adapters:
%s
    providers:
      - id: "prov-a"
        type: "vllm"
        category: "api"
        adapter: "vllm-gpu"
        models: ["served-qwen"]
        health: "available"
        capacity: 2
        max_queue: 4
        queue_timeout_ms: 5000
`, serverAddr, openAIAddr, providerBlock)
}

const validAdapterBlock = `      openai_compat_instances:
        - name: "vllm-gpu"
          enabled: true
          provider: "vllm"
          endpoint: "http://127.0.0.1:8000/v1"`

// invalidAdapterBlock enables an openai_compat instance but leaves endpoint
// empty, which ValidateEdgeConfig rejects.
const invalidAdapterBlock = `      openai_compat_instances:
        - name: "vllm-gpu"
          enabled: true
          provider: "vllm"
          endpoint: ""`

// modelsResponse represents the JSON response from /v1/models.
type modelsResponse struct {
	Object string      `json:"object"`
	Data   []modelItem `json:"data"`
}

// modelItem represents a single model entry in the /v1/models response.
type modelItem struct {
	ID      string `json:"id"`
	Object  string `json:"object"`
	Created int64  `json:"created"`
	OwnedBy string `json:"owned_by"`
}

// modelIDs returns the sorted list of model IDs from /v1/models.
// This avoids flaky timestamp comparisons on the `created` field.
func modelIDs(t *testing.T, client *http.Client, openAIAddr string) []string {
	t.Helper()
	resp, err := client.Get("http://" + openAIAddr + "/v1/models")
	if err != nil {
		t.Fatalf("GET /v1/models: %v", err)
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("GET /v1/models status=%d", resp.StatusCode)
	}
	var body modelsResponse
	if err := json.NewDecoder(resp.Body).Decode(&body); err != nil {
		t.Fatalf("decode /v1/models body: %v", err)
	}
	ids := make([]string, 0, len(body.Data))
	for _, m := range body.Data {
		ids = append(ids, m.ID)
	}
	sort.Strings(ids)
	return ids
}

// TestRefreshRejectedPreservesModels verifies SDD S10: an invalid candidate
// apply is rejected without mutating the running runtime config or the
// /v1/models surface. Uses stable model ID comparison to avoid timestamp flakiness.
func TestRefreshRejectedPreservesModels(t *testing.T) {
	dir := t.TempDir()
	serverAddr := freeTCPAddr(t)
	openAIAddr := freeTCPAddr(t)

	basePath := filepath.Join(dir, "base.yaml")
	if err := os.WriteFile(basePath, []byte(providerPoolConfigYAML(serverAddr, openAIAddr, validAdapterBlock)), 0o600); err != nil {
		t.Fatalf("write base: %v", err)
	}
	candidatePath := filepath.Join(dir, "invalid.yaml")
	if err := os.WriteFile(candidatePath, []byte(providerPoolConfigYAML(serverAddr, openAIAddr, invalidAdapterBlock)), 0o600); err != nil {
		t.Fatalf("write candidate: %v", err)
	}

	rt, err := NewRuntime(loadServeNormalizedConfig(t, basePath))
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	if err := rt.Start(context.Background()); err != nil {
		t.Fatalf("Start: %v", err)
	}
	defer func() {
		if err := rt.Stop(); err != nil {
			t.Fatalf("Stop: %v", err)
		}
	}()

	client := &http.Client{Timeout: 5 * time.Second}
	beforeIDs := modelIDs(t, client, openAIAddr)
	if len(beforeIDs) == 0 {
		t.Fatal("expected at least one model before refresh")
	}
	cfgBefore := rt.Cfg

	result, err := rt.RefreshConfig(context.Background(), configrefresh.Request{
		Mode:       configrefresh.ModeApply,
		ConfigPath: candidatePath,
		RequestID:  "reject-1",
	})
	if err != nil {
		t.Fatalf("RefreshConfig: %v", err)
	}
	if result.Status != configrefresh.StatusRejected {
		t.Fatalf("expected status=rejected for invalid candidate, got %q (summary=%s)", result.Status, result.Summary)
	}

	if rt.Cfg != cfgBefore {
		t.Fatal("runtime config snapshot mutated after rejected apply")
	}
	afterIDs := modelIDs(t, client, openAIAddr)
	if !reflect.DeepEqual(afterIDs, beforeIDs) {
		t.Fatalf("/v1/models changed after rejected apply:\nbefore=%v\nafter=%v", beforeIDs, afterIDs)
	}
}

// TestRefreshApplyDoesNotMutateOnRestartRequired verifies that a candidate with
// a restart_required change leaves the running runtime snapshot and /v1/models
// untouched. Uses stable model ID comparison.
func TestRefreshApplyDoesNotMutateOnRestartRequired(t *testing.T) {
	dir := t.TempDir()
	serverAddr := freeTCPAddr(t)
	openAIAddr := freeTCPAddr(t)

	basePath := filepath.Join(dir, "base.yaml")
	if err := os.WriteFile(basePath, []byte(providerPoolConfigYAML(serverAddr, openAIAddr, validAdapterBlock)), 0o600); err != nil {
		t.Fatalf("write base: %v", err)
	}
	// Candidate changes server.listen, which is classified restart_required.
	candidateYAML := providerPoolConfigYAML(freeTCPAddr(t), openAIAddr, validAdapterBlock)
	candidatePath := filepath.Join(dir, "restart.yaml")
	if err := os.WriteFile(candidatePath, []byte(candidateYAML), 0o600); err != nil {
		t.Fatalf("write candidate: %v", err)
	}

	rt, err := NewRuntime(loadServeNormalizedConfig(t, basePath))
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	if err := rt.Start(context.Background()); err != nil {
		t.Fatalf("Start: %v", err)
	}
	defer func() {
		if err := rt.Stop(); err != nil {
			t.Fatalf("Stop: %v", err)
		}
	}()

	client := &http.Client{Timeout: 5 * time.Second}
	beforeIDs := modelIDs(t, client, openAIAddr)
	if len(beforeIDs) == 0 {
		t.Fatal("expected at least one model before refresh")
	}
	cfgBefore := rt.Cfg

	result, err := rt.RefreshConfig(context.Background(), configrefresh.Request{
		Mode:       configrefresh.ModeApply,
		ConfigPath: candidatePath,
		RequestID:  "restart-1",
	})
	if err != nil {
		t.Fatalf("RefreshConfig: %v", err)
	}
	if result.Status != configrefresh.StatusRestartRequired {
		t.Fatalf("expected status=restart_required, got %q (summary=%s)", result.Status, result.Summary)
	}
	if len(result.RestartRequiredPaths) == 0 {
		t.Fatal("expected restart_required_paths to be reported")
	}
	if rt.Cfg != cfgBefore {
		t.Fatal("runtime config snapshot mutated after restart_required apply")
	}
	afterIDs := modelIDs(t, client, openAIAddr)
	if !reflect.DeepEqual(afterIDs, beforeIDs) {
		t.Fatalf("/v1/models changed after restart_required apply:\nbefore=%v\nafter=%v", beforeIDs, afterIDs)
	}
}

// TestRefreshAPIReportIncludesChangedItems verifies SDD S11: the refresh API
// response exposes the changed provider/model items so an operator can confirm
// what a refresh touched.
func TestRefreshAPIReportIncludesChangedItems(t *testing.T) {
	dir := t.TempDir()
	serverAddr := freeTCPAddr(t)
	openAIAddr := freeTCPAddr(t)

	basePath := filepath.Join(dir, "base.yaml")
	if err := os.WriteFile(basePath, []byte(providerPoolConfigYAML(serverAddr, openAIAddr, validAdapterBlock)), 0o600); err != nil {
		t.Fatalf("write base: %v", err)
	}
	// Candidate raises provider capacity (applied) and renames the model display
	// name (applied), so both a provider and a model item change.
	candidateYAML := strings.Replace(
		strings.Replace(providerPoolConfigYAML(serverAddr, openAIAddr, validAdapterBlock), "capacity: 2", "capacity: 8", 1),
		`display_name: "Qwen"`, `display_name: "Qwen Refreshed"`, 1)
	candidatePath := filepath.Join(dir, "candidate.yaml")
	if err := os.WriteFile(candidatePath, []byte(candidateYAML), 0o600); err != nil {
		t.Fatalf("write candidate: %v", err)
	}

	rt, err := NewRuntime(loadServeNormalizedConfig(t, basePath))
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	if err := rt.Start(context.Background()); err != nil {
		t.Fatalf("Start: %v", err)
	}
	defer func() {
		if err := rt.Stop(); err != nil {
			t.Fatalf("Stop: %v", err)
		}
	}()

	adminURL := "http://" + rt.RefreshAdmin.Addr() + "/refresh"
	client := &http.Client{Timeout: 5 * time.Second}
	body, _ := json.Marshal(configrefresh.Request{
		Mode:       configrefresh.ModeDryRun,
		ConfigPath: candidatePath,
		RequestID:  "report-1",
	})
	resp, err := client.Post(adminURL, "application/json", bytes.NewReader(body))
	if err != nil {
		t.Fatalf("POST /refresh: %v", err)
	}
	defer resp.Body.Close()
	var result configrefresh.Result
	if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
		t.Fatalf("decode response: %v", err)
	}

	if result.Status != configrefresh.StatusApplied {
		t.Fatalf("expected status=applied, got %q (summary=%s)", result.Status, result.Summary)
	}
	if !containsString(result.ChangedProviders, "prov-a") {
		t.Errorf("expected changed_providers to include prov-a, got %v", result.ChangedProviders)
	}
	if !containsString(result.ChangedModels, "qwen3.6:35b") {
		t.Errorf("expected changed_models to include qwen3.6:35b, got %v", result.ChangedModels)
	}
}

func containsString(s []string, want string) bool {
	for _, v := range s {
		if v == want {
			return true
		}
	}
	return false
}

// TestRefreshConfigApplyIncludesNodeResults verifies SDD S12: when a refresh
// apply succeeds, the result includes per-node refresh outcomes. Uses a
// net.Pipe-based fake node that registers with the edge and responds applied.
func TestRefreshConfigApplyIncludesNodeResults(t *testing.T) {
	dir := t.TempDir()
	serverAddr := freeTCPAddr(t)

	writeRefreshConfig := func(name string, capacity int) string {
		path := filepath.Join(dir, name)
		yaml := fmt.Sprintf(`
server:
  listen: %q
bootstrap:
  listen: "0.0.0.0:18090"
  artifact_dir: "artifacts"
logging:
  level: "error"
refresh:
  enabled: false
  listen: "127.0.0.1:0"
openai:
  listen: "0.0.0.0:18091"
a2a:
  listen: "0.0.0.0:8082"
metrics:
  port: 0
nodes:
  - id: "node-refresh-1"
    alias: "refresh-node"
    token: "tok-refresh"
    adapters:
      cli:
        enabled: true
      openai_compat_instances:
        - name: "primary"
          enabled: true
          provider: "vllm"
          endpoint: "http://127.0.0.1:9000/v1"
          capacity: 5
          max_queue: 4
          queue_timeout_ms: 5000
    runtime:
      concurrency: 7
    providers:
      - id: "prov-r"
        type: "ollama"
        category: "local_inference"
        models: ["llama3"]
        capacity: %d
        max_queue: 4
        queue_timeout_ms: 5000
`, serverAddr, capacity)
		if err := os.WriteFile(path, []byte(yaml), 0o600); err != nil {
			t.Fatalf("write %s: %v", name, err)
		}
		return path
	}

	basePath := writeRefreshConfig("base.yaml", 2)
	candidatePath := writeRefreshConfig("candidate.yaml", 8)

	rt, err := NewRuntime(loadServeNormalizedConfig(t, basePath))
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	if err := rt.Start(context.Background()); err != nil {
		t.Fatalf("Start: %v", err)
	}
	defer func() {
		if err := rt.Stop(); err != nil {
			t.Fatalf("Stop: %v", err)
		}
	}()

	// Build a fake node parser that can decode NodeConfigRefreshRequest (edge push)
	// and RegisterResponse (registration).
	nodeParserMap := toki.ParserMap{
		toki.TypeNameOf(&iop.RegisterResponse{}): func(b []byte) (proto.Message, error) {
			m := &iop.RegisterResponse{}
			return m, proto.Unmarshal(b, m)
		},
		toki.TypeNameOf(&iop.NodeConfigRefreshRequest{}): func(b []byte) (proto.Message, error) {
			m := &iop.NodeConfigRefreshRequest{}
			return m, proto.Unmarshal(b, m)
		},
	}

	host, portStr, _ := net.SplitHostPort(serverAddr)
	port := 0
	fmt.Sscanf(portStr, "%d", &port)
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	defer cancel()

	fakeNode, err := toki.DialTcp(ctx, host, port, 30, 10, nodeParserMap)
	if err != nil {
		t.Fatalf("dial edge: %v", err)
	}
	defer fakeNode.Close()

	// Fake node captures the refresh request and responds restart_required
	// (current Node behavior for any non-empty payload/change).
	var capturedRefreshReq *iop.NodeConfigRefreshRequest
	var capturedRefreshMu sync.Mutex
	toki.AddRequestListenerTyped[*iop.NodeConfigRefreshRequest, *iop.NodeConfigRefreshResponse](
		&fakeNode.Communicator,
		func(req *iop.NodeConfigRefreshRequest) (*iop.NodeConfigRefreshResponse, error) {
			capturedRefreshMu.Lock()
			capturedRefreshReq = req
			capturedRefreshMu.Unlock()
			return &iop.NodeConfigRefreshResponse{
				RequestId:            req.GetRequestId(),
				Status:               iop.NodeConfigRefreshStatus_NODE_CONFIG_REFRESH_STATUS_RESTART_REQUIRED,
				RestartRequiredPaths: req.GetChangedPaths(),
			}, nil
		},
	)

	// Register the fake node.
	regResp, err := toki.SendRequestTyped[*iop.RegisterRequest, *iop.RegisterResponse](
		&fakeNode.Communicator,
		&iop.RegisterRequest{Token: "tok-refresh"},
		2*time.Second,
	)
	if err != nil {
		t.Fatalf("register: %v", err)
	}
	if !regResp.GetAccepted() {
		t.Fatalf("expected accepted, got reason %q", regResp.GetReason())
	}

	// Wait for the node to appear in the registry.
	deadline := time.Now().Add(2 * time.Second)
	for time.Now().Before(deadline) {
		if _, ok := rt.Registry.Get("node-refresh-1"); ok {
			break
		}
		time.Sleep(25 * time.Millisecond)
	}
	if _, ok := rt.Registry.Get("node-refresh-1"); !ok {
		t.Fatal("fake node not registered in edge registry")
	}

	result, err := rt.RefreshConfig(context.Background(), configrefresh.Request{
		Mode:       configrefresh.ModeApply,
		ConfigPath: candidatePath,
		RequestID:  "node-result-test",
	})
	if err != nil {
		t.Fatalf("RefreshConfig: %v", err)
	}
	if result.Status != configrefresh.StatusApplied {
		t.Fatalf("expected status=applied, got %q (summary=%s)", result.Status, result.Summary)
	}
	if len(result.NodeResults) == 0 {
		t.Fatal("expected node_results to be populated after apply")
	}
	found := false
	for _, nr := range result.NodeResults {
		if nr.NodeID == "node-refresh-1" {
			found = true
			if nr.Status != "restart_required" {
				t.Errorf("expected node status=restart_required (current behavior for non-empty payload), got %q (error=%q)", nr.Status, nr.Error)
			}
		}
	}
	if !found {
		t.Fatalf("node-refresh-1 not found in node_results: %+v", result.NodeResults)
	}

	// Verify request_id propagation and payload contents on the fake node side.
	capturedRefreshMu.Lock()
	captured := capturedRefreshReq
	capturedRefreshMu.Unlock()
	if captured == nil {
		t.Fatal("fake node never received a NodeConfigRefreshRequest")
	}
	if captured.GetRequestId() != "node-result-test" {
		t.Errorf("request_id: got %q, want %q", captured.GetRequestId(), "node-result-test")
	}
	// Payload must carry the node-specific config edge builds from the NodeStore
	// record. Note: providers[].capacity is the refresh *trigger* (it drives
	// changed_paths), not a NodeConfigPayload field; the payload itself carries
	// the node's adapters + runtime, so we assert those concrete typed fields.
	payload := captured.GetConfig()
	if payload == nil {
		t.Fatal("expected non-nil NodeConfigPayload in refresh request")
	}
	// Runtime tuning is pushed from the node record.
	if got := payload.GetRuntime().GetConcurrency(); got != 7 {
		t.Errorf("runtime concurrency: got %d, want 7", got)
	}
	// The configured openai_compat adapter instance must be present with its
	// typed fields (provider/endpoint/capacity) intact, and the CLI adapter too.
	var oai *iop.OpenAICompatAdapterConfig
	hasCLI := false
	for _, a := range payload.GetAdapters() {
		switch a.GetType() {
		case "openai_compat":
			if a.GetName() == "primary" {
				oai = a.GetOpenaiCompat()
			}
		case "cli":
			hasCLI = true
		}
	}
	if oai == nil {
		t.Fatalf("expected openai_compat adapter %q in refresh payload, got %+v", "primary", payload.GetAdapters())
	}
	if oai.GetProvider() != "vllm" {
		t.Errorf("openai_compat provider: got %q, want %q", oai.GetProvider(), "vllm")
	}
	if oai.GetEndpoint() != "http://127.0.0.1:9000/v1" {
		t.Errorf("openai_compat endpoint: got %q, want %q", oai.GetEndpoint(), "http://127.0.0.1:9000/v1")
	}
	if oai.GetCapacity() != 5 {
		t.Errorf("openai_compat capacity: got %d, want 5", oai.GetCapacity())
	}
	if !hasCLI {
		t.Errorf("expected cli adapter in refresh payload, got %+v", payload.GetAdapters())
	}
}

// TestRefreshConfigApplySkipsDisconnectedConfiguredNode verifies that a node
// that is in the NodeStore but not in the live registry (post-disconnect state)
// appears as "skipped" in node_results alongside connected nodes.
func TestRefreshConfigApplySkipsDisconnectedConfiguredNode(t *testing.T) {
	dir := t.TempDir()
	serverAddr := freeTCPAddr(t)

	writeConfig := func(name string, capacity int) string {
		path := filepath.Join(dir, name)
		yaml := fmt.Sprintf(`
server:
  listen: %q
bootstrap:
  listen: "0.0.0.0:18092"
  artifact_dir: "artifacts"
logging:
  level: "error"
refresh:
  enabled: false
  listen: "127.0.0.1:0"
openai:
  listen: "0.0.0.0:18093"
a2a:
  listen: "0.0.0.0:8083"
metrics:
  port: 0
nodes:
  - id: "node-connected"
    alias: "conn-node"
    token: "tok-conn"
    adapters:
      cli:
        enabled: true
    providers:
      - id: "prov-skip"
        type: "ollama"
        category: "local_inference"
        models: ["llama3"]
        capacity: %d
        max_queue: 4
        queue_timeout_ms: 5000
  - id: "node-disconnected"
    alias: "disc-node"
    token: "tok-disc"
    adapters:
      cli:
        enabled: true
`, serverAddr, capacity)
		if err := os.WriteFile(path, []byte(yaml), 0o600); err != nil {
			t.Fatalf("write %s: %v", name, err)
		}
		return path
	}

	basePath := writeConfig("base.yaml", 2)
	candidatePath := writeConfig("candidate.yaml", 8)

	rt, err := NewRuntime(loadServeNormalizedConfig(t, basePath))
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	if err := rt.Start(context.Background()); err != nil {
		t.Fatalf("Start: %v", err)
	}
	defer func() {
		if err := rt.Stop(); err != nil {
			t.Fatalf("Stop: %v", err)
		}
	}()

	nodeParserMap := toki.ParserMap{
		toki.TypeNameOf(&iop.RegisterResponse{}): func(b []byte) (proto.Message, error) {
			m := &iop.RegisterResponse{}
			return m, proto.Unmarshal(b, m)
		},
		toki.TypeNameOf(&iop.NodeConfigRefreshRequest{}): func(b []byte) (proto.Message, error) {
			m := &iop.NodeConfigRefreshRequest{}
			return m, proto.Unmarshal(b, m)
		},
	}

	host, portStr, _ := net.SplitHostPort(serverAddr)
	port := 0
	fmt.Sscanf(portStr, "%d", &port)
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	defer cancel()

	// Only connect node-connected; node-disconnected never dials in.
	fakeNode, err := toki.DialTcp(ctx, host, port, 30, 10, nodeParserMap)
	if err != nil {
		t.Fatalf("dial edge: %v", err)
	}
	defer fakeNode.Close()

	toki.AddRequestListenerTyped[*iop.NodeConfigRefreshRequest, *iop.NodeConfigRefreshResponse](
		&fakeNode.Communicator,
		func(req *iop.NodeConfigRefreshRequest) (*iop.NodeConfigRefreshResponse, error) {
			return &iop.NodeConfigRefreshResponse{
				RequestId:            req.GetRequestId(),
				Status:               iop.NodeConfigRefreshStatus_NODE_CONFIG_REFRESH_STATUS_RESTART_REQUIRED,
				RestartRequiredPaths: req.GetChangedPaths(),
			}, nil
		},
	)

	regResp, err := toki.SendRequestTyped[*iop.RegisterRequest, *iop.RegisterResponse](
		&fakeNode.Communicator,
		&iop.RegisterRequest{Token: "tok-conn"},
		2*time.Second,
	)
	if err != nil {
		t.Fatalf("register: %v", err)
	}
	if !regResp.GetAccepted() {
		t.Fatalf("expected accepted, got reason %q", regResp.GetReason())
	}

	deadline := time.Now().Add(2 * time.Second)
	for time.Now().Before(deadline) {
		if _, ok := rt.Registry.Get("node-connected"); ok {
			break
		}
		time.Sleep(25 * time.Millisecond)
	}
	if _, ok := rt.Registry.Get("node-connected"); !ok {
		t.Fatal("fake node not registered in edge registry")
	}

	result, err := rt.RefreshConfig(context.Background(), configrefresh.Request{
		Mode:       configrefresh.ModeApply,
		ConfigPath: candidatePath,
		RequestID:  "skip-test",
	})
	if err != nil {
		t.Fatalf("RefreshConfig: %v", err)
	}
	if result.Status != configrefresh.StatusApplied {
		t.Fatalf("expected status=applied, got %q (summary=%s)", result.Status, result.Summary)
	}
	if len(result.NodeResults) != 2 {
		t.Fatalf("expected 2 node_results (connected + disconnected), got %d: %+v", len(result.NodeResults), result.NodeResults)
	}

	statusByID := make(map[string]string)
	for _, nr := range result.NodeResults {
		statusByID[nr.NodeID] = nr.Status
	}
	if statusByID["node-connected"] != "restart_required" {
		t.Errorf("node-connected: expected restart_required, got %q", statusByID["node-connected"])
	}
	if statusByID["node-disconnected"] != "skipped" {
		t.Errorf("node-disconnected: expected skipped, got %q", statusByID["node-disconnected"])
	}
}

// TestRefreshConfigApplyNoChangeSkipsNodePush verifies that an identical config
// apply (no-change) does NOT push a refresh request to connected nodes,
// preventing spurious restart_required responses.
func TestRefreshConfigApplyNoChangeSkipsNodePush(t *testing.T) {
	dir := t.TempDir()
	serverAddr := freeTCPAddr(t)

	writeConfig := func(name string) string {
		path := filepath.Join(dir, name)
		yaml := fmt.Sprintf(`
server:
  listen: %q
bootstrap:
  listen: "0.0.0.0:18094"
  artifact_dir: "artifacts"
logging:
  level: "error"
refresh:
  enabled: false
  listen: "127.0.0.1:0"
openai:
  listen: "0.0.0.0:18095"
a2a:
  listen: "0.0.0.0:8084"
metrics:
  port: 0
nodes:
  - id: "node-noc-change"
    alias: "noc-change-node"
    token: "tok-noc-change"
    adapters:
      cli:
        enabled: true
    providers:
      - id: "prov-nc"
        type: "ollama"
        category: "local_inference"
        models: ["llama3"]
        capacity: 2
        max_queue: 4
        queue_timeout_ms: 5000
`, serverAddr)
		if err := os.WriteFile(path, []byte(yaml), 0o600); err != nil {
			t.Fatalf("write %s: %v", name, err)
		}
		return path
	}

	basePath := writeConfig("base.yaml")
	// candidate is identical to base (same file).
	candidatePath := basePath

	rt, err := NewRuntime(loadServeNormalizedConfig(t, basePath))
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	if err := rt.Start(context.Background()); err != nil {
		t.Fatalf("Start: %v", err)
	}
	defer func() {
		if err := rt.Stop(); err != nil {
			t.Fatalf("Stop: %v", err)
		}
	}()

	// Build a fake node parser.
	nodeParserMap := toki.ParserMap{
		toki.TypeNameOf(&iop.RegisterResponse{}): func(b []byte) (proto.Message, error) {
			m := &iop.RegisterResponse{}
			return m, proto.Unmarshal(b, m)
		},
		toki.TypeNameOf(&iop.NodeConfigRefreshRequest{}): func(b []byte) (proto.Message, error) {
			m := &iop.NodeConfigRefreshRequest{}
			return m, proto.Unmarshal(b, m)
		},
	}

	host, portStr, _ := net.SplitHostPort(serverAddr)
	port := 0
	fmt.Sscanf(portStr, "%d", &port)
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	defer cancel()

	fakeNode, err := toki.DialTcp(ctx, host, port, 30, 10, nodeParserMap)
	if err != nil {
		t.Fatalf("dial edge: %v", err)
	}
	defer fakeNode.Close()

	// Track whether NodeConfigRefreshRequest is received.
	var receivedRefresh atomic.Bool
	toki.AddRequestListenerTyped[*iop.NodeConfigRefreshRequest, *iop.NodeConfigRefreshResponse](
		&fakeNode.Communicator,
		func(req *iop.NodeConfigRefreshRequest) (*iop.NodeConfigRefreshResponse, error) {
			receivedRefresh.Store(true)
			return &iop.NodeConfigRefreshResponse{
				RequestId: req.GetRequestId(),
				Status:    iop.NodeConfigRefreshStatus_NODE_CONFIG_REFRESH_STATUS_APPLIED,
			}, nil
		},
	)

	// Register the fake node.
	regResp, err := toki.SendRequestTyped[*iop.RegisterRequest, *iop.RegisterResponse](
		&fakeNode.Communicator,
		&iop.RegisterRequest{Token: "tok-noc-change"},
		2*time.Second,
	)
	if err != nil {
		t.Fatalf("register: %v", err)
	}
	if !regResp.GetAccepted() {
		t.Fatalf("expected accepted, got reason %q", regResp.GetReason())
	}

	// Wait for the node to appear in the registry.
	deadline := time.Now().Add(2 * time.Second)
	for time.Now().Before(deadline) {
		if _, ok := rt.Registry.Get("node-noc-change"); ok {
			break
		}
		time.Sleep(25 * time.Millisecond)
	}
	if _, ok := rt.Registry.Get("node-noc-change"); !ok {
		t.Fatal("fake node not registered in edge registry")
	}

	// Apply identical config (no-change).
	result, err := rt.RefreshConfig(context.Background(), configrefresh.Request{
		Mode:       configrefresh.ModeApply,
		ConfigPath: candidatePath,
		RequestID:  "no-change-1",
	})
	if err != nil {
		t.Fatalf("RefreshConfig: %v", err)
	}
	if result.Status != configrefresh.StatusApplied {
		t.Fatalf("expected status=applied, got %q (summary=%s)", result.Status, result.Summary)
	}
	// No-change apply should produce no node_results (no push).
	if len(result.NodeResults) != 0 {
		t.Errorf("expected no node_results for no-change apply, got %d: %+v", len(result.NodeResults), result.NodeResults)
	}
	// The fake node must NOT have received a NodeConfigRefreshRequest.
	if receivedRefresh.Load() {
		t.Error("expected no NodeConfigRefreshRequest to be sent for no-change apply, but it was received")
	}
}

func freeTCPAddr(t *testing.T) string {
	t.Helper()
	ln, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatalf("listen free tcp addr: %v", err)
	}
	addr := ln.Addr().String()
	if err := ln.Close(); err != nil {
		t.Fatalf("close free tcp addr listener: %v", err)
	}
	return addr
}

// TestNewRuntimeWiresControlPlaneConnector verifies that NewRuntime always
// wires a non-nil ControlPlane connector regardless of config.
func TestNewRuntimeWiresControlPlaneConnector(t *testing.T) {
	cfg := newTestConfig()
	rt, err := NewRuntime(cfg)
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	if rt.ControlPlane == nil {
		t.Fatal("expected non-nil ControlPlane connector")
	}
}

// TestNewRuntimeWiresStatusProviderIntoConnector verifies that NewRuntime wires
// the edge service as the Control Plane connector's status provider, so status
// requests are answered from the Edge-owned node snapshot surface.
func TestNewRuntimeWiresStatusProviderIntoConnector(t *testing.T) {
	cfg := newTestConfig()
	rt, err := NewRuntime(cfg)
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	if !rt.ControlPlane.StatusProviderConfigured() {
		t.Fatal("expected NewRuntime to wire a status provider into the connector")
	}
}

// TestNewRuntimeWiresNodeEventBusIntoConnector verifies that node lifecycle
// events are relayed through the shared events.Bus boundary.
func TestNewRuntimeWiresNodeEventBusIntoConnector(t *testing.T) {
	cfg := newTestConfig()
	rt, err := NewRuntime(cfg)
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	if !rt.ControlPlane.NodeEventBusConfigured() {
		t.Fatal("expected NewRuntime to wire the node event bus into the connector")
	}
}

// TestRuntimeStartsControlPlaneConnectorWhenEnabled verifies that Start does
// not fail when a Control Plane connector is configured with a local fake
// server accepting hello. Uses a local TCP fake endpoint.
func TestRuntimeStartsControlPlaneConnectorWhenEnabled(t *testing.T) {
	// Start a minimal fake Control Plane server.
	ln, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatalf("listen fake cp: %v", err)
	}
	cpAddr := ln.Addr().String()
	go func() {
		// Accept and immediately close; connector will fail hello and retry,
		// but Start() itself must succeed (connector errors are async).
		conn, err := ln.Accept()
		if err != nil {
			return
		}
		_ = conn.Close()
		_ = ln.Close()
	}()

	cfg := newTestConfig()
	cfg.ControlPlane = config.EdgeControlPlaneConf{
		Enabled:              true,
		WireAddr:             cpAddr,
		ReconnectIntervalSec: 60,
	}

	rt, err := NewRuntime(cfg)
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	if !rt.ControlPlane.IsEnabled() {
		t.Fatal("expected connector IsEnabled()==true")
	}

	ctx := context.Background()
	if err := rt.Start(ctx); err != nil {
		t.Fatalf("Start with enabled connector: %v", err)
	}
	if err := rt.Stop(); err != nil {
		t.Fatalf("Stop: %v", err)
	}
}

// TestRuntimeStopsControlPlaneConnector verifies that Stop() reaches the
// ControlPlane connector. A disabled connector is used so no real network
// activity occurs.
func TestRuntimeStopsControlPlaneConnector(t *testing.T) {
	cfg := newTestConfig()
	// disabled connector; Start/Stop must be no-ops on the connector side.
	cfg.ControlPlane = config.EdgeControlPlaneConf{
		Enabled:  false,
		WireAddr: "",
	}

	rt, err := NewRuntime(cfg)
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}

	ctx := context.Background()
	if err := rt.Start(ctx); err != nil {
		t.Fatalf("Start: %v", err)
	}
	if err := rt.Stop(); err != nil {
		t.Fatalf("Stop: %v", err)
	}
	// After Stop, the connector must be in a stopped-or-no-op state.
	// For disabled connector, CurrentState is Disconnected (never started).
	// Calling Stop again must not panic.
	rt.ControlPlane.Stop()
}

// TestRefreshRejectedPreservesProviderDispatch verifies SDD S10: a rejected
// refresh must preserve provider-pool dispatch state. It uses a net.Pipe-based
// fake node to capture the RunRequest sent by SubmitRun(ProviderPool=true)
// before and after a rejected config refresh.
func TestRefreshRejectedPreservesProviderDispatch(t *testing.T) {
	dir := t.TempDir()
	serverAddr := freeTCPAddr(t)
	openAIAddr := freeTCPAddr(t)

	basePath := filepath.Join(dir, "base.yaml")
	if err := os.WriteFile(basePath, []byte(providerPoolConfigYAML(serverAddr, openAIAddr, validAdapterBlock)), 0o600); err != nil {
		t.Fatalf("write base: %v", err)
	}
	candidatePath := filepath.Join(dir, "invalid.yaml")
	if err := os.WriteFile(candidatePath, []byte(providerPoolConfigYAML(serverAddr, openAIAddr, invalidAdapterBlock)), 0o600); err != nil {
		t.Fatalf("write candidate: %v", err)
	}

	rt, err := NewRuntime(loadServeNormalizedConfig(t, basePath))
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	if err := rt.Start(context.Background()); err != nil {
		t.Fatalf("Start: %v", err)
	}
	defer func() {
		if err := rt.Stop(); err != nil {
			t.Fatalf("Stop: %v", err)
		}
	}()

	// --- Dispatch capture setup using net.Pipe ---
	edgeConn, nodeConn := net.Pipe()
	defer edgeConn.Close()
	defer nodeConn.Close()

	parserMap := toki.ParserMap{
		toki.TypeNameOf(&iop.RunRequest{}): func(b []byte) (proto.Message, error) {
			m := &iop.RunRequest{}
			return m, proto.Unmarshal(b, m)
		},
	}

	edgeClient := toki.NewTcpClient(edgeConn, 0, 0, parserMap)
	nodeClient := toki.NewTcpClient(nodeConn, 0, 0, parserMap)

	var capturedReq *iop.RunRequest
	var capturedMu sync.Mutex
	toki.AddListenerTyped[*iop.RunRequest](&nodeClient.Communicator, func(req *iop.RunRequest) {
		capturedMu.Lock()
		capturedReq = req
		capturedMu.Unlock()
	})

	// Add a provider-pool node record to the NodeStore so provider-pool
	// resolution finds a dispatchable provider. The provider config must
	// match the adapter (vllm-gpu) and served target (served-qwen) expected
	// by the model catalog entry (qwen3.6:35b -> prov-a).
	rt.NodeStore.Add(&edgenode.NodeRecord{
		ID:      "node-pool-dispatch",
		Alias:   "pool-alias",
		Runtime: config.RuntimeConf{Concurrency: 4},
		Providers: []config.NodeProviderConf{
			{
				ID:       "prov-a",
				Type:     "vllm",
				Category: "api",
				Adapter:  "vllm-gpu",
				Models:   []string{"served-qwen"},
				Health:   "available",
				Capacity: 2,
			},
		},
	})

	// Register the fake node into the runtime registry.
	rt.Registry.Register(&edgenode.NodeEntry{
		NodeID:         "node-pool-dispatch",
		LifecycleState: edgenode.LifecycleConnected,
		Client:         edgeClient,
	})

	// Submit RunRequest via the runtime Service before refresh.
	dispatchAndCapture := func() (*iop.RunRequest, error) {
		capturedMu.Lock()
		capturedReq = nil
		capturedMu.Unlock()

		result, err := rt.Service.SubmitRun(context.Background(), edgeservice.SubmitRunRequest{
			RunID:         "dispatch-test-" + time.Now().Format("150405"),
			ModelGroupKey: "qwen3.6:35b",
			ProviderPool:  true,
			Background:    true,
		})
		if err != nil {
			return nil, err
		}
		// Drain the result handle to release resources.
		if result != nil {
			result.Close()
		}

		// Wait for the fake node to receive the request.
		time.Sleep(100 * time.Millisecond)

		capturedMu.Lock()
		defer capturedMu.Unlock()
		return capturedReq, nil
	}

	// --- Before rejected refresh: dispatch must succeed ---
	beforeReq, err := dispatchAndCapture()
	if err != nil {
		t.Fatalf("dispatch before refresh: %v", err)
	}
	if beforeReq == nil {
		t.Fatal("expected dispatch before refresh to produce a RunRequest")
	}
	if beforeReq.GetAdapter() != "vllm-gpu" {
		t.Errorf("before refresh adapter: got %q, want %q", beforeReq.GetAdapter(), "vllm-gpu")
	}
	if beforeReq.GetTarget() != "served-qwen" {
		t.Errorf("before refresh target: got %q, want %q", beforeReq.GetTarget(), "served-qwen")
	}

	// --- Execute rejected refresh ---
	result, err := rt.RefreshConfig(context.Background(), configrefresh.Request{
		Mode:       configrefresh.ModeApply,
		ConfigPath: candidatePath,
		RequestID:  "reject-dispatch-1",
	})
	if err != nil {
		t.Fatalf("RefreshConfig: %v", err)
	}
	if result.Status != configrefresh.StatusRejected {
		t.Fatalf("expected status=rejected, got %q (summary=%s)", result.Status, result.Summary)
	}

	// --- After rejected refresh: dispatch must preserve state ---
	afterReq, err := dispatchAndCapture()
	if err != nil {
		t.Fatalf("dispatch after refresh: %v", err)
	}
	if afterReq == nil {
		t.Fatal("expected dispatch after rejected refresh to produce a RunRequest")
	}
	if afterReq.GetAdapter() != "vllm-gpu" {
		t.Errorf("after rejected refresh adapter: got %q, want %q", afterReq.GetAdapter(), "vllm-gpu")
	}
	if afterReq.GetTarget() != "served-qwen" {
		t.Errorf("after rejected refresh target: got %q, want %q", afterReq.GetTarget(), "served-qwen")
	}
}

// TestRefreshConfigNodeRuntimeConcurrencyApplied verifies the S15 boundary at
// the running-edge level: a node runtime.concurrency change is applied live
// and pushed to the connected node.
func TestRefreshConfigNodeRuntimeConcurrencyApplied(t *testing.T) {
	dir := t.TempDir()
	serverAddr := freeTCPAddr(t)

	writeCfg := func(name string, concurrency int) string {
		path := filepath.Join(dir, name)
		yaml := fmt.Sprintf(`
server:
  listen: %q
bootstrap:
  listen: "0.0.0.0:18190"
  artifact_dir: "artifacts"
logging:
  level: "error"
refresh:
  enabled: false
  listen: "127.0.0.1:0"
openai:
  listen: "0.0.0.0:18191"
a2a:
  listen: "0.0.0.0:8182"
metrics:
  port: 0
nodes:
  - id: "node-rt-1"
    alias: "rt-node"
    token: "tok-rt"
    adapters:
      cli:
        enabled: true
    runtime:
      concurrency: %d
`, serverAddr, concurrency)
		if err := os.WriteFile(path, []byte(yaml), 0o600); err != nil {
			t.Fatalf("write %s: %v", name, err)
		}
		return path
	}

	basePath := writeCfg("base.yaml", 2)
	concurrencyPath := writeCfg("concurrency.yaml", 4)

	rt, err := NewRuntime(loadServeNormalizedConfig(t, basePath))
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	if err := rt.Start(context.Background()); err != nil {
		t.Fatalf("Start: %v", err)
	}
	defer func() {
		if err := rt.Stop(); err != nil {
			t.Fatalf("Stop: %v", err)
		}
	}()

	nodeParserMap := toki.ParserMap{
		toki.TypeNameOf(&iop.RegisterResponse{}): func(b []byte) (proto.Message, error) {
			m := &iop.RegisterResponse{}
			return m, proto.Unmarshal(b, m)
		},
		toki.TypeNameOf(&iop.NodeConfigRefreshRequest{}): func(b []byte) (proto.Message, error) {
			m := &iop.NodeConfigRefreshRequest{}
			return m, proto.Unmarshal(b, m)
		},
	}

	host, portStr, _ := net.SplitHostPort(serverAddr)
	port := 0
	fmt.Sscanf(portStr, "%d", &port)
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	defer cancel()

	fakeNode, err := toki.DialTcp(ctx, host, port, 30, 10, nodeParserMap)
	if err != nil {
		t.Fatalf("dial edge: %v", err)
	}
	defer fakeNode.Close()

	var capturedConcurrency int32 = -1
	var capturedMu sync.Mutex
	toki.AddRequestListenerTyped[*iop.NodeConfigRefreshRequest, *iop.NodeConfigRefreshResponse](
		&fakeNode.Communicator,
		func(req *iop.NodeConfigRefreshRequest) (*iop.NodeConfigRefreshResponse, error) {
			capturedMu.Lock()
			capturedConcurrency = req.GetConfig().GetRuntime().GetConcurrency()
			capturedMu.Unlock()
			return &iop.NodeConfigRefreshResponse{
				RequestId: req.GetRequestId(),
				Status:    iop.NodeConfigRefreshStatus_NODE_CONFIG_REFRESH_STATUS_APPLIED,
			}, nil
		},
	)

	regResp, err := toki.SendRequestTyped[*iop.RegisterRequest, *iop.RegisterResponse](
		&fakeNode.Communicator,
		&iop.RegisterRequest{Token: "tok-rt"},
		2*time.Second,
	)
	if err != nil {
		t.Fatalf("register: %v", err)
	}
	if !regResp.GetAccepted() {
		t.Fatalf("expected accepted, got reason %q", regResp.GetReason())
	}

	deadline := time.Now().Add(2 * time.Second)
	for time.Now().Before(deadline) {
		if _, ok := rt.Registry.Get("node-rt-1"); ok {
			break
		}
		time.Sleep(25 * time.Millisecond)
	}
	if _, ok := rt.Registry.Get("node-rt-1"); !ok {
		t.Fatal("fake node not registered in edge registry")
	}

	// concurrency change → applied, pushed to node, node result reported.
	concResult, err := rt.RefreshConfig(context.Background(), configrefresh.Request{
		Mode:       configrefresh.ModeApply,
		ConfigPath: concurrencyPath,
		RequestID:  "rt-conc",
	})
	if err != nil {
		t.Fatalf("RefreshConfig concurrency: %v", err)
	}
	if concResult.Status != configrefresh.StatusApplied {
		t.Fatalf("expected concurrency change status=applied, got %q (summary=%s)", concResult.Status, concResult.Summary)
	}
	found := false
	for _, nr := range concResult.NodeResults {
		if nr.NodeID == "node-rt-1" {
			found = true
			if nr.Status != "applied" {
				t.Errorf("expected node status=applied, got %q (error=%q)", nr.Status, nr.Error)
			}
		}
	}
	if !found {
		t.Fatalf("node-rt-1 not found in node_results: %+v", concResult.NodeResults)
	}

	capturedMu.Lock()
	got := capturedConcurrency
	capturedMu.Unlock()
	if got != 4 {
		t.Errorf("pushed runtime concurrency: got %d, want 4", got)
	}
}