iop/apps/edge/internal/bootstrap/runtime_refresh_node_test.go

package bootstrap

import (
	"context"
	"fmt"
	"net"
	"os"
	"path/filepath"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	toki "git.toki-labs.com/toki/proto-socket/go"
	"google.golang.org/protobuf/proto"

	"iop/apps/edge/internal/configrefresh"
	edgenode "iop/apps/edge/internal/node"
	edgeservice "iop/apps/edge/internal/service"
	"iop/packages/go/config"
	iop "iop/proto/gen/iop"
)

// readyFakeNode drives the NodeReadyRequest/ack handshake for a fake node that
// has already registered, so the edge marks its connection dispatch-ready. Fake
// nodes exercising config-refresh push must call it or the push skips them as
// pending.
// fakeNodeHandshakeTimeout bounds test-only register/ready requests, and
// fakeNodeReadyAttempts bounds retries of the idempotent ready handshake. The
// reconnect verification runs -count=10 -race and can delay a fresh TCP
// handshake beyond the old two-second test ceiling. Production constants are
// untouched.
const (
	fakeNodeHandshakeTimeout = 5 * time.Second
	fakeNodeReadyAttempts    = 4
)

func readyFakeNode(t *testing.T, fakeNode *toki.TcpClient, nodeID string) {
	t.Helper()
	var lastErr error
	for attempt := 1; attempt <= fakeNodeReadyAttempts; attempt++ {
		resp, err := toki.SendRequestTyped[*iop.NodeReadyRequest, *iop.NodeReadyResponse](
			&fakeNode.Communicator, &iop.NodeReadyRequest{NodeId: nodeID}, fakeNodeHandshakeTimeout)
		if err != nil {
			lastErr = err
			if !fakeNode.IsAlive() {
				break
			}
			continue // a duplicate ready is idempotent server side, so retry is safe
		}
		if !resp.GetReady() {
			t.Fatalf("expected ready ack, got reason %q", resp.GetReason())
		}
		return
	}
	t.Fatalf("ready request failed after %d attempts: %v", fakeNodeReadyAttempts, lastErr)
}

// TestRefreshConfigApplyIncludesNodeResults verifies SDD S12: when a refresh
// apply succeeds, the result includes per-node refresh outcomes. Uses a
// net.Pipe-based fake node that registers with the edge and responds applied.
func TestRefreshConfigApplyIncludesNodeResults(t *testing.T) {
	dir := t.TempDir()
	serverAddr := freeTCPAddr(t)

	writeRefreshConfig := func(name string, capacity int) string {
		path := filepath.Join(dir, name)
		yaml := fmt.Sprintf(`
server:
  listen: %q
bootstrap:
  listen: "0.0.0.0:18090"
  artifact_dir: "artifacts"
logging:
  level: "error"
refresh:
  enabled: false
  listen: "127.0.0.1:0"
openai:
  listen: "0.0.0.0:18091"
a2a:
  listen: "0.0.0.0:8082"
metrics:
  port: 0
nodes:
  - id: "node-refresh-1"
    alias: "refresh-node"
    token: "tok-refresh"
    adapters:
      cli:
        enabled: true
      openai_compat_instances:
        - name: "primary"
          enabled: true
          provider: "vllm"
          endpoint: "http://127.0.0.1:9000/v1"
          capacity: 5
          max_queue: 4
          queue_timeout_ms: 5000
    runtime:
      concurrency: 7
    providers:
      - id: "prov-r"
        type: "ollama"
        category: "local_inference"
        adapter: "cli"
        models: ["llama3"]
        capacity: %d
        max_queue: 4
        queue_timeout_ms: 5000
`, serverAddr, capacity)
		if err := os.WriteFile(path, []byte(yaml), 0o600); err != nil {
			t.Fatalf("write %s: %v", name, err)
		}
		return path
	}

	basePath := writeRefreshConfig("base.yaml", 2)
	candidatePath := writeRefreshConfig("candidate.yaml", 8)

	rt, err := NewRuntime(loadServeNormalizedConfig(t, basePath))
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	if err := rt.Start(context.Background()); err != nil {
		t.Fatalf("Start: %v", err)
	}
	defer func() {
		if err := rt.Stop(); err != nil {
			t.Fatalf("Stop: %v", err)
		}
	}()

	// Build a fake node parser that can decode NodeConfigRefreshRequest (edge push)
	// and RegisterResponse (registration).
	nodeParserMap := toki.ParserMap{
		toki.TypeNameOf(&iop.RegisterResponse{}): func(b []byte) (proto.Message, error) {
			m := &iop.RegisterResponse{}
			return m, proto.Unmarshal(b, m)
		},
		toki.TypeNameOf(&iop.NodeReadyResponse{}): func(b []byte) (proto.Message, error) {
			m := &iop.NodeReadyResponse{}
			return m, proto.Unmarshal(b, m)
		},
		toki.TypeNameOf(&iop.NodeConfigRefreshRequest{}): func(b []byte) (proto.Message, error) {
			m := &iop.NodeConfigRefreshRequest{}
			return m, proto.Unmarshal(b, m)
		},
	}

	host, portStr, _ := net.SplitHostPort(serverAddr)
	port := 0
	fmt.Sscanf(portStr, "%d", &port)
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	defer cancel()

	fakeNode, err := toki.DialTcp(ctx, host, port, 30, 10, nodeParserMap)
	if err != nil {
		t.Fatalf("dial edge: %v", err)
	}
	defer fakeNode.Close()

	// Fake node captures the refresh request and responds restart_required
	// (current Node behavior for any non-empty payload/change).
	var capturedRefreshReq *iop.NodeConfigRefreshRequest
	var capturedRefreshMu sync.Mutex
	toki.AddRequestListenerTyped[*iop.NodeConfigRefreshRequest, *iop.NodeConfigRefreshResponse](
		&fakeNode.Communicator,
		func(req *iop.NodeConfigRefreshRequest) (*iop.NodeConfigRefreshResponse, error) {
			capturedRefreshMu.Lock()
			capturedRefreshReq = req
			capturedRefreshMu.Unlock()
			return &iop.NodeConfigRefreshResponse{
				RequestId:            req.GetRequestId(),
				Status:               iop.NodeConfigRefreshStatus_NODE_CONFIG_REFRESH_STATUS_RESTART_REQUIRED,
				RestartRequiredPaths: req.GetChangedPaths(),
			}, nil
		},
	)

	// Register the fake node.
	regResp, err := toki.SendRequestTyped[*iop.RegisterRequest, *iop.RegisterResponse](
		&fakeNode.Communicator,
		&iop.RegisterRequest{Token: "tok-refresh"},
		2*time.Second,
	)
	if err != nil {
		t.Fatalf("register: %v", err)
	}
	if !regResp.GetAccepted() {
		t.Fatalf("expected accepted, got reason %q", regResp.GetReason())
	}
	// Registration only claims ownership; the ready signal makes the node
	// dispatch-ready so config-refresh push targets it instead of skipping it.
	readyFakeNode(t, fakeNode, regResp.GetNodeId())

	// Wait for the node to appear in the registry.
	deadline := time.Now().Add(2 * time.Second)
	for time.Now().Before(deadline) {
		if _, ok := rt.Registry.Get("node-refresh-1"); ok {
			break
		}
		time.Sleep(25 * time.Millisecond)
	}
	if _, ok := rt.Registry.Get("node-refresh-1"); !ok {
		t.Fatal("fake node not registered in edge registry")
	}

	result, err := rt.RefreshConfig(context.Background(), configrefresh.Request{
		Mode:       configrefresh.ModeApply,
		ConfigPath: candidatePath,
		RequestID:  "node-result-test",
	})
	if err != nil {
		t.Fatalf("RefreshConfig: %v", err)
	}
	if result.Status != configrefresh.StatusApplied {
		t.Fatalf("expected status=applied, got %q (summary=%s)", result.Status, result.Summary)
	}
	if len(result.NodeResults) == 0 {
		t.Fatal("expected node_results to be populated after apply")
	}
	found := false
	for _, nr := range result.NodeResults {
		if nr.NodeID == "node-refresh-1" {
			found = true
			if nr.Status != "restart_required" {
				t.Errorf("expected node status=restart_required (current behavior for non-empty payload), got %q (error=%q)", nr.Status, nr.Error)
			}
		}
	}
	if !found {
		t.Fatalf("node-refresh-1 not found in node_results: %+v", result.NodeResults)
	}

	// Verify request_id propagation and payload contents on the fake node side.
	capturedRefreshMu.Lock()
	captured := capturedRefreshReq
	capturedRefreshMu.Unlock()
	if captured == nil {
		t.Fatal("fake node never received a NodeConfigRefreshRequest")
	}
	if captured.GetRequestId() != "node-result-test" {
		t.Errorf("request_id: got %q, want %q", captured.GetRequestId(), "node-result-test")
	}
	// Payload must carry the node-specific config edge builds from the NodeStore
	// record. Note: providers[].capacity is the refresh *trigger* (it drives
	// changed_paths), not a NodeConfigPayload field; the payload itself carries
	// the node's adapters + runtime, so we assert those concrete typed fields.
	payload := captured.GetConfig()
	if payload == nil {
		t.Fatal("expected non-nil NodeConfigPayload in refresh request")
	}
	// Runtime tuning is pushed from the node record.
	if got := payload.GetRuntime().GetConcurrency(); got != 7 {
		t.Errorf("runtime concurrency: got %d, want 7", got)
	}
	// The configured openai_compat adapter instance must be present with its
	// typed fields (provider/endpoint/capacity) intact, and the CLI adapter too.
	var oai *iop.OpenAICompatAdapterConfig
	hasCLI := false
	for _, a := range payload.GetAdapters() {
		switch a.GetType() {
		case "openai_compat":
			if a.GetName() == "primary" {
				oai = a.GetOpenaiCompat()
			}
		case "cli":
			hasCLI = true
		}
	}
	if oai == nil {
		t.Fatalf("expected openai_compat adapter %q in refresh payload, got %+v", "primary", payload.GetAdapters())
	}
	if oai.GetProvider() != "vllm" {
		t.Errorf("openai_compat provider: got %q, want %q", oai.GetProvider(), "vllm")
	}
	if oai.GetEndpoint() != "http://127.0.0.1:9000/v1" {
		t.Errorf("openai_compat endpoint: got %q, want %q", oai.GetEndpoint(), "http://127.0.0.1:9000/v1")
	}
	if oai.GetCapacity() != 5 {
		t.Errorf("openai_compat capacity: got %d, want 5", oai.GetCapacity())
	}
	if !hasCLI {
		t.Errorf("expected cli adapter in refresh payload, got %+v", payload.GetAdapters())
	}
}

// TestRefreshConfigApplySkipsDisconnectedConfiguredNode verifies that a node
// that is in the NodeStore but not in the live registry (post-disconnect state)
// appears as "skipped" in node_results alongside connected nodes.
func TestRefreshConfigApplySkipsDisconnectedConfiguredNode(t *testing.T) {
	dir := t.TempDir()
	serverAddr := freeTCPAddr(t)

	writeConfig := func(name string, capacity int) string {
		path := filepath.Join(dir, name)
		yaml := fmt.Sprintf(`
server:
  listen: %q
bootstrap:
  listen: "0.0.0.0:18092"
  artifact_dir: "artifacts"
logging:
  level: "error"
refresh:
  enabled: false
  listen: "127.0.0.1:0"
openai:
  listen: "0.0.0.0:18093"
a2a:
  listen: "0.0.0.0:8083"
metrics:
  port: 0
nodes:
  - id: "node-connected"
    alias: "conn-node"
    token: "tok-conn"
    adapters:
      cli:
        enabled: true
    providers:
      - id: "prov-skip"
        type: "ollama"
        category: "local_inference"
        adapter: "cli"
        models: ["llama3"]
        capacity: %d
        max_queue: 4
        queue_timeout_ms: 5000
  - id: "node-disconnected"
    alias: "disc-node"
    token: "tok-disc"
    adapters:
      cli:
        enabled: true
`, serverAddr, capacity)
		if err := os.WriteFile(path, []byte(yaml), 0o600); err != nil {
			t.Fatalf("write %s: %v", name, err)
		}
		return path
	}

	basePath := writeConfig("base.yaml", 2)
	candidatePath := writeConfig("candidate.yaml", 8)

	rt, err := NewRuntime(loadServeNormalizedConfig(t, basePath))
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	if err := rt.Start(context.Background()); err != nil {
		t.Fatalf("Start: %v", err)
	}
	defer func() {
		if err := rt.Stop(); err != nil {
			t.Fatalf("Stop: %v", err)
		}
	}()

	nodeParserMap := toki.ParserMap{
		toki.TypeNameOf(&iop.RegisterResponse{}): func(b []byte) (proto.Message, error) {
			m := &iop.RegisterResponse{}
			return m, proto.Unmarshal(b, m)
		},
		toki.TypeNameOf(&iop.NodeReadyResponse{}): func(b []byte) (proto.Message, error) {
			m := &iop.NodeReadyResponse{}
			return m, proto.Unmarshal(b, m)
		},
		toki.TypeNameOf(&iop.NodeConfigRefreshRequest{}): func(b []byte) (proto.Message, error) {
			m := &iop.NodeConfigRefreshRequest{}
			return m, proto.Unmarshal(b, m)
		},
	}

	host, portStr, _ := net.SplitHostPort(serverAddr)
	port := 0
	fmt.Sscanf(portStr, "%d", &port)
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	defer cancel()

	// Only connect node-connected; node-disconnected never dials in.
	fakeNode, err := toki.DialTcp(ctx, host, port, 30, 10, nodeParserMap)
	if err != nil {
		t.Fatalf("dial edge: %v", err)
	}
	defer fakeNode.Close()

	toki.AddRequestListenerTyped[*iop.NodeConfigRefreshRequest, *iop.NodeConfigRefreshResponse](
		&fakeNode.Communicator,
		func(req *iop.NodeConfigRefreshRequest) (*iop.NodeConfigRefreshResponse, error) {
			return &iop.NodeConfigRefreshResponse{
				RequestId:            req.GetRequestId(),
				Status:               iop.NodeConfigRefreshStatus_NODE_CONFIG_REFRESH_STATUS_RESTART_REQUIRED,
				RestartRequiredPaths: req.GetChangedPaths(),
			}, nil
		},
	)

	regResp, err := toki.SendRequestTyped[*iop.RegisterRequest, *iop.RegisterResponse](
		&fakeNode.Communicator,
		&iop.RegisterRequest{Token: "tok-conn"},
		2*time.Second,
	)
	if err != nil {
		t.Fatalf("register: %v", err)
	}
	if !regResp.GetAccepted() {
		t.Fatalf("expected accepted, got reason %q", regResp.GetReason())
	}
	// Registration only claims ownership; the ready signal makes the node
	// dispatch-ready so config-refresh push targets it instead of skipping it.
	readyFakeNode(t, fakeNode, regResp.GetNodeId())

	deadline := time.Now().Add(2 * time.Second)
	for time.Now().Before(deadline) {
		if _, ok := rt.Registry.Get("node-connected"); ok {
			break
		}
		time.Sleep(25 * time.Millisecond)
	}
	if _, ok := rt.Registry.Get("node-connected"); !ok {
		t.Fatal("fake node not registered in edge registry")
	}

	result, err := rt.RefreshConfig(context.Background(), configrefresh.Request{
		Mode:       configrefresh.ModeApply,
		ConfigPath: candidatePath,
		RequestID:  "skip-test",
	})
	if err != nil {
		t.Fatalf("RefreshConfig: %v", err)
	}
	if result.Status != configrefresh.StatusApplied {
		t.Fatalf("expected status=applied, got %q (summary=%s)", result.Status, result.Summary)
	}
	if len(result.NodeResults) != 2 {
		t.Fatalf("expected 2 node_results (connected + disconnected), got %d: %+v", len(result.NodeResults), result.NodeResults)
	}

	statusByID := make(map[string]string)
	for _, nr := range result.NodeResults {
		statusByID[nr.NodeID] = nr.Status
	}
	if statusByID["node-connected"] != "restart_required" {
		t.Errorf("node-connected: expected restart_required, got %q", statusByID["node-connected"])
	}
	if statusByID["node-disconnected"] != "skipped" {
		t.Errorf("node-disconnected: expected skipped, got %q", statusByID["node-disconnected"])
	}
}

// TestRefreshConfigApplyNoChangeSkipsNodePush verifies that an identical config
// apply (no-change) does NOT push a refresh request to connected nodes,
// preventing spurious restart_required responses.
func TestRefreshConfigApplyNoChangeSkipsNodePush(t *testing.T) {
	dir := t.TempDir()
	serverAddr := freeTCPAddr(t)

	writeConfig := func(name string) string {
		path := filepath.Join(dir, name)
		yaml := fmt.Sprintf(`
server:
  listen: %q
bootstrap:
  listen: "0.0.0.0:18094"
  artifact_dir: "artifacts"
logging:
  level: "error"
refresh:
  enabled: false
  listen: "127.0.0.1:0"
openai:
  listen: "0.0.0.0:18095"
a2a:
  listen: "0.0.0.0:8084"
metrics:
  port: 0
nodes:
  - id: "node-noc-change"
    alias: "noc-change-node"
    token: "tok-noc-change"
    adapters:
      cli:
        enabled: true
    providers:
      - id: "prov-nc"
        type: "ollama"
        category: "local_inference"
        adapter: "cli"
        models: ["llama3"]
        capacity: 2
        max_queue: 4
        queue_timeout_ms: 5000
`, serverAddr)
		if err := os.WriteFile(path, []byte(yaml), 0o600); err != nil {
			t.Fatalf("write %s: %v", name, err)
		}
		return path
	}

	basePath := writeConfig("base.yaml")
	// candidate is identical to base (same file).
	candidatePath := basePath

	rt, err := NewRuntime(loadServeNormalizedConfig(t, basePath))
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	if err := rt.Start(context.Background()); err != nil {
		t.Fatalf("Start: %v", err)
	}
	defer func() {
		if err := rt.Start(context.Background()); err == nil {
			t.Fatal("expected already started error")
		} else {
			// runtime is running, stop it
			if err := rt.Stop(); err != nil {
				t.Fatalf("Stop: %v", err)
			}
		}
	}()

	// Build a fake node parser.
	nodeParserMap := toki.ParserMap{
		toki.TypeNameOf(&iop.RegisterResponse{}): func(b []byte) (proto.Message, error) {
			m := &iop.RegisterResponse{}
			return m, proto.Unmarshal(b, m)
		},
		toki.TypeNameOf(&iop.NodeReadyResponse{}): func(b []byte) (proto.Message, error) {
			m := &iop.NodeReadyResponse{}
			return m, proto.Unmarshal(b, m)
		},
		toki.TypeNameOf(&iop.NodeConfigRefreshRequest{}): func(b []byte) (proto.Message, error) {
			m := &iop.NodeConfigRefreshRequest{}
			return m, proto.Unmarshal(b, m)
		},
	}

	host, portStr, _ := net.SplitHostPort(serverAddr)
	port := 0
	fmt.Sscanf(portStr, "%d", &port)
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	defer cancel()

	fakeNode, err := toki.DialTcp(ctx, host, port, 30, 10, nodeParserMap)
	if err != nil {
		t.Fatalf("dial edge: %v", err)
	}
	defer fakeNode.Close()

	// Track whether NodeConfigRefreshRequest is received.
	var receivedRefresh atomic.Bool
	toki.AddRequestListenerTyped[*iop.NodeConfigRefreshRequest, *iop.NodeConfigRefreshResponse](
		&fakeNode.Communicator,
		func(req *iop.NodeConfigRefreshRequest) (*iop.NodeConfigRefreshResponse, error) {
			receivedRefresh.Store(true)
			return &iop.NodeConfigRefreshResponse{
				RequestId: req.GetRequestId(),
				Status:    iop.NodeConfigRefreshStatus_NODE_CONFIG_REFRESH_STATUS_APPLIED,
			}, nil
		},
	)

	// Register the fake node.
	regResp, err := toki.SendRequestTyped[*iop.RegisterRequest, *iop.RegisterResponse](
		&fakeNode.Communicator,
		&iop.RegisterRequest{Token: "tok-noc-change"},
		2*time.Second,
	)
	if err != nil {
		t.Fatalf("register: %v", err)
	}
	if !regResp.GetAccepted() {
		t.Fatalf("expected accepted, got reason %q", regResp.GetReason())
	}
	// Registration only claims ownership; the ready signal makes the node
	// dispatch-ready so config-refresh push targets it instead of skipping it.
	readyFakeNode(t, fakeNode, regResp.GetNodeId())

	// Wait for the node to appear in the registry.
	deadline := time.Now().Add(2 * time.Second)
	for time.Now().Before(deadline) {
		if _, ok := rt.Registry.Get("node-noc-change"); ok {
			break
		}
		time.Sleep(25 * time.Millisecond)
	}
	if _, ok := rt.Registry.Get("node-noc-change"); !ok {
		t.Fatal("fake node not registered in edge registry")
	}

	// Apply identical config (no-change).
	result, err := rt.RefreshConfig(context.Background(), configrefresh.Request{
		Mode:       configrefresh.ModeApply,
		ConfigPath: candidatePath,
		RequestID:  "no-change-1",
	})
	if err != nil {
		t.Fatalf("RefreshConfig: %v", err)
	}
	if result.Status != configrefresh.StatusApplied {
		t.Fatalf("expected status=applied, got %q (summary=%s)", result.Status, result.Summary)
	}
	// No-change apply should produce no node_results (no push).
	if len(result.NodeResults) != 0 {
		t.Errorf("expected no node_results for no-change apply, got %d: %+v", len(result.NodeResults), result.NodeResults)
	}
	// The fake node must NOT have received a NodeConfigRefreshRequest.
	if receivedRefresh.Load() {
		t.Error("expected no NodeConfigRefreshRequest to be sent for no-change apply, but it was received")
	}
}

// TestRefreshRejectedPreservesProviderDispatch verifies SDD S10: a rejected
// refresh must preserve provider-pool dispatch state. It uses a net.Pipe-based
// fake node to capture the RunRequest sent by SubmitRun(ProviderPool=true)
// before and after a rejected config refresh.
func TestRefreshRejectedPreservesProviderDispatch(t *testing.T) {
	dir := t.TempDir()
	serverAddr := freeTCPAddr(t)
	openAIAddr := freeTCPAddr(t)

	basePath := filepath.Join(dir, "base.yaml")
	if err := os.WriteFile(basePath, []byte(providerPoolConfigYAML(serverAddr, openAIAddr, validAdapterBlock)), 0o600); err != nil {
		t.Fatalf("write base: %v", err)
	}
	candidatePath := filepath.Join(dir, "invalid.yaml")
	if err := os.WriteFile(candidatePath, []byte(providerPoolConfigYAML(serverAddr, openAIAddr, invalidAdapterBlock)), 0o600); err != nil {
		t.Fatalf("write candidate: %v", err)
	}

	rt, err := NewRuntime(loadServeNormalizedConfig(t, basePath))
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	if err := rt.Start(context.Background()); err != nil {
		t.Fatalf("Start: %v", err)
	}
	defer func() {
		if err := rt.Stop(); err != nil {
			t.Fatalf("Stop: %v", err)
		}
	}()

	// --- Dispatch capture setup using net.Pipe ---
	edgeConn, nodeConn := net.Pipe()
	defer edgeConn.Close()
	defer nodeConn.Close()

	parserMap := toki.ParserMap{
		toki.TypeNameOf(&iop.RunRequest{}): func(b []byte) (proto.Message, error) {
			m := &iop.RunRequest{}
			return m, proto.Unmarshal(b, m)
		},
	}

	edgeClient := toki.NewTcpClient(edgeConn, 0, 0, parserMap)
	nodeClient := toki.NewTcpClient(nodeConn, 0, 0, parserMap)

	var capturedReq *iop.RunRequest
	var capturedMu sync.Mutex
	toki.AddListenerTyped[*iop.RunRequest](&nodeClient.Communicator, func(req *iop.RunRequest) {
		capturedMu.Lock()
		capturedReq = req
		capturedMu.Unlock()
	})

	// Add a provider-pool node record to the NodeStore so provider-pool
	// resolution finds a dispatchable provider. The provider config must
	// match the adapter (vllm-gpu) and served target (served-qwen) expected
	// by the model catalog entry (qwen3.6:35b -> prov-a).
	rt.NodeStore.Add(&edgenode.NodeRecord{
		ID:      "node-pool-dispatch",
		Alias:   "pool-alias",
		Runtime: config.RuntimeConf{Concurrency: 4},
		Adapters: config.AdaptersConf{
			OpenAICompatInstances: []config.OpenAICompatInstanceConf{
				{Name: "vllm-gpu", Enabled: true, Endpoint: "http://127.0.0.1:8000/v1"},
			},
		},
		Providers: []config.NodeProviderConf{
			{
				ID:       "prov-a",
				Type:     "vllm",
				Category: "api",
				Adapter:  "vllm-gpu",
				Models:   []string{"served-qwen"},
				Health:   "available",
				Capacity: 2,
			},
		},
	})

	// Register the fake node into the runtime registry.
	rt.Registry.Register(&edgenode.NodeEntry{
		NodeID:         "node-pool-dispatch",
		LifecycleState: edgenode.LifecycleConnected,
		Client:         edgeClient,
	})

	// Submit RunRequest via the runtime Service before refresh.
	dispatchAndCapture := func() (*iop.RunRequest, error) {
		capturedMu.Lock()
		capturedReq = nil
		capturedMu.Unlock()

		result, err := rt.Service.SubmitRun(context.Background(), edgeservice.SubmitRunRequest{
			RunID:         "dispatch-test-" + time.Now().Format("150405"),
			ModelGroupKey: "qwen3.6:35b",
			ProviderPool:  true,
			Background:    true,
		})
		if err != nil {
			return nil, err
		}
		// Drain the result handle to release resources.
		if result != nil {
			result.Close()
		}

		// Wait for the fake node to receive the request.
		time.Sleep(100 * time.Millisecond)

		capturedMu.Lock()
		defer capturedMu.Unlock()
		return capturedReq, nil
	}

	// --- Before rejected refresh: dispatch must succeed ---
	beforeReq, err := dispatchAndCapture()
	if err != nil {
		t.Fatalf("dispatch before refresh: %v", err)
	}
	if beforeReq == nil {
		t.Fatal("expected dispatch before refresh to produce a RunRequest")
	}
	if beforeReq.GetAdapter() != "vllm-gpu" {
		t.Errorf("before refresh adapter: got %q, want %q", beforeReq.GetAdapter(), "vllm-gpu")
	}
	if beforeReq.GetTarget() != "served-qwen" {
		t.Errorf("before refresh target: got %q, want %q", beforeReq.GetTarget(), "served-qwen")
	}

	// --- Execute rejected refresh ---
	result, err := rt.RefreshConfig(context.Background(), configrefresh.Request{
		Mode:       configrefresh.ModeApply,
		ConfigPath: candidatePath,
		RequestID:  "reject-dispatch-1",
	})
	if err != nil {
		t.Fatalf("RefreshConfig: %v", err)
	}
	if result.Status != configrefresh.StatusRejected {
		t.Fatalf("expected status=rejected, got %q (summary=%s)", result.Status, result.Summary)
	}

	// --- After rejected refresh: dispatch must preserve state ---
	afterReq, err := dispatchAndCapture()
	if err != nil {
		t.Fatalf("dispatch after refresh: %v", err)
	}
	if afterReq == nil {
		t.Fatal("expected dispatch after rejected refresh to produce a RunRequest")
	}
	if afterReq.GetAdapter() != "vllm-gpu" {
		t.Errorf("after rejected refresh adapter: got %q, want %q", afterReq.GetAdapter(), "vllm-gpu")
	}
	if afterReq.GetTarget() != "served-qwen" {
		t.Errorf("after rejected refresh target: got %q, want %q", afterReq.GetTarget(), "served-qwen")
	}
}

// TestRefreshConfigNodeRuntimeConcurrencyApplied verifies the S15 boundary at
// the running-edge level: a node runtime.concurrency change is applied live
// and pushed to the connected node.
func TestRefreshConfigNodeRuntimeConcurrencyApplied(t *testing.T) {
	dir := t.TempDir()
	serverAddr := freeTCPAddr(t)

	writeCfg := func(name string, concurrency int) string {
		path := filepath.Join(dir, name)
		yaml := fmt.Sprintf(`
server:
  listen: %q
bootstrap:
  listen: "0.0.0.0:18190"
  artifact_dir: "artifacts"
logging:
  level: "error"
refresh:
  enabled: false
  listen: "127.0.0.1:0"
openai:
  listen: "0.0.0.0:18191"
a2a:
  listen: "0.0.0.0:8182"
metrics:
  port: 0
nodes:
  - id: "node-rt-1"
    alias: "rt-node"
    token: "tok-rt"
    adapters:
      cli:
        enabled: true
    runtime:
      concurrency: %d
`, serverAddr, concurrency)
		if err := os.WriteFile(path, []byte(yaml), 0o600); err != nil {
			t.Fatalf("write %s: %v", name, err)
		}
		return path
	}

	basePath := writeCfg("base.yaml", 2)
	concurrencyPath := writeCfg("concurrency.yaml", 4)

	rt, err := NewRuntime(loadServeNormalizedConfig(t, basePath))
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	if err := rt.Start(context.Background()); err != nil {
		t.Fatalf("Start: %v", err)
	}
	defer func() {
		if err := rt.Stop(); err != nil {
			t.Fatalf("Stop: %v", err)
		}
	}()

	nodeParserMap := toki.ParserMap{
		toki.TypeNameOf(&iop.RegisterResponse{}): func(b []byte) (proto.Message, error) {
			m := &iop.RegisterResponse{}
			return m, proto.Unmarshal(b, m)
		},
		toki.TypeNameOf(&iop.NodeReadyResponse{}): func(b []byte) (proto.Message, error) {
			m := &iop.NodeReadyResponse{}
			return m, proto.Unmarshal(b, m)
		},
		toki.TypeNameOf(&iop.NodeConfigRefreshRequest{}): func(b []byte) (proto.Message, error) {
			m := &iop.NodeConfigRefreshRequest{}
			return m, proto.Unmarshal(b, m)
		},
	}

	host, portStr, _ := net.SplitHostPort(serverAddr)
	port := 0
	fmt.Sscanf(portStr, "%d", &port)
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	defer cancel()

	fakeNode, err := toki.DialTcp(ctx, host, port, 30, 10, nodeParserMap)
	if err != nil {
		t.Fatalf("dial edge: %v", err)
	}
	defer fakeNode.Close()

	var capturedConcurrency int32 = -1
	var capturedMu sync.Mutex
	toki.AddRequestListenerTyped[*iop.NodeConfigRefreshRequest, *iop.NodeConfigRefreshResponse](
		&fakeNode.Communicator,
		func(req *iop.NodeConfigRefreshRequest) (*iop.NodeConfigRefreshResponse, error) {
			capturedMu.Lock()
			capturedConcurrency = req.GetConfig().GetRuntime().GetConcurrency()
			capturedMu.Unlock()
			return &iop.NodeConfigRefreshResponse{
				RequestId: req.GetRequestId(),
				Status:    iop.NodeConfigRefreshStatus_NODE_CONFIG_REFRESH_STATUS_APPLIED,
			}, nil
		},
	)

	regResp, err := toki.SendRequestTyped[*iop.RegisterRequest, *iop.RegisterResponse](
		&fakeNode.Communicator,
		&iop.RegisterRequest{Token: "tok-rt"},
		2*time.Second,
	)
	if err != nil {
		t.Fatalf("register: %v", err)
	}
	if !regResp.GetAccepted() {
		t.Fatalf("expected accepted, got reason %q", regResp.GetReason())
	}
	// Registration only claims ownership; the ready signal makes the node
	// dispatch-ready so config-refresh push targets it instead of skipping it.
	readyFakeNode(t, fakeNode, regResp.GetNodeId())

	deadline := time.Now().Add(2 * time.Second)
	for time.Now().Before(deadline) {
		if _, ok := rt.Registry.Get("node-rt-1"); ok {
			break
		}
		time.Sleep(25 * time.Millisecond)
	}
	if _, ok := rt.Registry.Get("node-rt-1"); !ok {
		t.Fatal("fake node not registered in edge registry")
	}

	// concurrency change → applied, pushed to node, node result reported.
	concResult, err := rt.RefreshConfig(context.Background(), configrefresh.Request{
		Mode:       configrefresh.ModeApply,
		ConfigPath: concurrencyPath,
		RequestID:  "rt-conc",
	})
	if err != nil {
		t.Fatalf("RefreshConfig concurrency: %v", err)
	}
	if concResult.Status != configrefresh.StatusApplied {
		t.Fatalf("expected concurrency change status=applied, got %q (summary=%s)", concResult.Status, concResult.Summary)
	}
	found := false
	for _, nr := range concResult.NodeResults {
		if nr.NodeID == "node-rt-1" {
			found = true
			if nr.Status != "applied" {
				t.Errorf("expected node status=applied, got %q (error=%q)", nr.Status, nr.Error)
			}
		}
	}
	if !found {
		t.Fatalf("node-rt-1 not found in node_results: %+v", concResult.NodeResults)
	}

	capturedMu.Lock()
	got := capturedConcurrency
	capturedMu.Unlock()
	if got != 4 {
		t.Errorf("pushed runtime concurrency: got %d, want 4", got)
	}
}