package bootstrap import ( "context" "fmt" "net" "os" "path/filepath" "sync" "sync/atomic" "testing" "time" toki "git.toki-labs.com/toki/proto-socket/go" "google.golang.org/protobuf/proto" "iop/apps/edge/internal/configrefresh" edgenode "iop/apps/edge/internal/node" edgeservice "iop/apps/edge/internal/service" "iop/packages/go/config" iop "iop/proto/gen/iop" ) // readyFakeNode drives the NodeReadyRequest/ack handshake for a fake node that // has already registered, so the edge marks its connection dispatch-ready. Fake // nodes exercising config-refresh push must call it or the push skips them as // pending. // fakeNodeHandshakeTimeout bounds test-only register/ready requests, and // fakeNodeReadyAttempts bounds retries of the idempotent ready handshake. The // reconnect verification runs -count=10 -race and can delay a fresh TCP // handshake beyond the old two-second test ceiling. Production constants are // untouched. const ( fakeNodeHandshakeTimeout = 5 * time.Second fakeNodeReadyAttempts = 4 ) func readyFakeNode(t *testing.T, fakeNode *toki.TcpClient, nodeID string) { t.Helper() var lastErr error for attempt := 1; attempt <= fakeNodeReadyAttempts; attempt++ { resp, err := toki.SendRequestTyped[*iop.NodeReadyRequest, *iop.NodeReadyResponse]( &fakeNode.Communicator, &iop.NodeReadyRequest{NodeId: nodeID}, fakeNodeHandshakeTimeout) if err != nil { lastErr = err if !fakeNode.IsAlive() { break } continue // a duplicate ready is idempotent server side, so retry is safe } if !resp.GetReady() { t.Fatalf("expected ready ack, got reason %q", resp.GetReason()) } return } t.Fatalf("ready request failed after %d attempts: %v", fakeNodeReadyAttempts, lastErr) } // TestRefreshConfigApplyIncludesNodeResults verifies SDD S12: when a refresh // apply succeeds, the result includes per-node refresh outcomes. Uses a // net.Pipe-based fake node that registers with the edge and responds applied. func TestRefreshConfigApplyIncludesNodeResults(t *testing.T) { dir := t.TempDir() serverAddr := freeTCPAddr(t) writeRefreshConfig := func(name string, capacity int) string { path := filepath.Join(dir, name) yaml := fmt.Sprintf(` server: listen: %q bootstrap: listen: "0.0.0.0:18090" artifact_dir: "artifacts" logging: level: "error" refresh: enabled: false listen: "127.0.0.1:0" openai: listen: "0.0.0.0:18091" a2a: listen: "0.0.0.0:8082" metrics: port: 0 nodes: - id: "node-refresh-1" alias: "refresh-node" token: "tok-refresh" adapters: cli: enabled: true openai_compat_instances: - name: "primary" enabled: true provider: "vllm" endpoint: "http://127.0.0.1:9000/v1" capacity: 5 max_queue: 4 queue_timeout_ms: 5000 runtime: concurrency: 7 providers: - id: "prov-r" type: "ollama" category: "local_inference" adapter: "cli" models: ["llama3"] capacity: %d max_queue: 4 queue_timeout_ms: 5000 `, serverAddr, capacity) if err := os.WriteFile(path, []byte(yaml), 0o600); err != nil { t.Fatalf("write %s: %v", name, err) } return path } basePath := writeRefreshConfig("base.yaml", 2) candidatePath := writeRefreshConfig("candidate.yaml", 8) rt, err := NewRuntime(loadServeNormalizedConfig(t, basePath)) if err != nil { t.Fatalf("NewRuntime: %v", err) } if err := rt.Start(context.Background()); err != nil { t.Fatalf("Start: %v", err) } defer func() { if err := rt.Stop(); err != nil { t.Fatalf("Stop: %v", err) } }() // Build a fake node parser that can decode NodeConfigRefreshRequest (edge push) // and RegisterResponse (registration). nodeParserMap := toki.ParserMap{ toki.TypeNameOf(&iop.RegisterResponse{}): func(b []byte) (proto.Message, error) { m := &iop.RegisterResponse{} return m, proto.Unmarshal(b, m) }, toki.TypeNameOf(&iop.NodeReadyResponse{}): func(b []byte) (proto.Message, error) { m := &iop.NodeReadyResponse{} return m, proto.Unmarshal(b, m) }, toki.TypeNameOf(&iop.NodeConfigRefreshRequest{}): func(b []byte) (proto.Message, error) { m := &iop.NodeConfigRefreshRequest{} return m, proto.Unmarshal(b, m) }, } host, portStr, _ := net.SplitHostPort(serverAddr) port := 0 fmt.Sscanf(portStr, "%d", &port) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() fakeNode, err := toki.DialTcp(ctx, host, port, 30, 10, nodeParserMap) if err != nil { t.Fatalf("dial edge: %v", err) } defer fakeNode.Close() // Fake node captures the refresh request and responds restart_required // (current Node behavior for any non-empty payload/change). var capturedRefreshReq *iop.NodeConfigRefreshRequest var capturedRefreshMu sync.Mutex toki.AddRequestListenerTyped[*iop.NodeConfigRefreshRequest, *iop.NodeConfigRefreshResponse]( &fakeNode.Communicator, func(req *iop.NodeConfigRefreshRequest) (*iop.NodeConfigRefreshResponse, error) { capturedRefreshMu.Lock() capturedRefreshReq = req capturedRefreshMu.Unlock() return &iop.NodeConfigRefreshResponse{ RequestId: req.GetRequestId(), Status: iop.NodeConfigRefreshStatus_NODE_CONFIG_REFRESH_STATUS_RESTART_REQUIRED, RestartRequiredPaths: req.GetChangedPaths(), }, nil }, ) // Register the fake node. regResp, err := toki.SendRequestTyped[*iop.RegisterRequest, *iop.RegisterResponse]( &fakeNode.Communicator, &iop.RegisterRequest{Token: "tok-refresh"}, 2*time.Second, ) if err != nil { t.Fatalf("register: %v", err) } if !regResp.GetAccepted() { t.Fatalf("expected accepted, got reason %q", regResp.GetReason()) } // Registration only claims ownership; the ready signal makes the node // dispatch-ready so config-refresh push targets it instead of skipping it. readyFakeNode(t, fakeNode, regResp.GetNodeId()) // Wait for the node to appear in the registry. deadline := time.Now().Add(2 * time.Second) for time.Now().Before(deadline) { if _, ok := rt.Registry.Get("node-refresh-1"); ok { break } time.Sleep(25 * time.Millisecond) } if _, ok := rt.Registry.Get("node-refresh-1"); !ok { t.Fatal("fake node not registered in edge registry") } result, err := rt.RefreshConfig(context.Background(), configrefresh.Request{ Mode: configrefresh.ModeApply, ConfigPath: candidatePath, RequestID: "node-result-test", }) if err != nil { t.Fatalf("RefreshConfig: %v", err) } if result.Status != configrefresh.StatusApplied { t.Fatalf("expected status=applied, got %q (summary=%s)", result.Status, result.Summary) } if len(result.NodeResults) == 0 { t.Fatal("expected node_results to be populated after apply") } found := false for _, nr := range result.NodeResults { if nr.NodeID == "node-refresh-1" { found = true if nr.Status != "restart_required" { t.Errorf("expected node status=restart_required (current behavior for non-empty payload), got %q (error=%q)", nr.Status, nr.Error) } } } if !found { t.Fatalf("node-refresh-1 not found in node_results: %+v", result.NodeResults) } // Verify request_id propagation and payload contents on the fake node side. capturedRefreshMu.Lock() captured := capturedRefreshReq capturedRefreshMu.Unlock() if captured == nil { t.Fatal("fake node never received a NodeConfigRefreshRequest") } if captured.GetRequestId() != "node-result-test" { t.Errorf("request_id: got %q, want %q", captured.GetRequestId(), "node-result-test") } // Payload must carry the node-specific config edge builds from the NodeStore // record. Note: providers[].capacity is the refresh *trigger* (it drives // changed_paths), not a NodeConfigPayload field; the payload itself carries // the node's adapters + runtime, so we assert those concrete typed fields. payload := captured.GetConfig() if payload == nil { t.Fatal("expected non-nil NodeConfigPayload in refresh request") } // Runtime tuning is pushed from the node record. if got := payload.GetRuntime().GetConcurrency(); got != 7 { t.Errorf("runtime concurrency: got %d, want 7", got) } // The configured openai_compat adapter instance must be present with its // typed fields (provider/endpoint/capacity) intact, and the CLI adapter too. var oai *iop.OpenAICompatAdapterConfig hasCLI := false for _, a := range payload.GetAdapters() { switch a.GetType() { case "openai_compat": if a.GetName() == "primary" { oai = a.GetOpenaiCompat() } case "cli": hasCLI = true } } if oai == nil { t.Fatalf("expected openai_compat adapter %q in refresh payload, got %+v", "primary", payload.GetAdapters()) } if oai.GetProvider() != "vllm" { t.Errorf("openai_compat provider: got %q, want %q", oai.GetProvider(), "vllm") } if oai.GetEndpoint() != "http://127.0.0.1:9000/v1" { t.Errorf("openai_compat endpoint: got %q, want %q", oai.GetEndpoint(), "http://127.0.0.1:9000/v1") } if oai.GetCapacity() != 5 { t.Errorf("openai_compat capacity: got %d, want 5", oai.GetCapacity()) } if !hasCLI { t.Errorf("expected cli adapter in refresh payload, got %+v", payload.GetAdapters()) } } // TestRefreshConfigApplySkipsDisconnectedConfiguredNode verifies that a node // that is in the NodeStore but not in the live registry (post-disconnect state) // appears as "skipped" in node_results alongside connected nodes. func TestRefreshConfigApplySkipsDisconnectedConfiguredNode(t *testing.T) { dir := t.TempDir() serverAddr := freeTCPAddr(t) writeConfig := func(name string, capacity int) string { path := filepath.Join(dir, name) yaml := fmt.Sprintf(` server: listen: %q bootstrap: listen: "0.0.0.0:18092" artifact_dir: "artifacts" logging: level: "error" refresh: enabled: false listen: "127.0.0.1:0" openai: listen: "0.0.0.0:18093" a2a: listen: "0.0.0.0:8083" metrics: port: 0 nodes: - id: "node-connected" alias: "conn-node" token: "tok-conn" adapters: cli: enabled: true providers: - id: "prov-skip" type: "ollama" category: "local_inference" adapter: "cli" models: ["llama3"] capacity: %d max_queue: 4 queue_timeout_ms: 5000 - id: "node-disconnected" alias: "disc-node" token: "tok-disc" adapters: cli: enabled: true `, serverAddr, capacity) if err := os.WriteFile(path, []byte(yaml), 0o600); err != nil { t.Fatalf("write %s: %v", name, err) } return path } basePath := writeConfig("base.yaml", 2) candidatePath := writeConfig("candidate.yaml", 8) rt, err := NewRuntime(loadServeNormalizedConfig(t, basePath)) if err != nil { t.Fatalf("NewRuntime: %v", err) } if err := rt.Start(context.Background()); err != nil { t.Fatalf("Start: %v", err) } defer func() { if err := rt.Stop(); err != nil { t.Fatalf("Stop: %v", err) } }() nodeParserMap := toki.ParserMap{ toki.TypeNameOf(&iop.RegisterResponse{}): func(b []byte) (proto.Message, error) { m := &iop.RegisterResponse{} return m, proto.Unmarshal(b, m) }, toki.TypeNameOf(&iop.NodeReadyResponse{}): func(b []byte) (proto.Message, error) { m := &iop.NodeReadyResponse{} return m, proto.Unmarshal(b, m) }, toki.TypeNameOf(&iop.NodeConfigRefreshRequest{}): func(b []byte) (proto.Message, error) { m := &iop.NodeConfigRefreshRequest{} return m, proto.Unmarshal(b, m) }, } host, portStr, _ := net.SplitHostPort(serverAddr) port := 0 fmt.Sscanf(portStr, "%d", &port) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() // Only connect node-connected; node-disconnected never dials in. fakeNode, err := toki.DialTcp(ctx, host, port, 30, 10, nodeParserMap) if err != nil { t.Fatalf("dial edge: %v", err) } defer fakeNode.Close() toki.AddRequestListenerTyped[*iop.NodeConfigRefreshRequest, *iop.NodeConfigRefreshResponse]( &fakeNode.Communicator, func(req *iop.NodeConfigRefreshRequest) (*iop.NodeConfigRefreshResponse, error) { return &iop.NodeConfigRefreshResponse{ RequestId: req.GetRequestId(), Status: iop.NodeConfigRefreshStatus_NODE_CONFIG_REFRESH_STATUS_RESTART_REQUIRED, RestartRequiredPaths: req.GetChangedPaths(), }, nil }, ) regResp, err := toki.SendRequestTyped[*iop.RegisterRequest, *iop.RegisterResponse]( &fakeNode.Communicator, &iop.RegisterRequest{Token: "tok-conn"}, 2*time.Second, ) if err != nil { t.Fatalf("register: %v", err) } if !regResp.GetAccepted() { t.Fatalf("expected accepted, got reason %q", regResp.GetReason()) } // Registration only claims ownership; the ready signal makes the node // dispatch-ready so config-refresh push targets it instead of skipping it. readyFakeNode(t, fakeNode, regResp.GetNodeId()) deadline := time.Now().Add(2 * time.Second) for time.Now().Before(deadline) { if _, ok := rt.Registry.Get("node-connected"); ok { break } time.Sleep(25 * time.Millisecond) } if _, ok := rt.Registry.Get("node-connected"); !ok { t.Fatal("fake node not registered in edge registry") } result, err := rt.RefreshConfig(context.Background(), configrefresh.Request{ Mode: configrefresh.ModeApply, ConfigPath: candidatePath, RequestID: "skip-test", }) if err != nil { t.Fatalf("RefreshConfig: %v", err) } if result.Status != configrefresh.StatusApplied { t.Fatalf("expected status=applied, got %q (summary=%s)", result.Status, result.Summary) } if len(result.NodeResults) != 2 { t.Fatalf("expected 2 node_results (connected + disconnected), got %d: %+v", len(result.NodeResults), result.NodeResults) } statusByID := make(map[string]string) for _, nr := range result.NodeResults { statusByID[nr.NodeID] = nr.Status } if statusByID["node-connected"] != "restart_required" { t.Errorf("node-connected: expected restart_required, got %q", statusByID["node-connected"]) } if statusByID["node-disconnected"] != "skipped" { t.Errorf("node-disconnected: expected skipped, got %q", statusByID["node-disconnected"]) } } // TestRefreshConfigApplyNoChangeSkipsNodePush verifies that an identical config // apply (no-change) does NOT push a refresh request to connected nodes, // preventing spurious restart_required responses. func TestRefreshConfigApplyNoChangeSkipsNodePush(t *testing.T) { dir := t.TempDir() serverAddr := freeTCPAddr(t) writeConfig := func(name string) string { path := filepath.Join(dir, name) yaml := fmt.Sprintf(` server: listen: %q bootstrap: listen: "0.0.0.0:18094" artifact_dir: "artifacts" logging: level: "error" refresh: enabled: false listen: "127.0.0.1:0" openai: listen: "0.0.0.0:18095" a2a: listen: "0.0.0.0:8084" metrics: port: 0 nodes: - id: "node-noc-change" alias: "noc-change-node" token: "tok-noc-change" adapters: cli: enabled: true providers: - id: "prov-nc" type: "ollama" category: "local_inference" adapter: "cli" models: ["llama3"] capacity: 2 max_queue: 4 queue_timeout_ms: 5000 `, serverAddr) if err := os.WriteFile(path, []byte(yaml), 0o600); err != nil { t.Fatalf("write %s: %v", name, err) } return path } basePath := writeConfig("base.yaml") // candidate is identical to base (same file). candidatePath := basePath rt, err := NewRuntime(loadServeNormalizedConfig(t, basePath)) if err != nil { t.Fatalf("NewRuntime: %v", err) } if err := rt.Start(context.Background()); err != nil { t.Fatalf("Start: %v", err) } defer func() { if err := rt.Start(context.Background()); err == nil { t.Fatal("expected already started error") } else { // runtime is running, stop it if err := rt.Stop(); err != nil { t.Fatalf("Stop: %v", err) } } }() // Build a fake node parser. nodeParserMap := toki.ParserMap{ toki.TypeNameOf(&iop.RegisterResponse{}): func(b []byte) (proto.Message, error) { m := &iop.RegisterResponse{} return m, proto.Unmarshal(b, m) }, toki.TypeNameOf(&iop.NodeReadyResponse{}): func(b []byte) (proto.Message, error) { m := &iop.NodeReadyResponse{} return m, proto.Unmarshal(b, m) }, toki.TypeNameOf(&iop.NodeConfigRefreshRequest{}): func(b []byte) (proto.Message, error) { m := &iop.NodeConfigRefreshRequest{} return m, proto.Unmarshal(b, m) }, } host, portStr, _ := net.SplitHostPort(serverAddr) port := 0 fmt.Sscanf(portStr, "%d", &port) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() fakeNode, err := toki.DialTcp(ctx, host, port, 30, 10, nodeParserMap) if err != nil { t.Fatalf("dial edge: %v", err) } defer fakeNode.Close() // Track whether NodeConfigRefreshRequest is received. var receivedRefresh atomic.Bool toki.AddRequestListenerTyped[*iop.NodeConfigRefreshRequest, *iop.NodeConfigRefreshResponse]( &fakeNode.Communicator, func(req *iop.NodeConfigRefreshRequest) (*iop.NodeConfigRefreshResponse, error) { receivedRefresh.Store(true) return &iop.NodeConfigRefreshResponse{ RequestId: req.GetRequestId(), Status: iop.NodeConfigRefreshStatus_NODE_CONFIG_REFRESH_STATUS_APPLIED, }, nil }, ) // Register the fake node. regResp, err := toki.SendRequestTyped[*iop.RegisterRequest, *iop.RegisterResponse]( &fakeNode.Communicator, &iop.RegisterRequest{Token: "tok-noc-change"}, 2*time.Second, ) if err != nil { t.Fatalf("register: %v", err) } if !regResp.GetAccepted() { t.Fatalf("expected accepted, got reason %q", regResp.GetReason()) } // Registration only claims ownership; the ready signal makes the node // dispatch-ready so config-refresh push targets it instead of skipping it. readyFakeNode(t, fakeNode, regResp.GetNodeId()) // Wait for the node to appear in the registry. deadline := time.Now().Add(2 * time.Second) for time.Now().Before(deadline) { if _, ok := rt.Registry.Get("node-noc-change"); ok { break } time.Sleep(25 * time.Millisecond) } if _, ok := rt.Registry.Get("node-noc-change"); !ok { t.Fatal("fake node not registered in edge registry") } // Apply identical config (no-change). result, err := rt.RefreshConfig(context.Background(), configrefresh.Request{ Mode: configrefresh.ModeApply, ConfigPath: candidatePath, RequestID: "no-change-1", }) if err != nil { t.Fatalf("RefreshConfig: %v", err) } if result.Status != configrefresh.StatusApplied { t.Fatalf("expected status=applied, got %q (summary=%s)", result.Status, result.Summary) } // No-change apply should produce no node_results (no push). if len(result.NodeResults) != 0 { t.Errorf("expected no node_results for no-change apply, got %d: %+v", len(result.NodeResults), result.NodeResults) } // The fake node must NOT have received a NodeConfigRefreshRequest. if receivedRefresh.Load() { t.Error("expected no NodeConfigRefreshRequest to be sent for no-change apply, but it was received") } } // TestRefreshRejectedPreservesProviderDispatch verifies SDD S10: a rejected // refresh must preserve provider-pool dispatch state. It uses a net.Pipe-based // fake node to capture the RunRequest sent by SubmitRun(ProviderPool=true) // before and after a rejected config refresh. func TestRefreshRejectedPreservesProviderDispatch(t *testing.T) { dir := t.TempDir() serverAddr := freeTCPAddr(t) openAIAddr := freeTCPAddr(t) basePath := filepath.Join(dir, "base.yaml") if err := os.WriteFile(basePath, []byte(providerPoolConfigYAML(serverAddr, openAIAddr, validAdapterBlock)), 0o600); err != nil { t.Fatalf("write base: %v", err) } candidatePath := filepath.Join(dir, "invalid.yaml") if err := os.WriteFile(candidatePath, []byte(providerPoolConfigYAML(serverAddr, openAIAddr, invalidAdapterBlock)), 0o600); err != nil { t.Fatalf("write candidate: %v", err) } rt, err := NewRuntime(loadServeNormalizedConfig(t, basePath)) if err != nil { t.Fatalf("NewRuntime: %v", err) } if err := rt.Start(context.Background()); err != nil { t.Fatalf("Start: %v", err) } defer func() { if err := rt.Stop(); err != nil { t.Fatalf("Stop: %v", err) } }() // --- Dispatch capture setup using net.Pipe --- edgeConn, nodeConn := net.Pipe() defer edgeConn.Close() defer nodeConn.Close() parserMap := toki.ParserMap{ toki.TypeNameOf(&iop.RunRequest{}): func(b []byte) (proto.Message, error) { m := &iop.RunRequest{} return m, proto.Unmarshal(b, m) }, } edgeClient := toki.NewTcpClient(edgeConn, 0, 0, parserMap) nodeClient := toki.NewTcpClient(nodeConn, 0, 0, parserMap) var capturedReq *iop.RunRequest var capturedMu sync.Mutex toki.AddListenerTyped[*iop.RunRequest](&nodeClient.Communicator, func(req *iop.RunRequest) { capturedMu.Lock() capturedReq = req capturedMu.Unlock() }) // Add a provider-pool node record to the NodeStore so provider-pool // resolution finds a dispatchable provider. The provider config must // match the adapter (vllm-gpu) and served target (served-qwen) expected // by the model catalog entry (qwen3.6:35b -> prov-a). rt.NodeStore.Add(&edgenode.NodeRecord{ ID: "node-pool-dispatch", Alias: "pool-alias", Runtime: config.RuntimeConf{Concurrency: 4}, Adapters: config.AdaptersConf{ OpenAICompatInstances: []config.OpenAICompatInstanceConf{ {Name: "vllm-gpu", Enabled: true, Endpoint: "http://127.0.0.1:8000/v1"}, }, }, Providers: []config.NodeProviderConf{ { ID: "prov-a", Type: "vllm", Category: "api", Adapter: "vllm-gpu", Models: []string{"served-qwen"}, Health: "available", Capacity: 2, }, }, }) // Register the fake node into the runtime registry. rt.Registry.Register(&edgenode.NodeEntry{ NodeID: "node-pool-dispatch", LifecycleState: edgenode.LifecycleConnected, Client: edgeClient, }) // Submit RunRequest via the runtime Service before refresh. dispatchAndCapture := func() (*iop.RunRequest, error) { capturedMu.Lock() capturedReq = nil capturedMu.Unlock() result, err := rt.Service.SubmitRun(context.Background(), edgeservice.SubmitRunRequest{ RunID: "dispatch-test-" + time.Now().Format("150405"), ModelGroupKey: "qwen3.6:35b", ProviderPool: true, Background: true, }) if err != nil { return nil, err } // Drain the result handle to release resources. if result != nil { result.Close() } // Wait for the fake node to receive the request. time.Sleep(100 * time.Millisecond) capturedMu.Lock() defer capturedMu.Unlock() return capturedReq, nil } // --- Before rejected refresh: dispatch must succeed --- beforeReq, err := dispatchAndCapture() if err != nil { t.Fatalf("dispatch before refresh: %v", err) } if beforeReq == nil { t.Fatal("expected dispatch before refresh to produce a RunRequest") } if beforeReq.GetAdapter() != "vllm-gpu" { t.Errorf("before refresh adapter: got %q, want %q", beforeReq.GetAdapter(), "vllm-gpu") } if beforeReq.GetTarget() != "served-qwen" { t.Errorf("before refresh target: got %q, want %q", beforeReq.GetTarget(), "served-qwen") } // --- Execute rejected refresh --- result, err := rt.RefreshConfig(context.Background(), configrefresh.Request{ Mode: configrefresh.ModeApply, ConfigPath: candidatePath, RequestID: "reject-dispatch-1", }) if err != nil { t.Fatalf("RefreshConfig: %v", err) } if result.Status != configrefresh.StatusRejected { t.Fatalf("expected status=rejected, got %q (summary=%s)", result.Status, result.Summary) } // --- After rejected refresh: dispatch must preserve state --- afterReq, err := dispatchAndCapture() if err != nil { t.Fatalf("dispatch after refresh: %v", err) } if afterReq == nil { t.Fatal("expected dispatch after rejected refresh to produce a RunRequest") } if afterReq.GetAdapter() != "vllm-gpu" { t.Errorf("after rejected refresh adapter: got %q, want %q", afterReq.GetAdapter(), "vllm-gpu") } if afterReq.GetTarget() != "served-qwen" { t.Errorf("after rejected refresh target: got %q, want %q", afterReq.GetTarget(), "served-qwen") } } // TestRefreshConfigNodeRuntimeConcurrencyApplied verifies the S15 boundary at // the running-edge level: a node runtime.concurrency change is applied live // and pushed to the connected node. func TestRefreshConfigNodeRuntimeConcurrencyApplied(t *testing.T) { dir := t.TempDir() serverAddr := freeTCPAddr(t) writeCfg := func(name string, concurrency int) string { path := filepath.Join(dir, name) yaml := fmt.Sprintf(` server: listen: %q bootstrap: listen: "0.0.0.0:18190" artifact_dir: "artifacts" logging: level: "error" refresh: enabled: false listen: "127.0.0.1:0" openai: listen: "0.0.0.0:18191" a2a: listen: "0.0.0.0:8182" metrics: port: 0 nodes: - id: "node-rt-1" alias: "rt-node" token: "tok-rt" adapters: cli: enabled: true runtime: concurrency: %d `, serverAddr, concurrency) if err := os.WriteFile(path, []byte(yaml), 0o600); err != nil { t.Fatalf("write %s: %v", name, err) } return path } basePath := writeCfg("base.yaml", 2) concurrencyPath := writeCfg("concurrency.yaml", 4) rt, err := NewRuntime(loadServeNormalizedConfig(t, basePath)) if err != nil { t.Fatalf("NewRuntime: %v", err) } if err := rt.Start(context.Background()); err != nil { t.Fatalf("Start: %v", err) } defer func() { if err := rt.Stop(); err != nil { t.Fatalf("Stop: %v", err) } }() nodeParserMap := toki.ParserMap{ toki.TypeNameOf(&iop.RegisterResponse{}): func(b []byte) (proto.Message, error) { m := &iop.RegisterResponse{} return m, proto.Unmarshal(b, m) }, toki.TypeNameOf(&iop.NodeReadyResponse{}): func(b []byte) (proto.Message, error) { m := &iop.NodeReadyResponse{} return m, proto.Unmarshal(b, m) }, toki.TypeNameOf(&iop.NodeConfigRefreshRequest{}): func(b []byte) (proto.Message, error) { m := &iop.NodeConfigRefreshRequest{} return m, proto.Unmarshal(b, m) }, } host, portStr, _ := net.SplitHostPort(serverAddr) port := 0 fmt.Sscanf(portStr, "%d", &port) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() fakeNode, err := toki.DialTcp(ctx, host, port, 30, 10, nodeParserMap) if err != nil { t.Fatalf("dial edge: %v", err) } defer fakeNode.Close() var capturedConcurrency int32 = -1 var capturedMu sync.Mutex toki.AddRequestListenerTyped[*iop.NodeConfigRefreshRequest, *iop.NodeConfigRefreshResponse]( &fakeNode.Communicator, func(req *iop.NodeConfigRefreshRequest) (*iop.NodeConfigRefreshResponse, error) { capturedMu.Lock() capturedConcurrency = req.GetConfig().GetRuntime().GetConcurrency() capturedMu.Unlock() return &iop.NodeConfigRefreshResponse{ RequestId: req.GetRequestId(), Status: iop.NodeConfigRefreshStatus_NODE_CONFIG_REFRESH_STATUS_APPLIED, }, nil }, ) regResp, err := toki.SendRequestTyped[*iop.RegisterRequest, *iop.RegisterResponse]( &fakeNode.Communicator, &iop.RegisterRequest{Token: "tok-rt"}, 2*time.Second, ) if err != nil { t.Fatalf("register: %v", err) } if !regResp.GetAccepted() { t.Fatalf("expected accepted, got reason %q", regResp.GetReason()) } // Registration only claims ownership; the ready signal makes the node // dispatch-ready so config-refresh push targets it instead of skipping it. readyFakeNode(t, fakeNode, regResp.GetNodeId()) deadline := time.Now().Add(2 * time.Second) for time.Now().Before(deadline) { if _, ok := rt.Registry.Get("node-rt-1"); ok { break } time.Sleep(25 * time.Millisecond) } if _, ok := rt.Registry.Get("node-rt-1"); !ok { t.Fatal("fake node not registered in edge registry") } // concurrency change → applied, pushed to node, node result reported. concResult, err := rt.RefreshConfig(context.Background(), configrefresh.Request{ Mode: configrefresh.ModeApply, ConfigPath: concurrencyPath, RequestID: "rt-conc", }) if err != nil { t.Fatalf("RefreshConfig concurrency: %v", err) } if concResult.Status != configrefresh.StatusApplied { t.Fatalf("expected concurrency change status=applied, got %q (summary=%s)", concResult.Status, concResult.Summary) } found := false for _, nr := range concResult.NodeResults { if nr.NodeID == "node-rt-1" { found = true if nr.Status != "applied" { t.Errorf("expected node status=applied, got %q (error=%q)", nr.Status, nr.Error) } } } if !found { t.Fatalf("node-rt-1 not found in node_results: %+v", concResult.NodeResults) } capturedMu.Lock() got := capturedConcurrency capturedMu.Unlock() if got != 4 { t.Errorf("pushed runtime concurrency: got %d, want 4", got) } }