iop/apps/edge/internal/bootstrap/runtime_refresh_test.go

package bootstrap

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"sync"
	"testing"
	"time"

	toki "git.toki-labs.com/toki/proto-socket/go"
	"google.golang.org/protobuf/proto"
	"iop/apps/edge/internal/configrefresh"
	edgenode "iop/apps/edge/internal/node"
	edgeservice "iop/apps/edge/internal/service"
	"iop/packages/go/config"
	iop "iop/proto/gen/iop"
)

// TestRefreshAdminDryRunAndApplyReachRunningRuntime verifies S06: a running
// Edge process can receive dry-run and apply refresh requests via the admin API
// and returns mode-specific results.
func TestRefreshAdminDryRunAndApplyReachRunningRuntime(t *testing.T) {
	dir := t.TempDir()

	serverAddr := freeTCPAddr(t)

	// Write base and candidate YAMLs with explicit immutable field values so
	// that config.LoadEdge produces configs with identical defaults on both sides.
	// Only capacity differs between base and candidate.
	baseYAML := fmt.Sprintf(`
server:
  listen: %q
bootstrap:
  listen: "0.0.0.0:18080"
  artifact_dir: "artifacts"
logging:
  level: "error"
refresh:
  enabled: true
  listen: "127.0.0.1:0"
openai:
  listen: "0.0.0.0:18081"
a2a:
  listen: "0.0.0.0:8081"
metrics:
  port: 0
nodes:
  - id: "node-1"
    alias: "n1"
    token: "tok-1"
    adapters:
      cli:
        enabled: true
    providers:
      - id: "prov-a"
        type: "ollama"
        category: "local_inference"
        adapter: "cli"
        models: ["llama3.1"]
        capacity: 2
        max_queue: 4
        queue_timeout_ms: 5000
`, serverAddr)

	candidateYAML := fmt.Sprintf(`
server:
  listen: %q
bootstrap:
  listen: "0.0.0.0:18080"
  artifact_dir: "artifacts"
logging:
  level: "error"
refresh:
  enabled: true
  listen: "127.0.0.1:0"
openai:
  listen: "0.0.0.0:18081"
a2a:
  listen: "0.0.0.0:8081"
metrics:
  port: 0
nodes:
  - id: "node-1"
    alias: "n1"
    token: "tok-1"
    adapters:
      cli:
        enabled: true
    providers:
      - id: "prov-a"
        type: "ollama"
        category: "local_inference"
        adapter: "cli"
        models: ["llama3.1"]
        capacity: 8
        max_queue: 4
        queue_timeout_ms: 5000
`, serverAddr)

	basePath := filepath.Join(dir, "base.yaml")
	if err := os.WriteFile(basePath, []byte(baseYAML), 0o600); err != nil {
		t.Fatalf("write base: %v", err)
	}
	candidatePath := filepath.Join(dir, "candidate.yaml")
	if err := os.WriteFile(candidatePath, []byte(candidateYAML), 0o600); err != nil {
		t.Fatalf("write candidate: %v", err)
	}

	currentCfg, err := config.LoadEdge(basePath)
	if err != nil {
		t.Fatalf("load base config: %v", err)
	}
	// Apply serve-equivalent normalization so the current config matches
	// what iop-edge serve would hold after loadRuntimeConfig.
	currentCfg.Logging.Path = serveNormalizedLogPath()
	currentCfg.Bootstrap.ArtifactDir = serveNormalizedArtifactDir(currentCfg.Bootstrap.ArtifactDir)
	// Override fields that don't need to match between runtime and candidate.
	currentCfg.Refresh.Enabled = true
	currentCfg.Refresh.Listen = "127.0.0.1:0"
	currentCfg.Logging.Level = "error"

	rt, err := NewRuntime(currentCfg)
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	ctx := context.Background()
	if err := rt.Start(ctx); err != nil {
		t.Fatalf("Start: %v", err)
	}
	defer func() {
		if err := rt.Stop(); err != nil {
			t.Fatalf("Stop: %v", err)
		}
	}()

	adminAddr := rt.RefreshAdmin.Addr()
	if adminAddr == "" {
		t.Fatal("refresh admin addr is empty after Start")
	}
	adminURL := "http://" + adminAddr + "/refresh"

	client := &http.Client{Timeout: 5 * time.Second}

	postRefresh := func(t *testing.T, mode configrefresh.Mode) configrefresh.Result {
		t.Helper()
		body, _ := json.Marshal(configrefresh.Request{
			Mode:       mode,
			ConfigPath: candidatePath,
			RequestID:  string(mode) + "-req",
		})
		resp, err := client.Post(adminURL, "application/json", bytes.NewReader(body))
		if err != nil {
			t.Fatalf("POST /refresh mode=%s: %v", mode, err)
		}
		defer resp.Body.Close()
		var result configrefresh.Result
		if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
			t.Fatalf("decode response: %v", err)
		}
		return result
	}

	// dry-run: process is reached, result has mode=dry_run.
	dryResult := postRefresh(t, configrefresh.ModeDryRun)
	if dryResult.Mode != configrefresh.ModeDryRun {
		t.Errorf("dry-run: expected mode=%q, got %q", configrefresh.ModeDryRun, dryResult.Mode)
	}
	if dryResult.Status != configrefresh.StatusApplied {
		t.Errorf("dry-run: expected status=%q (capacity change), got %q", configrefresh.StatusApplied, dryResult.Status)
	}

	// apply: process is reached, result has mode=apply and status=applied.
	applyResult := postRefresh(t, configrefresh.ModeApply)
	if applyResult.Mode != configrefresh.ModeApply {
		t.Errorf("apply: expected mode=%q, got %q", configrefresh.ModeApply, applyResult.Mode)
	}
	if applyResult.Status != configrefresh.StatusApplied {
		t.Errorf("apply: expected status=%q, got %q", configrefresh.StatusApplied, applyResult.Status)
	}

	// verify dry-run and apply have distinct observable results: the apply
	// committed the capacity change, so the next dry-run shows no changes.
	dryResult2 := postRefresh(t, configrefresh.ModeDryRun)
	if len(dryResult2.Changes) != 0 {
		t.Errorf("after apply, expected no diff changes on second dry-run, got %+v", dryResult2.Changes)
	}
}

func TestRefreshApplyConcurrentRuntimeReaders(t *testing.T) {
	dir := t.TempDir()
	serverAddr := freeTCPAddr(t)
	openAIAddr := freeTCPAddr(t)

	writeConfig := func(t *testing.T, name string, capacity int, displayName string) string {
		t.Helper()
		path := filepath.Join(dir, name)
		yaml := fmt.Sprintf(`
server:
  listen: %q
bootstrap:
  listen: "0.0.0.0:18080"
  artifact_dir: "artifacts"
logging:
  level: "error"
refresh:
  enabled: false
  listen: "127.0.0.1:19093"
openai:
  enabled: true
  listen: %q
  provider_id: "test-provider"
  adapter: "openai_compat"
  target: ""
a2a:
  listen: "0.0.0.0:8081"
metrics:
  port: 0
models:
  - id: "qwen3.6:35b"
    display_name: %q
    providers:
      prov-a: "served-qwen"
nodes:
  - id: "node-1"
    alias: "n1"
    token: "tok-1"
    adapters:
      openai_compat_instances:
        - name: "vllm-gpu"
          enabled: true
          provider: "vllm"
          endpoint: "http://127.0.0.1:8000/v1"
    providers:
      - id: "prov-a"
        type: "vllm"
        category: "api"
        adapter: "vllm-gpu"
        models: ["served-qwen"]
        health: "available"
        capacity: %d
        max_queue: 4
        queue_timeout_ms: 5000
`, serverAddr, openAIAddr, displayName, capacity)
		if err := os.WriteFile(path, []byte(yaml), 0o600); err != nil {
			t.Fatalf("write %s: %v", name, err)
		}
		return path
	}

	basePath := writeConfig(t, "base.yaml", 2, "Qwen Base")
	candidatePath := writeConfig(t, "candidate.yaml", 8, "Qwen Candidate")

	currentCfg, err := config.LoadEdge(basePath)
	if err != nil {
		t.Fatalf("load base config: %v", err)
	}
	// Apply serve-equivalent normalization.
	currentCfg.Logging.Path = serveNormalizedLogPath()
	currentCfg.Bootstrap.ArtifactDir = serveNormalizedArtifactDir(currentCfg.Bootstrap.ArtifactDir)
	rt, err := NewRuntime(currentCfg)
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	if err := rt.Start(context.Background()); err != nil {
		t.Fatalf("Start: %v", err)
	}
	defer func() {
		if err := rt.Stop(); err != nil {
			t.Fatalf("Stop: %v", err)
		}
	}()

	client := &http.Client{Timeout: 5 * time.Second}
	start := make(chan struct{})
	var wg sync.WaitGroup
	errCh := make(chan error, 3)

	wg.Add(1)
	go func() {
		defer wg.Done()
		<-start
		for i := 0; i < 50; i++ {
			result, err := rt.RefreshConfig(context.Background(), configrefresh.Request{
				Mode:       configrefresh.ModeApply,
				ConfigPath: candidatePath,
				RequestID:  fmt.Sprintf("apply-%d", i),
			})
			if err != nil {
				errCh <- err
				return
			}
			if result.Status != configrefresh.StatusApplied {
				errCh <- fmt.Errorf("refresh status=%s summary=%s changes=%+v", result.Status, result.Summary, result.Changes)
				return
			}
		}
	}()

	wg.Add(1)
	go func() {
		defer wg.Done()
		<-start
		for i := 0; i < 100; i++ {
			_ = rt.Service.ListNodeSnapshots()
		}
	}()

	wg.Add(1)
	go func() {
		defer wg.Done()
		<-start
		url := "http://" + openAIAddr + "/v1/models"
		for i := 0; i < 100; i++ {
			resp, err := client.Get(url)
			if err != nil {
				errCh <- err
				return
			}
			resp.Body.Close()
			if resp.StatusCode != http.StatusOK {
				errCh <- fmt.Errorf("GET /v1/models status=%d", resp.StatusCode)
				return
			}
		}
	}()

	close(start)
	wg.Wait()
	close(errCh)
	for err := range errCh {
		if err != nil {
			t.Fatalf("concurrent refresh reader failed: %v", err)
		}
	}
}

// TestRefreshRejectedPreservesModels verifies SDD S10: an invalid candidate
// apply is rejected without mutating the running runtime config or the
// /v1/models surface. Uses stable model ID comparison to avoid timestamp flakiness.
func TestRefreshRejectedPreservesModels(t *testing.T) {
	dir := t.TempDir()
	serverAddr := freeTCPAddr(t)
	openAIAddr := freeTCPAddr(t)

	basePath := filepath.Join(dir, "base.yaml")
	if err := os.WriteFile(basePath, []byte(providerPoolConfigYAML(serverAddr, openAIAddr, validAdapterBlock)), 0o600); err != nil {
		t.Fatalf("write base: %v", err)
	}
	candidatePath := filepath.Join(dir, "invalid.yaml")
	if err := os.WriteFile(candidatePath, []byte(providerPoolConfigYAML(serverAddr, openAIAddr, invalidAdapterBlock)), 0o600); err != nil {
		t.Fatalf("write candidate: %v", err)
	}

	rt, err := NewRuntime(loadServeNormalizedConfig(t, basePath))
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	if err := rt.Start(context.Background()); err != nil {
		t.Fatalf("Start: %v", err)
	}
	defer func() {
		if err := rt.Stop(); err != nil {
			t.Fatalf("Stop: %v", err)
		}
	}()

	client := &http.Client{Timeout: 5 * time.Second}
	beforeIDs := modelIDs(t, client, openAIAddr)
	if len(beforeIDs) == 0 {
		t.Fatal("expected at least one model before refresh")
	}
	cfgBefore := rt.Cfg

	result, err := rt.RefreshConfig(context.Background(), configrefresh.Request{
		Mode:       configrefresh.ModeApply,
		ConfigPath: candidatePath,
		RequestID:  "reject-1",
	})
	if err != nil {
		t.Fatalf("RefreshConfig: %v", err)
	}
	if result.Status != configrefresh.StatusRejected {
		t.Fatalf("expected status=rejected for invalid candidate, got %q (summary=%s)", result.Status, result.Summary)
	}

	if rt.Cfg != cfgBefore {
		t.Fatal("runtime config snapshot mutated after rejected apply")
	}
	afterIDs := modelIDs(t, client, openAIAddr)
	if !reflect.DeepEqual(afterIDs, beforeIDs) {
		t.Fatalf("/v1/models changed after rejected apply:\nbefore=%v\nafter=%v", beforeIDs, afterIDs)
	}
}

// TestRefreshApplyDoesNotMutateOnRestartRequired verifies that a candidate with
// a restart_required change leaves the running runtime snapshot and /v1/models
// untouched. Uses stable model ID comparison.
func TestRefreshApplyDoesNotMutateOnRestartRequired(t *testing.T) {
	dir := t.TempDir()
	serverAddr := freeTCPAddr(t)
	openAIAddr := freeTCPAddr(t)

	basePath := filepath.Join(dir, "base.yaml")
	if err := os.WriteFile(basePath, []byte(providerPoolConfigYAML(serverAddr, openAIAddr, validAdapterBlock)), 0o600); err != nil {
		t.Fatalf("write base: %v", err)
	}
	// Candidate changes server.listen, which is classified restart_required.
	candidateYAML := providerPoolConfigYAML(freeTCPAddr(t), openAIAddr, validAdapterBlock)
	candidatePath := filepath.Join(dir, "restart.yaml")
	if err := os.WriteFile(candidatePath, []byte(candidateYAML), 0o600); err != nil {
		t.Fatalf("write candidate: %v", err)
	}

	rt, err := NewRuntime(loadServeNormalizedConfig(t, basePath))
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	if err := rt.Start(context.Background()); err != nil {
		t.Fatalf("Start: %v", err)
	}
	defer func() {
		if err := rt.Stop(); err != nil {
			t.Fatalf("Stop: %v", err)
		}
	}()

	client := &http.Client{Timeout: 5 * time.Second}
	beforeIDs := modelIDs(t, client, openAIAddr)
	if len(beforeIDs) == 0 {
		t.Fatal("expected at least one model before refresh")
	}
	cfgBefore := rt.Cfg

	result, err := rt.RefreshConfig(context.Background(), configrefresh.Request{
		Mode:       configrefresh.ModeApply,
		ConfigPath: candidatePath,
		RequestID:  "restart-1",
	})
	if err != nil {
		t.Fatalf("RefreshConfig: %v", err)
	}
	if result.Status != configrefresh.StatusRestartRequired {
		t.Fatalf("expected status=restart_required, got %q (summary=%s)", result.Status, result.Summary)
	}
	if len(result.RestartRequiredPaths) == 0 {
		t.Fatal("expected restart_required_paths to be reported")
	}
	if rt.Cfg != cfgBefore {
		t.Fatal("runtime config snapshot mutated after restart_required apply")
	}
	afterIDs := modelIDs(t, client, openAIAddr)
	if !reflect.DeepEqual(afterIDs, beforeIDs) {
		t.Fatalf("/v1/models changed after restart_required apply:\nbefore=%v\nafter=%v", beforeIDs, afterIDs)
	}
}

// TestRefreshAPIReportIncludesChangedItems verifies SDD S11: the refresh API
// response exposes the changed provider/model items so an operator can confirm
// what a refresh touched.
func TestRefreshAPIReportIncludesChangedItems(t *testing.T) {
	dir := t.TempDir()
	serverAddr := freeTCPAddr(t)
	openAIAddr := freeTCPAddr(t)

	basePath := filepath.Join(dir, "base.yaml")
	if err := os.WriteFile(basePath, []byte(providerPoolConfigYAML(serverAddr, openAIAddr, validAdapterBlock)), 0o600); err != nil {
		t.Fatalf("write base: %v", err)
	}
	// Candidate raises provider capacity (applied) and renames the model display
	// name (applied), so both a provider and a model item change.
	candidateYAML := strings.Replace(
		strings.Replace(providerPoolConfigYAML(serverAddr, openAIAddr, validAdapterBlock), "capacity: 2", "capacity: 8", 1),
		`display_name: "Qwen"`, `display_name: "Qwen Refreshed"`, 1)
	candidatePath := filepath.Join(dir, "candidate.yaml")
	if err := os.WriteFile(candidatePath, []byte(candidateYAML), 0o600); err != nil {
		t.Fatalf("write candidate: %v", err)
	}

	rt, err := NewRuntime(loadServeNormalizedConfig(t, basePath))
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}
	if err := rt.Start(context.Background()); err != nil {
		t.Fatalf("Start: %v", err)
	}
	defer func() {
		if err := rt.Stop(); err != nil {
			t.Fatalf("Stop: %v", err)
		}
	}()

	adminURL := "http://" + rt.RefreshAdmin.Addr() + "/refresh"
	client := &http.Client{Timeout: 5 * time.Second}
	body, _ := json.Marshal(configrefresh.Request{
		Mode:       configrefresh.ModeDryRun,
		ConfigPath: candidatePath,
		RequestID:  "report-1",
	})
	resp, err := client.Post(adminURL, "application/json", bytes.NewReader(body))
	if err != nil {
		t.Fatalf("POST /refresh: %v", err)
	}
	defer resp.Body.Close()
	var result configrefresh.Result
	if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
		t.Fatalf("decode response: %v", err)
	}

	if result.Status != configrefresh.StatusApplied {
		t.Fatalf("expected status=applied, got %q (summary=%s)", result.Status, result.Summary)
	}
	if !containsString(result.ChangedProviders, "prov-a") {
		t.Errorf("expected changed_providers to include prov-a, got %v", result.ChangedProviders)
	}
	if !containsString(result.ChangedModels, "qwen3.6:35b") {
		t.Errorf("expected changed_models to include qwen3.6:35b, got %v", result.ChangedModels)
	}
}

// TestRefreshProviderPoolPolicy verifies that NewRuntime initial assembly and
// RefreshConfig apply both propagate the root provider-pool policy into the
// queue manager. The initial SetRuntimeConfig call in NewRuntime must use the
// same atomic handoff as the apply path, so the root max_queue and timeout
// are observable on the queue immediately after NewRuntime returns.
func TestRefreshProviderPoolPolicy(t *testing.T) {
	dir := t.TempDir()
	serverAddr := freeTCPAddr(t)

	cfgYAML := fmt.Sprintf(`
server:
  listen: %q
bootstrap:
  listen: "0.0.0.0:18080"
  artifact_dir: "artifacts"
logging:
  level: "error"
refresh:
  enabled: false
  listen: "127.0.0.1:0"
metrics:
  port: 0
models:
  - id: "qwen3.6:35b"
    display_name: "Qwen"
    providers:
      prov-a: "served-qwen"
nodes:
  - id: "node-rp"
    alias: "n-rp"
    token: "tok-rp"
    adapters:
      openai_compat_instances:
        - name: "vllm-gpu"
          enabled: true
          provider: "vllm"
          endpoint: "http://127.0.0.1:8000/v1"
    providers:
      - id: "prov-a"
        type: "vllm"
        category: "api"
        adapter: "vllm-gpu"
        models: ["served-qwen"]
        health: "available"
        capacity: 2
        max_queue: 4
        queue_timeout_ms: 5000
`, serverAddr)

	cfgPath := filepath.Join(dir, "base.yaml")
	if err := os.WriteFile(cfgPath, []byte(cfgYAML), 0o600); err != nil {
		t.Fatalf("write config: %v", err)
	}

	cfg, err := config.LoadEdge(cfgPath)
	if err != nil {
		t.Fatalf("load config: %v", err)
	}
	cfg.Logging.Path = serveNormalizedLogPath()
	cfg.Bootstrap.ArtifactDir = serveNormalizedArtifactDir(cfg.Bootstrap.ArtifactDir)

	rt, err := NewRuntime(cfg)
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}

	// Initial handoff: the root policy must be propagated to the queue.
	initialPolicy := rt.Service.ProviderPoolPolicy()
	if initialPolicy.MaxQueue != 4 {
		t.Errorf("expected initial maxQueue=4, got %d", initialPolicy.MaxQueue)
	}
	if initialPolicy.QueueTimeoutSet != true {
		t.Errorf("expected initial queueTimeoutSet=true")
	}
	if initialPolicy.QueueTimeout != 5*time.Second {
		t.Errorf("expected initial queueTimeout=5s, got %v", initialPolicy.QueueTimeout)
	}

	// Apply a new policy via RefreshConfig and verify it replaces the root.
	candidatePath := filepath.Join(dir, "candidate.yaml")
	candidateYAML := strings.Replace(cfgYAML, "capacity: 2", "capacity: 4", 1)
	candidateYAML = strings.Replace(candidateYAML, "max_queue: 4", "max_queue: 2", 1)
	candidateYAML = strings.Replace(candidateYAML, "queue_timeout_ms: 5000", "queue_timeout_ms: 2000", 1)
	if err := os.WriteFile(candidatePath, []byte(candidateYAML), 0o600); err != nil {
		t.Fatalf("write candidate: %v", err)
	}

	result, err := rt.RefreshConfig(context.Background(), configrefresh.Request{
		Mode:       configrefresh.ModeApply,
		ConfigPath: candidatePath,
		RequestID:  "rp-apply-1",
	})
	if err != nil {
		t.Fatalf("RefreshConfig: %v", err)
	}
	if result.Status != configrefresh.StatusApplied {
		t.Fatalf("expected status=applied, got %q (summary=%s)", result.Status, result.Summary)
	}

	afterPolicy := rt.Service.ProviderPoolPolicy()
	if afterPolicy.MaxQueue != 2 {
		t.Errorf("expected maxQueue=2 after apply, got %d", afterPolicy.MaxQueue)
	}
	if afterPolicy.QueueTimeoutSet != true {
		t.Errorf("expected queueTimeoutSet=true after apply")
	}
	if afterPolicy.QueueTimeout != 2*time.Second {
		t.Errorf("expected queueTimeout=2s after apply, got %v", afterPolicy.QueueTimeout)
	}
}

// TestRefreshRuntimeSnapshotPreservesLeaseAndAppliesPolicyAtomically verifies
// that a single Runtime.RefreshConfig apply updates provider attributes
// (capacity) and the root provider-pool queue policy from the same config
// snapshot. The observable contract is: after a refresh that changes both
// capacity and policy, both reflect the new values in a single apply cycle,
// existing leases survive the shrink, and the queued waiter remains blocked
// until occupancy falls below the refreshed capacity.
func TestRefreshRuntimeSnapshotPreservesLeaseAndAppliesPolicyAtomically(t *testing.T) {
	dir := t.TempDir()
	serverAddr := freeTCPAddr(t)

	cfgYAML := fmt.Sprintf(`
server:
  listen: %q
bootstrap:
  listen: "0.0.0.0:18080"
  artifact_dir: "artifacts"
logging:
  level: "error"
refresh:
  enabled: false
  listen: "127.0.0.1:0"
metrics:
  port: 0
provider_pool:
  max_queue: 8
  queue_timeout_ms: 5000
models:
  - id: "group-atomic"
    providers:
      prov-atomic: "served-atomic"
nodes:
  - id: "node-atomic"
    alias: "n-atm"
    token: "tok-atm"
    adapters:
      openai_compat_instances:
        - name: "vllm-gpu"
          enabled: true
          provider: "vllm"
          endpoint: "http://127.0.0.1:0/v1"
    providers:
      - id: "prov-atomic"
        type: "vllm"
        category: "api"
        adapter: "vllm-gpu"
        models: ["served-atomic"]
        health: "available"
        capacity: 2
`, serverAddr)

	cfgPath := filepath.Join(dir, "base.yaml")
	if err := os.WriteFile(cfgPath, []byte(cfgYAML), 0o600); err != nil {
		t.Fatalf("write config: %v", err)
	}

	cfg, err := config.LoadEdge(cfgPath)
	if err != nil {
		t.Fatalf("load config: %v", err)
	}
	cfg.Logging.Path = serveNormalizedLogPath()
	cfg.Bootstrap.ArtifactDir = serveNormalizedArtifactDir(cfg.Bootstrap.ArtifactDir)

	rt, err := NewRuntime(cfg)
	if err != nil {
		t.Fatalf("NewRuntime: %v", err)
	}

	initialPolicy := rt.Service.ProviderPoolPolicy()
	if initialPolicy.MaxQueue != 8 {
		t.Fatalf("expected initial maxQueue=8, got %d", initialPolicy.MaxQueue)
	}
	if initialPolicy.QueueTimeout != 5*time.Second {
		t.Fatalf("expected initial queueTimeout=5s, got %v", initialPolicy.QueueTimeout)
	}

	edgeConn, nodeConn := net.Pipe()
	t.Cleanup(func() {
		edgeConn.Close()
		nodeConn.Close()
	})
	parserMap := toki.ParserMap{
		toki.TypeNameOf(&iop.RunRequest{}): func(b []byte) (proto.Message, error) {
			m := &iop.RunRequest{}
			return m, proto.Unmarshal(b, m)
		},
		toki.TypeNameOf(&iop.NodeConfigRefreshRequest{}): func(b []byte) (proto.Message, error) {
			m := &iop.NodeConfigRefreshRequest{}
			return m, proto.Unmarshal(b, m)
		},
		toki.TypeNameOf(&iop.NodeConfigRefreshResponse{}): func(b []byte) (proto.Message, error) {
			m := &iop.NodeConfigRefreshResponse{}
			return m, proto.Unmarshal(b, m)
		},
	}
	edgeClient := toki.NewTcpClient(edgeConn, 0, 0, parserMap)
	nodeClient := toki.NewTcpClient(nodeConn, 0, 0, parserMap)
	toki.AddListenerTyped[*iop.RunRequest](&nodeClient.Communicator, func(*iop.RunRequest) {})
	toki.AddRequestListenerTyped[*iop.NodeConfigRefreshRequest, *iop.NodeConfigRefreshResponse](
		&nodeClient.Communicator,
		func(req *iop.NodeConfigRefreshRequest) (*iop.NodeConfigRefreshResponse, error) {
			return &iop.NodeConfigRefreshResponse{
				RequestId: req.GetRequestId(),
				Status:    iop.NodeConfigRefreshStatus_NODE_CONFIG_REFRESH_STATUS_APPLIED,
			}, nil
		},
	)
	rt.Registry.Register(&edgenode.NodeEntry{
		NodeID:         "node-atomic",
		LifecycleState: edgenode.LifecycleConnected,
		Client:         edgeClient,
	})

	submit := func(runID string) edgeservice.RunResult {
		t.Helper()
		run, submitErr := rt.Service.SubmitRun(t.Context(), edgeservice.SubmitRunRequest{
			RunID:         runID,
			ModelGroupKey: "group-atomic",
			ProviderPool:  true,
			Background:    true,
		})
		if submitErr != nil {
			t.Fatalf("SubmitRun %q: %v", runID, submitErr)
		}
		return run
	}
	held1 := submit("atomic-held-1")
	held2 := submit("atomic-held-2")
	type submitResult struct {
		run edgeservice.RunResult
		err error
	}
	waiterCh := make(chan submitResult, 1)
	go func() {
		run, submitErr := rt.Service.SubmitRun(t.Context(), edgeservice.SubmitRunRequest{
			RunID:         "atomic-waiter",
			ModelGroupKey: "group-atomic",
			ProviderPool:  true,
			Background:    true,
		})
		waiterCh <- submitResult{run: run, err: submitErr}
	}()

	providerSnapshot := func() *iop.ProviderSnapshot {
		t.Helper()
		for _, node := range rt.Service.ListNodeSnapshots() {
			if node.NodeID != "node-atomic" {
				continue
			}
			for _, provider := range node.ProviderSnapshots {
				if provider.GetId() == "prov-atomic" {
					return provider
				}
			}
		}
		t.Fatal("provider prov-atomic snapshot not found")
		return nil
	}
	queueDeadline := time.Now().Add(2 * time.Second)
	for time.Now().Before(queueDeadline) {
		if providerSnapshot().GetQueued() == 1 {
			break
		}
		time.Sleep(time.Millisecond)
	}
	beforeRefresh := providerSnapshot()
	if beforeRefresh.GetInFlight() != 2 || beforeRefresh.GetQueued() != 1 {
		t.Fatalf("expected pre-refresh inFlight=2 queued=1, got inFlight=%d queued=%d", beforeRefresh.GetInFlight(), beforeRefresh.GetQueued())
	}

	// Apply a candidate that shrinks capacity AND changes policy atomically.
	candidatePath := filepath.Join(dir, "candidate.yaml")
	candidateYAML := strings.Replace(cfgYAML, "capacity: 2", "capacity: 1", 1)
	candidateYAML = strings.Replace(candidateYAML, "max_queue: 8", "max_queue: 4", 1)
	candidateYAML = strings.Replace(candidateYAML, "queue_timeout_ms: 5000", "queue_timeout_ms: 1000", 1)
	if err := os.WriteFile(candidatePath, []byte(candidateYAML), 0o600); err != nil {
		t.Fatalf("write candidate: %v", err)
	}

	result, err := rt.RefreshConfig(context.Background(), configrefresh.Request{
		Mode:       configrefresh.ModeApply,
		ConfigPath: candidatePath,
		RequestID:  "atomic-apply-1",
	})
	if err != nil {
		t.Fatalf("RefreshConfig: %v", err)
	}
	if result.Status != configrefresh.StatusApplied {
		t.Fatalf("expected status=applied, got %q", result.Status)
	}

	// Verify the policy snapshot reflects the new config.
	afterPolicy := rt.Service.ProviderPoolPolicy()
	if afterPolicy.MaxQueue != 4 {
		t.Errorf("expected maxQueue=4 after atomic apply, got %d", afterPolicy.MaxQueue)
	}
	if afterPolicy.QueueTimeout != 1*time.Second {
		t.Errorf("expected queueTimeout=1s after atomic apply, got %v", afterPolicy.QueueTimeout)
	}
	afterRefresh := providerSnapshot()
	if afterRefresh.GetCapacity() != 1 || afterRefresh.GetInFlight() != 2 || afterRefresh.GetQueued() != 1 {
		t.Fatalf(
			"expected atomic snapshot capacity=1 inFlight=2 queued=1, got capacity=%d inFlight=%d queued=%d",
			afterRefresh.GetCapacity(), afterRefresh.GetInFlight(), afterRefresh.GetQueued(),
		)
	}

	// Verify the node store reflects the new capacity.
	store := rt.Service.NodeStore()
	records := store.All()
	if len(records) != 1 {
		t.Fatalf("expected 1 node record, got %d", len(records))
	}
	rec := records[0]
	found := false
	for _, p := range rec.Providers {
		if p.ID == "prov-atomic" {
			if p.Capacity != 1 {
				t.Errorf("expected capacity=1 after atomic apply, got %d", p.Capacity)
			}
			found = true
		}
	}
	if !found {
		t.Fatal("provider prov-atomic not found in node record")
	}

	// The capacity shrink preserves both leases and blocks the waiter until
	// occupancy falls below the new capacity.
	rt.Service.HandleRunLifecycleEvent(&iop.RunEvent{RunId: held1.Dispatch().RunID, Type: "complete"})
	held1.Close()
	afterFirstRelease := providerSnapshot()
	if afterFirstRelease.GetInFlight() != 1 || afterFirstRelease.GetQueued() != 1 {
		t.Fatalf(
			"expected waiter blocked at shrunken capacity after one release, got inFlight=%d queued=%d",
			afterFirstRelease.GetInFlight(), afterFirstRelease.GetQueued(),
		)
	}
	rt.Service.HandleRunLifecycleEvent(&iop.RunEvent{RunId: held2.Dispatch().RunID, Type: "complete"})
	held2.Close()

	var waiter edgeservice.RunResult
	select {
	case got := <-waiterCh:
		if got.err != nil {
			t.Fatalf("queued SubmitRun after second release: %v", got.err)
		}
		waiter = got.run
	case <-time.After(2 * time.Second):
		t.Fatal("queued waiter was not dispatched after occupancy fell below refreshed capacity")
	}
	rt.Service.HandleRunLifecycleEvent(&iop.RunEvent{RunId: waiter.Dispatch().RunID, Type: "complete"})
	waiter.Close()
	finalSnapshot := providerSnapshot()
	if finalSnapshot.GetInFlight() != 0 || finalSnapshot.GetQueued() != 0 {
		t.Errorf("expected final drain, got inFlight=%d queued=%d", finalSnapshot.GetInFlight(), finalSnapshot.GetQueued())
	}

	finalPolicy := rt.Service.ProviderPoolPolicy()
	if finalPolicy.MaxQueue != 4 || finalPolicy.QueueTimeout != 1*time.Second {
		t.Errorf("final snapshot inconsistent: maxQueue=%d queueTimeout=%v", finalPolicy.MaxQueue, finalPolicy.QueueTimeout)
	}
}