iop/apps/edge/internal/service/provider_scheduling_test.go

package service

import (
	"context"
	"errors"
	"reflect"
	"testing"
	"time"

	edgenode "iop/apps/edge/internal/node"
	"iop/packages/go/config"
)

// TestModelQueueUsesProviderCapacity verifies that the capacity supplied in the
// candidateNode (derived from adapter config) overrides the default of 1.
func TestModelQueueUsesProviderCapacity(t *testing.T) {
	entry := &edgenode.NodeEntry{NodeID: "node-pc1"}
	// Provider capacity = 2 (two concurrent slots on this node).
	cands := []candidateNode{{entry: entry, capacity: 2}}
	defPolicy := groupPolicy{}

	m := newModelQueueManager(nil)

	// First admit: inflight=0 < cap=2 → dispatched immediately.
	n1, err := m.admit(context.Background(), "g-pc", "", "", cands, defPolicy, nil, false, false)
	if err != nil || n1 == nil {
		t.Fatalf("first admit: %v", err)
	}

	// Second admit: inflight=1 < cap=2 → still dispatched (not queued).
	n2, err := m.admit(context.Background(), "g-pc", "", "", cands, defPolicy, nil, false, false)
	if err != nil || n2 == nil {
		t.Fatalf("second admit (capacity=2 should allow): %v", err)
	}

	// Third request must queue because inflight=2 == cap=2.
	item := &queueItem{
		candidates: cands,
		waitCh:     make(chan admitResult, 1),
		deadline:   time.Now().Add(200 * time.Millisecond),
	}
	m.mu.Lock()
	g := m.getOrCreateGroupLocked("g-pc", groupPolicy{})
	g.queue = append(g.queue, item)
	m.mu.Unlock()

	// Releasing one slot should dispatch the queued item.
	m.releaseSlot("g-pc", "node-pc1")

	select {
	case res := <-item.waitCh:
		if res.err != nil {
			t.Fatalf("expected dispatch after slot release, got: %v", res.err)
		}
		if res.candidate == nil || res.candidate.entry.NodeID != "node-pc1" {
			t.Fatalf("unexpected candidate: %v", res.candidate)
		}
	case <-time.After(200 * time.Millisecond):
		t.Fatal("timeout: item not dispatched after slot release")
	}
}

// TestRefreshProviderQueuePolicyUpdatesExistingGroup verifies that a tightened
// queue policy (e.g. provider max_queue lowered via config refresh) propagates
// to an already-created group on the next admission, while in-flight slots are
// preserved.
func TestRefreshProviderQueuePolicyUpdatesExistingGroup(t *testing.T) {
	entry := &edgenode.NodeEntry{NodeID: "node-rp"}
	cands := []candidateNode{{entry: entry, capacity: 2}}
	m := newModelQueueManager(nil)

	// Create the group with a generous policy and take one slot.
	initial := groupPolicy{maxQueue: 8, queueTimeout: 5 * time.Second}
	if _, err := m.admit(context.Background(), "g-rp", "", "", cands, initial, nil, false, false); err != nil {
		t.Fatalf("first admit: %v", err)
	}
	m.mu.Lock()
	if got := m.groups["g-rp"].policy.maxQueue; got != 8 {
		m.mu.Unlock()
		t.Fatalf("group created with maxQueue=%d, want 8", got)
	}
	m.mu.Unlock()

	// Config refresh tightens the policy to maxQueue=1; the recomputed admit
	// carries it and must update the existing group's policy. capacity=2 means
	// this second admit is dispatched (not queued).
	refreshed := groupPolicy{maxQueue: 1, queueTimeout: 5 * time.Second}
	if _, err := m.admit(context.Background(), "g-rp", "", "", cands, refreshed, nil, false, false); err != nil {
		t.Fatalf("second admit: %v", err)
	}

	m.mu.Lock()
	g := m.groups["g-rp"]
	if g.policy.maxQueue != 1 {
		m.mu.Unlock()
		t.Fatalf("existing group policy not refreshed: maxQueue=%d, want 1", g.policy.maxQueue)
	}
	// Both slots now in-flight (cap=2). Queue one item to reach the refreshed
	// maxQueue=1 so the next admission must overflow.
	g.queue = append(g.queue, &queueItem{
		candidates: cands,
		waitCh:     make(chan admitResult, 1),
		deadline:   time.Now().Add(5 * time.Second),
	})
	m.mu.Unlock()

	if _, err := m.admit(context.Background(), "g-rp", "", "", cands, refreshed, nil, false, false); !errors.Is(err, errQueueFull) {
		t.Fatalf("expected errQueueFull under refreshed maxQueue=1, got: %v", err)
	}
}

// TestModelQueueUsesProviderQueuePolicy verifies that max_queue and
// queue_timeout from the policy parameter are respected.
func TestModelQueueUsesProviderQueuePolicy(t *testing.T) {
	t.Run("maxQueue enforced", func(t *testing.T) {
		entry := &edgenode.NodeEntry{NodeID: "node-pq1"}
		cands := []candidateNode{{entry: entry, capacity: 1}}
		// Provider policy: maxQueue=1.
		policy := groupPolicy{maxQueue: 1, queueTimeout: 5 * time.Second}

		m := newModelQueueManager(nil)

		// Fill capacity and queue one item manually.
		m.mu.Lock()
		g := m.getOrCreateGroupLocked("g-pq-max", policy)
		g.inflight["node-pq1"] = 1
		g.queue = []*queueItem{{
			candidates: cands,
			waitCh:     make(chan admitResult, 1),
			deadline:   time.Now().Add(5 * time.Second),
		}}
		m.mu.Unlock()

		// Second admit should fail: queue already at maxQueue=1.
		_, err := m.admit(context.Background(), "g-pq-max", "", "", cands, policy, nil, false, false)
		if !errors.Is(err, errQueueFull) {
			t.Fatalf("expected errQueueFull, got: %v", err)
		}
	})

	t.Run("queueTimeout enforced", func(t *testing.T) {
		entry := &edgenode.NodeEntry{NodeID: "node-pq2"}
		cands := []candidateNode{{entry: entry, capacity: 1}}
		// Provider policy: very short timeout.
		policy := groupPolicy{maxQueue: 16, queueTimeout: 20 * time.Millisecond}

		m := newModelQueueManager(nil)

		// Fill capacity.
		m.mu.Lock()
		g := m.getOrCreateGroupLocked("g-pq-to", policy)
		g.inflight["node-pq2"] = 1
		m.mu.Unlock()

		start := time.Now()
		_, err := m.admit(context.Background(), "g-pq-to", "", "", cands, policy, nil, false, false)
		elapsed := time.Since(start)

		if !errors.Is(err, errQueueTimeout) {
			t.Fatalf("expected errQueueTimeout, got: %v", err)
		}
		if elapsed < 15*time.Millisecond {
			t.Fatalf("timed out too fast: %v", elapsed)
		}
	})

	t.Run("zeroQueueTimeoutWaitsForContextCancellation", func(t *testing.T) {
		entry := &edgenode.NodeEntry{NodeID: "node-pq3"}
		cands := []candidateNode{{entry: entry, capacity: 1}}
		policy := groupPolicy{maxQueue: 16, queueTimeoutSet: true}

		m := newModelQueueManager(nil)

		m.mu.Lock()
		g := m.getOrCreateGroupLocked("g-pq-no-timeout", policy)
		g.inflight["node-pq3"] = 1
		m.mu.Unlock()

		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond)
		defer cancel()

		start := time.Now()
		_, err := m.admit(ctx, "g-pq-no-timeout", "", "", cands, policy, nil, false, false)
		elapsed := time.Since(start)

		if !errors.Is(err, context.DeadlineExceeded) {
			t.Fatalf("expected context deadline, got: %v", err)
		}
		if errors.Is(err, errQueueTimeout) {
			t.Fatalf("queue timeout should be disabled, got: %v", err)
		}
		if elapsed < 25*time.Millisecond {
			t.Fatalf("returned before context deadline: %v", elapsed)
		}
	})
}

func TestGroupPolicyFromStoreZeroQueueTimeoutDisablesTimeout(t *testing.T) {
	store := edgenode.NewNodeStore()
	store.Add(&edgenode.NodeRecord{
		ID: "node-policy-zero-timeout",
		Adapters: config.AdaptersConf{
			OpenAICompatInstances: []config.OpenAICompatInstanceConf{
				{
					Name:           "provider-zero-timeout",
					Enabled:        true,
					Capacity:       1,
					MaxQueue:       16,
					QueueTimeoutMS: 0,
				},
			},
		},
	})
	entries := []*edgenode.NodeEntry{{NodeID: "node-policy-zero-timeout"}}

	policy := groupPolicyFromStore(store, entries, "provider-zero-timeout", "")

	if policy.maxQueue != 16 {
		t.Fatalf("maxQueue: got %d, want 16", policy.maxQueue)
	}
	if !policy.queueTimeoutSet {
		t.Fatalf("queueTimeoutSet: got false, want true")
	}
	if policy.queueTimeout != 0 {
		t.Fatalf("queueTimeout: got %v, want no timeout", policy.queueTimeout)
	}
}

// TestModelQueueProviderInflightSelectionBeatsPriority verifies that in_flight
// level is the primary selection criterion: a candidate with lower in_flight is
// preferred even if it has a higher (worse) priority value.
func TestModelQueueProviderInflightSelectionBeatsPriority(t *testing.T) {
	entryA := &edgenode.NodeEntry{NodeID: "node-ip-a"}
	entryB := &edgenode.NodeEntry{NodeID: "node-ip-b"}
	// A has priority=1 (better) but inflight=3.
	// B has priority=10 (worse) but inflight=0.
	cands := []candidateNode{
		{entry: entryA, capacity: 5, priority: 1, providerID: "prov-a"},
		{entry: entryB, capacity: 5, priority: 10, providerID: "prov-b"},
	}
	m := newModelQueueManager(nil)

	// Put A at inflight=3, B at inflight=0.
	m.mu.Lock()
	g := m.getOrCreateGroupLocked("g-ip", groupPolicy{})
	g.inflight["node-ip-a:prov-a"] = 3
	m.mu.Unlock()

	// Admit should pick B (inflight=0 < inflight=3), not A.
	sel, err := m.admit(context.Background(), "g-ip", "", "", cands, groupPolicy{}, nil, false, false)
	if err != nil {
		t.Fatalf("admit: %v", err)
	}
	if sel == nil || sel.entry.NodeID != "node-ip-b" {
		t.Fatalf("expected node-ip-b (lower in_flight), got %v", sel)
	}
}

// TestModelQueueProviderFullPriorityFallsThrough verifies that priority does
// not override capacity: a full high-priority provider is skipped and the next
// available priority tier receives the request.
func TestModelQueueProviderFullPriorityFallsThrough(t *testing.T) {
	entryA := &edgenode.NodeEntry{NodeID: "node-full-a"}
	entryB := &edgenode.NodeEntry{NodeID: "node-next-b"}
	cands := []candidateNode{
		{entry: entryA, capacity: 4, priority: 1, providerID: "prov-fast"},
		{entry: entryB, capacity: 3, priority: 10, providerID: "prov-slow"},
	}
	m := newModelQueueManager(nil)

	m.mu.Lock()
	g := m.getOrCreateGroupLocked("g-full-priority", groupPolicy{})
	g.inflight["node-full-a:prov-fast"] = 4
	m.mu.Unlock()

	sel, err := m.admit(context.Background(), "g-full-priority", "", "", cands, groupPolicy{}, nil, false, false)
	if err != nil {
		t.Fatalf("admit: %v", err)
	}
	if sel == nil || sel.providerID != "prov-slow" {
		t.Fatalf("expected prov-slow after high-priority provider is full, got %v", sel)
	}
}

// TestModelQueueProviderLevelingPriorityAdmissionOrder fixes the intended
// provider-pool order for differently-sized providers: fill each in_flight
// level before moving to the next level, and use priority within the same
// level. With capacities 4/3/2 and priorities 0/1/2, the first nine admits are
// gx10, onex, mac, gx10, onex, mac, gx10, onex, gx10. The tenth has no
// available candidate until a slot is released.
func TestModelQueueProviderLevelingPriorityAdmissionOrder(t *testing.T) {
	cands := []candidateNode{
		{entry: &edgenode.NodeEntry{NodeID: "node-gx10"}, capacity: 4, priority: 0, providerID: "gx10-vllm"},
		{entry: &edgenode.NodeEntry{NodeID: "node-onex"}, capacity: 3, priority: 1, providerID: "onexplayer-lemonade"},
		{entry: &edgenode.NodeEntry{NodeID: "node-mac"}, capacity: 2, priority: 2, providerID: "mac-mlx-vllm"},
	}
	m := newModelQueueManager(nil)

	var got []string
	for i := 0; i < 9; i++ {
		sel, err := m.admit(context.Background(), "qwen3.6:35b", "", "", cands, groupPolicy{}, nil, false, false)
		if err != nil {
			t.Fatalf("admit %d: %v", i+1, err)
		}
		got = append(got, sel.providerID)
	}
	want := []string{
		"gx10-vllm",
		"onexplayer-lemonade",
		"mac-mlx-vllm",
		"gx10-vllm",
		"onexplayer-lemonade",
		"mac-mlx-vllm",
		"gx10-vllm",
		"onexplayer-lemonade",
		"gx10-vllm",
	}
	if !reflect.DeepEqual(got, want) {
		t.Fatalf("admission order mismatch:\n got: %v\nwant: %v", got, want)
	}

	m.mu.Lock()
	g := m.getOrCreateGroupLocked("qwen3.6:35b", groupPolicy{})
	if candidate := m.findAvailableNodeLocked(g, cands, false); candidate != nil {
		t.Fatalf("expected no available candidate after filling capacity, got %s", candidate.providerID)
	}
	m.mu.Unlock()
}

// TestModelQueueProviderPriorityBreaksEqualInflightTie verifies that when
// in_flight counts are equal, the candidate with the lower priority value
// is selected.
func TestModelQueueProviderPriorityBreaksEqualInflightTie(t *testing.T) {
	entryA := &edgenode.NodeEntry{NodeID: "node-pt-a"}
	entryB := &edgenode.NodeEntry{NodeID: "node-pt-b"}
	// Both have inflight=2, but A has priority=1 (better) than B (priority=5).
	cands := []candidateNode{
		{entry: entryA, capacity: 5, priority: 1, providerID: "prov-a"},
		{entry: entryB, capacity: 5, priority: 5, providerID: "prov-b"},
	}
	m := newModelQueueManager(nil)

	m.mu.Lock()
	g := m.getOrCreateGroupLocked("g-pt", groupPolicy{})
	g.inflight["node-pt-a:prov-a"] = 2
	g.inflight["node-pt-b:prov-b"] = 2
	m.mu.Unlock()

	// Admit should pick A (lower priority=1 when inflight is equal).
	sel, err := m.admit(context.Background(), "g-pt", "", "", cands, groupPolicy{}, nil, false, false)
	if err != nil {
		t.Fatalf("admit: %v", err)
	}
	if sel == nil || sel.entry.NodeID != "node-pt-a" {
		t.Fatalf("expected node-pt-a (lower priority tie-break), got %v", sel)
	}
}

// TestModelQueueProviderEqualInflightPriorityRotates verifies that when both
// in_flight and priority are equal, the rotation logic selects the next slot
// after the last selected one.
func TestModelQueueProviderEqualInflightPriorityRotates(t *testing.T) {
	entryA := &edgenode.NodeEntry{NodeID: "node-r-a"}
	entryB := &edgenode.NodeEntry{NodeID: "node-r-b"}
	// Both have inflight=0 and priority=0.
	cands := []candidateNode{
		{entry: entryA, capacity: 5, priority: 0, providerID: "prov-a"},
		{entry: entryB, capacity: 5, priority: 0, providerID: "prov-b"},
	}
	m := newModelQueueManager(nil)

	// First admit: A wins deterministically.
	sel1, err := m.admit(context.Background(), "g-r", "", "", cands, groupPolicy{}, nil, false, false)
	if err != nil || sel1 == nil {
		t.Fatalf("first admit: %v", err)
	}
	if sel1.providerID != "prov-a" {
		t.Fatalf("expected prov-a on first admit, got %q", sel1.providerID)
	}

	// Release prov-a's slot.
	m.releaseSlot("g-r", "node-r-a", "prov-a")

	// Second admit: rotation should pick B (next after last selected A).
	sel2, err := m.admit(context.Background(), "g-r", "", "", cands, groupPolicy{}, nil, false, false)
	if err != nil || sel2 == nil {
		t.Fatalf("second admit: %v", err)
	}
	if sel2.providerID != "prov-b" {
		t.Fatalf("expected prov-b on second admit (rotation), got %q", sel2.providerID)
	}

	// Release prov-b's slot.
	m.releaseSlot("g-r", "node-r-b", "prov-b")

	// Third admit: rotation should cycle back to A.
	sel3, err := m.admit(context.Background(), "g-r", "", "", cands, groupPolicy{}, nil, false, false)
	if err != nil || sel3 == nil {
		t.Fatalf("third admit: %v", err)
	}
	if sel3.providerID != "prov-a" {
		t.Fatalf("expected prov-a on third admit (rotation cycle), got %q", sel3.providerID)
	}
}

func TestModelQueueProviderEqualIdleTieRotates(t *testing.T) {
	gx10 := &edgenode.NodeEntry{NodeID: "node-gx10-idle"}
	onex := &edgenode.NodeEntry{NodeID: "node-onex-idle"}
	cands := []candidateNode{
		{entry: gx10, capacity: 4, providerID: "gx10-vllm"},
		{entry: onex, capacity: 3, providerID: "onexplayer-lemonade"},
	}
	m := newModelQueueManager(nil)

	sel1, err := m.admit(context.Background(), "g-idle-rotate", "", "", cands, groupPolicy{}, nil, false, false)
	if err != nil {
		t.Fatalf("first admit: %v", err)
	}
	if sel1.providerID != "gx10-vllm" {
		t.Fatalf("first admit provider: got %q want gx10-vllm", sel1.providerID)
	}
	m.releaseSlot("g-idle-rotate", sel1.entry.NodeID, sel1.providerID)

	sel2, err := m.admit(context.Background(), "g-idle-rotate", "", "", cands, groupPolicy{}, nil, false, false)
	if err != nil {
		t.Fatalf("second admit: %v", err)
	}
	if sel2.providerID != "onexplayer-lemonade" {
		t.Fatalf("second admit provider: got %q want onexplayer-lemonade", sel2.providerID)
	}
	m.releaseSlot("g-idle-rotate", sel2.entry.NodeID, sel2.providerID)

	sel3, err := m.admit(context.Background(), "g-idle-rotate", "", "", cands, groupPolicy{}, nil, false, false)
	if err != nil {
		t.Fatalf("third admit: %v", err)
	}
	if sel3.providerID != "gx10-vllm" {
		t.Fatalf("third admit provider: got %q want gx10-vllm", sel3.providerID)
	}
}

// TestModelQueueProviderServedTargetRewrite verifies that the selected
// candidateNode carries its servedTarget so callers can rewrite req.Target.
func TestModelQueueProviderServedTargetRewrite(t *testing.T) {
	entry := &edgenode.NodeEntry{NodeID: "node-tr-rewrite"}
	cands := []candidateNode{{
		entry:        entry,
		capacity:     2,
		providerID:   "prov-vllm",
		servedTarget: "qwen3-72b-instruct",
	}}
	m := newModelQueueManager(nil)

	sel, err := m.admit(context.Background(), "g-tr-rewrite", "", "", cands, groupPolicy{}, nil, false, false)
	if err != nil {
		t.Fatalf("admit: %v", err)
	}
	if sel == nil {
		t.Fatal("expected non-nil candidate")
	}
	if sel.servedTarget != "qwen3-72b-instruct" {
		t.Errorf("servedTarget: got %q, want %q", sel.servedTarget, "qwen3-72b-instruct")
	}
	if sel.providerID != "prov-vllm" {
		t.Errorf("providerID: got %q, want %q", sel.providerID, "prov-vllm")
	}
}

// TestProviderStatusInflightTracking verifies that getStatsForProviderLocked
// correctly counts in-flight runs keyed by (nodeID, providerID).
func TestProviderStatusInflightTracking(t *testing.T) {
	m := newModelQueueManager(nil)

	m.mu.Lock()
	key1 := providerResourceKey{nodeID: "node-x", providerID: "prov-1"}
	m.resources[key1] = &providerResourceState{nodeID: "node-x", providerID: "prov-1", capacity: 5, enabled: true, inFlight: 2}
	key2 := providerResourceKey{nodeID: "node-x", providerID: "prov-2"}
	m.resources[key2] = &providerResourceState{nodeID: "node-x", providerID: "prov-2", capacity: 5, enabled: true, inFlight: 1}
	m.mu.Unlock()

	m.mu.Lock()
	inf1, q1 := m.getStatsForProviderLocked("node-x", "prov-1")
	inf2, q2 := m.getStatsForProviderLocked("node-x", "prov-2")
	inf3, q3 := m.getStatsForProviderLocked("node-y", "prov-1")
	m.mu.Unlock()

	if inf1 != 2 || q1 != 0 {
		t.Errorf("prov-1 on node-x: inflight=%d queued=%d, want 2/0", inf1, q1)
	}
	if inf2 != 1 || q2 != 0 {
		t.Errorf("prov-2 on node-x: inflight=%d queued=%d, want 1/0", inf2, q2)
	}
	if inf3 != 0 || q3 != 0 {
		t.Errorf("prov-1 on node-y: inflight=%d queued=%d, want 0/0", inf3, q3)
	}
}

// TestModelQueueProviderCapacityIsPerProviderSlot verifies that same-node
// multiple provider candidates each have independent capacity/in-flight
// accounting per REVIEW_API-2.
func TestModelQueueProviderCapacityIsPerProviderSlot(t *testing.T) {
	entry := &edgenode.NodeEntry{NodeID: "node-same"}
	// Two provider candidates on the same node, each capacity=1.
	cands := []candidateNode{
		{entry: entry, capacity: 1, providerID: "prov-a", adapter: "vllm"},
		{entry: entry, capacity: 1, providerID: "prov-b", adapter: "vllm"},
	}
	m := newModelQueueManager(nil)

	// First admit: pick prov-a (deterministic tie-break by providerID).
	sel1, err := m.admit(context.Background(), "g-same", "", "", cands, groupPolicy{}, nil, false, false)
	if err != nil || sel1 == nil {
		t.Fatalf("first admit: %v", err)
	}
	if sel1.providerID != "prov-a" {
		t.Fatalf("expected prov-a, got %s", sel1.providerID)
	}

	// Second admit: should pick prov-b independently (prov-a is full, prov-b has capacity).
	sel2, err := m.admit(context.Background(), "g-same", "", "", cands, groupPolicy{}, nil, false, false)
	if err != nil || sel2 == nil {
		t.Fatalf("second admit (prov-b should be available): %v", err)
	}
	if sel2.providerID != "prov-b" {
		t.Fatalf("expected prov-b, got %s", sel2.providerID)
	}

	// Third admit: both providers at capacity, should queue.
	ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
	defer cancel()
	_, err = m.admit(ctx, "g-same", "", "", cands, groupPolicy{}, nil, false, false)
	if !errors.Is(err, errQueueTimeout) && !errors.Is(err, context.DeadlineExceeded) {
		t.Errorf("expected queue timeout or deadline exceeded, got: %v", err)
	}
}

// TestResolveProviderPoolCandidatesFiltersInvalidProviders verifies that
// unavailable providers, served model mismatch, empty adapter providers,
// and capacity zero/unknown providers are excluded from dispatch candidates.
// This test calls the actual Service.resolveProviderPoolCandidates method.
func TestResolveProviderPoolCandidatesFiltersInvalidProviders(t *testing.T) {
	// Build model catalog entry for "qwen3.6:35b".
	catalog := []config.ModelCatalogEntry{
		{
			ID: "qwen3.6:35b",
			Providers: map[string]string{
				"prov-available":   "served-qwen",
				"prov-unavailable": "served-qwen",
				"prov-mismatch":    "served-qwen",
				"prov-no-adapter":  "served-qwen",
				"prov-cap-zero":    "served-qwen",
				"prov-cap-unknown": "served-qwen",
			},
		},
	}

	// Build NodeStore with multiple providers.
	store := edgenode.NewNodeStore()

	// Valid provider: available, has adapter, has capacity, served model matches provider's models.
	store.Add(&edgenode.NodeRecord{
		ID:      "node-valid",
		Runtime: config.RuntimeConf{Concurrency: 4},
		Adapters: config.AdaptersConf{
			VllmInstances: []config.VllmInstanceConf{
				{Name: "vllm-gpu", Enabled: true, Endpoint: "http://127.0.0.1:8000/v1"},
			},
		},
		Providers: []config.NodeProviderConf{
			{
				ID:       "prov-available",
				Adapter:  "vllm-gpu",
				Models:   []string{"served-qwen", "served-llama"},
				Health:   "available",
				Capacity: 2,
			},
		},
	})

	// Invalid: health = "unavailable".
	store.Add(&edgenode.NodeRecord{
		ID:      "node-bad-health",
		Runtime: config.RuntimeConf{Concurrency: 4},
		Adapters: config.AdaptersConf{
			VllmInstances: []config.VllmInstanceConf{
				{Name: "vllm-gpu", Enabled: true, Endpoint: "http://127.0.0.1:8000/v1"},
			},
		},
		Providers: []config.NodeProviderConf{
			{
				ID:       "prov-unavailable",
				Adapter:  "vllm-gpu",
				Models:   []string{"served-qwen"},
				Health:   "unavailable",
				Capacity: 2,
			},
		},
	})

	// Invalid: served model not in provider's own models list.
	store.Add(&edgenode.NodeRecord{
		ID:      "node-mismatch",
		Runtime: config.RuntimeConf{Concurrency: 4},
		Adapters: config.AdaptersConf{
			VllmInstances: []config.VllmInstanceConf{
				{Name: "vllm-gpu", Enabled: true, Endpoint: "http://127.0.0.1:8000/v1"},
			},
		},
		Providers: []config.NodeProviderConf{
			{
				ID:       "prov-mismatch",
				Adapter:  "vllm-gpu",
				Models:   []string{"served-llama"}, // does NOT include "served-qwen"
				Health:   "available",
				Capacity: 2,
			},
		},
	})

	// Provider-first (empty adapter, health available, capacity > 0): now a valid candidate.
	// adapter key = provider ID "prov-no-adapter".
	store.Add(&edgenode.NodeRecord{
		ID:      "node-no-adapter",
		Runtime: config.RuntimeConf{Concurrency: 4},
		Adapters: config.AdaptersConf{
			VllmInstances: []config.VllmInstanceConf{
				{Name: "vllm-gpu", Enabled: true, Endpoint: "http://127.0.0.1:8000/v1"},
			},
		},
		Providers: []config.NodeProviderConf{
			{
				ID:       "prov-no-adapter",
				Type:     "vllm",
				Adapter:  "", // provider-first: adapter key derived from ID
				Models:   []string{"served-qwen"},
				Health:   "available",
				Capacity: 2,
			},
		},
	})

	// Invalid: capacity = 0.
	store.Add(&edgenode.NodeRecord{
		ID:      "node-cap-zero",
		Runtime: config.RuntimeConf{Concurrency: 4},
		Adapters: config.AdaptersConf{
			VllmInstances: []config.VllmInstanceConf{
				{Name: "vllm-gpu", Enabled: true, Endpoint: "http://127.0.0.1:8000/v1"},
			},
		},
		Providers: []config.NodeProviderConf{
			{
				ID:       "prov-cap-zero",
				Adapter:  "vllm-gpu",
				Models:   []string{"served-qwen"},
				Health:   "available",
				Capacity: 0,
			},
		},
	})

	// Invalid: capacity < 0 (negative/unknown).
	store.Add(&edgenode.NodeRecord{
		ID:      "node-cap-unknown",
		Runtime: config.RuntimeConf{Concurrency: 4},
		Adapters: config.AdaptersConf{
			VllmInstances: []config.VllmInstanceConf{
				{Name: "vllm-gpu", Enabled: true, Endpoint: "http://127.0.0.1:8000/v1"},
			},
		},
		Providers: []config.NodeProviderConf{
			{
				ID:       "prov-cap-unknown",
				Adapter:  "vllm-gpu",
				Models:   []string{"served-qwen"},
				Health:   "available",
				Capacity: -1,
			},
		},
	})

	// Build a fake registry with all nodes.
	reg := edgenode.NewRegistry()
	allRecs := store.All()
	for _, rec := range allRecs {
		entry := &edgenode.NodeEntry{
			NodeID:         rec.ID,
			LifecycleState: edgenode.LifecycleConnected,
		}
		reg.Register(entry)
	}

	// Create Service with catalog and node store.
	svc := New(reg, nil)
	svc.SetNodeStore(store)
	svc.SetModelCatalog(catalog)

	// Call the actual resolveProviderPoolCandidates.
	req := SubmitRunRequest{
		ModelGroupKey: "qwen3.6:35b",
		ProviderPool:  true,
	}
	storeSnapshot, catalogSnapshot, _ := svc.runtimeConfigSnapshot()
	candidates, policy, err := svc.resolveProviderPoolCandidates(req, storeSnapshot, catalogSnapshot)
	if err != nil {
		t.Fatalf("resolveProviderPoolCandidates: %v", err)
	}

	// prov-available (legacy/compat) and prov-no-adapter (provider-first) both pass.
	// All others are filtered: unavailable health, model mismatch, cap=0, cap<0.
	if len(candidates) != 2 {
		ids := make([]string, len(candidates))
		for i, c := range candidates {
			ids[i] = c.providerID
		}
		t.Fatalf("expected 2 candidates, got %d: %v", len(candidates), ids)
	}

	byID := map[string]*candidateNode{}
	for i := range candidates {
		byID[candidates[i].providerID] = &candidates[i]
	}

	// Legacy/compat candidate uses explicit adapter key.
	avail := byID["prov-available"]
	if avail == nil {
		t.Fatal("prov-available not in candidates")
	}
	if avail.adapter != "vllm-gpu" {
		t.Errorf("prov-available adapter: got %q, want %q", avail.adapter, "vllm-gpu")
	}
	if avail.servedTarget != "served-qwen" {
		t.Errorf("prov-available servedTarget: got %q", avail.servedTarget)
	}

	// Provider-first candidate uses provider ID as adapter key.
	noAdp := byID["prov-no-adapter"]
	if noAdp == nil {
		t.Fatal("prov-no-adapter not in candidates (provider-first dispatch must use provider ID as adapter key)")
	}
	if noAdp.adapter != "prov-no-adapter" {
		t.Errorf("prov-no-adapter adapter: got %q, want provider ID %q", noAdp.adapter, "prov-no-adapter")
	}
	if noAdp.servedTarget != "served-qwen" {
		t.Errorf("prov-no-adapter servedTarget: got %q", noAdp.servedTarget)
	}

	// Policy is always zero for provider-pool resolution: the canonical policy
	// is owned by the atomic runtime snapshot, not by the resolution path.
	if policy.maxQueue != 0 {
		t.Errorf("policy.maxQueue: got %d, expected 0 (policy is owned by runtime snapshot)", policy.maxQueue)
	}
}

// TestGlobalPumpPreservesProviderTieBreakWithinSequence verifies that ordering
// waiters globally does not disturb provider selection: among equally
// dispatchable waiters the global enqueue sequence decides who goes first, and
// for the waiter that goes, the existing priority tie-break still decides which
// provider it lands on.
func TestGlobalPumpPreservesProviderTieBreakWithinSequence(t *testing.T) {
	entry := &edgenode.NodeEntry{NodeID: "node-tb"}
	// Same in-flight state on both providers; the lower priority value wins.
	cands := []candidateNode{
		{entry: entry, capacity: 1, providerID: "prov-tb-low", priority: 5},
		{entry: entry, capacity: 1, providerID: "prov-tb-high", priority: 1},
	}
	m := newModelQueueManager(nil)

	// Fill both providers so the waiters queue, then free them one at a time.
	firstHold, err := m.admit(context.Background(), "g-tb-a", "", "", cands, groupPolicy{}, nil, false, false)
	if err != nil {
		t.Fatalf("first fill: %v", err)
	}
	secondHold, err := m.admit(context.Background(), "g-tb-a", "", "", cands, groupPolicy{}, nil, false, false)
	if err != nil {
		t.Fatalf("second fill: %v", err)
	}
	if firstHold.providerID != "prov-tb-high" {
		t.Fatalf("priority tie-break broken on the direct path: first admit took %q, want prov-tb-high", firstHold.providerID)
	}

	// Arrival order spans two groups: earlier in group B, later in group A.
	earlier := queueItemForTest(cands, false)
	enqueueForTest(m, "g-tb-b", earlier, nil)
	later := queueItemForTest(cands, false)
	enqueueForTest(m, "g-tb-a", later, nil)

	// Free the higher-priority provider: the earliest waiter takes it.
	m.releaseLease(firstHold.leaseID, "complete")
	select {
	case res := <-earlier.waitCh:
		if res.err != nil {
			t.Fatalf("earlier waiter dispatch error: %v", res.err)
		}
		if res.candidate.providerID != "prov-tb-high" {
			t.Fatalf("earlier waiter took %q, want the higher-priority prov-tb-high", res.candidate.providerID)
		}
	default:
		t.Fatal("earliest waiter was not dispatched first")
	}
	select {
	case <-later.waitCh:
		t.Fatal("later waiter dispatched out of global enqueue order")
	default:
	}

	// Free the remaining provider: the later waiter takes what is left.
	m.releaseLease(secondHold.leaseID, "complete")
	select {
	case res := <-later.waitCh:
		if res.err != nil {
			t.Fatalf("later waiter dispatch error: %v", res.err)
		}
		if res.candidate.providerID != "prov-tb-low" {
			t.Fatalf("later waiter took %q, want the remaining prov-tb-low", res.candidate.providerID)
		}
	default:
		t.Fatal("later waiter was not dispatched after the second release")
	}
}