package service import ( "context" "errors" "reflect" "runtime" "testing" "time" edgenode "iop/apps/edge/internal/node" "iop/packages/go/config" ) // TestModelQueueUsesProviderCapacity verifies that the capacity supplied in the // candidateNode (derived from adapter config) overrides the default of 1. func TestModelQueueUsesProviderCapacity(t *testing.T) { entry := &edgenode.NodeEntry{NodeID: "node-pc1"} // Provider capacity = 2 (two concurrent slots on this node). cands := []candidateNode{{entry: entry, capacity: 2}} defPolicy := groupPolicy{} m := newModelQueueManager(nil) // First admit: inflight=0 < cap=2 → dispatched immediately. n1, err := m.admit(context.Background(), "g-pc", "", "", cands, defPolicy, nil, false, false) if err != nil || n1 == nil { t.Fatalf("first admit: %v", err) } // Second admit: inflight=1 < cap=2 → still dispatched (not queued). n2, err := m.admit(context.Background(), "g-pc", "", "", cands, defPolicy, nil, false, false) if err != nil || n2 == nil { t.Fatalf("second admit (capacity=2 should allow): %v", err) } // Third request must queue because inflight=2 == cap=2. item := &queueItem{ candidates: cands, waitCh: make(chan admitResult, 1), deadline: time.Now().Add(200 * time.Millisecond), } m.mu.Lock() g := m.getOrCreateGroupLocked("g-pc", groupPolicy{}) g.queue = append(g.queue, item) m.mu.Unlock() // Releasing one slot should dispatch the queued item. m.releaseSlot("g-pc", "node-pc1") select { case res := <-item.waitCh: if res.err != nil { t.Fatalf("expected dispatch after slot release, got: %v", res.err) } if res.candidate == nil || res.candidate.entry.NodeID != "node-pc1" { t.Fatalf("unexpected candidate: %v", res.candidate) } case <-time.After(200 * time.Millisecond): t.Fatal("timeout: item not dispatched after slot release") } } // TestRefreshProviderQueuePolicyUpdatesExistingGroup verifies that a tightened // queue policy (e.g. provider max_queue lowered via config refresh) propagates // to an already-created group on the next admission, while in-flight slots are // preserved. func TestRefreshProviderQueuePolicyUpdatesExistingGroup(t *testing.T) { entry := &edgenode.NodeEntry{NodeID: "node-rp"} cands := []candidateNode{{entry: entry, capacity: 2}} m := newModelQueueManager(nil) // Create the group with a generous policy and take one slot. initial := groupPolicy{maxQueue: 8, queueTimeout: 5 * time.Second} if _, err := m.admit(context.Background(), "g-rp", "", "", cands, initial, nil, false, false); err != nil { t.Fatalf("first admit: %v", err) } m.mu.Lock() if got := m.groups["g-rp"].policy.maxQueue; got != 8 { m.mu.Unlock() t.Fatalf("group created with maxQueue=%d, want 8", got) } m.mu.Unlock() // Config refresh tightens the policy to maxQueue=1; the recomputed admit // carries it and must update the existing group's policy. capacity=2 means // this second admit is dispatched (not queued). refreshed := groupPolicy{maxQueue: 1, queueTimeout: 5 * time.Second} if _, err := m.admit(context.Background(), "g-rp", "", "", cands, refreshed, nil, false, false); err != nil { t.Fatalf("second admit: %v", err) } m.mu.Lock() g := m.groups["g-rp"] if g.policy.maxQueue != 1 { m.mu.Unlock() t.Fatalf("existing group policy not refreshed: maxQueue=%d, want 1", g.policy.maxQueue) } // Both slots now in-flight (cap=2). Queue one item to reach the refreshed // maxQueue=1 so the next admission must overflow. g.queue = append(g.queue, &queueItem{ candidates: cands, waitCh: make(chan admitResult, 1), deadline: time.Now().Add(5 * time.Second), }) m.mu.Unlock() if _, err := m.admit(context.Background(), "g-rp", "", "", cands, refreshed, nil, false, false); !errors.Is(err, errQueueFull) { t.Fatalf("expected errQueueFull under refreshed maxQueue=1, got: %v", err) } } // TestModelQueueUsesProviderQueuePolicy verifies that max_queue and // queue_timeout from the policy parameter are respected. func TestModelQueueUsesProviderQueuePolicy(t *testing.T) { t.Run("maxQueue enforced", func(t *testing.T) { entry := &edgenode.NodeEntry{NodeID: "node-pq1"} cands := []candidateNode{{entry: entry, capacity: 1}} // Provider policy: maxQueue=1. policy := groupPolicy{maxQueue: 1, queueTimeout: 5 * time.Second} m := newModelQueueManager(nil) // Fill capacity and queue one item manually. m.mu.Lock() g := m.getOrCreateGroupLocked("g-pq-max", policy) g.inflight["node-pq1"] = 1 g.queue = []*queueItem{{ candidates: cands, waitCh: make(chan admitResult, 1), deadline: time.Now().Add(5 * time.Second), }} m.mu.Unlock() // Second admit should fail: queue already at maxQueue=1. _, err := m.admit(context.Background(), "g-pq-max", "", "", cands, policy, nil, false, false) if !errors.Is(err, errQueueFull) { t.Fatalf("expected errQueueFull, got: %v", err) } }) t.Run("queueTimeout enforced", func(t *testing.T) { entry := &edgenode.NodeEntry{NodeID: "node-pq2"} cands := []candidateNode{{entry: entry, capacity: 1}} // Provider policy: very short timeout. policy := groupPolicy{maxQueue: 16, queueTimeout: 20 * time.Millisecond} m := newModelQueueManager(nil) // Fill capacity. m.mu.Lock() g := m.getOrCreateGroupLocked("g-pq-to", policy) g.inflight["node-pq2"] = 1 m.mu.Unlock() start := time.Now() _, err := m.admit(context.Background(), "g-pq-to", "", "", cands, policy, nil, false, false) elapsed := time.Since(start) if !errors.Is(err, errQueueTimeout) { t.Fatalf("expected errQueueTimeout, got: %v", err) } if elapsed < 15*time.Millisecond { t.Fatalf("timed out too fast: %v", elapsed) } }) t.Run("zeroQueueTimeoutWaitsForContextCancellation", func(t *testing.T) { entry := &edgenode.NodeEntry{NodeID: "node-pq3"} cands := []candidateNode{{entry: entry, capacity: 1}} policy := groupPolicy{maxQueue: 16, queueTimeoutSet: true} m := newModelQueueManager(nil) m.mu.Lock() g := m.getOrCreateGroupLocked("g-pq-no-timeout", policy) g.inflight["node-pq3"] = 1 m.mu.Unlock() ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond) defer cancel() start := time.Now() _, err := m.admit(ctx, "g-pq-no-timeout", "", "", cands, policy, nil, false, false) elapsed := time.Since(start) if !errors.Is(err, context.DeadlineExceeded) { t.Fatalf("expected context deadline, got: %v", err) } if errors.Is(err, errQueueTimeout) { t.Fatalf("queue timeout should be disabled, got: %v", err) } if elapsed < 25*time.Millisecond { t.Fatalf("returned before context deadline: %v", elapsed) } }) } func TestGroupPolicyFromStoreZeroQueueTimeoutDisablesTimeout(t *testing.T) { store := edgenode.NewNodeStore() store.Add(&edgenode.NodeRecord{ ID: "node-policy-zero-timeout", Adapters: config.AdaptersConf{ OpenAICompatInstances: []config.OpenAICompatInstanceConf{ { Name: "provider-zero-timeout", Enabled: true, Capacity: 1, MaxQueue: 16, QueueTimeoutMS: 0, }, }, }, }) entries := []*edgenode.NodeEntry{{NodeID: "node-policy-zero-timeout"}} policy := groupPolicyFromStore(store, entries, "provider-zero-timeout", "") if policy.maxQueue != 16 { t.Fatalf("maxQueue: got %d, want 16", policy.maxQueue) } if !policy.queueTimeoutSet { t.Fatalf("queueTimeoutSet: got false, want true") } if policy.queueTimeout != 0 { t.Fatalf("queueTimeout: got %v, want no timeout", policy.queueTimeout) } } // TestModelQueueProviderInflightSelectionBeatsPriority verifies that in_flight // level is the primary selection criterion: a candidate with lower in_flight is // preferred even if it has a higher (worse) priority value. func TestModelQueueProviderInflightSelectionBeatsPriority(t *testing.T) { entryA := &edgenode.NodeEntry{NodeID: "node-ip-a"} entryB := &edgenode.NodeEntry{NodeID: "node-ip-b"} // A has priority=1 (better) but inflight=3. // B has priority=10 (worse) but inflight=0. cands := []candidateNode{ {entry: entryA, capacity: 5, priority: 1, providerID: "prov-a"}, {entry: entryB, capacity: 5, priority: 10, providerID: "prov-b"}, } m := newModelQueueManager(nil) // Put A at inflight=3, B at inflight=0. m.mu.Lock() g := m.getOrCreateGroupLocked("g-ip", groupPolicy{}) g.inflight["node-ip-a:prov-a"] = 3 m.mu.Unlock() // Admit should pick B (inflight=0 < inflight=3), not A. sel, err := m.admit(context.Background(), "g-ip", "", "", cands, groupPolicy{}, nil, false, false) if err != nil { t.Fatalf("admit: %v", err) } if sel == nil || sel.entry.NodeID != "node-ip-b" { t.Fatalf("expected node-ip-b (lower in_flight), got %v", sel) } } // TestModelQueueProviderFullPriorityFallsThrough verifies that priority does // not override capacity: a full high-priority provider is skipped and the next // available priority tier receives the request. func TestModelQueueProviderFullPriorityFallsThrough(t *testing.T) { entryA := &edgenode.NodeEntry{NodeID: "node-full-a"} entryB := &edgenode.NodeEntry{NodeID: "node-next-b"} cands := []candidateNode{ {entry: entryA, capacity: 4, priority: 1, providerID: "prov-fast"}, {entry: entryB, capacity: 3, priority: 10, providerID: "prov-slow"}, } m := newModelQueueManager(nil) m.mu.Lock() g := m.getOrCreateGroupLocked("g-full-priority", groupPolicy{}) g.inflight["node-full-a:prov-fast"] = 4 m.mu.Unlock() sel, err := m.admit(context.Background(), "g-full-priority", "", "", cands, groupPolicy{}, nil, false, false) if err != nil { t.Fatalf("admit: %v", err) } if sel == nil || sel.providerID != "prov-slow" { t.Fatalf("expected prov-slow after high-priority provider is full, got %v", sel) } } // TestModelQueueProviderLevelingPriorityAdmissionOrder fixes the intended // provider-pool order for differently-sized providers: fill each in_flight // level before moving to the next level, and use priority within the same // level. With capacities 4/3/2 and priorities 0/1/2, the first nine admits are // gx10, onex, mac, gx10, onex, mac, gx10, onex, gx10. The tenth has no // available candidate until a slot is released. func TestModelQueueProviderLevelingPriorityAdmissionOrder(t *testing.T) { cands := []candidateNode{ {entry: &edgenode.NodeEntry{NodeID: "node-gx10"}, capacity: 4, priority: 0, providerID: "gx10-vllm"}, {entry: &edgenode.NodeEntry{NodeID: "node-onex"}, capacity: 3, priority: 1, providerID: "onexplayer-lemonade"}, {entry: &edgenode.NodeEntry{NodeID: "node-mac"}, capacity: 2, priority: 2, providerID: "mac-mlx-vllm"}, } m := newModelQueueManager(nil) var got []string for i := 0; i < 9; i++ { sel, err := m.admit(context.Background(), "qwen3.6:35b", "", "", cands, groupPolicy{}, nil, false, false) if err != nil { t.Fatalf("admit %d: %v", i+1, err) } got = append(got, sel.providerID) } want := []string{ "gx10-vllm", "onexplayer-lemonade", "mac-mlx-vllm", "gx10-vllm", "onexplayer-lemonade", "mac-mlx-vllm", "gx10-vllm", "onexplayer-lemonade", "gx10-vllm", } if !reflect.DeepEqual(got, want) { t.Fatalf("admission order mismatch:\n got: %v\nwant: %v", got, want) } m.mu.Lock() g := m.getOrCreateGroupLocked("qwen3.6:35b", groupPolicy{}) if candidate := m.findAvailableNodeLocked(g, cands, false); candidate != nil { t.Fatalf("expected no available candidate after filling capacity, got %s", candidate.providerID) } m.mu.Unlock() } // TestModelQueueProviderPriorityBreaksEqualInflightTie verifies that when // in_flight counts are equal, the candidate with the lower priority value // is selected. func TestModelQueueProviderPriorityBreaksEqualInflightTie(t *testing.T) { entryA := &edgenode.NodeEntry{NodeID: "node-pt-a"} entryB := &edgenode.NodeEntry{NodeID: "node-pt-b"} // Both have inflight=2, but A has priority=1 (better) than B (priority=5). cands := []candidateNode{ {entry: entryA, capacity: 5, priority: 1, providerID: "prov-a"}, {entry: entryB, capacity: 5, priority: 5, providerID: "prov-b"}, } m := newModelQueueManager(nil) m.mu.Lock() g := m.getOrCreateGroupLocked("g-pt", groupPolicy{}) g.inflight["node-pt-a:prov-a"] = 2 g.inflight["node-pt-b:prov-b"] = 2 m.mu.Unlock() // Admit should pick A (lower priority=1 when inflight is equal). sel, err := m.admit(context.Background(), "g-pt", "", "", cands, groupPolicy{}, nil, false, false) if err != nil { t.Fatalf("admit: %v", err) } if sel == nil || sel.entry.NodeID != "node-pt-a" { t.Fatalf("expected node-pt-a (lower priority tie-break), got %v", sel) } } // TestModelQueueProviderEqualInflightPriorityRotates verifies that when both // in_flight and priority are equal, the rotation logic selects the next slot // after the last selected one. func TestModelQueueProviderEqualInflightPriorityRotates(t *testing.T) { entryA := &edgenode.NodeEntry{NodeID: "node-r-a"} entryB := &edgenode.NodeEntry{NodeID: "node-r-b"} // Both have inflight=0 and priority=0. cands := []candidateNode{ {entry: entryA, capacity: 5, priority: 0, providerID: "prov-a"}, {entry: entryB, capacity: 5, priority: 0, providerID: "prov-b"}, } m := newModelQueueManager(nil) // First admit: A wins deterministically. sel1, err := m.admit(context.Background(), "g-r", "", "", cands, groupPolicy{}, nil, false, false) if err != nil || sel1 == nil { t.Fatalf("first admit: %v", err) } if sel1.providerID != "prov-a" { t.Fatalf("expected prov-a on first admit, got %q", sel1.providerID) } // Release prov-a's slot. m.releaseSlot("g-r", "node-r-a", "prov-a") // Second admit: rotation should pick B (next after last selected A). sel2, err := m.admit(context.Background(), "g-r", "", "", cands, groupPolicy{}, nil, false, false) if err != nil || sel2 == nil { t.Fatalf("second admit: %v", err) } if sel2.providerID != "prov-b" { t.Fatalf("expected prov-b on second admit (rotation), got %q", sel2.providerID) } // Release prov-b's slot. m.releaseSlot("g-r", "node-r-b", "prov-b") // Third admit: rotation should cycle back to A. sel3, err := m.admit(context.Background(), "g-r", "", "", cands, groupPolicy{}, nil, false, false) if err != nil || sel3 == nil { t.Fatalf("third admit: %v", err) } if sel3.providerID != "prov-a" { t.Fatalf("expected prov-a on third admit (rotation cycle), got %q", sel3.providerID) } } func TestModelQueueProviderEqualIdleTieRotates(t *testing.T) { gx10 := &edgenode.NodeEntry{NodeID: "node-gx10-idle"} onex := &edgenode.NodeEntry{NodeID: "node-onex-idle"} cands := []candidateNode{ {entry: gx10, capacity: 4, providerID: "gx10-vllm"}, {entry: onex, capacity: 3, providerID: "onexplayer-lemonade"}, } m := newModelQueueManager(nil) sel1, err := m.admit(context.Background(), "g-idle-rotate", "", "", cands, groupPolicy{}, nil, false, false) if err != nil { t.Fatalf("first admit: %v", err) } if sel1.providerID != "gx10-vllm" { t.Fatalf("first admit provider: got %q want gx10-vllm", sel1.providerID) } m.releaseSlot("g-idle-rotate", sel1.entry.NodeID, sel1.providerID) sel2, err := m.admit(context.Background(), "g-idle-rotate", "", "", cands, groupPolicy{}, nil, false, false) if err != nil { t.Fatalf("second admit: %v", err) } if sel2.providerID != "onexplayer-lemonade" { t.Fatalf("second admit provider: got %q want onexplayer-lemonade", sel2.providerID) } m.releaseSlot("g-idle-rotate", sel2.entry.NodeID, sel2.providerID) sel3, err := m.admit(context.Background(), "g-idle-rotate", "", "", cands, groupPolicy{}, nil, false, false) if err != nil { t.Fatalf("third admit: %v", err) } if sel3.providerID != "gx10-vllm" { t.Fatalf("third admit provider: got %q want gx10-vllm", sel3.providerID) } } // TestModelQueueProviderServedTargetRewrite verifies that the selected // candidateNode carries its servedTarget so callers can rewrite req.Target. func TestModelQueueProviderServedTargetRewrite(t *testing.T) { entry := &edgenode.NodeEntry{NodeID: "node-tr-rewrite"} cands := []candidateNode{{ entry: entry, capacity: 2, providerID: "prov-vllm", servedTarget: "qwen3-72b-instruct", }} m := newModelQueueManager(nil) sel, err := m.admit(context.Background(), "g-tr-rewrite", "", "", cands, groupPolicy{}, nil, false, false) if err != nil { t.Fatalf("admit: %v", err) } if sel == nil { t.Fatal("expected non-nil candidate") } if sel.servedTarget != "qwen3-72b-instruct" { t.Errorf("servedTarget: got %q, want %q", sel.servedTarget, "qwen3-72b-instruct") } if sel.providerID != "prov-vllm" { t.Errorf("providerID: got %q, want %q", sel.providerID, "prov-vllm") } } // TestProviderStatusInflightTracking verifies that getStatsForProviderLocked // correctly counts in-flight runs keyed by (nodeID, providerID). func TestProviderStatusInflightTracking(t *testing.T) { m := newModelQueueManager(nil) m.mu.Lock() key1 := providerResourceKey{nodeID: "node-x", providerID: "prov-1"} m.resources[key1] = &providerResourceState{nodeID: "node-x", providerID: "prov-1", capacity: 5, enabled: true, inFlight: 2} key2 := providerResourceKey{nodeID: "node-x", providerID: "prov-2"} m.resources[key2] = &providerResourceState{nodeID: "node-x", providerID: "prov-2", capacity: 5, enabled: true, inFlight: 1} m.mu.Unlock() m.mu.Lock() inf1, q1 := m.getStatsForProviderLocked("node-x", "prov-1") inf2, q2 := m.getStatsForProviderLocked("node-x", "prov-2") inf3, q3 := m.getStatsForProviderLocked("node-y", "prov-1") m.mu.Unlock() if inf1 != 2 || q1 != 0 { t.Errorf("prov-1 on node-x: inflight=%d queued=%d, want 2/0", inf1, q1) } if inf2 != 1 || q2 != 0 { t.Errorf("prov-2 on node-x: inflight=%d queued=%d, want 1/0", inf2, q2) } if inf3 != 0 || q3 != 0 { t.Errorf("prov-1 on node-y: inflight=%d queued=%d, want 0/0", inf3, q3) } } // TestModelQueueProviderCapacityIsPerProviderSlot verifies that same-node // multiple provider candidates each have independent capacity/in-flight // accounting per REVIEW_API-2. func TestModelQueueProviderCapacityIsPerProviderSlot(t *testing.T) { entry := &edgenode.NodeEntry{NodeID: "node-same"} // Two provider candidates on the same node, each capacity=1. cands := []candidateNode{ {entry: entry, capacity: 1, providerID: "prov-a", adapter: "vllm"}, {entry: entry, capacity: 1, providerID: "prov-b", adapter: "vllm"}, } m := newModelQueueManager(nil) // First admit: pick prov-a (deterministic tie-break by providerID). sel1, err := m.admit(context.Background(), "g-same", "", "", cands, groupPolicy{}, nil, false, false) if err != nil || sel1 == nil { t.Fatalf("first admit: %v", err) } if sel1.providerID != "prov-a" { t.Fatalf("expected prov-a, got %s", sel1.providerID) } // Second admit: should pick prov-b independently (prov-a is full, prov-b has capacity). sel2, err := m.admit(context.Background(), "g-same", "", "", cands, groupPolicy{}, nil, false, false) if err != nil || sel2 == nil { t.Fatalf("second admit (prov-b should be available): %v", err) } if sel2.providerID != "prov-b" { t.Fatalf("expected prov-b, got %s", sel2.providerID) } // Third admit: both providers at capacity, should queue. ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) defer cancel() _, err = m.admit(ctx, "g-same", "", "", cands, groupPolicy{}, nil, false, false) if !errors.Is(err, errQueueTimeout) && !errors.Is(err, context.DeadlineExceeded) { t.Errorf("expected queue timeout or deadline exceeded, got: %v", err) } } // TestResolveProviderPoolCandidatesFiltersInvalidProviders verifies that // unavailable providers, served model mismatch, empty adapter providers, // and capacity zero/unknown providers are excluded from dispatch candidates. // This test calls the actual Service.resolveProviderPoolCandidates method. func TestResolveProviderPoolCandidatesFiltersInvalidProviders(t *testing.T) { // Build model catalog entry for "qwen3.6:35b". catalog := []config.ModelCatalogEntry{ { ID: "qwen3.6:35b", Providers: map[string]string{ "prov-available": "served-qwen", "prov-unavailable": "served-qwen", "prov-mismatch": "served-qwen", "prov-no-adapter": "served-qwen", "prov-cap-zero": "served-qwen", "prov-cap-unknown": "served-qwen", }, }, } // Build NodeStore with multiple providers. store := edgenode.NewNodeStore() // Valid provider: available, has adapter, has capacity, served model matches provider's models. store.Add(&edgenode.NodeRecord{ ID: "node-valid", Runtime: config.RuntimeConf{Concurrency: 4}, Adapters: config.AdaptersConf{ VllmInstances: []config.VllmInstanceConf{ {Name: "vllm-gpu", Enabled: true, Endpoint: "http://127.0.0.1:8000/v1"}, }, }, Providers: []config.NodeProviderConf{ { ID: "prov-available", Adapter: "vllm-gpu", Models: []string{"served-qwen", "served-llama"}, Health: "available", Capacity: 2, }, }, }) // Invalid: health = "unavailable". store.Add(&edgenode.NodeRecord{ ID: "node-bad-health", Runtime: config.RuntimeConf{Concurrency: 4}, Adapters: config.AdaptersConf{ VllmInstances: []config.VllmInstanceConf{ {Name: "vllm-gpu", Enabled: true, Endpoint: "http://127.0.0.1:8000/v1"}, }, }, Providers: []config.NodeProviderConf{ { ID: "prov-unavailable", Adapter: "vllm-gpu", Models: []string{"served-qwen"}, Health: "unavailable", Capacity: 2, }, }, }) // Invalid: served model not in provider's own models list. store.Add(&edgenode.NodeRecord{ ID: "node-mismatch", Runtime: config.RuntimeConf{Concurrency: 4}, Adapters: config.AdaptersConf{ VllmInstances: []config.VllmInstanceConf{ {Name: "vllm-gpu", Enabled: true, Endpoint: "http://127.0.0.1:8000/v1"}, }, }, Providers: []config.NodeProviderConf{ { ID: "prov-mismatch", Adapter: "vllm-gpu", Models: []string{"served-llama"}, // does NOT include "served-qwen" Health: "available", Capacity: 2, }, }, }) // Provider-first (empty adapter, health available, capacity > 0): now a valid candidate. // adapter key = provider ID "prov-no-adapter". store.Add(&edgenode.NodeRecord{ ID: "node-no-adapter", Runtime: config.RuntimeConf{Concurrency: 4}, Adapters: config.AdaptersConf{ VllmInstances: []config.VllmInstanceConf{ {Name: "vllm-gpu", Enabled: true, Endpoint: "http://127.0.0.1:8000/v1"}, }, }, Providers: []config.NodeProviderConf{ { ID: "prov-no-adapter", Type: "vllm", Adapter: "", // provider-first: adapter key derived from ID Models: []string{"served-qwen"}, Health: "available", Capacity: 2, }, }, }) // Invalid: capacity = 0. store.Add(&edgenode.NodeRecord{ ID: "node-cap-zero", Runtime: config.RuntimeConf{Concurrency: 4}, Adapters: config.AdaptersConf{ VllmInstances: []config.VllmInstanceConf{ {Name: "vllm-gpu", Enabled: true, Endpoint: "http://127.0.0.1:8000/v1"}, }, }, Providers: []config.NodeProviderConf{ { ID: "prov-cap-zero", Adapter: "vllm-gpu", Models: []string{"served-qwen"}, Health: "available", Capacity: 0, }, }, }) // Invalid: capacity < 0 (negative/unknown). store.Add(&edgenode.NodeRecord{ ID: "node-cap-unknown", Runtime: config.RuntimeConf{Concurrency: 4}, Adapters: config.AdaptersConf{ VllmInstances: []config.VllmInstanceConf{ {Name: "vllm-gpu", Enabled: true, Endpoint: "http://127.0.0.1:8000/v1"}, }, }, Providers: []config.NodeProviderConf{ { ID: "prov-cap-unknown", Adapter: "vllm-gpu", Models: []string{"served-qwen"}, Health: "available", Capacity: -1, }, }, }) // Build a fake registry with all nodes. reg := edgenode.NewRegistry() allRecs := store.All() for _, rec := range allRecs { entry := &edgenode.NodeEntry{ NodeID: rec.ID, LifecycleState: edgenode.LifecycleConnected, } reg.Register(entry) } // Create Service with catalog and node store. svc := New(reg, nil) svc.SetNodeStore(store) svc.SetModelCatalog(catalog) // Call the actual resolveProviderPoolCandidates. req := SubmitRunRequest{ ModelGroupKey: "qwen3.6:35b", ProviderPool: true, } storeSnapshot, catalogSnapshot, _ := svc.runtimeConfigSnapshot() candidates, policy, err := svc.resolveProviderPoolCandidates(req, storeSnapshot, catalogSnapshot) if err != nil { t.Fatalf("resolveProviderPoolCandidates: %v", err) } // prov-available (legacy/compat) and prov-no-adapter (provider-first) both pass. // All others are filtered: unavailable health, model mismatch, cap=0, cap<0. if len(candidates) != 2 { ids := make([]string, len(candidates)) for i, c := range candidates { ids[i] = c.providerID } t.Fatalf("expected 2 candidates, got %d: %v", len(candidates), ids) } byID := map[string]*candidateNode{} for i := range candidates { byID[candidates[i].providerID] = &candidates[i] } // Legacy/compat candidate uses explicit adapter key. avail := byID["prov-available"] if avail == nil { t.Fatal("prov-available not in candidates") } if avail.adapter != "vllm-gpu" { t.Errorf("prov-available adapter: got %q, want %q", avail.adapter, "vllm-gpu") } if avail.servedTarget != "served-qwen" { t.Errorf("prov-available servedTarget: got %q", avail.servedTarget) } // Provider-first candidate uses provider ID as adapter key. noAdp := byID["prov-no-adapter"] if noAdp == nil { t.Fatal("prov-no-adapter not in candidates (provider-first dispatch must use provider ID as adapter key)") } if noAdp.adapter != "prov-no-adapter" { t.Errorf("prov-no-adapter adapter: got %q, want provider ID %q", noAdp.adapter, "prov-no-adapter") } if noAdp.servedTarget != "served-qwen" { t.Errorf("prov-no-adapter servedTarget: got %q", noAdp.servedTarget) } // Policy is always zero for provider-pool resolution: the canonical policy // is owned by the atomic runtime snapshot, not by the resolution path. if policy.maxQueue != 0 { t.Errorf("policy.maxQueue: got %d, expected 0 (policy is owned by runtime snapshot)", policy.maxQueue) } } // TestGlobalPumpPreservesProviderTieBreakWithinSequence verifies that ordering // waiters globally does not disturb provider selection: among equally // dispatchable waiters the global enqueue sequence decides who goes first, and // for the waiter that goes, the existing priority tie-break still decides which // provider it lands on. func TestGlobalPumpPreservesProviderTieBreakWithinSequence(t *testing.T) { entry := &edgenode.NodeEntry{NodeID: "node-tb"} // Same in-flight state on both providers; the lower priority value wins. cands := []candidateNode{ {entry: entry, capacity: 1, providerID: "prov-tb-low", priority: 5}, {entry: entry, capacity: 1, providerID: "prov-tb-high", priority: 1}, } m := newModelQueueManager(nil) // Fill both providers so the waiters queue, then free them one at a time. firstHold, err := m.admit(context.Background(), "g-tb-a", "", "", cands, groupPolicy{}, nil, false, false) if err != nil { t.Fatalf("first fill: %v", err) } secondHold, err := m.admit(context.Background(), "g-tb-a", "", "", cands, groupPolicy{}, nil, false, false) if err != nil { t.Fatalf("second fill: %v", err) } if firstHold.providerID != "prov-tb-high" { t.Fatalf("priority tie-break broken on the direct path: first admit took %q, want prov-tb-high", firstHold.providerID) } // Arrival order spans two groups: earlier in group B, later in group A. earlier := queueItemForTest(cands, false) enqueueForTest(m, "g-tb-b", earlier, nil) later := queueItemForTest(cands, false) enqueueForTest(m, "g-tb-a", later, nil) // Free the higher-priority provider: the earliest waiter takes it. m.releaseLease(firstHold.leaseID, "complete") select { case res := <-earlier.waitCh: if res.err != nil { t.Fatalf("earlier waiter dispatch error: %v", res.err) } if res.candidate.providerID != "prov-tb-high" { t.Fatalf("earlier waiter took %q, want the higher-priority prov-tb-high", res.candidate.providerID) } default: t.Fatal("earliest waiter was not dispatched first") } select { case <-later.waitCh: t.Fatal("later waiter dispatched out of global enqueue order") default: } // Free the remaining provider: the later waiter takes what is left. m.releaseLease(secondHold.leaseID, "complete") select { case res := <-later.waitCh: if res.err != nil { t.Fatalf("later waiter dispatch error: %v", res.err) } if res.candidate.providerID != "prov-tb-low" { t.Fatalf("later waiter took %q, want the remaining prov-tb-low", res.candidate.providerID) } default: t.Fatal("later waiter was not dispatched after the second release") } } // applyProviderPoolPolicy drives the same locked setter used by // Service.SetRuntimeConfig. The test helper owns only lock acquisition; deadline // rebase and queue pumping remain production behavior. func applyProviderPoolPolicy(m *modelQueueManager, policy groupPolicy) { m.mu.Lock() m.setProviderPoolPolicyLocked(m.store, policy) m.mu.Unlock() } // TestProviderPoolPolicyRefreshShortensExistingWaiterTimeout verifies that // shortening the root provider-pool queue timeout via refresh actually shortens // the wait of an already-queued item. func TestProviderPoolPolicyRefreshShortensExistingWaiterTimeout(t *testing.T) { m := newModelQueueManager(nil) applyProviderPoolPolicy(m, NewGroupPolicy(8, 2*time.Second)) entry := &edgenode.NodeEntry{NodeID: "node-shorten"} cands := []candidateNode{{entry: entry, capacity: 1, providerID: "prov-shorten"}} m.mu.Lock() g := m.getOrCreateGroupLocked("g-shorten", groupPolicy{maxQueue: 8, queueTimeout: 2 * time.Second}) g.inflight["node-shorten:prov-shorten"] = 1 m.mu.Unlock() start := time.Now() resultCh := waitForAdmissionInPool(m, t, "g-shorten", cands) // Shorten timeout from 2s to 150ms. applyProviderPoolPolicy(m, NewGroupPolicy(8, 150*time.Millisecond)) select { case res := <-resultCh: elapsed := time.Since(start) if res.err == nil { t.Fatalf("waiter expected timeout, got dispatched (elapsed=%v)", elapsed) } if !errors.Is(res.err, errQueueTimeout) { t.Fatalf("expected errQueueTimeout, got: %v", res.err) } if elapsed > 500*time.Millisecond { t.Fatalf("timeout took %v, expected near 150ms", elapsed) } if elapsed < 50*time.Millisecond { t.Fatalf("timeout too fast: %v", elapsed) } case <-time.After(2 * time.Second): t.Fatal("waiter did not timeout after shortening") } } // TestProviderPoolPolicyRefreshExtendsExistingWaiterTimeout verifies that // extending the root provider-pool queue timeout keeps a queued waiter alive // past the original deadline. func TestProviderPoolPolicyRefreshExtendsExistingWaiterTimeout(t *testing.T) { m := newModelQueueManager(nil) applyProviderPoolPolicy(m, NewGroupPolicy(8, 100*time.Millisecond)) entry := &edgenode.NodeEntry{NodeID: "node-extend"} cands := []candidateNode{{entry: entry, capacity: 1, providerID: "prov-extend"}} m.mu.Lock() g := m.getOrCreateGroupLocked("g-extend", groupPolicy{maxQueue: 8, queueTimeout: 100 * time.Millisecond}) g.inflight["node-extend:prov-extend"] = 1 m.mu.Unlock() resultCh := waitForAdmissionInPool(m, t, "g-extend", cands) // Extend the timeout to 3s right after the goroutine enters the select loop. applyProviderPoolPolicy(m, NewGroupPolicy(8, 3*time.Second)) // Wait past the original 100ms deadline. The goroutine should still be // waiting because the deadline was extended to 3s. time.Sleep(200 * time.Millisecond) select { case res := <-resultCh: t.Fatalf("waiter should still be queued at T+200ms, got: %v", res.err) default: } // Verify the deadline was extended. m.mu.Lock() deadline := m.groups["g-extend"].queue[0].deadline m.mu.Unlock() if time.Until(deadline) < 2*time.Second { t.Fatalf("expected extended deadline, got %v until deadline", time.Until(deadline)) } // Free the slot to dispatch. m.mu.Lock() m.releaseSlotLocked("g-extend", "node-extend", "prov-extend", false) m.mu.Unlock() select { case res := <-resultCh: if res.err != nil { t.Fatalf("waiter error after release: %v", res.err) } case <-time.After(1 * time.Second): t.Fatal("waiter did not dispatch after release") } } // TestProviderPoolPolicyRefreshDisablesExistingWaiterTimeout verifies that // setting the queue timeout to zero (no timeout) on an already-queued item // keeps the waiter alive indefinitely. func TestProviderPoolPolicyRefreshDisablesExistingWaiterTimeout(t *testing.T) { m := newModelQueueManager(nil) applyProviderPoolPolicy(m, NewGroupPolicy(8, 100*time.Millisecond)) entry := &edgenode.NodeEntry{NodeID: "node-disable"} cands := []candidateNode{{entry: entry, capacity: 1, providerID: "prov-disable"}} m.mu.Lock() g := m.getOrCreateGroupLocked("g-disable", groupPolicy{maxQueue: 8, queueTimeout: 100 * time.Millisecond}) g.inflight["node-disable:prov-disable"] = 1 m.mu.Unlock() resultCh := waitForAdmissionInPool(m, t, "g-disable", cands) // Disable the queue timeout. applyProviderPoolPolicy(m, groupPolicy{maxQueue: 8, queueTimeout: 0, queueTimeoutSet: true}) // The waiter should still be queued and have a zero deadline. m.mu.Lock() count := len(m.groups["g-disable"].queue) item := m.groups["g-disable"].queue[0] m.mu.Unlock() if count == 0 { t.Fatal("waiter was removed after timeout was disabled") } if !item.deadline.IsZero() { t.Fatalf("expected zero deadline after disabling timeout, got %v", item.deadline) } // Wait longer to ensure it's not a spurious timeout. time.Sleep(300 * time.Millisecond) select { case res := <-resultCh: t.Fatalf("waiter should still be queued after disabling, got: %v", res.err) default: } // Free the slot to dispatch. m.mu.Lock() m.releaseSlotLocked("g-disable", "node-disable", "prov-disable", false) m.mu.Unlock() select { case res := <-resultCh: if res.err != nil { t.Fatalf("waiter error after release: %v", res.err) } case <-time.After(1 * time.Second): t.Fatal("waiter did not dispatch after release") } } // TestProviderPoolPolicyRefreshExtensionIgnoresStaleTimer verifies that when a // refresh extends the deadline while the old timer is about to fire, the // waiter does NOT receive a timeout result. func TestProviderPoolPolicyRefreshExtensionIgnoresStaleTimer(t *testing.T) { m := newModelQueueManager(nil) applyProviderPoolPolicy(m, NewGroupPolicy(8, 50*time.Millisecond)) entry := &edgenode.NodeEntry{NodeID: "node-stale"} cands := []candidateNode{{entry: entry, capacity: 1, providerID: "prov-stale"}} m.mu.Lock() g := m.getOrCreateGroupLocked("g-stale", groupPolicy{maxQueue: 8, queueTimeout: 50 * time.Millisecond}) g.inflight["node-stale:prov-stale"] = 1 m.mu.Unlock() resultCh := waitForAdmissionInPool(m, t, "g-stale", cands) // Extend the timeout right after the goroutine enters the select loop // (well before the old 50ms timer fires). applyProviderPoolPolicy(m, NewGroupPolicy(8, 5*time.Second)) // Wait past the original 50ms deadline. time.Sleep(100 * time.Millisecond) select { case res := <-resultCh: t.Fatalf("waiter should not have timed out after extension, got: %v", res.err) default: } // Free the slot to dispatch the waiter. m.mu.Lock() m.releaseSlotLocked("g-stale", "node-stale", "prov-stale", false) m.mu.Unlock() select { case res := <-resultCh: if res.err != nil { t.Fatalf("waiter got error after extension + release: %v", res.err) } if res.candidate == nil || res.candidate.providerID != "prov-stale" { t.Fatalf("waiter dispatched to unexpected candidate: %+v", res.candidate) } case <-time.After(1 * time.Second): t.Fatal("waiter did not dispatch after extension + release") } } // TestProviderPoolMaxQueueShrinkPreservesExistingWaiters verifies that // shrinking the root max_queue below the number of currently-queued items does // not evict them. func TestProviderPoolMaxQueueShrinkPreservesExistingWaiters(t *testing.T) { m := newModelQueueManager(nil) applyProviderPoolPolicy(m, NewGroupPolicy(8, 5*time.Second)) entry := &edgenode.NodeEntry{NodeID: "node-shrink"} cands := []candidateNode{{entry: entry, capacity: 1, providerID: "prov-shrink"}} m.mu.Lock() // Set inflight=1 so capacity=1 is fully used up and every admit goes into the queue. m.getOrCreateGroupLocked("g-shrink", groupPolicy{maxQueue: 8, queueTimeout: 5 * time.Second}) m.groups["g-shrink"].inflight["node-shrink:prov-shrink"] = 1 m.mu.Unlock() // Queue 3 waiters via the real admit path. waitForAdmissionInPool confirms // each new queue entry under m.mu before the next goroutine starts. results := make([]chan admitResult, 3) for i := 0; i < 3; i++ { results[i] = waitForAdmissionInPool(m, t, "g-shrink", cands) } // Verify all 3 are queued. m.mu.Lock() count := len(m.groups["g-shrink"].queue) m.mu.Unlock() if count != 3 { t.Fatalf("expected 3 waiters queued, got %d", count) } // Shrink max_queue to 1. applyProviderPoolPolicy(m, NewGroupPolicy(1, 5*time.Second)) // Existing waiters must still be in the queue. m.mu.Lock() count = len(m.groups["g-shrink"].queue) m.mu.Unlock() if count != 3 { t.Fatalf("expected 3 waiters preserved after shrink, got %d", count) } // New admission must be blocked by errQueueFull. _, _, err := m.admitWithReason(t.Context(), "g-shrink", "", "", cands, groupPolicy{}, nil, false, true) if !errors.Is(err, errQueueFull) { t.Fatalf("expected errQueueFull after max_queue shrink, got: %v", err) } // Release the slot three times to drain all three waiters (capacity=1). for i := 0; i < 3; i++ { m.mu.Lock() m.releaseSlotLocked("g-shrink", "node-shrink", "prov-shrink", false) m.mu.Unlock() select { case res := <-results[i]: if res.err != nil { t.Fatalf("waiter %d error: %v", i, res.err) } case <-time.After(1 * time.Second): t.Fatalf("waiter %d did not dispatch", i) } } } // waitForAdmissionInPool starts a goroutine that enters the provider-pool // queue via admitWithReason and returns the result channel. It uses a // bounded poll under m.mu to confirm the goroutine has enqueued, so tests // are deterministic and race-free. func waitForAdmissionInPool(m *modelQueueManager, t *testing.T, groupKey string, cands []candidateNode) chan admitResult { t.Helper() m.mu.Lock() initialQueueLen := 0 if g := m.groups[groupKey]; g != nil { initialQueueLen = len(g.queue) } m.mu.Unlock() ch := make(chan admitResult, 1) go func() { c, _, err := m.admitWithReason(t.Context(), groupKey, "", "", cands, groupPolicy{}, nil, false, true) ch <- admitResult{candidate: c, err: err} }() deadline := time.After(2 * time.Second) for { select { case result := <-ch: ch <- result return ch case <-deadline: t.Fatalf("waiter did not enter queue select loop") default: m.mu.Lock() var queueLen int if g := m.groups[groupKey]; g != nil { queueLen = len(g.queue) } m.mu.Unlock() if queueLen > initialQueueLen { return ch } runtime.Gosched() } } } // TestProviderPoolPolicyRefreshReenablesExistingWaiterTimeout verifies that // re-enabling a queue timeout (transitioning from zero to a positive value) // on an already-disabled waiter restores the deadline derived from its // original enqueue time and eventually times it out. func TestProviderPoolPolicyRefreshReenablesExistingWaiterTimeout(t *testing.T) { m := newModelQueueManager(nil) applyProviderPoolPolicy(m, NewGroupPolicy(8, 200*time.Millisecond)) entry := &edgenode.NodeEntry{NodeID: "node-reenable"} cands := []candidateNode{{entry: entry, capacity: 1, providerID: "prov-reenable"}} m.mu.Lock() g := m.getOrCreateGroupLocked("g-reenable", groupPolicy{maxQueue: 8, queueTimeout: 200 * time.Millisecond}) g.inflight["node-reenable:prov-reenable"] = 1 m.mu.Unlock() resultCh := waitForAdmissionInPool(m, t, "g-reenable", cands) // Disable the timeout (set to 0). applyProviderPoolPolicy(m, groupPolicy{maxQueue: 8, queueTimeout: 0, queueTimeoutSet: true}) // Verify deadline became zero. m.mu.Lock() item := m.groups["g-reenable"].queue[0] if !item.deadline.IsZero() { m.mu.Unlock() t.Fatalf("expected zero deadline after disable, got %v", item.deadline) } m.mu.Unlock() // Wait past what would have been the old deadline to confirm no spurious timeout. time.Sleep(300 * time.Millisecond) select { case res := <-resultCh: t.Fatalf("waiter should still be queued after disable, got: %v", res.err) default: } // Re-enable with a 650ms timeout measured from the original enqueue time. reenabledAt := time.Now() applyProviderPoolPolicy(m, NewGroupPolicy(8, 650*time.Millisecond)) // Verify deadline was re-assigned based on enqueuedAt + new timeout. m.mu.Lock() newDeadline := m.groups["g-reenable"].queue[0].deadline m.mu.Unlock() if newDeadline.IsZero() { t.Fatal("expected non-zero deadline after re-enable") } if remaining := time.Until(newDeadline); remaining < 200*time.Millisecond || remaining > 500*time.Millisecond { t.Fatalf("expected re-enabled deadline based on original enqueue time, remaining=%v", remaining) } select { case res := <-resultCh: if !errors.Is(res.err, errQueueTimeout) { t.Fatalf("expected timeout after re-enable, got: %v", res.err) } if elapsed := time.Since(reenabledAt); elapsed < 150*time.Millisecond || elapsed > 700*time.Millisecond { t.Fatalf("re-enabled timeout fired outside expected window: %v", elapsed) } case <-time.After(1 * time.Second): t.Fatal("waiter did not adopt the re-enabled timeout") } } // TestProviderPoolPolicyRefreshEnablesInitiallyDisabledWaiterTimeout verifies // that a waiter enqueued while timeout=0 continues observing policy changes and // arms a timer when a positive timeout is applied later. func TestProviderPoolPolicyRefreshEnablesInitiallyDisabledWaiterTimeout(t *testing.T) { m := newModelQueueManager(nil) applyProviderPoolPolicy(m, groupPolicy{maxQueue: 8, queueTimeoutSet: true}) entry := &edgenode.NodeEntry{NodeID: "node-initial-zero"} cands := []candidateNode{{entry: entry, capacity: 1, providerID: "prov-initial-zero"}} m.mu.Lock() g := m.getOrCreateGroupLocked("g-initial-zero", groupPolicy{maxQueue: 8, queueTimeoutSet: true}) g.inflight["node-initial-zero:prov-initial-zero"] = 1 m.mu.Unlock() resultCh := waitForAdmissionInPool(m, t, "g-initial-zero", cands) enabledAt := time.Now() applyProviderPoolPolicy(m, NewGroupPolicy(8, 200*time.Millisecond)) select { case res := <-resultCh: if !errors.Is(res.err, errQueueTimeout) { t.Fatalf("expected timeout after enabling initial-zero waiter, got: %v", res.err) } if elapsed := time.Since(enabledAt); elapsed < 50*time.Millisecond || elapsed > 500*time.Millisecond { t.Fatalf("initial-zero waiter timeout outside expected window: %v", elapsed) } case <-time.After(1 * time.Second): t.Fatal("initial-zero waiter did not adopt the enabled timeout") } } // TestProviderPoolPolicyRefreshDoesNotChangeLegacyWaiter verifies that the // deadline rebase path only touches provider-pool items (providerPool=true). // Legacy queue items (providerPool=false) must retain their original deadline // even after a policy refresh that shortens the timeout. func TestProviderPoolPolicyRefreshDoesNotChangeLegacyWaiter(t *testing.T) { m := newModelQueueManager(nil) applyProviderPoolPolicy(m, NewGroupPolicy(8, 2*time.Second)) entry := &edgenode.NodeEntry{NodeID: "node-legacy"} cands := []candidateNode{{entry: entry, capacity: 1, providerID: "prov-legacy"}} m.mu.Lock() g := m.getOrCreateGroupLocked("g-legacy", groupPolicy{maxQueue: 8, queueTimeout: 2 * time.Second}) g.inflight["node-legacy:prov-legacy"] = 1 m.mu.Unlock() resultCh := make(chan admitResult, 1) go func() { candidate, _, err := m.admitWithReason( t.Context(), "g-legacy", "", "", cands, groupPolicy{maxQueue: 8, queueTimeout: 2 * time.Second}, nil, false, false, ) resultCh <- admitResult{candidate: candidate, err: err} }() waitForQueueLen(t, m, "g-legacy", 1) m.mu.Lock() originalDeadline := m.groups["g-legacy"].queue[0].deadline m.mu.Unlock() applyProviderPoolPolicy(m, NewGroupPolicy(8, 100*time.Millisecond)) m.mu.Lock() updatedDeadline := m.groups["g-legacy"].queue[0].deadline m.mu.Unlock() if !updatedDeadline.Equal(originalDeadline) { t.Fatalf("legacy waiter deadline changed to %v, expected %v", updatedDeadline, originalDeadline) } m.mu.Lock() m.releaseSlotLocked("g-legacy", "node-legacy", "prov-legacy", false) m.mu.Unlock() select { case res := <-resultCh: if res.err != nil { t.Fatalf("legacy waiter error after release: %v", res.err) } case <-time.After(1 * time.Second): t.Fatal("legacy waiter did not dispatch after release") } }