package service import ( "context" "errors" "testing" "time" edgenode "iop/apps/edge/internal/node" "iop/packages/go/config" ) // longSlotCount reads the group's long in-flight counter for a provider slot. func longSlotCount(m *modelQueueManager, groupKey, nodeID, providerID string) int { m.mu.Lock() defer m.mu.Unlock() if providerID != "" { key := providerResourceKey{nodeID: nodeID, providerID: providerID} if res, ok := m.resources[key]; ok { return res.longInFlight } return 0 } g, ok := m.groups[groupKey] if !ok { return -1 } return g.longInflight[nodeID] } func inflightSlotCount(m *modelQueueManager, groupKey, nodeID, providerID string) int { m.mu.Lock() defer m.mu.Unlock() if providerID != "" { key := providerResourceKey{nodeID: nodeID, providerID: providerID} if res, ok := m.resources[key]; ok { return res.inFlight } return 0 } g, ok := m.groups[groupKey] if !ok { return -1 } return g.inflight[nodeID] } // TestModelQueueLongDispatchReservesBothCounters verifies S04: a long request // occupies both the normal in-flight counter and the long in-flight counter, and // every terminal reason (complete/error/cancelled) recovers both. func TestModelQueueLongDispatchReservesBothCounters(t *testing.T) { for _, reason := range []string{"complete", "error", "cancelled"} { t.Run(reason, func(t *testing.T) { entry := &edgenode.NodeEntry{NodeID: "node-long"} cands := []candidateNode{{entry: entry, capacity: 3, providerID: "prov-long", longContextCapacity: 2}} m := newModelQueueManager(nil) sel, err := m.admit(context.Background(), "g-long", "", "", cands, groupPolicy{}, nil, true, false) if err != nil || sel == nil { t.Fatalf("long admit: %v", err) } if got := inflightSlotCount(m, "g-long", "node-long", "prov-long"); got != 1 { t.Errorf("normal in-flight after admit = %d, want 1", got) } if got := longSlotCount(m, "g-long", "node-long", "prov-long"); got != 1 { t.Errorf("long in-flight after admit = %d, want 1", got) } m.trackLease(sel.leaseID, "run-long-1") if got := m.longInFlightForProvider("node-long", "prov-long"); got != 1 { t.Errorf("longInFlightForProvider after track = %d, want 1", got) } m.releaseRun("run-long-1", reason) if got := inflightSlotCount(m, "g-long", "node-long", "prov-long"); got != 0 { t.Errorf("normal in-flight after %s = %d, want 0", reason, got) } if got := longSlotCount(m, "g-long", "node-long", "prov-long"); got != 0 { t.Errorf("long in-flight after %s = %d, want 0", reason, got) } if got := m.longInFlightForProvider("node-long", "prov-long"); got != 0 { t.Errorf("longInFlightForProvider after %s = %d, want 0", reason, got) } }) } } // TestQueueSkipsLongHeadWhenNormalBehindCanDispatch verifies the queue-skip // contract (S06 / queue-skip): when the head of the queue is a long request // whose provider long slots are full, a normal request queued behind it is // still dispatched on ordinary capacity, and the blocked long head stays queued. func TestQueueSkipsLongHeadWhenNormalBehindCanDispatch(t *testing.T) { entry := &edgenode.NodeEntry{NodeID: "node-skip"} // Provider has normal capacity 2 and a single long slot. cands := []candidateNode{{entry: entry, capacity: 2, providerID: "prov-skip", longContextCapacity: 1}} m := newModelQueueManager(nil) m.mu.Lock() g := m.getOrCreateGroupLocked("g-skip", groupPolicy{}) // One long request already in flight: the single long slot is full while one // of the two normal slots remains free. g.inflight["node-skip:prov-skip"] = 1 g.longInflight["node-skip:prov-skip"] = 1 longItem := &queueItem{ candidates: cands, waitCh: make(chan admitResult, 1), deadline: time.Now().Add(2 * time.Second), long: true, } normalItem := &queueItem{ candidates: cands, waitCh: make(chan admitResult, 1), deadline: time.Now().Add(2 * time.Second), } // long head first, normal request queued behind it. g.queue = []*queueItem{longItem, normalItem} m.pumpAllLocked() m.mu.Unlock() // The normal request behind the blocked long head must be dispatched. select { case res := <-normalItem.waitCh: if res.err != nil { t.Fatalf("normal item expected dispatch, got error: %v", res.err) } if res.candidate == nil || res.candidate.providerID != "prov-skip" { t.Fatalf("normal item: unexpected candidate %v", res.candidate) } default: t.Fatal("normal request behind long head was not dispatched (head-of-line blocking)") } // The long head must remain queued because its long slot is still full. select { case <-longItem.waitCh: t.Fatal("long head should not dispatch while its long slot is full") default: } m.mu.Lock() if len(g.queue) != 1 || g.queue[0] != longItem { n := len(g.queue) m.mu.Unlock() t.Fatalf("expected only the long head to remain queued, got %d item(s)", n) } m.mu.Unlock() } // TestQueueSkipPreservesFIFOAmongDispatchableNormalItems verifies that the // front-scan dispatch keeps FIFO order among equally-dispatchable items: with a // single normal slot, the earlier queued item is dispatched first and the later // one only after another slot frees. func TestQueueSkipPreservesFIFOAmongDispatchableNormalItems(t *testing.T) { entry := &edgenode.NodeEntry{NodeID: "node-fifo2"} cands := []candidateNode{{entry: entry, capacity: 1, providerID: "prov-fifo2"}} m := newModelQueueManager(nil) m.mu.Lock() g := m.getOrCreateGroupLocked("g-fifo2", groupPolicy{}) // Fill the single normal slot so both queued items must wait. g.inflight["node-fifo2:prov-fifo2"] = 1 first := &queueItem{candidates: cands, waitCh: make(chan admitResult, 1), deadline: time.Now().Add(2 * time.Second)} second := &queueItem{candidates: cands, waitCh: make(chan admitResult, 1), deadline: time.Now().Add(2 * time.Second)} g.queue = []*queueItem{first, second} m.mu.Unlock() // Release one slot: only the earlier (first) item may dispatch. m.mu.Lock() m.releaseSlotLocked("g-fifo2", "node-fifo2", "prov-fifo2", false) m.mu.Unlock() select { case res := <-first.waitCh: if res.err != nil { t.Fatalf("first item dispatch error: %v", res.err) } default: t.Fatal("first (earliest) item was not dispatched first") } select { case <-second.waitCh: t.Fatal("second item should still wait (only one slot freed)") default: } // Release again → the second item dispatches. m.mu.Lock() m.releaseSlotLocked("g-fifo2", "node-fifo2", "prov-fifo2", false) m.mu.Unlock() select { case res := <-second.waitCh: if res.err != nil { t.Fatalf("second item dispatch error: %v", res.err) } default: t.Fatal("second item not dispatched after second slot release") } } // TestModelQueueNormalRequestDoesNotReserveLongSlot verifies a normal request // occupies only the normal in-flight counter, never the long counter, even on a // provider that declares long capacity. func TestModelQueueNormalRequestDoesNotReserveLongSlot(t *testing.T) { entry := &edgenode.NodeEntry{NodeID: "node-n"} cands := []candidateNode{{entry: entry, capacity: 3, providerID: "prov-n", longContextCapacity: 2}} m := newModelQueueManager(nil) sel, err := m.admit(context.Background(), "g-n", "", "", cands, groupPolicy{}, nil, false, false) // long omitted → normal if err != nil || sel == nil { t.Fatalf("normal admit: %v", err) } if got := inflightSlotCount(m, "g-n", "node-n", "prov-n"); got != 1 { t.Errorf("normal in-flight = %d, want 1", got) } if got := longSlotCount(m, "g-n", "node-n", "prov-n"); got != 0 { t.Errorf("long in-flight for normal request = %d, want 0", got) } } // TestModelQueueLongSlotFullExcludesLongKeepsNormal verifies S05 and D06: a // provider whose long slots are full is excluded from long candidates (the long // request moves to another provider or queues), but the same provider remains a // valid candidate for normal requests while its normal capacity has room. func TestModelQueueLongSlotFullExcludesLongKeepsNormal(t *testing.T) { entryA := &edgenode.NodeEntry{NodeID: "node-a"} entryB := &edgenode.NodeEntry{NodeID: "node-b"} cands := []candidateNode{ {entry: entryA, capacity: 3, priority: 0, providerID: "prov-a", longContextCapacity: 1}, {entry: entryB, capacity: 3, priority: 1, providerID: "prov-b", longContextCapacity: 1}, } m := newModelQueueManager(nil) // Long #1 → prov-a (priority 0 preferred), filling prov-a's single long slot. sel1, err := m.admit(context.Background(), "g-x", "", "", cands, groupPolicy{}, nil, true, false) if err != nil || sel1 == nil { t.Fatalf("long1 admit: %v", err) } if sel1.providerID != "prov-a" { t.Fatalf("long1 provider = %q, want prov-a", sel1.providerID) } // Long #2 must skip the long-full prov-a and land on prov-b. sel2, err := m.admit(context.Background(), "g-x", "", "", cands, groupPolicy{}, nil, true, false) if err != nil || sel2 == nil { t.Fatalf("long2 admit: %v", err) } if sel2.providerID != "prov-b" { t.Fatalf("long2 provider = %q, want prov-b (prov-a long slot full)", sel2.providerID) } // A normal request restricted to the long-full prov-a is still admitted, // because normal admission ignores long slot state (D06). normalCands := []candidateNode{cands[0]} selN, err := m.admit(context.Background(), "g-x", "", "", normalCands, groupPolicy{}, nil, false, false) if err != nil || selN == nil { t.Fatalf("normal admit on long-full provider: %v", err) } if selN.providerID != "prov-a" { t.Fatalf("normal provider = %q, want prov-a", selN.providerID) } // A further long request restricted to the long-full prov-a cannot be // admitted immediately; it queues and times out. longOnlyA := []candidateNode{cands[0]} _, err = m.admit(context.Background(), "g-x", "", "", longOnlyA, groupPolicy{maxQueue: 1, queueTimeout: 20 * time.Millisecond}, nil, true, false) if !errors.Is(err, errQueueTimeout) { t.Fatalf("long request on long-full provider: expected queue timeout, got %v", err) } } func TestModelQueueReasonLongContextCapacityFull(t *testing.T) { entryA := &edgenode.NodeEntry{NodeID: "node-a"} cands := []candidateNode{ {entry: entryA, capacity: 3, priority: 0, providerID: "prov-a", longContextCapacity: 1}, } m := newModelQueueManager(nil) // Long #1 → prov-a, filling prov-a's single long slot. _, _, err := m.admitWithReason(context.Background(), "g-x", "", "", cands, groupPolicy{}, nil, true, false) if err != nil { t.Fatalf("first admit: %v", err) } // Long #2 → prov-a. It should block because longContextCapacity is full. done := make(chan struct{}) go func() { defer close(done) // The queue timeout has to outlast the observation window below: the // assertion can only read the item's reason while the item is queued. _, _, err := m.admitWithReason(context.Background(), "g-x", "", "", cands, groupPolicy{maxQueue: 1, queueTimeout: 500 * time.Millisecond}, nil, true, false) if !errors.Is(err, errQueueTimeout) { t.Errorf("expected timeout, got %v", err) } }() // Poll rather than sleeping a fixed slice: under load the goroutine can need // more than a few milliseconds to reach the queue. waitForQueueLen(t, m, "g-x", 1) m.mu.Lock() g, ok := m.groups["g-x"] if !ok || len(g.queue) != 1 { m.mu.Unlock() t.Fatalf("expected 1 queued item, got group ok=%t, len=%d", ok, len(g.queue)) } reason := g.queue[0].reason m.mu.Unlock() if reason != "long_context_capacity_full" { t.Errorf("expected queue reason 'long_context_capacity_full', got %q", reason) } <-done } func TestModelQueueReasonCapacityFull(t *testing.T) { entryA := &edgenode.NodeEntry{NodeID: "node-a"} cands := []candidateNode{ {entry: entryA, capacity: 1, priority: 0, providerID: "prov-a"}, } m := newModelQueueManager(nil) // #1 → prov-a, filling capacity. _, _, err := m.admitWithReason(context.Background(), "g-y", "", "", cands, groupPolicy{}, nil, false, false) if err != nil { t.Fatalf("first admit: %v", err) } // #2 → prov-a. It should block because capacity is full. done := make(chan struct{}) go func() { defer close(done) // As above: the item must stay queued long enough to be observed. _, _, err := m.admitWithReason(context.Background(), "g-y", "", "", cands, groupPolicy{maxQueue: 1, queueTimeout: 500 * time.Millisecond}, nil, false, false) if !errors.Is(err, errQueueTimeout) { t.Errorf("expected timeout, got %v", err) } }() waitForQueueLen(t, m, "g-y", 1) m.mu.Lock() g, ok := m.groups["g-y"] if !ok || len(g.queue) != 1 { m.mu.Unlock() t.Fatalf("expected 1 queued item") } reason := g.queue[0].reason m.mu.Unlock() if reason != "capacity_full" { t.Errorf("expected queue reason 'capacity_full', got %q", reason) } <-done } func TestProviderLongCapacitySharedAcrossOrnithModelAliases(t *testing.T) { const ( nodeID = "rtx5090-lemonade-node" providerID = "rtx5090-lemonade" ) modelAliases := []string{"ornith:35b", "ornith-fast"} entry := &edgenode.NodeEntry{NodeID: nodeID} cands := []candidateNode{ {entry: entry, capacity: 5, providerID: providerID, longContextCapacity: 1}, } store := edgenode.NewNodeStore() store.Add(&edgenode.NodeRecord{ ID: nodeID, Providers: []config.NodeProviderConf{ { ID: providerID, Capacity: 5, LongContextCapacity: 1, Adapter: "vllm", }, }, }) m := newModelQueueManager(store) m.setStore(store) start := make(chan struct{}) resCh := make(chan concurrentAdmitResult, 2) for i := 0; i < 2; i++ { go func(id int) { <-start ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) defer cancel() groupKey := modelAliases[id] sel, err := m.admit(ctx, groupKey, "", "", cands, groupPolicy{maxQueue: 1, queueTimeout: 30 * time.Millisecond}, nil, true, false) resCh <- concurrentAdmitResult{candidate: sel, err: err} }(i) } close(start) var results []concurrentAdmitResult for i := 0; i < 2; i++ { results = append(results, <-resCh) } successCount := 0 failCount := 0 var successSel *candidateNode for _, res := range results { if res.err == nil { successCount++ successSel = res.candidate } else { failCount++ } } if successCount != 1 || failCount != 1 { t.Fatalf("expected exactly 1 success and 1 failure under concurrent long admission (capacity=1), got success=%d fail=%d", successCount, failCount) } // Verify long peak was 1 if got := m.longInFlightForProvider(nodeID, providerID); got != 1 { t.Errorf("expected long peak to be 1, got %d", got) } // Release successful slot m.trackLease(successSel.leaseID, "run-shared-1") m.releaseRun("run-shared-1", "complete") // Verify long counts recovered to 0 if got := m.longInFlightForProvider(nodeID, providerID); got != 0 { t.Errorf("expected long counts to recover to 0 after release, got %d", got) } } func TestProviderResourceLongReleaseAfterCapacityDisabled(t *testing.T) { entry := &edgenode.NodeEntry{NodeID: "node-ref-1"} cands := []candidateNode{{entry: entry, capacity: 1, providerID: "prov-ref-1", longContextCapacity: 1}} store := edgenode.NewNodeStore() rec := &edgenode.NodeRecord{ ID: "node-ref-1", Providers: []config.NodeProviderConf{ {ID: "prov-ref-1", Capacity: 1, LongContextCapacity: 1, Adapter: "vllm"}, }, } store.Add(rec) m := newModelQueueManager(store) m.setStore(store) sel, err := m.admit(context.Background(), "g-ref", "", "", cands, groupPolicy{}, nil, true, false) if err != nil || sel == nil { t.Fatalf("admit: %v", err) } if got := m.longInFlightForProvider("node-ref-1", "prov-ref-1"); got != 1 { t.Fatalf("expected longInFlight=1, got %d", got) } rec.Providers[0].Capacity = 0 rec.Providers[0].LongContextCapacity = 0 m.setStore(store) m.trackLease(sel.leaseID, "run-ref-1") m.releaseRun("run-ref-1", "complete") if got := m.longInFlightForProvider("node-ref-1", "prov-ref-1"); got != 0 { t.Errorf("longInFlight after capacity disabled release = %d, want 0", got) } } func TestProviderResourceLongReservationUsesEffectiveRefreshState(t *testing.T) { entry := &edgenode.NodeEntry{NodeID: "node-eff"} cands := []candidateNode{{entry: entry, capacity: 1, providerID: "prov-eff", longContextCapacity: 1}} store := edgenode.NewNodeStore() rec := &edgenode.NodeRecord{ ID: "node-eff", Providers: []config.NodeProviderConf{ {ID: "prov-eff", Capacity: 1, LongContextCapacity: 1, Adapter: "vllm"}, }, } store.Add(rec) m := newModelQueueManager(store) m.setStore(store) _, err := m.admit(context.Background(), "g-eff", "", "", cands, groupPolicy{}, nil, true, false) if err != nil { t.Fatalf("first admit: %v", err) } done := make(chan struct{}) go func() { defer close(done) _, err := m.admit(context.Background(), "g-eff", "", "", cands, groupPolicy{maxQueue: 1}, nil, true, false) if err != nil { t.Errorf("queued admit failed: %v", err) } }() waitForQueueLen(t, m, "g-eff", 1) rec.Providers[0].LongContextCapacity = 0 m.setStore(store) m.releaseSlotWithLong("g-eff", "node-eff", "prov-eff", true) <-done if got := m.longInFlightForProvider("node-eff", "prov-eff"); got != 0 { t.Errorf("expected longInFlight=0 for the dispatched queued run under disabled capacity, got %d", got) } } func TestProviderResourceOrphanRemovedAfterLastRelease(t *testing.T) { entry := &edgenode.NodeEntry{NodeID: "node-orph"} cands := []candidateNode{{entry: entry, capacity: 1, providerID: "prov-orph", longContextCapacity: 1}} store := edgenode.NewNodeStore() rec := &edgenode.NodeRecord{ ID: "node-orph", Providers: []config.NodeProviderConf{ {ID: "prov-orph", Capacity: 1, LongContextCapacity: 1, Adapter: "vllm"}, }, } store.Add(rec) m := newModelQueueManager(store) m.setStore(store) _, err := m.admit(context.Background(), "g-orph", "", "", cands, groupPolicy{}, nil, true, false) if err != nil { t.Fatalf("admit: %v", err) } key := providerResourceKey{nodeID: "node-orph", providerID: "prov-orph"} if res := m.resources[key]; res == nil || res.orphan { t.Fatal("expected active non-orphan resource") } rec.Providers = nil m.setStore(store) m.mu.Lock() res, ok := m.resources[key] m.mu.Unlock() if !ok || res == nil || !res.orphan { t.Fatalf("expected resource to be kept as orphan, ok=%t, res=%v", ok, res) } m.releaseSlotWithLong("g-orph", "node-orph", "prov-orph", true) m.mu.Lock() _, ok = m.resources[key] m.mu.Unlock() if ok { t.Fatal("expected orphan resource to be removed from map after last release") } } // TestGlobalPumpLongHeadDoesNotBlockNormalOtherGroup verifies that head-of-line // avoidance holds across model groups, not only inside one queue: a long request // that arrived first and is stuck on a full long slot must not hold up a normal // request that arrived later in a different group and fits ordinary capacity. func TestGlobalPumpLongHeadDoesNotBlockNormalOtherGroup(t *testing.T) { entry := &edgenode.NodeEntry{NodeID: "node-xg"} // Two normal slots, a single long slot. cands := []candidateNode{{entry: entry, capacity: 2, providerID: "prov-xg", longContextCapacity: 1}} m := newModelQueueManager(nil) // One long request in flight fills the only long slot; one normal slot is free. if _, err := m.admit(context.Background(), "g-xg-a", "", "", cands, groupPolicy{}, nil, true, false); err != nil { t.Fatalf("long admit: %v", err) } // Arrival order: the blocked long request first, the normal one behind it in // another group. longItem := queueItemForTest(cands, true) enqueueForTest(m, "g-xg-a", longItem, nil) normalItem := queueItemForTest(cands, false) enqueueForTest(m, "g-xg-b", normalItem, nil) m.mu.Lock() m.pumpAllLocked() m.mu.Unlock() select { case res := <-normalItem.waitCh: if res.err != nil { t.Fatalf("normal item in the other group expected dispatch, got error: %v", res.err) } default: t.Fatal("normal request in another group was blocked behind a long head (cross-group head-of-line blocking)") } select { case <-longItem.waitCh: t.Fatal("long head must stay queued while its long slot is full") default: } if got := pendingItemCount(m); got != 1 { t.Fatalf("expected only the long head to remain pending, got %d item(s)", got) } } // TestGlobalPumpNoLeakAfterMixedLongAndNormalDrain verifies that repeated mixed // long/normal traffic across groups leaves every counter at zero: a scheduler // that reserves in one place and releases in another is exactly where long-slot // leaks hide. func TestGlobalPumpNoLeakAfterMixedLongAndNormalDrain(t *testing.T) { entry := &edgenode.NodeEntry{NodeID: "node-drain"} cands := []candidateNode{{entry: entry, capacity: 2, providerID: "prov-drain", longContextCapacity: 1}} m := newModelQueueManager(nil) for round := 0; round < 20; round++ { long, err := m.admit(context.Background(), "g-drain-a", "", "", cands, groupPolicy{}, nil, true, false) if err != nil || long == nil { t.Fatalf("round %d long admit: %v", round, err) } normal, err := m.admit(context.Background(), "g-drain-b", "", "", cands, groupPolicy{}, nil, false, false) if err != nil || normal == nil { t.Fatalf("round %d normal admit: %v", round, err) } m.releaseLease(long.leaseID, "complete") m.releaseLease(normal.leaseID, "complete") } inFlight, longInFlight := providerResourceCounts(m, "node-drain", "prov-drain") if inFlight != 0 || longInFlight != 0 { t.Fatalf("expected zero counters after drain, got inflight=%d long=%d", inFlight, longInFlight) } if got := leaseCount(m); got != 0 { t.Fatalf("expected zero live leases after drain, got %d", got) } if got := pendingItemCount(m); got != 0 { t.Fatalf("expected zero pending items after drain, got %d", got) } }