toki
0ffcb88db0
- Archive provider-resource-admission-ownership milestone/SDD - Align contract: CP-edge wire, runtime refresh, node runtime, OpenAI surface - Update roadmap: phase state, priority queue - Update specs: control-plane ops, OpenAI surface, edge execution, provider pool refresh - Add node runtime supervisor bootstrapping and unit tests - Fix control-plane edge registry handler and http_views - Fix edge model queue admission and long context queue tests
634 lines
21 KiB
Go
634 lines
21 KiB
Go
package service
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"testing"
|
|
"time"
|
|
|
|
edgenode "iop/apps/edge/internal/node"
|
|
"iop/packages/go/config"
|
|
)
|
|
|
|
// longSlotCount reads the group's long in-flight counter for a provider slot.
|
|
func longSlotCount(m *modelQueueManager, groupKey, nodeID, providerID string) int {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
if providerID != "" {
|
|
key := providerResourceKey{nodeID: nodeID, providerID: providerID}
|
|
if res, ok := m.resources[key]; ok {
|
|
return res.longInFlight
|
|
}
|
|
return 0
|
|
}
|
|
g, ok := m.groups[groupKey]
|
|
if !ok {
|
|
return -1
|
|
}
|
|
return g.longInflight[nodeID]
|
|
}
|
|
|
|
func inflightSlotCount(m *modelQueueManager, groupKey, nodeID, providerID string) int {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
if providerID != "" {
|
|
key := providerResourceKey{nodeID: nodeID, providerID: providerID}
|
|
if res, ok := m.resources[key]; ok {
|
|
return res.inFlight
|
|
}
|
|
return 0
|
|
}
|
|
g, ok := m.groups[groupKey]
|
|
if !ok {
|
|
return -1
|
|
}
|
|
return g.inflight[nodeID]
|
|
}
|
|
|
|
// TestModelQueueLongDispatchReservesBothCounters verifies S04: a long request
|
|
// occupies both the normal in-flight counter and the long in-flight counter, and
|
|
// every terminal reason (complete/error/cancelled) recovers both.
|
|
func TestModelQueueLongDispatchReservesBothCounters(t *testing.T) {
|
|
for _, reason := range []string{"complete", "error", "cancelled"} {
|
|
t.Run(reason, func(t *testing.T) {
|
|
entry := &edgenode.NodeEntry{NodeID: "node-long"}
|
|
cands := []candidateNode{{entry: entry, capacity: 3, providerID: "prov-long", longContextCapacity: 2}}
|
|
m := newModelQueueManager(nil)
|
|
|
|
sel, err := m.admit(context.Background(), "g-long", "", "", cands, groupPolicy{}, nil, true, false)
|
|
if err != nil || sel == nil {
|
|
t.Fatalf("long admit: %v", err)
|
|
}
|
|
if got := inflightSlotCount(m, "g-long", "node-long", "prov-long"); got != 1 {
|
|
t.Errorf("normal in-flight after admit = %d, want 1", got)
|
|
}
|
|
if got := longSlotCount(m, "g-long", "node-long", "prov-long"); got != 1 {
|
|
t.Errorf("long in-flight after admit = %d, want 1", got)
|
|
}
|
|
|
|
m.trackLease(sel.leaseID, "run-long-1")
|
|
if got := m.longInFlightForProvider("node-long", "prov-long"); got != 1 {
|
|
t.Errorf("longInFlightForProvider after track = %d, want 1", got)
|
|
}
|
|
|
|
m.releaseRun("run-long-1", reason)
|
|
if got := inflightSlotCount(m, "g-long", "node-long", "prov-long"); got != 0 {
|
|
t.Errorf("normal in-flight after %s = %d, want 0", reason, got)
|
|
}
|
|
if got := longSlotCount(m, "g-long", "node-long", "prov-long"); got != 0 {
|
|
t.Errorf("long in-flight after %s = %d, want 0", reason, got)
|
|
}
|
|
if got := m.longInFlightForProvider("node-long", "prov-long"); got != 0 {
|
|
t.Errorf("longInFlightForProvider after %s = %d, want 0", reason, got)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestQueueSkipsLongHeadWhenNormalBehindCanDispatch verifies the queue-skip
|
|
// contract (S06 / queue-skip): when the head of the queue is a long request
|
|
// whose provider long slots are full, a normal request queued behind it is
|
|
// still dispatched on ordinary capacity, and the blocked long head stays queued.
|
|
func TestQueueSkipsLongHeadWhenNormalBehindCanDispatch(t *testing.T) {
|
|
entry := &edgenode.NodeEntry{NodeID: "node-skip"}
|
|
// Provider has normal capacity 2 and a single long slot.
|
|
cands := []candidateNode{{entry: entry, capacity: 2, providerID: "prov-skip", longContextCapacity: 1}}
|
|
m := newModelQueueManager(nil)
|
|
|
|
m.mu.Lock()
|
|
g := m.getOrCreateGroupLocked("g-skip", groupPolicy{})
|
|
// One long request already in flight: the single long slot is full while one
|
|
// of the two normal slots remains free.
|
|
g.inflight["node-skip:prov-skip"] = 1
|
|
g.longInflight["node-skip:prov-skip"] = 1
|
|
|
|
longItem := &queueItem{
|
|
candidates: cands,
|
|
waitCh: make(chan admitResult, 1),
|
|
deadline: time.Now().Add(2 * time.Second),
|
|
long: true,
|
|
}
|
|
normalItem := &queueItem{
|
|
candidates: cands,
|
|
waitCh: make(chan admitResult, 1),
|
|
deadline: time.Now().Add(2 * time.Second),
|
|
}
|
|
// long head first, normal request queued behind it.
|
|
g.queue = []*queueItem{longItem, normalItem}
|
|
m.pumpAllLocked()
|
|
m.mu.Unlock()
|
|
|
|
// The normal request behind the blocked long head must be dispatched.
|
|
select {
|
|
case res := <-normalItem.waitCh:
|
|
if res.err != nil {
|
|
t.Fatalf("normal item expected dispatch, got error: %v", res.err)
|
|
}
|
|
if res.candidate == nil || res.candidate.providerID != "prov-skip" {
|
|
t.Fatalf("normal item: unexpected candidate %v", res.candidate)
|
|
}
|
|
default:
|
|
t.Fatal("normal request behind long head was not dispatched (head-of-line blocking)")
|
|
}
|
|
|
|
// The long head must remain queued because its long slot is still full.
|
|
select {
|
|
case <-longItem.waitCh:
|
|
t.Fatal("long head should not dispatch while its long slot is full")
|
|
default:
|
|
}
|
|
|
|
m.mu.Lock()
|
|
if len(g.queue) != 1 || g.queue[0] != longItem {
|
|
n := len(g.queue)
|
|
m.mu.Unlock()
|
|
t.Fatalf("expected only the long head to remain queued, got %d item(s)", n)
|
|
}
|
|
m.mu.Unlock()
|
|
}
|
|
|
|
// TestQueueSkipPreservesFIFOAmongDispatchableNormalItems verifies that the
|
|
// front-scan dispatch keeps FIFO order among equally-dispatchable items: with a
|
|
// single normal slot, the earlier queued item is dispatched first and the later
|
|
// one only after another slot frees.
|
|
func TestQueueSkipPreservesFIFOAmongDispatchableNormalItems(t *testing.T) {
|
|
entry := &edgenode.NodeEntry{NodeID: "node-fifo2"}
|
|
cands := []candidateNode{{entry: entry, capacity: 1, providerID: "prov-fifo2"}}
|
|
m := newModelQueueManager(nil)
|
|
|
|
m.mu.Lock()
|
|
g := m.getOrCreateGroupLocked("g-fifo2", groupPolicy{})
|
|
// Fill the single normal slot so both queued items must wait.
|
|
g.inflight["node-fifo2:prov-fifo2"] = 1
|
|
first := &queueItem{candidates: cands, waitCh: make(chan admitResult, 1), deadline: time.Now().Add(2 * time.Second)}
|
|
second := &queueItem{candidates: cands, waitCh: make(chan admitResult, 1), deadline: time.Now().Add(2 * time.Second)}
|
|
g.queue = []*queueItem{first, second}
|
|
m.mu.Unlock()
|
|
|
|
// Release one slot: only the earlier (first) item may dispatch.
|
|
m.mu.Lock()
|
|
m.releaseSlotLocked("g-fifo2", "node-fifo2", "prov-fifo2", false)
|
|
m.mu.Unlock()
|
|
|
|
select {
|
|
case res := <-first.waitCh:
|
|
if res.err != nil {
|
|
t.Fatalf("first item dispatch error: %v", res.err)
|
|
}
|
|
default:
|
|
t.Fatal("first (earliest) item was not dispatched first")
|
|
}
|
|
select {
|
|
case <-second.waitCh:
|
|
t.Fatal("second item should still wait (only one slot freed)")
|
|
default:
|
|
}
|
|
|
|
// Release again → the second item dispatches.
|
|
m.mu.Lock()
|
|
m.releaseSlotLocked("g-fifo2", "node-fifo2", "prov-fifo2", false)
|
|
m.mu.Unlock()
|
|
select {
|
|
case res := <-second.waitCh:
|
|
if res.err != nil {
|
|
t.Fatalf("second item dispatch error: %v", res.err)
|
|
}
|
|
default:
|
|
t.Fatal("second item not dispatched after second slot release")
|
|
}
|
|
}
|
|
|
|
// TestModelQueueNormalRequestDoesNotReserveLongSlot verifies a normal request
|
|
// occupies only the normal in-flight counter, never the long counter, even on a
|
|
// provider that declares long capacity.
|
|
func TestModelQueueNormalRequestDoesNotReserveLongSlot(t *testing.T) {
|
|
entry := &edgenode.NodeEntry{NodeID: "node-n"}
|
|
cands := []candidateNode{{entry: entry, capacity: 3, providerID: "prov-n", longContextCapacity: 2}}
|
|
m := newModelQueueManager(nil)
|
|
|
|
sel, err := m.admit(context.Background(), "g-n", "", "", cands, groupPolicy{}, nil, false, false) // long omitted → normal
|
|
if err != nil || sel == nil {
|
|
t.Fatalf("normal admit: %v", err)
|
|
}
|
|
if got := inflightSlotCount(m, "g-n", "node-n", "prov-n"); got != 1 {
|
|
t.Errorf("normal in-flight = %d, want 1", got)
|
|
}
|
|
if got := longSlotCount(m, "g-n", "node-n", "prov-n"); got != 0 {
|
|
t.Errorf("long in-flight for normal request = %d, want 0", got)
|
|
}
|
|
}
|
|
|
|
// TestModelQueueLongSlotFullExcludesLongKeepsNormal verifies S05 and D06: a
|
|
// provider whose long slots are full is excluded from long candidates (the long
|
|
// request moves to another provider or queues), but the same provider remains a
|
|
// valid candidate for normal requests while its normal capacity has room.
|
|
func TestModelQueueLongSlotFullExcludesLongKeepsNormal(t *testing.T) {
|
|
entryA := &edgenode.NodeEntry{NodeID: "node-a"}
|
|
entryB := &edgenode.NodeEntry{NodeID: "node-b"}
|
|
cands := []candidateNode{
|
|
{entry: entryA, capacity: 3, priority: 0, providerID: "prov-a", longContextCapacity: 1},
|
|
{entry: entryB, capacity: 3, priority: 1, providerID: "prov-b", longContextCapacity: 1},
|
|
}
|
|
m := newModelQueueManager(nil)
|
|
|
|
// Long #1 → prov-a (priority 0 preferred), filling prov-a's single long slot.
|
|
sel1, err := m.admit(context.Background(), "g-x", "", "", cands, groupPolicy{}, nil, true, false)
|
|
if err != nil || sel1 == nil {
|
|
t.Fatalf("long1 admit: %v", err)
|
|
}
|
|
if sel1.providerID != "prov-a" {
|
|
t.Fatalf("long1 provider = %q, want prov-a", sel1.providerID)
|
|
}
|
|
|
|
// Long #2 must skip the long-full prov-a and land on prov-b.
|
|
sel2, err := m.admit(context.Background(), "g-x", "", "", cands, groupPolicy{}, nil, true, false)
|
|
if err != nil || sel2 == nil {
|
|
t.Fatalf("long2 admit: %v", err)
|
|
}
|
|
if sel2.providerID != "prov-b" {
|
|
t.Fatalf("long2 provider = %q, want prov-b (prov-a long slot full)", sel2.providerID)
|
|
}
|
|
|
|
// A normal request restricted to the long-full prov-a is still admitted,
|
|
// because normal admission ignores long slot state (D06).
|
|
normalCands := []candidateNode{cands[0]}
|
|
selN, err := m.admit(context.Background(), "g-x", "", "", normalCands, groupPolicy{}, nil, false, false)
|
|
if err != nil || selN == nil {
|
|
t.Fatalf("normal admit on long-full provider: %v", err)
|
|
}
|
|
if selN.providerID != "prov-a" {
|
|
t.Fatalf("normal provider = %q, want prov-a", selN.providerID)
|
|
}
|
|
|
|
// A further long request restricted to the long-full prov-a cannot be
|
|
// admitted immediately; it queues and times out.
|
|
longOnlyA := []candidateNode{cands[0]}
|
|
_, err = m.admit(context.Background(), "g-x", "", "", longOnlyA, groupPolicy{maxQueue: 1, queueTimeout: 20 * time.Millisecond}, nil, true, false)
|
|
if !errors.Is(err, errQueueTimeout) {
|
|
t.Fatalf("long request on long-full provider: expected queue timeout, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestModelQueueReasonLongContextCapacityFull(t *testing.T) {
|
|
entryA := &edgenode.NodeEntry{NodeID: "node-a"}
|
|
cands := []candidateNode{
|
|
{entry: entryA, capacity: 3, priority: 0, providerID: "prov-a", longContextCapacity: 1},
|
|
}
|
|
m := newModelQueueManager(nil)
|
|
|
|
// Long #1 → prov-a, filling prov-a's single long slot.
|
|
_, _, err := m.admitWithReason(context.Background(), "g-x", "", "", cands, groupPolicy{}, nil, true, false)
|
|
if err != nil {
|
|
t.Fatalf("first admit: %v", err)
|
|
}
|
|
|
|
// Long #2 → prov-a. It should block because longContextCapacity is full.
|
|
done := make(chan struct{})
|
|
go func() {
|
|
defer close(done)
|
|
// The queue timeout has to outlast the observation window below: the
|
|
// assertion can only read the item's reason while the item is queued.
|
|
_, _, err := m.admitWithReason(context.Background(), "g-x", "", "", cands, groupPolicy{maxQueue: 1, queueTimeout: 500 * time.Millisecond}, nil, true, false)
|
|
if !errors.Is(err, errQueueTimeout) {
|
|
t.Errorf("expected timeout, got %v", err)
|
|
}
|
|
}()
|
|
|
|
// Poll rather than sleeping a fixed slice: under load the goroutine can need
|
|
// more than a few milliseconds to reach the queue.
|
|
waitForQueueLen(t, m, "g-x", 1)
|
|
|
|
m.mu.Lock()
|
|
g, ok := m.groups["g-x"]
|
|
if !ok || len(g.queue) != 1 {
|
|
m.mu.Unlock()
|
|
t.Fatalf("expected 1 queued item, got group ok=%t, len=%d", ok, len(g.queue))
|
|
}
|
|
reason := g.queue[0].reason
|
|
m.mu.Unlock()
|
|
|
|
if reason != "long_context_capacity_full" {
|
|
t.Errorf("expected queue reason 'long_context_capacity_full', got %q", reason)
|
|
}
|
|
|
|
<-done
|
|
}
|
|
|
|
func TestModelQueueReasonCapacityFull(t *testing.T) {
|
|
entryA := &edgenode.NodeEntry{NodeID: "node-a"}
|
|
cands := []candidateNode{
|
|
{entry: entryA, capacity: 1, priority: 0, providerID: "prov-a"},
|
|
}
|
|
m := newModelQueueManager(nil)
|
|
|
|
// #1 → prov-a, filling capacity.
|
|
_, _, err := m.admitWithReason(context.Background(), "g-y", "", "", cands, groupPolicy{}, nil, false, false)
|
|
if err != nil {
|
|
t.Fatalf("first admit: %v", err)
|
|
}
|
|
|
|
// #2 → prov-a. It should block because capacity is full.
|
|
done := make(chan struct{})
|
|
go func() {
|
|
defer close(done)
|
|
// As above: the item must stay queued long enough to be observed.
|
|
_, _, err := m.admitWithReason(context.Background(), "g-y", "", "", cands, groupPolicy{maxQueue: 1, queueTimeout: 500 * time.Millisecond}, nil, false, false)
|
|
if !errors.Is(err, errQueueTimeout) {
|
|
t.Errorf("expected timeout, got %v", err)
|
|
}
|
|
}()
|
|
|
|
waitForQueueLen(t, m, "g-y", 1)
|
|
|
|
m.mu.Lock()
|
|
g, ok := m.groups["g-y"]
|
|
if !ok || len(g.queue) != 1 {
|
|
m.mu.Unlock()
|
|
t.Fatalf("expected 1 queued item")
|
|
}
|
|
reason := g.queue[0].reason
|
|
m.mu.Unlock()
|
|
|
|
if reason != "capacity_full" {
|
|
t.Errorf("expected queue reason 'capacity_full', got %q", reason)
|
|
}
|
|
|
|
<-done
|
|
}
|
|
|
|
func TestProviderLongCapacitySharedAcrossOrnithModelAliases(t *testing.T) {
|
|
const (
|
|
nodeID = "rtx5090-lemonade-node"
|
|
providerID = "rtx5090-lemonade"
|
|
)
|
|
modelAliases := []string{"ornith:35b", "ornith-fast"}
|
|
entry := &edgenode.NodeEntry{NodeID: nodeID}
|
|
cands := []candidateNode{
|
|
{entry: entry, capacity: 5, providerID: providerID, longContextCapacity: 1},
|
|
}
|
|
store := edgenode.NewNodeStore()
|
|
store.Add(&edgenode.NodeRecord{
|
|
ID: nodeID,
|
|
Providers: []config.NodeProviderConf{
|
|
{
|
|
ID: providerID,
|
|
Capacity: 5,
|
|
LongContextCapacity: 1,
|
|
Adapter: "vllm",
|
|
},
|
|
},
|
|
})
|
|
m := newModelQueueManager(store)
|
|
m.setStore(store)
|
|
|
|
start := make(chan struct{})
|
|
resCh := make(chan concurrentAdmitResult, 2)
|
|
|
|
for i := 0; i < 2; i++ {
|
|
go func(id int) {
|
|
<-start
|
|
ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
|
|
defer cancel()
|
|
groupKey := modelAliases[id]
|
|
sel, err := m.admit(ctx, groupKey, "", "", cands, groupPolicy{maxQueue: 1, queueTimeout: 30 * time.Millisecond}, nil, true, false)
|
|
resCh <- concurrentAdmitResult{candidate: sel, err: err}
|
|
}(i)
|
|
}
|
|
|
|
close(start)
|
|
|
|
var results []concurrentAdmitResult
|
|
for i := 0; i < 2; i++ {
|
|
results = append(results, <-resCh)
|
|
}
|
|
|
|
successCount := 0
|
|
failCount := 0
|
|
var successSel *candidateNode
|
|
for _, res := range results {
|
|
if res.err == nil {
|
|
successCount++
|
|
successSel = res.candidate
|
|
} else {
|
|
failCount++
|
|
}
|
|
}
|
|
|
|
if successCount != 1 || failCount != 1 {
|
|
t.Fatalf("expected exactly 1 success and 1 failure under concurrent long admission (capacity=1), got success=%d fail=%d", successCount, failCount)
|
|
}
|
|
|
|
// Verify long peak was 1
|
|
if got := m.longInFlightForProvider(nodeID, providerID); got != 1 {
|
|
t.Errorf("expected long peak to be 1, got %d", got)
|
|
}
|
|
|
|
// Release successful slot
|
|
m.trackLease(successSel.leaseID, "run-shared-1")
|
|
m.releaseRun("run-shared-1", "complete")
|
|
|
|
// Verify long counts recovered to 0
|
|
if got := m.longInFlightForProvider(nodeID, providerID); got != 0 {
|
|
t.Errorf("expected long counts to recover to 0 after release, got %d", got)
|
|
}
|
|
}
|
|
|
|
func TestProviderResourceLongReleaseAfterCapacityDisabled(t *testing.T) {
|
|
entry := &edgenode.NodeEntry{NodeID: "node-ref-1"}
|
|
cands := []candidateNode{{entry: entry, capacity: 1, providerID: "prov-ref-1", longContextCapacity: 1}}
|
|
store := edgenode.NewNodeStore()
|
|
rec := &edgenode.NodeRecord{
|
|
ID: "node-ref-1",
|
|
Providers: []config.NodeProviderConf{
|
|
{ID: "prov-ref-1", Capacity: 1, LongContextCapacity: 1, Adapter: "vllm"},
|
|
},
|
|
}
|
|
store.Add(rec)
|
|
m := newModelQueueManager(store)
|
|
m.setStore(store)
|
|
|
|
sel, err := m.admit(context.Background(), "g-ref", "", "", cands, groupPolicy{}, nil, true, false)
|
|
if err != nil || sel == nil {
|
|
t.Fatalf("admit: %v", err)
|
|
}
|
|
|
|
if got := m.longInFlightForProvider("node-ref-1", "prov-ref-1"); got != 1 {
|
|
t.Fatalf("expected longInFlight=1, got %d", got)
|
|
}
|
|
|
|
rec.Providers[0].Capacity = 0
|
|
rec.Providers[0].LongContextCapacity = 0
|
|
m.setStore(store)
|
|
|
|
m.trackLease(sel.leaseID, "run-ref-1")
|
|
m.releaseRun("run-ref-1", "complete")
|
|
|
|
if got := m.longInFlightForProvider("node-ref-1", "prov-ref-1"); got != 0 {
|
|
t.Errorf("longInFlight after capacity disabled release = %d, want 0", got)
|
|
}
|
|
}
|
|
|
|
func TestProviderResourceLongReservationUsesEffectiveRefreshState(t *testing.T) {
|
|
entry := &edgenode.NodeEntry{NodeID: "node-eff"}
|
|
cands := []candidateNode{{entry: entry, capacity: 1, providerID: "prov-eff", longContextCapacity: 1}}
|
|
store := edgenode.NewNodeStore()
|
|
rec := &edgenode.NodeRecord{
|
|
ID: "node-eff",
|
|
Providers: []config.NodeProviderConf{
|
|
{ID: "prov-eff", Capacity: 1, LongContextCapacity: 1, Adapter: "vllm"},
|
|
},
|
|
}
|
|
store.Add(rec)
|
|
m := newModelQueueManager(store)
|
|
m.setStore(store)
|
|
|
|
_, err := m.admit(context.Background(), "g-eff", "", "", cands, groupPolicy{}, nil, true, false)
|
|
if err != nil {
|
|
t.Fatalf("first admit: %v", err)
|
|
}
|
|
|
|
done := make(chan struct{})
|
|
go func() {
|
|
defer close(done)
|
|
_, err := m.admit(context.Background(), "g-eff", "", "", cands, groupPolicy{maxQueue: 1}, nil, true, false)
|
|
if err != nil {
|
|
t.Errorf("queued admit failed: %v", err)
|
|
}
|
|
}()
|
|
|
|
waitForQueueLen(t, m, "g-eff", 1)
|
|
|
|
rec.Providers[0].LongContextCapacity = 0
|
|
m.setStore(store)
|
|
|
|
m.releaseSlotWithLong("g-eff", "node-eff", "prov-eff", true)
|
|
|
|
<-done
|
|
|
|
if got := m.longInFlightForProvider("node-eff", "prov-eff"); got != 0 {
|
|
t.Errorf("expected longInFlight=0 for the dispatched queued run under disabled capacity, got %d", got)
|
|
}
|
|
}
|
|
|
|
func TestProviderResourceOrphanRemovedAfterLastRelease(t *testing.T) {
|
|
entry := &edgenode.NodeEntry{NodeID: "node-orph"}
|
|
cands := []candidateNode{{entry: entry, capacity: 1, providerID: "prov-orph", longContextCapacity: 1}}
|
|
store := edgenode.NewNodeStore()
|
|
rec := &edgenode.NodeRecord{
|
|
ID: "node-orph",
|
|
Providers: []config.NodeProviderConf{
|
|
{ID: "prov-orph", Capacity: 1, LongContextCapacity: 1, Adapter: "vllm"},
|
|
},
|
|
}
|
|
store.Add(rec)
|
|
m := newModelQueueManager(store)
|
|
m.setStore(store)
|
|
|
|
_, err := m.admit(context.Background(), "g-orph", "", "", cands, groupPolicy{}, nil, true, false)
|
|
if err != nil {
|
|
t.Fatalf("admit: %v", err)
|
|
}
|
|
|
|
key := providerResourceKey{nodeID: "node-orph", providerID: "prov-orph"}
|
|
if res := m.resources[key]; res == nil || res.orphan {
|
|
t.Fatal("expected active non-orphan resource")
|
|
}
|
|
|
|
rec.Providers = nil
|
|
m.setStore(store)
|
|
|
|
m.mu.Lock()
|
|
res, ok := m.resources[key]
|
|
m.mu.Unlock()
|
|
if !ok || res == nil || !res.orphan {
|
|
t.Fatalf("expected resource to be kept as orphan, ok=%t, res=%v", ok, res)
|
|
}
|
|
|
|
m.releaseSlotWithLong("g-orph", "node-orph", "prov-orph", true)
|
|
|
|
m.mu.Lock()
|
|
_, ok = m.resources[key]
|
|
m.mu.Unlock()
|
|
if ok {
|
|
t.Fatal("expected orphan resource to be removed from map after last release")
|
|
}
|
|
}
|
|
|
|
// TestGlobalPumpLongHeadDoesNotBlockNormalOtherGroup verifies that head-of-line
|
|
// avoidance holds across model groups, not only inside one queue: a long request
|
|
// that arrived first and is stuck on a full long slot must not hold up a normal
|
|
// request that arrived later in a different group and fits ordinary capacity.
|
|
func TestGlobalPumpLongHeadDoesNotBlockNormalOtherGroup(t *testing.T) {
|
|
entry := &edgenode.NodeEntry{NodeID: "node-xg"}
|
|
// Two normal slots, a single long slot.
|
|
cands := []candidateNode{{entry: entry, capacity: 2, providerID: "prov-xg", longContextCapacity: 1}}
|
|
m := newModelQueueManager(nil)
|
|
|
|
// One long request in flight fills the only long slot; one normal slot is free.
|
|
if _, err := m.admit(context.Background(), "g-xg-a", "", "", cands, groupPolicy{}, nil, true, false); err != nil {
|
|
t.Fatalf("long admit: %v", err)
|
|
}
|
|
|
|
// Arrival order: the blocked long request first, the normal one behind it in
|
|
// another group.
|
|
longItem := queueItemForTest(cands, true)
|
|
enqueueForTest(m, "g-xg-a", longItem, nil)
|
|
normalItem := queueItemForTest(cands, false)
|
|
enqueueForTest(m, "g-xg-b", normalItem, nil)
|
|
|
|
m.mu.Lock()
|
|
m.pumpAllLocked()
|
|
m.mu.Unlock()
|
|
|
|
select {
|
|
case res := <-normalItem.waitCh:
|
|
if res.err != nil {
|
|
t.Fatalf("normal item in the other group expected dispatch, got error: %v", res.err)
|
|
}
|
|
default:
|
|
t.Fatal("normal request in another group was blocked behind a long head (cross-group head-of-line blocking)")
|
|
}
|
|
select {
|
|
case <-longItem.waitCh:
|
|
t.Fatal("long head must stay queued while its long slot is full")
|
|
default:
|
|
}
|
|
|
|
if got := pendingItemCount(m); got != 1 {
|
|
t.Fatalf("expected only the long head to remain pending, got %d item(s)", got)
|
|
}
|
|
}
|
|
|
|
// TestGlobalPumpNoLeakAfterMixedLongAndNormalDrain verifies that repeated mixed
|
|
// long/normal traffic across groups leaves every counter at zero: a scheduler
|
|
// that reserves in one place and releases in another is exactly where long-slot
|
|
// leaks hide.
|
|
func TestGlobalPumpNoLeakAfterMixedLongAndNormalDrain(t *testing.T) {
|
|
entry := &edgenode.NodeEntry{NodeID: "node-drain"}
|
|
cands := []candidateNode{{entry: entry, capacity: 2, providerID: "prov-drain", longContextCapacity: 1}}
|
|
m := newModelQueueManager(nil)
|
|
|
|
for round := 0; round < 20; round++ {
|
|
long, err := m.admit(context.Background(), "g-drain-a", "", "", cands, groupPolicy{}, nil, true, false)
|
|
if err != nil || long == nil {
|
|
t.Fatalf("round %d long admit: %v", round, err)
|
|
}
|
|
normal, err := m.admit(context.Background(), "g-drain-b", "", "", cands, groupPolicy{}, nil, false, false)
|
|
if err != nil || normal == nil {
|
|
t.Fatalf("round %d normal admit: %v", round, err)
|
|
}
|
|
m.releaseLease(long.leaseID, "complete")
|
|
m.releaseLease(normal.leaseID, "complete")
|
|
}
|
|
|
|
inFlight, longInFlight := providerResourceCounts(m, "node-drain", "prov-drain")
|
|
if inFlight != 0 || longInFlight != 0 {
|
|
t.Fatalf("expected zero counters after drain, got inflight=%d long=%d", inFlight, longInFlight)
|
|
}
|
|
if got := leaseCount(m); got != 0 {
|
|
t.Fatalf("expected zero live leases after drain, got %d", got)
|
|
}
|
|
if got := pendingItemCount(m); got != 0 {
|
|
t.Fatalf("expected zero pending items after drain, got %d", got)
|
|
}
|
|
}
|