toki
2f560e3f3b
- Archive completed subtask plans/code reviews (04, 05+03,04, 07+03) - Add provider_pool_admission_test.go - Update edge config types, load, catalog validation - Update runtime, config refresh, service layers for admission - Update test docs and inventory - Update provider scheduling, resolution, tunnel, status modules
575 lines
19 KiB
Go
575 lines
19 KiB
Go
package service
|
|
|
|
import (
|
|
"context"
|
|
"sync"
|
|
"testing"
|
|
|
|
edgenode "iop/apps/edge/internal/node"
|
|
"iop/packages/go/config"
|
|
)
|
|
|
|
// reservationFixture admits one slot on a single-capacity provider and returns
|
|
// the manager, the reservation for that admission, and the slot key it holds.
|
|
func reservationFixture(t *testing.T, long bool, longCapacity int) (*modelQueueManager, *queueReservation, string) {
|
|
t.Helper()
|
|
store := edgenode.NewNodeStore()
|
|
store.Add(&edgenode.NodeRecord{
|
|
ID: "node-res",
|
|
Runtime: config.RuntimeConf{Concurrency: 1},
|
|
Providers: []config.NodeProviderConf{
|
|
// The node record is the live source for capacity and long-context
|
|
// capacity, so it must declare the same long limit the candidate does:
|
|
// the scheduler re-resolves these fields from the record at dispatch.
|
|
{ID: "prov-1", Capacity: 1, Adapter: "vllm", LongContextCapacity: longCapacity},
|
|
},
|
|
})
|
|
|
|
entry := &edgenode.NodeEntry{NodeID: "node-res"}
|
|
selected := &candidateNode{
|
|
entry: entry,
|
|
capacity: 1,
|
|
providerID: "prov-1",
|
|
longContextCapacity: longCapacity,
|
|
}
|
|
cands := []candidateNode{*selected}
|
|
|
|
m := newModelQueueManager(store)
|
|
admitted, _, err := m.admitWithReason(context.Background(), "g-res", "", "", cands, groupPolicy{}, nil, long, false)
|
|
if err != nil || admitted == nil {
|
|
t.Fatalf("admit: %v", err)
|
|
}
|
|
|
|
return m, newQueueReservation(m, admitted), "node-res:prov-1"
|
|
}
|
|
|
|
func inflightCount(t *testing.T, m *modelQueueManager, groupKey, slot string) (int, int) {
|
|
t.Helper()
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
|
|
var nodeID, providerID string
|
|
if colonIdx := findLastColon(slot); colonIdx > 0 {
|
|
nodeID = slot[:colonIdx]
|
|
providerID = slot[colonIdx+1:]
|
|
} else {
|
|
nodeID = slot
|
|
}
|
|
|
|
if providerID != "" {
|
|
key := providerResourceKey{nodeID: nodeID, providerID: providerID}
|
|
if res, ok := m.resources[key]; ok {
|
|
return res.inFlight, res.longInFlight
|
|
}
|
|
return 0, 0
|
|
}
|
|
|
|
group, ok := m.groups[groupKey]
|
|
if !ok {
|
|
t.Fatalf("group %q not found", groupKey)
|
|
}
|
|
return group.inflight[slot], group.longInflight[slot]
|
|
}
|
|
|
|
// TestQueueReservationReleasesExactlyOnce pins the release contract at each
|
|
// boundary a dispatch can fail on: the slot is freed once, and a repeated
|
|
// release cannot free a second slot.
|
|
func TestQueueReservationReleasesExactlyOnce(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
// track reports whether the dispatch got as far as registering the run
|
|
// inflight before failing.
|
|
track bool
|
|
reason string
|
|
}{
|
|
{name: "prepare tunnel error before track", reason: "prepare-tunnel-error"},
|
|
{name: "prepare run error before track", reason: "prepare-run-error"},
|
|
{name: "build error before track", reason: "build-error"},
|
|
{name: "unknown execution path before track", reason: "unknown-execution-path"},
|
|
{name: "send error after track", track: true, reason: "send-error"},
|
|
{name: "no event bus after track", track: true, reason: "no-event-bus"},
|
|
}
|
|
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
m, reservation, slot := reservationFixture(t, false, 0)
|
|
|
|
if inflight, _ := inflightCount(t, m, "g-res", slot); inflight != 1 {
|
|
t.Fatalf("expected 1 reserved slot after admission, got %d", inflight)
|
|
}
|
|
|
|
if tc.track {
|
|
reservation.track("run-res")
|
|
}
|
|
|
|
reservation.release(tc.reason)
|
|
if inflight, _ := inflightCount(t, m, "g-res", slot); inflight != 0 {
|
|
t.Fatalf("expected the slot to be released, got %d in flight", inflight)
|
|
}
|
|
|
|
// A second release must not decrement a slot that now belongs to
|
|
// another request.
|
|
reservation.release(tc.reason)
|
|
if inflight, _ := inflightCount(t, m, "g-res", slot); inflight != 0 {
|
|
t.Fatalf("expected release to be idempotent, got %d in flight", inflight)
|
|
}
|
|
if tc.track {
|
|
m.mu.Lock()
|
|
_, stillTracked := m.leaseByRun["run-res"]
|
|
m.mu.Unlock()
|
|
if stillTracked {
|
|
t.Fatal("expected the run to be removed from the lease index")
|
|
}
|
|
}
|
|
if got := leaseCount(m); got != 0 {
|
|
t.Fatalf("expected the lease record to be dropped, got %d live leases", got)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestQueueReservationHandOffKeepsSlotHeld pins that a dispatched request keeps
|
|
// its slot: after hand-off the run/tunnel lifecycle owns the release, so a late
|
|
// release on the reservation must not free it.
|
|
func TestQueueReservationHandOffKeepsSlotHeld(t *testing.T) {
|
|
m, reservation, slot := reservationFixture(t, false, 0)
|
|
reservation.track("run-res")
|
|
reservation.handOff()
|
|
|
|
reservation.release("send-error")
|
|
|
|
if inflight, _ := inflightCount(t, m, "g-res", slot); inflight != 1 {
|
|
t.Fatalf("expected the dispatched run to keep its slot, got %d in flight", inflight)
|
|
}
|
|
|
|
// The lifecycle owner releases it exactly once, via the run registration.
|
|
m.releaseRun("run-res", "complete")
|
|
if inflight, _ := inflightCount(t, m, "g-res", slot); inflight != 0 {
|
|
t.Fatalf("expected the terminal event to release the slot, got %d in flight", inflight)
|
|
}
|
|
}
|
|
|
|
// TestQueueReservationLongSlotMatchesReservation pins that the long-context
|
|
// slot is released only when it was actually reserved: a long request on a
|
|
// provider that declares a long limit.
|
|
func TestQueueReservationLongSlotMatchesReservation(t *testing.T) {
|
|
t.Run("long request on a long-capacity provider releases the long slot", func(t *testing.T) {
|
|
m, reservation, slot := reservationFixture(t, true, 1)
|
|
if _, longInflight := inflightCount(t, m, "g-res", slot); longInflight != 1 {
|
|
t.Fatalf("expected 1 reserved long slot, got %d", longInflight)
|
|
}
|
|
|
|
reservation.release("send-error")
|
|
|
|
inflight, longInflight := inflightCount(t, m, "g-res", slot)
|
|
if inflight != 0 || longInflight != 0 {
|
|
t.Fatalf("expected both slots released, got inflight=%d long=%d", inflight, longInflight)
|
|
}
|
|
})
|
|
|
|
t.Run("long request without a declared long limit reserves no long slot", func(t *testing.T) {
|
|
m, reservation, slot := reservationFixture(t, true, 0)
|
|
if _, longInflight := inflightCount(t, m, "g-res", slot); longInflight != 0 {
|
|
t.Fatalf("expected no long slot to be reserved, got %d", longInflight)
|
|
}
|
|
if leaseLong(t, m, reservation.leaseID) {
|
|
t.Fatal("expected the lease not to claim a long slot")
|
|
}
|
|
|
|
reservation.release("send-error")
|
|
|
|
if inflight, _ := inflightCount(t, m, "g-res", slot); inflight != 0 {
|
|
t.Fatalf("expected the slot to be released, got %d in flight", inflight)
|
|
}
|
|
})
|
|
}
|
|
|
|
// leaseRaceFixture admits two leases on one capacity-2 provider and returns the
|
|
// manager, both lease ids, and the shared slot key. Holding a second lease is
|
|
// what makes an over-release observable: the resource counters clamp at zero, so
|
|
// a lone lease could absorb a double decrement silently, while here the extra
|
|
// decrement would steal the surviving lease's slot.
|
|
func leaseRaceFixture(t *testing.T, long bool, longCapacity int) (m *modelQueueManager, first, second uint64, slot string) {
|
|
t.Helper()
|
|
store := edgenode.NewNodeStore()
|
|
store.Add(&edgenode.NodeRecord{
|
|
ID: "node-race",
|
|
Runtime: config.RuntimeConf{Concurrency: 2},
|
|
Providers: []config.NodeProviderConf{
|
|
{ID: "prov-race", Capacity: 2, LongContextCapacity: longCapacity, Adapter: "vllm"},
|
|
},
|
|
})
|
|
|
|
entry := &edgenode.NodeEntry{NodeID: "node-race"}
|
|
cands := []candidateNode{{
|
|
entry: entry,
|
|
capacity: 2,
|
|
providerID: "prov-race",
|
|
longContextCapacity: longCapacity,
|
|
}}
|
|
|
|
m = newModelQueueManager(store)
|
|
m.setStore(store)
|
|
|
|
ids := make([]uint64, 0, 2)
|
|
for i := 0; i < 2; i++ {
|
|
admitted, _, err := m.admitWithReason(context.Background(), "g-race", "", "", cands, groupPolicy{}, nil, long, false)
|
|
if err != nil || admitted == nil {
|
|
t.Fatalf("admit %d: %v", i, err)
|
|
}
|
|
if admitted.leaseID == 0 {
|
|
t.Fatalf("admit %d returned no lease id", i)
|
|
}
|
|
ids = append(ids, admitted.leaseID)
|
|
}
|
|
if ids[0] == ids[1] {
|
|
t.Fatalf("expected unique lease ids per admission, both were %d", ids[0])
|
|
}
|
|
return m, ids[0], ids[1], "node-race:prov-race"
|
|
}
|
|
|
|
// TestProviderLeaseConcurrentReleaseExactlyOnce pins the core lease invariant:
|
|
// however many terminal causes race on one lease — send failure, run terminal,
|
|
// tunnel terminal — the slot is returned exactly once and the other live lease
|
|
// keeps what it holds.
|
|
func TestProviderLeaseConcurrentReleaseExactlyOnce(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
long bool
|
|
longCapacity int
|
|
wantLong int
|
|
}{
|
|
{name: "normal slots"},
|
|
{name: "long slots", long: true, longCapacity: 2, wantLong: 1},
|
|
}
|
|
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
m, first, second, slot := leaseRaceFixture(t, tc.long, tc.longCapacity)
|
|
m.trackLease(first, "run-race-1")
|
|
|
|
const racers = 32
|
|
var wg sync.WaitGroup
|
|
start := make(chan struct{})
|
|
for i := 0; i < racers; i++ {
|
|
wg.Add(1)
|
|
go func(i int) {
|
|
defer wg.Done()
|
|
<-start
|
|
switch i % 4 {
|
|
case 0:
|
|
m.releaseLease(first, "send-error")
|
|
case 1:
|
|
m.releaseRun("run-race-1", "complete")
|
|
case 2:
|
|
m.releaseRun("run-race-1", "cancelled")
|
|
default:
|
|
m.releaseLease(first, "tunnel-END")
|
|
}
|
|
}(i)
|
|
}
|
|
close(start)
|
|
wg.Wait()
|
|
|
|
inflight, longInflight := inflightCount(t, m, "g-race", slot)
|
|
if inflight != 1 {
|
|
t.Fatalf("expected the surviving lease to keep 1 slot, got %d in flight", inflight)
|
|
}
|
|
if longInflight != tc.wantLong {
|
|
t.Fatalf("expected %d long slots held, got %d", tc.wantLong, longInflight)
|
|
}
|
|
if got := leaseCount(m); got != 1 {
|
|
t.Fatalf("expected only the surviving lease to remain, got %d leases", got)
|
|
}
|
|
|
|
m.mu.Lock()
|
|
_, stillIndexed := m.leaseByRun["run-race-1"]
|
|
_, survivorLive := m.leases[second]
|
|
m.mu.Unlock()
|
|
if stillIndexed {
|
|
t.Error("released lease is still in the run index")
|
|
}
|
|
if !survivorLive {
|
|
t.Error("the untouched lease was released by the racing calls")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestProviderLeaseTrackAndSendFailureRace covers the window the old bool guard
|
|
// could not: a terminal cause arriving while the dispatch is still binding the
|
|
// run id. Whichever side wins, the slot is freed once and no run id is left
|
|
// pointing at a lease that no longer exists.
|
|
func TestProviderLeaseTrackAndSendFailureRace(t *testing.T) {
|
|
for i := 0; i < 16; i++ {
|
|
m, first, second, slot := leaseRaceFixture(t, false, 0)
|
|
|
|
var wg sync.WaitGroup
|
|
start := make(chan struct{})
|
|
wg.Add(2)
|
|
go func() {
|
|
defer wg.Done()
|
|
<-start
|
|
m.trackLease(first, "run-track-race")
|
|
}()
|
|
go func() {
|
|
defer wg.Done()
|
|
<-start
|
|
m.releaseLease(first, "send-error")
|
|
}()
|
|
close(start)
|
|
wg.Wait()
|
|
|
|
// The terminal path that lost the race still runs afterwards; it must
|
|
// not free a second slot.
|
|
m.releaseRun("run-track-race", "complete")
|
|
m.releaseLease(first, "tunnel-closed")
|
|
|
|
if inflight, _ := inflightCount(t, m, "g-race", slot); inflight != 1 {
|
|
t.Fatalf("iteration %d: expected the surviving lease to keep 1 slot, got %d", i, inflight)
|
|
}
|
|
m.mu.Lock()
|
|
_, stillIndexed := m.leaseByRun["run-track-race"]
|
|
_, survivorLive := m.leases[second]
|
|
m.mu.Unlock()
|
|
if stillIndexed {
|
|
t.Fatalf("iteration %d: run id still indexed after the lease was released", i)
|
|
}
|
|
if !survivorLive {
|
|
t.Fatalf("iteration %d: the untouched lease was released", i)
|
|
}
|
|
}
|
|
}
|
|
|
|
// leaseDisconnectFixture admits a primary lease on one node and a control lease
|
|
// on another, returning the manager, both lease ids, the primary slot key, and
|
|
// the control slot key. Two separate (node, provider) resources let us observe
|
|
// that a node disconnect only touches the disconnected node's counters.
|
|
func leaseDisconnectFixture(t *testing.T, long bool, longCapacity int) (m *modelQueueManager, primaryLeaseID, controlLeaseID uint64, primarySlot, controlSlot string) {
|
|
t.Helper()
|
|
store := edgenode.NewNodeStore()
|
|
store.Add(&edgenode.NodeRecord{
|
|
ID: "node-disconn-primary",
|
|
Runtime: config.RuntimeConf{Concurrency: 2},
|
|
Providers: []config.NodeProviderConf{
|
|
{ID: "prov-disconn", Capacity: 2, LongContextCapacity: longCapacity, Adapter: "vllm"},
|
|
},
|
|
})
|
|
store.Add(&edgenode.NodeRecord{
|
|
ID: "node-disconn-control",
|
|
Runtime: config.RuntimeConf{Concurrency: 1},
|
|
Providers: []config.NodeProviderConf{
|
|
{ID: "prov-disconn", Capacity: 1, Adapter: "vllm"},
|
|
},
|
|
})
|
|
|
|
primaryEntry := &edgenode.NodeEntry{NodeID: "node-disconn-primary"}
|
|
controlEntry := &edgenode.NodeEntry{NodeID: "node-disconn-control"}
|
|
primaryCands := []candidateNode{{
|
|
entry: primaryEntry,
|
|
capacity: 2,
|
|
providerID: "prov-disconn",
|
|
longContextCapacity: longCapacity,
|
|
}}
|
|
controlCands := []candidateNode{{
|
|
entry: controlEntry,
|
|
capacity: 1,
|
|
providerID: "prov-disconn",
|
|
longContextCapacity: 0,
|
|
}}
|
|
|
|
m = newModelQueueManager(store)
|
|
m.setStore(store)
|
|
|
|
// Admit primary lease.
|
|
admitted, _, err := m.admitWithReason(context.Background(), "g-disconn", "", "", primaryCands, groupPolicy{}, nil, long, false)
|
|
if err != nil || admitted == nil {
|
|
t.Fatalf("admit primary: %v", err)
|
|
}
|
|
primaryLeaseID = admitted.leaseID
|
|
primarySlot = primaryEntry.NodeID + ":" + "prov-disconn"
|
|
|
|
// Admit control lease on a different node.
|
|
admitted2, _, err := m.admitWithReason(context.Background(), "g-disconn-ctrl", "", "", controlCands, groupPolicy{}, nil, false, false)
|
|
if err != nil || admitted2 == nil {
|
|
t.Fatalf("admit control: %v", err)
|
|
}
|
|
controlLeaseID = admitted2.leaseID
|
|
controlSlot = controlEntry.NodeID + ":" + "prov-disconn"
|
|
|
|
return m, primaryLeaseID, controlLeaseID, primarySlot, controlSlot
|
|
}
|
|
|
|
// TestProviderLeaseNodeDisconnectRaceExactlyOnce pins that when the current-owner
|
|
// node disconnects, the race between releaseNode and any other terminal cause
|
|
// (releaseLease, releaseRun) frees the disconnected slot exactly once, and that
|
|
// a later re-admission on the same (node, provider) is isolated from stale
|
|
// releases of the old lease identity. Both normal and long-context cases are
|
|
// verified.
|
|
func TestProviderLeaseNodeDisconnectRaceExactlyOnce(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
long bool
|
|
longCapacity int
|
|
wantLong int
|
|
}{
|
|
{name: "normal slots"},
|
|
{name: "long slots", long: true, longCapacity: 2, wantLong: 1},
|
|
}
|
|
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
m, primaryLeaseID, controlLeaseID, primarySlot, controlSlot := leaseDisconnectFixture(t, tc.long, tc.longCapacity)
|
|
|
|
// Track the primary lease so releaseRun has a run id to resolve.
|
|
primaryRunID := "run-disconn-race"
|
|
m.trackLease(primaryLeaseID, primaryRunID)
|
|
|
|
// Barrier: race releaseNode against releaseLease and releaseRun.
|
|
const racers = 32
|
|
var wg sync.WaitGroup
|
|
start := make(chan struct{})
|
|
for i := 0; i < racers; i++ {
|
|
wg.Add(1)
|
|
go func(i int) {
|
|
defer wg.Done()
|
|
<-start
|
|
switch i % 3 {
|
|
case 0:
|
|
m.releaseNode("node-disconn-primary", "transport-closed")
|
|
case 1:
|
|
m.releaseRun(primaryRunID, "complete")
|
|
default:
|
|
m.releaseLease(primaryLeaseID, "tunnel-END")
|
|
}
|
|
}(i)
|
|
}
|
|
close(start)
|
|
wg.Wait()
|
|
|
|
// After the race, the disconnected primary resource must be zeroed.
|
|
inflight, longInflight := inflightCount(t, m, "g-disconn", primarySlot)
|
|
if inflight != 0 {
|
|
t.Fatalf("expected disconnected primary inflight=0, got %d", inflight)
|
|
}
|
|
if longInflight != 0 {
|
|
t.Fatalf("expected disconnected primary longInflight=0, got %d", longInflight)
|
|
}
|
|
|
|
// The control lease on the other node must be untouched.
|
|
ctrlInflight, ctrlLongInflight := inflightCount(t, m, "g-disconn-ctrl", controlSlot)
|
|
if ctrlInflight != 1 {
|
|
t.Fatalf("expected control inflight=1, got %d", ctrlInflight)
|
|
}
|
|
if ctrlLongInflight != 0 {
|
|
t.Fatalf("expected control longInflight=0, got %d", ctrlLongInflight)
|
|
}
|
|
|
|
// The primary lease must be gone from the lease map.
|
|
m.mu.Lock()
|
|
_, primaryStillLive := m.leases[primaryLeaseID]
|
|
_, controlStillLive := m.leases[controlLeaseID]
|
|
_, stillIndexed := m.leaseByRun[primaryRunID]
|
|
m.mu.Unlock()
|
|
if primaryStillLive {
|
|
t.Error("primary lease still in lease map after disconnect race")
|
|
}
|
|
if !controlStillLive {
|
|
t.Error("control lease was released by the racing calls")
|
|
}
|
|
if stillIndexed {
|
|
t.Error("primary run id still indexed after disconnect race")
|
|
}
|
|
|
|
// Re-admit on the same (node, provider) that just disconnected.
|
|
primaryEntry := &edgenode.NodeEntry{NodeID: "node-disconn-primary"}
|
|
primaryCands := []candidateNode{{
|
|
entry: primaryEntry,
|
|
capacity: 2,
|
|
providerID: "prov-disconn",
|
|
longContextCapacity: tc.longCapacity,
|
|
}}
|
|
newAdmitted, _, err := m.admitWithReason(context.Background(), "g-disconn-re", "", "", primaryCands, groupPolicy{}, nil, tc.long, false)
|
|
if err != nil || newAdmitted == nil {
|
|
t.Fatalf("re-admit: %v", err)
|
|
}
|
|
newLeaseID := newAdmitted.leaseID
|
|
|
|
// New lease must hold exactly the expected counters.
|
|
newInflight, newLongInflight := inflightCount(t, m, "g-disconn-re", primarySlot)
|
|
if newInflight != 1 {
|
|
t.Fatalf("expected re-admitted inflight=1, got %d", newInflight)
|
|
}
|
|
if newLongInflight != tc.wantLong {
|
|
t.Fatalf("expected re-admitted longInflight=%d, got %d", tc.wantLong, newLongInflight)
|
|
}
|
|
|
|
// Now issue late releases of the OLD lease identity. They must be
|
|
// no-ops: the new lease counters must not decrement.
|
|
m.releaseLease(primaryLeaseID, "late-send-error")
|
|
m.releaseRun(primaryRunID, "late-complete")
|
|
|
|
finalInflight, finalLongInflight := inflightCount(t, m, "g-disconn-re", primarySlot)
|
|
if finalInflight != 1 {
|
|
t.Fatalf("expected late old-identity release to be a no-op, inflight=%d (want 1)", finalInflight)
|
|
}
|
|
if finalLongInflight != tc.wantLong {
|
|
t.Fatalf("expected late old-identity release to be a no-op, longInflight=%d (want %d)", finalLongInflight, tc.wantLong)
|
|
}
|
|
|
|
// The new lease must still be live.
|
|
m.mu.Lock()
|
|
_, newStillLive := m.leases[newLeaseID]
|
|
m.mu.Unlock()
|
|
if !newStillLive {
|
|
t.Error("new lease was released by stale old-identity calls")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestQueueReservationReleaseWithCapacityDisabled(t *testing.T) {
|
|
store := edgenode.NewNodeStore()
|
|
rec := &edgenode.NodeRecord{
|
|
ID: "node-res-disabled",
|
|
Runtime: config.RuntimeConf{Concurrency: 1},
|
|
Providers: []config.NodeProviderConf{
|
|
{ID: "prov-1", Capacity: 1, LongContextCapacity: 1, Adapter: "vllm"},
|
|
},
|
|
}
|
|
store.Add(rec)
|
|
|
|
entry := &edgenode.NodeEntry{NodeID: "node-res-disabled"}
|
|
selected := &candidateNode{
|
|
entry: entry,
|
|
capacity: 1,
|
|
providerID: "prov-1",
|
|
longContextCapacity: 1,
|
|
}
|
|
cands := []candidateNode{*selected}
|
|
|
|
m := newModelQueueManager(store)
|
|
m.setStore(store)
|
|
|
|
admitted, _, err := m.admitWithReason(context.Background(), "g-res", "", "", cands, groupPolicy{}, nil, true, false)
|
|
if err != nil || admitted == nil {
|
|
t.Fatalf("admit: %v", err)
|
|
}
|
|
|
|
reservation := newQueueReservation(m, admitted)
|
|
if got := m.longInFlightForProvider("node-res-disabled", "prov-1"); got != 1 {
|
|
t.Fatalf("expected longInFlight=1, got %d", got)
|
|
}
|
|
|
|
// Disable capacity via refresh
|
|
rec.Providers[0].Capacity = 0
|
|
rec.Providers[0].LongContextCapacity = 0
|
|
m.setStore(store)
|
|
|
|
// Release
|
|
reservation.release("send-error")
|
|
|
|
if got := m.longInFlightForProvider("node-res-disabled", "prov-1"); got != 0 {
|
|
t.Errorf("longInFlight after capacity disabled release = %d, want 0", got)
|
|
}
|
|
}
|