iop/apps/edge/internal/service/queue_reservation_test.go

package service

import (
	"context"
	"sync"
	"testing"

	edgenode "iop/apps/edge/internal/node"
	"iop/packages/go/config"
)

// reservationFixture admits one slot on a single-capacity provider and returns
// the manager, the reservation for that admission, and the slot key it holds.
func reservationFixture(t *testing.T, long bool, longCapacity int) (*modelQueueManager, *queueReservation, string) {
	t.Helper()
	store := edgenode.NewNodeStore()
	store.Add(&edgenode.NodeRecord{
		ID:      "node-res",
		Runtime: config.RuntimeConf{Concurrency: 1},
		Providers: []config.NodeProviderConf{
			// The node record is the live source for capacity and long-context
			// capacity, so it must declare the same long limit the candidate does:
			// the scheduler re-resolves these fields from the record at dispatch.
			{ID: "prov-1", Capacity: 1, Adapter: "vllm", LongContextCapacity: longCapacity},
		},
	})

	entry := &edgenode.NodeEntry{NodeID: "node-res"}
	selected := &candidateNode{
		entry:               entry,
		capacity:            1,
		providerID:          "prov-1",
		longContextCapacity: longCapacity,
	}
	cands := []candidateNode{*selected}

	m := newModelQueueManager(store)
	admitted, _, err := m.admitWithReason(context.Background(), "g-res", "", "", cands, groupPolicy{}, nil, long, false)
	if err != nil || admitted == nil {
		t.Fatalf("admit: %v", err)
	}

	return m, newQueueReservation(m, admitted), "node-res:prov-1"
}

func inflightCount(t *testing.T, m *modelQueueManager, groupKey, slot string) (int, int) {
	t.Helper()
	m.mu.Lock()
	defer m.mu.Unlock()

	var nodeID, providerID string
	if colonIdx := findLastColon(slot); colonIdx > 0 {
		nodeID = slot[:colonIdx]
		providerID = slot[colonIdx+1:]
	} else {
		nodeID = slot
	}

	if providerID != "" {
		key := providerResourceKey{nodeID: nodeID, providerID: providerID}
		if res, ok := m.resources[key]; ok {
			return res.inFlight, res.longInFlight
		}
		return 0, 0
	}

	group, ok := m.groups[groupKey]
	if !ok {
		t.Fatalf("group %q not found", groupKey)
	}
	return group.inflight[slot], group.longInflight[slot]
}

// TestQueueReservationReleasesExactlyOnce pins the release contract at each
// boundary a dispatch can fail on: the slot is freed once, and a repeated
// release cannot free a second slot.
func TestQueueReservationReleasesExactlyOnce(t *testing.T) {
	cases := []struct {
		name string
		// track reports whether the dispatch got as far as registering the run
		// inflight before failing.
		track  bool
		reason string
	}{
		{name: "prepare tunnel error before track", reason: "prepare-tunnel-error"},
		{name: "prepare run error before track", reason: "prepare-run-error"},
		{name: "build error before track", reason: "build-error"},
		{name: "unknown execution path before track", reason: "unknown-execution-path"},
		{name: "send error after track", track: true, reason: "send-error"},
		{name: "no event bus after track", track: true, reason: "no-event-bus"},
	}

	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			m, reservation, slot := reservationFixture(t, false, 0)

			if inflight, _ := inflightCount(t, m, "g-res", slot); inflight != 1 {
				t.Fatalf("expected 1 reserved slot after admission, got %d", inflight)
			}

			if tc.track {
				reservation.track("run-res")
			}

			reservation.release(tc.reason)
			if inflight, _ := inflightCount(t, m, "g-res", slot); inflight != 0 {
				t.Fatalf("expected the slot to be released, got %d in flight", inflight)
			}

			// A second release must not decrement a slot that now belongs to
			// another request.
			reservation.release(tc.reason)
			if inflight, _ := inflightCount(t, m, "g-res", slot); inflight != 0 {
				t.Fatalf("expected release to be idempotent, got %d in flight", inflight)
			}
			if tc.track {
				m.mu.Lock()
				_, stillTracked := m.leaseByRun["run-res"]
				m.mu.Unlock()
				if stillTracked {
					t.Fatal("expected the run to be removed from the lease index")
				}
			}
			if got := leaseCount(m); got != 0 {
				t.Fatalf("expected the lease record to be dropped, got %d live leases", got)
			}
		})
	}
}

// TestQueueReservationHandOffKeepsSlotHeld pins that a dispatched request keeps
// its slot: after hand-off the run/tunnel lifecycle owns the release, so a late
// release on the reservation must not free it.
func TestQueueReservationHandOffKeepsSlotHeld(t *testing.T) {
	m, reservation, slot := reservationFixture(t, false, 0)
	reservation.track("run-res")
	reservation.handOff()

	reservation.release("send-error")

	if inflight, _ := inflightCount(t, m, "g-res", slot); inflight != 1 {
		t.Fatalf("expected the dispatched run to keep its slot, got %d in flight", inflight)
	}

	// The lifecycle owner releases it exactly once, via the run registration.
	m.releaseRun("run-res", "complete")
	if inflight, _ := inflightCount(t, m, "g-res", slot); inflight != 0 {
		t.Fatalf("expected the terminal event to release the slot, got %d in flight", inflight)
	}
}

// TestQueueReservationLongSlotMatchesReservation pins that the long-context
// slot is released only when it was actually reserved: a long request on a
// provider that declares a long limit.
func TestQueueReservationLongSlotMatchesReservation(t *testing.T) {
	t.Run("long request on a long-capacity provider releases the long slot", func(t *testing.T) {
		m, reservation, slot := reservationFixture(t, true, 1)
		if _, longInflight := inflightCount(t, m, "g-res", slot); longInflight != 1 {
			t.Fatalf("expected 1 reserved long slot, got %d", longInflight)
		}

		reservation.release("send-error")

		inflight, longInflight := inflightCount(t, m, "g-res", slot)
		if inflight != 0 || longInflight != 0 {
			t.Fatalf("expected both slots released, got inflight=%d long=%d", inflight, longInflight)
		}
	})

	t.Run("long request without a declared long limit reserves no long slot", func(t *testing.T) {
		m, reservation, slot := reservationFixture(t, true, 0)
		if _, longInflight := inflightCount(t, m, "g-res", slot); longInflight != 0 {
			t.Fatalf("expected no long slot to be reserved, got %d", longInflight)
		}
		if leaseLong(t, m, reservation.leaseID) {
			t.Fatal("expected the lease not to claim a long slot")
		}

		reservation.release("send-error")

		if inflight, _ := inflightCount(t, m, "g-res", slot); inflight != 0 {
			t.Fatalf("expected the slot to be released, got %d in flight", inflight)
		}
	})
}

// leaseRaceFixture admits two leases on one capacity-2 provider and returns the
// manager, both lease ids, and the shared slot key. Holding a second lease is
// what makes an over-release observable: the resource counters clamp at zero, so
// a lone lease could absorb a double decrement silently, while here the extra
// decrement would steal the surviving lease's slot.
func leaseRaceFixture(t *testing.T, long bool, longCapacity int) (m *modelQueueManager, first, second uint64, slot string) {
	t.Helper()
	store := edgenode.NewNodeStore()
	store.Add(&edgenode.NodeRecord{
		ID:      "node-race",
		Runtime: config.RuntimeConf{Concurrency: 2},
		Providers: []config.NodeProviderConf{
			{ID: "prov-race", Capacity: 2, LongContextCapacity: longCapacity, Adapter: "vllm"},
		},
	})

	entry := &edgenode.NodeEntry{NodeID: "node-race"}
	cands := []candidateNode{{
		entry:               entry,
		capacity:            2,
		providerID:          "prov-race",
		longContextCapacity: longCapacity,
	}}

	m = newModelQueueManager(store)
	m.setStore(store)

	ids := make([]uint64, 0, 2)
	for i := 0; i < 2; i++ {
		admitted, _, err := m.admitWithReason(context.Background(), "g-race", "", "", cands, groupPolicy{}, nil, long, false)
		if err != nil || admitted == nil {
			t.Fatalf("admit %d: %v", i, err)
		}
		if admitted.leaseID == 0 {
			t.Fatalf("admit %d returned no lease id", i)
		}
		ids = append(ids, admitted.leaseID)
	}
	if ids[0] == ids[1] {
		t.Fatalf("expected unique lease ids per admission, both were %d", ids[0])
	}
	return m, ids[0], ids[1], "node-race:prov-race"
}

// TestProviderLeaseConcurrentReleaseExactlyOnce pins the core lease invariant:
// however many terminal causes race on one lease — send failure, run terminal,
// tunnel terminal — the slot is returned exactly once and the other live lease
// keeps what it holds.
func TestProviderLeaseConcurrentReleaseExactlyOnce(t *testing.T) {
	cases := []struct {
		name         string
		long         bool
		longCapacity int
		wantLong     int
	}{
		{name: "normal slots"},
		{name: "long slots", long: true, longCapacity: 2, wantLong: 1},
	}

	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			m, first, second, slot := leaseRaceFixture(t, tc.long, tc.longCapacity)
			m.trackLease(first, "run-race-1")

			const racers = 32
			var wg sync.WaitGroup
			start := make(chan struct{})
			for i := 0; i < racers; i++ {
				wg.Add(1)
				go func(i int) {
					defer wg.Done()
					<-start
					switch i % 4 {
					case 0:
						m.releaseLease(first, "send-error")
					case 1:
						m.releaseRun("run-race-1", "complete")
					case 2:
						m.releaseRun("run-race-1", "cancelled")
					default:
						m.releaseLease(first, "tunnel-END")
					}
				}(i)
			}
			close(start)
			wg.Wait()

			inflight, longInflight := inflightCount(t, m, "g-race", slot)
			if inflight != 1 {
				t.Fatalf("expected the surviving lease to keep 1 slot, got %d in flight", inflight)
			}
			if longInflight != tc.wantLong {
				t.Fatalf("expected %d long slots held, got %d", tc.wantLong, longInflight)
			}
			if got := leaseCount(m); got != 1 {
				t.Fatalf("expected only the surviving lease to remain, got %d leases", got)
			}

			m.mu.Lock()
			_, stillIndexed := m.leaseByRun["run-race-1"]
			_, survivorLive := m.leases[second]
			m.mu.Unlock()
			if stillIndexed {
				t.Error("released lease is still in the run index")
			}
			if !survivorLive {
				t.Error("the untouched lease was released by the racing calls")
			}
		})
	}
}

// TestProviderLeaseTrackAndSendFailureRace covers the window the old bool guard
// could not: a terminal cause arriving while the dispatch is still binding the
// run id. Whichever side wins, the slot is freed once and no run id is left
// pointing at a lease that no longer exists.
func TestProviderLeaseTrackAndSendFailureRace(t *testing.T) {
	for i := 0; i < 16; i++ {
		m, first, second, slot := leaseRaceFixture(t, false, 0)

		var wg sync.WaitGroup
		start := make(chan struct{})
		wg.Add(2)
		go func() {
			defer wg.Done()
			<-start
			m.trackLease(first, "run-track-race")
		}()
		go func() {
			defer wg.Done()
			<-start
			m.releaseLease(first, "send-error")
		}()
		close(start)
		wg.Wait()

		// The terminal path that lost the race still runs afterwards; it must
		// not free a second slot.
		m.releaseRun("run-track-race", "complete")
		m.releaseLease(first, "tunnel-closed")

		if inflight, _ := inflightCount(t, m, "g-race", slot); inflight != 1 {
			t.Fatalf("iteration %d: expected the surviving lease to keep 1 slot, got %d", i, inflight)
		}
		m.mu.Lock()
		_, stillIndexed := m.leaseByRun["run-track-race"]
		_, survivorLive := m.leases[second]
		m.mu.Unlock()
		if stillIndexed {
			t.Fatalf("iteration %d: run id still indexed after the lease was released", i)
		}
		if !survivorLive {
			t.Fatalf("iteration %d: the untouched lease was released", i)
		}
	}
}

// leaseDisconnectFixture admits a primary lease on one node and a control lease
// on another, returning the manager, both lease ids, the primary slot key, and
// the control slot key. Two separate (node, provider) resources let us observe
// that a node disconnect only touches the disconnected node's counters.
func leaseDisconnectFixture(t *testing.T, long bool, longCapacity int) (m *modelQueueManager, primaryLeaseID, controlLeaseID uint64, primarySlot, controlSlot string) {
	t.Helper()
	store := edgenode.NewNodeStore()
	store.Add(&edgenode.NodeRecord{
		ID:      "node-disconn-primary",
		Runtime: config.RuntimeConf{Concurrency: 2},
		Providers: []config.NodeProviderConf{
			{ID: "prov-disconn", Capacity: 2, LongContextCapacity: longCapacity, Adapter: "vllm"},
		},
	})
	store.Add(&edgenode.NodeRecord{
		ID:      "node-disconn-control",
		Runtime: config.RuntimeConf{Concurrency: 1},
		Providers: []config.NodeProviderConf{
			{ID: "prov-disconn", Capacity: 1, Adapter: "vllm"},
		},
	})

	primaryEntry := &edgenode.NodeEntry{NodeID: "node-disconn-primary"}
	controlEntry := &edgenode.NodeEntry{NodeID: "node-disconn-control"}
	primaryCands := []candidateNode{{
		entry:               primaryEntry,
		capacity:            2,
		providerID:          "prov-disconn",
		longContextCapacity: longCapacity,
	}}
	controlCands := []candidateNode{{
		entry:               controlEntry,
		capacity:            1,
		providerID:          "prov-disconn",
		longContextCapacity: 0,
	}}

	m = newModelQueueManager(store)
	m.setStore(store)

	// Admit primary lease.
	admitted, _, err := m.admitWithReason(context.Background(), "g-disconn", "", "", primaryCands, groupPolicy{}, nil, long, false)
	if err != nil || admitted == nil {
		t.Fatalf("admit primary: %v", err)
	}
	primaryLeaseID = admitted.leaseID
	primarySlot = primaryEntry.NodeID + ":" + "prov-disconn"

	// Admit control lease on a different node.
	admitted2, _, err := m.admitWithReason(context.Background(), "g-disconn-ctrl", "", "", controlCands, groupPolicy{}, nil, false, false)
	if err != nil || admitted2 == nil {
		t.Fatalf("admit control: %v", err)
	}
	controlLeaseID = admitted2.leaseID
	controlSlot = controlEntry.NodeID + ":" + "prov-disconn"

	return m, primaryLeaseID, controlLeaseID, primarySlot, controlSlot
}

// TestProviderLeaseNodeDisconnectRaceExactlyOnce pins that when the current-owner
// node disconnects, the race between releaseNode and any other terminal cause
// (releaseLease, releaseRun) frees the disconnected slot exactly once, and that
// a later re-admission on the same (node, provider) is isolated from stale
// releases of the old lease identity. Both normal and long-context cases are
// verified.
func TestProviderLeaseNodeDisconnectRaceExactlyOnce(t *testing.T) {
	cases := []struct {
		name         string
		long         bool
		longCapacity int
		wantLong     int
	}{
		{name: "normal slots"},
		{name: "long slots", long: true, longCapacity: 2, wantLong: 1},
	}

	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			m, primaryLeaseID, controlLeaseID, primarySlot, controlSlot := leaseDisconnectFixture(t, tc.long, tc.longCapacity)

			// Track the primary lease so releaseRun has a run id to resolve.
			primaryRunID := "run-disconn-race"
			m.trackLease(primaryLeaseID, primaryRunID)

			// Barrier: race releaseNode against releaseLease and releaseRun.
			const racers = 32
			var wg sync.WaitGroup
			start := make(chan struct{})
			for i := 0; i < racers; i++ {
				wg.Add(1)
				go func(i int) {
					defer wg.Done()
					<-start
					switch i % 3 {
					case 0:
						m.releaseNode("node-disconn-primary", 0, "transport-closed")
					case 1:
						m.releaseRun(primaryRunID, "complete")
					default:
						m.releaseLease(primaryLeaseID, "tunnel-END")
					}
				}(i)
			}
			close(start)
			wg.Wait()

			// After the race, the disconnected primary resource must be zeroed.
			inflight, longInflight := inflightCount(t, m, "g-disconn", primarySlot)
			if inflight != 0 {
				t.Fatalf("expected disconnected primary inflight=0, got %d", inflight)
			}
			if longInflight != 0 {
				t.Fatalf("expected disconnected primary longInflight=0, got %d", longInflight)
			}

			// The control lease on the other node must be untouched.
			ctrlInflight, ctrlLongInflight := inflightCount(t, m, "g-disconn-ctrl", controlSlot)
			if ctrlInflight != 1 {
				t.Fatalf("expected control inflight=1, got %d", ctrlInflight)
			}
			if ctrlLongInflight != 0 {
				t.Fatalf("expected control longInflight=0, got %d", ctrlLongInflight)
			}

			// The primary lease must be gone from the lease map.
			m.mu.Lock()
			_, primaryStillLive := m.leases[primaryLeaseID]
			_, controlStillLive := m.leases[controlLeaseID]
			_, stillIndexed := m.leaseByRun[primaryRunID]
			m.mu.Unlock()
			if primaryStillLive {
				t.Error("primary lease still in lease map after disconnect race")
			}
			if !controlStillLive {
				t.Error("control lease was released by the racing calls")
			}
			if stillIndexed {
				t.Error("primary run id still indexed after disconnect race")
			}

			// Re-admit on the same (node, provider) that just disconnected.
			primaryEntry := &edgenode.NodeEntry{NodeID: "node-disconn-primary"}
			primaryCands := []candidateNode{{
				entry:               primaryEntry,
				capacity:            2,
				providerID:          "prov-disconn",
				longContextCapacity: tc.longCapacity,
			}}
			newAdmitted, _, err := m.admitWithReason(context.Background(), "g-disconn-re", "", "", primaryCands, groupPolicy{}, nil, tc.long, false)
			if err != nil || newAdmitted == nil {
				t.Fatalf("re-admit: %v", err)
			}
			newLeaseID := newAdmitted.leaseID

			// New lease must hold exactly the expected counters.
			newInflight, newLongInflight := inflightCount(t, m, "g-disconn-re", primarySlot)
			if newInflight != 1 {
				t.Fatalf("expected re-admitted inflight=1, got %d", newInflight)
			}
			if newLongInflight != tc.wantLong {
				t.Fatalf("expected re-admitted longInflight=%d, got %d", tc.wantLong, newLongInflight)
			}

			// Now issue late releases of the OLD lease identity. They must be
			// no-ops: the new lease counters must not decrement.
			m.releaseLease(primaryLeaseID, "late-send-error")
			m.releaseRun(primaryRunID, "late-complete")

			finalInflight, finalLongInflight := inflightCount(t, m, "g-disconn-re", primarySlot)
			if finalInflight != 1 {
				t.Fatalf("expected late old-identity release to be a no-op, inflight=%d (want 1)", finalInflight)
			}
			if finalLongInflight != tc.wantLong {
				t.Fatalf("expected late old-identity release to be a no-op, longInflight=%d (want %d)", finalLongInflight, tc.wantLong)
			}

			// The new lease must still be live.
			m.mu.Lock()
			_, newStillLive := m.leases[newLeaseID]
			m.mu.Unlock()
			if !newStillLive {
				t.Error("new lease was released by stale old-identity calls")
			}
		})
	}
}

// TestDisconnectFencesOnlyMatchingGeneration pins the manager-level generation
// fence: a disconnect carrying a stale (older) generation leaves the current
// owner's lease and provider resource untouched; only the current generation
// fences them; a late reserve carrying the fenced generation is rejected on the
// orphaned resource; and a reconnect with a strictly newer generation re-acquires
// the slot.
func TestDisconnectFencesOnlyMatchingGeneration(t *testing.T) {
	store := edgenode.NewNodeStore()
	store.Add(&edgenode.NodeRecord{
		ID:      "node-gen",
		Runtime: config.RuntimeConf{Concurrency: 2},
		Providers: []config.NodeProviderConf{
			{ID: "prov-gen", Capacity: 2, Adapter: "vllm"},
		},
	})

	m := newModelQueueManager(store)
	m.setStore(store)

	genCandidates := func(gen uint64) []candidateNode {
		return []candidateNode{{
			entry:      &edgenode.NodeEntry{NodeID: "node-gen", ConnectionGeneration: gen},
			capacity:   2,
			providerID: "prov-gen",
			generation: gen,
		}}
	}

	admitted, _, err := m.admitWithReason(context.Background(), "g-gen", "", "", genCandidates(5), groupPolicy{}, nil, false, false)
	if err != nil || admitted == nil {
		t.Fatalf("admit gen 5: %v", err)
	}
	gen5Lease := admitted.leaseID
	if gen5Lease == 0 {
		t.Fatal("admit gen 5 returned no lease id")
	}

	const slot = "node-gen:prov-gen"
	if inflight, _ := inflightCount(t, m, "g-gen", slot); inflight != 1 {
		t.Fatalf("expected 1 in-flight after admit, got %d", inflight)
	}

	// A disconnect for an older generation is a stale no-op: the current owner
	// (gen 5) keeps its lease and its resource slot.
	m.releaseNode("node-gen", 4, "stale-disconnect")
	if inflight, _ := inflightCount(t, m, "g-gen", slot); inflight != 1 {
		t.Fatalf("stale-generation disconnect changed in-flight to %d, want 1", inflight)
	}
	m.mu.Lock()
	_, gen5Live := m.leases[gen5Lease]
	m.mu.Unlock()
	if !gen5Live {
		t.Fatal("stale-generation disconnect released the current owner's lease")
	}

	// The matching generation fences: lease dropped, resource zeroed to offline.
	m.releaseNode("node-gen", 5, "current-disconnect")
	if inflight, _ := inflightCount(t, m, "g-gen", slot); inflight != 0 {
		t.Fatalf("current-generation disconnect left %d in-flight, want 0", inflight)
	}
	m.mu.Lock()
	_, gen5StillLive := m.leases[gen5Lease]
	m.mu.Unlock()
	if gen5StillLive {
		t.Fatal("current-generation disconnect did not release the lease")
	}

	// A reserve carrying the fenced (stale) generation must be rejected on the
	// orphaned resource: only a strictly newer connection may re-acquire it.
	m.mu.Lock()
	staleCand := genCandidates(5)[0]
	group := m.getOrCreateGroupLocked("g-gen", groupPolicy{})
	_, staleReserved := m.reserveCandidateLocked(group, &staleCand, false)
	m.mu.Unlock()
	if staleReserved {
		t.Fatal("stale (gen 5) candidate reserved on the orphaned resource")
	}

	// A reconnect with a strictly higher generation re-acquires the slot.
	reAdmit, _, err := m.admitWithReason(context.Background(), "g-gen-recon", "", "", genCandidates(6), groupPolicy{}, nil, false, false)
	if err != nil || reAdmit == nil {
		t.Fatalf("re-admit gen 6: %v", err)
	}
	if inflight, _ := inflightCount(t, m, "g-gen-recon", slot); inflight != 1 {
		t.Fatalf("expected re-admitted in-flight 1, got %d", inflight)
	}
	// A late disconnect carrying the old fenced generation must not touch the
	// reconnect's lease.
	m.releaseNode("node-gen", 5, "late-stale-disconnect")
	if inflight, _ := inflightCount(t, m, "g-gen-recon", slot); inflight != 1 {
		t.Fatalf("late stale-generation disconnect fenced the reconnect, in-flight=%d want 1", inflight)
	}
}

// TestDisconnectSettlesOldLeaseWhenNewGenerationAlreadyReserved pins the mixed-
// generation overlap the previous fence missed: when an old connection (gen1) and
// a reconnect (gen2) both hold a slot on the same capacity-2 provider resource, a
// disconnect of the old generation must return exactly its own slot through the
// per-lease decrement path — leaving the newer lease and the resource it still
// owns intact — and the resource must fully recover to zero only once the newer
// lease reaches its own terminal. Both normal and long-context occupancy are
// verified under the same ordering so the settlement is exact for either counter.
func TestDisconnectSettlesOldLeaseWhenNewGenerationAlreadyReserved(t *testing.T) {
	cases := []struct {
		name         string
		long         bool
		longCapacity int
		wantLong     int
	}{
		{name: "normal slots"},
		{name: "long slots", long: true, longCapacity: 2, wantLong: 1},
	}

	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			store := edgenode.NewNodeStore()
			store.Add(&edgenode.NodeRecord{
				ID:      "node-mix",
				Runtime: config.RuntimeConf{Concurrency: 2},
				Providers: []config.NodeProviderConf{
					{ID: "prov-mix", Capacity: 2, LongContextCapacity: tc.longCapacity, Adapter: "vllm"},
				},
			})

			m := newModelQueueManager(store)
			m.setStore(store)

			genCandidates := func(gen uint64) []candidateNode {
				return []candidateNode{{
					entry:               &edgenode.NodeEntry{NodeID: "node-mix", ConnectionGeneration: gen},
					capacity:            2,
					providerID:          "prov-mix",
					longContextCapacity: tc.longCapacity,
					generation:          gen,
				}}
			}

			// gen1 admits first and holds one slot on the capacity-2 resource.
			gen1Admit, _, err := m.admitWithReason(context.Background(), "g-mix", "", "", genCandidates(1), groupPolicy{}, nil, tc.long, false)
			if err != nil || gen1Admit == nil {
				t.Fatalf("admit gen1: %v", err)
			}
			gen1Lease := gen1Admit.leaseID
			const gen1RunID = "run-mix-gen1"
			m.trackLease(gen1Lease, gen1RunID)

			// gen2 is a reconnect that admits while gen1 is still in flight, so both
			// generations now co-occupy the same provider resource.
			gen2Admit, _, err := m.admitWithReason(context.Background(), "g-mix", "", "", genCandidates(2), groupPolicy{}, nil, tc.long, false)
			if err != nil || gen2Admit == nil {
				t.Fatalf("admit gen2: %v", err)
			}
			gen2Lease := gen2Admit.leaseID
			const gen2RunID = "run-mix-gen2"
			m.trackLease(gen2Lease, gen2RunID)

			const slot = "node-mix:prov-mix"
			if inflight, longInflight := inflightCount(t, m, "g-mix", slot); inflight != 2 || longInflight != 2*tc.wantLong {
				t.Fatalf("expected both leases in flight (inflight=2 long=%d), got inflight=%d long=%d", 2*tc.wantLong, inflight, longInflight)
			}

			// gen1 disconnects. Only its slot must be returned; the newer gen2 lease
			// and the resource counter it still holds must survive.
			m.releaseNode("node-mix", 1, "gen1-disconnect")

			inflight, longInflight := inflightCount(t, m, "g-mix", slot)
			if inflight != 1 {
				t.Fatalf("after gen1 disconnect: inflight=%d, want 1 (only gen1's slot returned)", inflight)
			}
			if longInflight != tc.wantLong {
				t.Fatalf("after gen1 disconnect: longInflight=%d, want %d", longInflight, tc.wantLong)
			}
			m.mu.Lock()
			_, gen1Live := m.leases[gen1Lease]
			_, gen2Live := m.leases[gen2Lease]
			_, gen1Indexed := m.leaseByRun[gen1RunID]
			m.mu.Unlock()
			if gen1Live {
				t.Error("gen1 lease survived its own disconnect")
			}
			if !gen2Live {
				t.Error("gen1 disconnect released the newer gen2 lease")
			}
			if gen1Indexed {
				t.Error("gen1 run id still indexed after disconnect")
			}

			// gen2 reaching its own terminal returns the last slot: the resource now
			// fully recovers to zero, proving no counter was stranded.
			m.releaseRun(gen2RunID, "complete")
			finalInflight, finalLong := inflightCount(t, m, "g-mix", slot)
			if finalInflight != 0 {
				t.Fatalf("after gen2 terminal: inflight=%d, want 0 (full recovery)", finalInflight)
			}
			if finalLong != 0 {
				t.Fatalf("after gen2 terminal: longInflight=%d, want 0", finalLong)
			}
			if got := leaseCount(m); got != 0 {
				t.Fatalf("expected every lease settled, got %d live", got)
			}
		})
	}
}

func TestQueueReservationReleaseWithCapacityDisabled(t *testing.T) {
	store := edgenode.NewNodeStore()
	rec := &edgenode.NodeRecord{
		ID:      "node-res-disabled",
		Runtime: config.RuntimeConf{Concurrency: 1},
		Providers: []config.NodeProviderConf{
			{ID: "prov-1", Capacity: 1, LongContextCapacity: 1, Adapter: "vllm"},
		},
	}
	store.Add(rec)

	entry := &edgenode.NodeEntry{NodeID: "node-res-disabled"}
	selected := &candidateNode{
		entry:               entry,
		capacity:            1,
		providerID:          "prov-1",
		longContextCapacity: 1,
	}
	cands := []candidateNode{*selected}

	m := newModelQueueManager(store)
	m.setStore(store)

	admitted, _, err := m.admitWithReason(context.Background(), "g-res", "", "", cands, groupPolicy{}, nil, true, false)
	if err != nil || admitted == nil {
		t.Fatalf("admit: %v", err)
	}

	reservation := newQueueReservation(m, admitted)
	if got := m.longInFlightForProvider("node-res-disabled", "prov-1"); got != 1 {
		t.Fatalf("expected longInFlight=1, got %d", got)
	}

	// Disable capacity via refresh
	rec.Providers[0].Capacity = 0
	rec.Providers[0].LongContextCapacity = 0
	m.setStore(store)

	// Release
	reservation.release("send-error")

	if got := m.longInFlightForProvider("node-res-disabled", "prov-1"); got != 0 {
		t.Errorf("longInFlight after capacity disabled release = %d, want 0", got)
	}
}