iop/apps/edge/internal/service/provider_scheduling_advanced_test.go

package service

import (
	"context"
	"net"
	"sync"
	"testing"
	"time"

	toki "git.toki-labs.com/toki/proto-socket/go"
	"google.golang.org/protobuf/proto"

	edgeevents "iop/apps/edge/internal/events"
	edgenode "iop/apps/edge/internal/node"
	"iop/packages/go/config"
	iop "iop/proto/gen/iop"
)

func TestSubmitRunProviderPoolRewritesAdapterAndTarget(t *testing.T) {
	// Use net.Pipe to create a fake node connection that captures the RunRequest.
	edgeConn, nodeConn := net.Pipe()
	defer edgeConn.Close()
	defer nodeConn.Close()

	parserMap := toki.ParserMap{
		toki.TypeNameOf(&iop.RunRequest{}): func(b []byte) (proto.Message, error) {
			m := &iop.RunRequest{}
			return m, proto.Unmarshal(b, m)
		},
	}

	edgeClient := toki.NewTcpClient(edgeConn, 0, 0, parserMap)
	nodeClient := toki.NewTcpClient(nodeConn, 0, 0, parserMap)

	// Capture the RunRequest received by the fake node.
	var capturedReq *iop.RunRequest
	var capturedMu sync.Mutex
	toki.AddListenerTyped[*iop.RunRequest](&nodeClient.Communicator, func(req *iop.RunRequest) {
		capturedMu.Lock()
		capturedReq = req
		capturedMu.Unlock()
	})

	// Build the model catalog with provider references.
	catalog := []config.ModelCatalogEntry{
		{
			ID: "qwen3.6:35b",
			Providers: map[string]string{
				"prov-vllm-01": "served-qwen",
			},
		},
	}

	// Build NodeStore with a provider-pool provider.
	store := edgenode.NewNodeStore()
	store.Add(&edgenode.NodeRecord{
		ID:      "node-pool",
		Runtime: config.RuntimeConf{Concurrency: 4},
		Adapters: config.AdaptersConf{
			VllmInstances: []config.VllmInstanceConf{
				{Name: "vllm-gpu", Enabled: true, Endpoint: "http://127.0.0.1:8000/v1"},
			},
		},
		Providers: []config.NodeProviderConf{
			{
				ID:       "prov-vllm-01",
				Adapter:  "vllm-gpu",
				Models:   []string{"served-qwen"},
				Health:   "available",
				Capacity: 2,
			},
		},
	})

	// Build registry with the fake node.
	reg := edgenode.NewRegistry()
	reg.Register(&edgenode.NodeEntry{
		NodeID:         "node-pool",
		LifecycleState: edgenode.LifecycleConnected,
		Client:         edgeClient,
	})

	// Create Service with queue and catalog.
	// events bus must be non-nil to activate the queue path for provider-pool.
	bus := edgeevents.NewBus()
	svc := New(reg, bus)
	svc.SetNodeStore(store)
	svc.SetModelCatalog(catalog)

	// SubmitRun with ProviderPool=true.
	result, err := svc.SubmitRun(context.Background(), SubmitRunRequest{
		RunID:         "run-pool-test-001",
		ModelGroupKey: "qwen3.6:35b",
		ProviderPool:  true,
		Background:    true,
	})
	if err != nil {
		t.Fatalf("SubmitRun: %v", err)
	}
	if result == nil {
		t.Fatal("expected non-nil RunResult")
	}

	// Wait for the fake node to receive the request.
	time.Sleep(50 * time.Millisecond)

	capturedMu.Lock()
	defer capturedMu.Unlock()

	if capturedReq == nil {
		t.Fatal("no RunRequest captured from fake node; SubmitRun did not send")
	}

	// Verify that the adapter and target were rewritten from the provider-pool candidate.
	if capturedReq.GetAdapter() != "vllm-gpu" {
		t.Errorf("adapter: got %q, want %q", capturedReq.GetAdapter(), "vllm-gpu")
	}
	if capturedReq.GetTarget() != "served-qwen" {
		t.Errorf("target: got %q, want %q", capturedReq.GetTarget(), "served-qwen")
	}
	if capturedReq.GetRunId() != "run-pool-test-001" {
		t.Errorf("runID: got %q, want %q", capturedReq.GetRunId(), "run-pool-test-001")
	}
}

// TestResolveProviderPoolCandidatesAdapterInstanceValidation verifies that the
// defensive isProviderAdapterInstanceValid check in resolveProviderPoolCandidates
// excludes providers whose adapter name resolves to a disabled exact instance,
// an ambiguous type route, a zero-instance type route, or an unknown adapter key,
// while valid named instances and single enabled type routes remain as candidates.
func TestResolveProviderPoolCandidatesAdapterInstanceValidation(t *testing.T) {
	catalog := []config.ModelCatalogEntry{
		{
			ID: "ai-group",
			Providers: map[string]string{
				"prov-disabled":  "ai-model",
				"prov-ambiguous": "ai-model",
				"prov-valid":     "ai-model",
				"prov-zero-type": "ai-model",
				"prov-missing":   "ai-model",
			},
		},
	}

	store := edgenode.NewNodeStore()

	// prov-disabled: adapter="vllm-gpu" but VllmInstance "vllm-gpu" is disabled.
	store.Add(&edgenode.NodeRecord{
		ID: "node-disabled",
		Adapters: config.AdaptersConf{
			VllmInstances: []config.VllmInstanceConf{
				{Name: "vllm-gpu", Enabled: false, Endpoint: "http://127.0.0.1:8000/v1"},
			},
		},
		Providers: []config.NodeProviderConf{
			{ID: "prov-disabled", Adapter: "vllm-gpu", Models: []string{"ai-model"}, Health: "available", Capacity: 2},
		},
	})

	// prov-ambiguous: adapter="openai_compat" type route with 2 enabled instances → ambiguous.
	store.Add(&edgenode.NodeRecord{
		ID: "node-ambiguous",
		Adapters: config.AdaptersConf{
			OpenAICompatInstances: []config.OpenAICompatInstanceConf{
				{Name: "compat-a", Enabled: true, Endpoint: "http://127.0.0.1:8000/v1"},
				{Name: "compat-b", Enabled: true, Endpoint: "http://127.0.0.1:8001/v1"},
			},
		},
		Providers: []config.NodeProviderConf{
			{ID: "prov-ambiguous", Adapter: "openai_compat", Models: []string{"ai-model"}, Health: "available", Capacity: 2},
		},
	})

	// prov-valid: adapter="vllm-gpu" and VllmInstance "vllm-gpu" is enabled.
	store.Add(&edgenode.NodeRecord{
		ID: "node-valid",
		Adapters: config.AdaptersConf{
			VllmInstances: []config.VllmInstanceConf{
				{Name: "vllm-gpu", Enabled: true, Endpoint: "http://127.0.0.1:8000/v1"},
			},
		},
		Providers: []config.NodeProviderConf{
			{ID: "prov-valid", Adapter: "vllm-gpu", Models: []string{"ai-model"}, Health: "available", Capacity: 2},
		},
	})

	// prov-zero-type: adapter="ollama" type route but no enabled ollama instances → zero-instance type route excluded.
	store.Add(&edgenode.NodeRecord{
		ID: "node-zero-type",
		Adapters: config.AdaptersConf{
			OllamaInstances: []config.OllamaInstanceConf{
				{Name: "ollama-disabled", Enabled: false, BaseURL: "http://127.0.0.1:11434"},
			},
		},
		Providers: []config.NodeProviderConf{
			{ID: "prov-zero-type", Adapter: "ollama", Models: []string{"ai-model"}, Health: "available", Capacity: 2},
		},
	})

	// prov-missing: adapter="unknown-missing" not in any instance list and not a known type key → unknown adapter excluded.
	store.Add(&edgenode.NodeRecord{
		ID: "node-missing",
		Adapters: config.AdaptersConf{
			VllmInstances: []config.VllmInstanceConf{
				{Name: "vllm-gpu", Enabled: true, Endpoint: "http://127.0.0.1:9000/v1"},
			},
		},
		Providers: []config.NodeProviderConf{
			{ID: "prov-missing", Adapter: "unknown-missing", Models: []string{"ai-model"}, Health: "available", Capacity: 2},
		},
	})

	reg := edgenode.NewRegistry()
	allRecs := store.All()
	for _, rec := range allRecs {
		reg.Register(&edgenode.NodeEntry{NodeID: rec.ID, LifecycleState: edgenode.LifecycleConnected})
	}

	svc := New(reg, nil)
	svc.SetNodeStore(store)
	svc.SetModelCatalog(catalog)

	req := SubmitRunRequest{ModelGroupKey: "ai-group", ProviderPool: true}
	storeSnap, catalogSnap, _ := svc.runtimeConfigSnapshot()
	candidates, _, err := svc.resolveProviderPoolCandidates(req, storeSnap, catalogSnap)
	if err != nil {
		t.Fatalf("resolveProviderPoolCandidates: %v", err)
	}

	// Only prov-valid should be a candidate.
	got := make(map[string]bool)
	for _, c := range candidates {
		got[c.providerID] = true
	}
	if len(candidates) != 1 {
		t.Fatalf("expected 1 candidate (prov-valid), got %d: %v", len(candidates), got)
	}
	if !got["prov-valid"] {
		t.Error("prov-valid (enabled exact instance) must be a candidate")
	}
	if got["prov-disabled"] {
		t.Error("prov-disabled (disabled exact instance) must be excluded")
	}
	if got["prov-ambiguous"] {
		t.Error("prov-ambiguous (ambiguous type route) must be excluded")
	}
	if got["prov-zero-type"] {
		t.Error("prov-zero-type (zero-instance type route) must be excluded")
	}
	if got["prov-missing"] {
		t.Error("prov-missing (unknown adapter) must be excluded")
	}
}

// TestGetSnapshotForNodeCatalogFirstNoDuplicates verifies that when a node has
// both adapter instances and a providers[] catalog, getSnapshotForNode returns
// only catalog snapshots (no adapter duplicates).
func TestGetSnapshotForNodeCatalogFirstNoDuplicates(t *testing.T) {
	rec := &edgenode.NodeRecord{
		ID: "node-catalog",
		Adapters: config.AdaptersConf{
			CLI: config.CLIConf{Enabled: true},
			OpenAICompatInstances: []config.OpenAICompatInstanceConf{
				{Name: "vllm-gpu", Enabled: true},
			},
		},
		Providers: []config.NodeProviderConf{
			{ID: "prov-a", Adapter: "vllm-gpu", Models: []string{"model-a"}, Health: "available", Capacity: 2},
			{ID: "prov-b", Adapter: "vllm-gpu", Models: []string{"model-b"}, Health: "available", Capacity: 3},
		},
		Runtime: config.RuntimeConf{Concurrency: 1},
	}
	store := edgenode.NewNodeStore()
	store.Add(rec)

	m := newModelQueueManager(store)

	snaps := m.getSnapshotForNode("node-catalog", rec)

	// Must return exactly the catalog providers — no CLI or openai_compat adapter duplicates.
	if len(snaps) != 2 {
		t.Fatalf("expected 2 catalog snapshots, got %d: %+v", len(snaps), snaps)
	}
	ids := map[string]bool{}
	for _, s := range snaps {
		ids[s.Id] = true
	}
	if !ids["prov-a"] || !ids["prov-b"] {
		t.Errorf("expected catalog provider IDs prov-a and prov-b, got: %v", ids)
	}
	// No adapter-only snapshots (no empty Id entries that correspond to adapter sections).
	for _, s := range snaps {
		if s.Id == "" {
			t.Errorf("unexpected adapter-only snapshot (empty Id) in catalog-first result: %+v", s)
		}
	}
}

// TestResolveProviderPoolCandidatesSkipsDisabledProvider verifies that a
// provider with enabled=false is excluded from dispatch candidates.
func TestResolveProviderPoolCandidatesSkipsDisabledProvider(t *testing.T) {
	disabled := false
	catalog := []config.ModelCatalogEntry{
		{
			ID: "model-x",
			Providers: map[string]string{
				"prov-active":   "served-x",
				"prov-disabled": "served-x",
			},
		},
	}
	store := edgenode.NewNodeStore()
	store.Add(&edgenode.NodeRecord{
		ID: "node-d",
		Providers: []config.NodeProviderConf{
			{ID: "prov-active", Type: "vllm", Adapter: "vllm-gpu", Models: []string{"served-x"}, Health: "available", Capacity: 2},
			{ID: "prov-disabled", Type: "vllm", Adapter: "vllm-gpu", Models: []string{"served-x"}, Health: "available", Capacity: 2, Enabled: &disabled},
		},
		Adapters: config.AdaptersConf{
			VllmInstances: []config.VllmInstanceConf{{Name: "vllm-gpu", Enabled: true, Endpoint: "http://127.0.0.1:8000/v1"}},
		},
	})
	reg := edgenode.NewRegistry()
	reg.Register(&edgenode.NodeEntry{NodeID: "node-d", LifecycleState: edgenode.LifecycleConnected})

	svc := New(reg, nil)
	svc.SetNodeStore(store)
	svc.SetModelCatalog(catalog)

	storeSnap, catalogSnap, _ := svc.runtimeConfigSnapshot()
	candidates, _, err := svc.resolveProviderPoolCandidates(SubmitRunRequest{ModelGroupKey: "model-x", ProviderPool: true}, storeSnap, catalogSnap)
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if len(candidates) != 1 {
		t.Fatalf("expected 1 candidate (disabled filtered), got %d", len(candidates))
	}
	if candidates[0].providerID != "prov-active" {
		t.Errorf("expected prov-active, got %q", candidates[0].providerID)
	}
}

// TestResolveProviderPoolCandidatesUsesProviderIDWhenAdapterOmitted verifies
// that a provider-first provider (empty Adapter field) uses its ID as the
// dispatch adapter key.
func TestResolveProviderPoolCandidatesUsesProviderIDWhenAdapterOmitted(t *testing.T) {
	catalog := []config.ModelCatalogEntry{
		{
			ID: "model-y",
			Providers: map[string]string{
				"prov-first": "served-y",
			},
		},
	}
	store := edgenode.NewNodeStore()
	store.Add(&edgenode.NodeRecord{
		ID: "node-pf",
		Providers: []config.NodeProviderConf{
			{ID: "prov-first", Type: "vllm", Adapter: "", Models: []string{"served-y"}, Health: "available", Capacity: 3},
		},
	})
	reg := edgenode.NewRegistry()
	reg.Register(&edgenode.NodeEntry{NodeID: "node-pf", LifecycleState: edgenode.LifecycleConnected})

	svc := New(reg, nil)
	svc.SetNodeStore(store)
	svc.SetModelCatalog(catalog)

	storeSnap, catalogSnap, _ := svc.runtimeConfigSnapshot()
	candidates, _, err := svc.resolveProviderPoolCandidates(SubmitRunRequest{ModelGroupKey: "model-y", ProviderPool: true}, storeSnap, catalogSnap)
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if len(candidates) != 1 {
		t.Fatalf("expected 1 candidate, got %d", len(candidates))
	}
	if candidates[0].adapter != "prov-first" {
		t.Errorf("provider-first adapter key: got %q, want provider ID %q", candidates[0].adapter, "prov-first")
	}
	if candidates[0].providerID != "prov-first" {
		t.Errorf("providerID: got %q", candidates[0].providerID)
	}
}

// TestProviderSnapshotsReportDisabledProvider verifies that a disabled
// provider appears in snapshots with status=disabled and effective capacity 0.
func TestProviderSnapshotsReportDisabledProvider(t *testing.T) {
	disabled := false
	rec := &edgenode.NodeRecord{
		ID: "node-snap",
		Providers: []config.NodeProviderConf{
			{ID: "prov-on", Type: "vllm", Health: "available", Capacity: 4},
			{ID: "prov-off", Type: "vllm", Health: "available", Capacity: 4, Enabled: &disabled},
		},
	}
	store := edgenode.NewNodeStore()
	store.Add(rec)
	m := newModelQueueManager(store)

	snaps := m.getSnapshotForNode("node-snap", rec)
	if len(snaps) != 2 {
		t.Fatalf("expected 2 snapshots, got %d", len(snaps))
	}

	byID := map[string]*iop.ProviderSnapshot{}
	for _, s := range snaps {
		byID[s.Id] = s
	}

	on := byID["prov-on"]
	if on == nil {
		t.Fatal("prov-on snapshot missing")
	}
	if on.Status != "available" {
		t.Errorf("prov-on status: got %q, want available", on.Status)
	}
	if on.Capacity != 4 {
		t.Errorf("prov-on capacity: got %d, want 4", on.Capacity)
	}

	off := byID["prov-off"]
	if off == nil {
		t.Fatal("prov-off snapshot missing")
	}
	if off.Status != "disabled" {
		t.Errorf("prov-off status: got %q, want disabled", off.Status)
	}
	if off.Health != "disabled" {
		t.Errorf("prov-off health: got %q, want disabled", off.Health)
	}
	if off.Capacity != 0 {
		t.Errorf("prov-off effective capacity: got %d, want 0", off.Capacity)
	}
}

// TestStatusProviderProviderFirstNoAdapterDuplicates verifies that provider-first
// providers (empty Adapter) appear once in snapshots with no adapter duplicate.
func TestStatusProviderProviderFirstNoAdapterDuplicates(t *testing.T) {
	rec := &edgenode.NodeRecord{
		ID: "node-pf-snap",
		Adapters: config.AdaptersConf{
			VllmInstances: []config.VllmInstanceConf{{Name: "vllm-gpu", Enabled: true}},
		},
		Providers: []config.NodeProviderConf{
			{ID: "prov-first-a", Type: "vllm", Adapter: "", Models: []string{"model-a"}, Health: "available", Capacity: 2},
			{ID: "prov-first-b", Type: "vllm", Adapter: "", Models: []string{"model-b"}, Health: "available", Capacity: 2},
		},
	}
	store := edgenode.NewNodeStore()
	store.Add(rec)
	m := newModelQueueManager(store)

	snaps := m.getSnapshotForNode("node-pf-snap", rec)
	// catalog-first: exactly 2 provider snapshots, no adapter duplicate.
	if len(snaps) != 2 {
		t.Fatalf("expected 2 provider-first snapshots, got %d: %+v", len(snaps), snaps)
	}
	for _, s := range snaps {
		if s.Id == "" {
			t.Errorf("unexpected adapter-only snapshot (empty Id): %+v", s)
		}
		if s.Adapter != "" {
			t.Errorf("provider-first snapshot should have empty Adapter, got %q for id=%q", s.Adapter, s.Id)
		}
	}
}

// TestResolveProviderPoolCandidatesPropagatesPriority verifies that the actual
// resolveProviderPoolCandidates path propagates config provider Priority into
// candidateNode.priority.
func TestResolveProviderPoolCandidatesPropagatesPriority(t *testing.T) {
	catalog := []config.ModelCatalogEntry{
		{
			ID: "model-prio",
			Providers: map[string]string{
				"prov-high-prio":   "served-model-prio",
				"prov-low-prio":    "served-model-prio",
				"prov-no-priority": "served-model-prio",
			},
		},
	}

	store := edgenode.NewNodeStore()
	// High-priority provider (priority=3).
	store.Add(&edgenode.NodeRecord{
		ID:      "node-prio",
		Runtime: config.RuntimeConf{Concurrency: 4},
		Adapters: config.AdaptersConf{
			VllmInstances: []config.VllmInstanceConf{
				{Name: "vllm-gpu", Enabled: true, Endpoint: "http://127.0.0.1:8000/v1"},
			},
		},
		Providers: []config.NodeProviderConf{
			{
				ID:       "prov-high-prio",
				Adapter:  "vllm-gpu",
				Models:   []string{"served-model-prio"},
				Health:   "available",
				Capacity: 2,
				Priority: 3,
			},
			{
				ID:       "prov-low-prio",
				Adapter:  "vllm-gpu",
				Models:   []string{"served-model-prio"},
				Health:   "available",
				Capacity: 2,
				Priority: 10,
			},
			{
				ID:       "prov-no-priority",
				Adapter:  "vllm-gpu",
				Models:   []string{"served-model-prio"},
				Health:   "available",
				Capacity: 2,
				// Priority omitted → defaults to 0.
			},
		},
	})

	reg := edgenode.NewRegistry()
	for _, rec := range store.All() {
		reg.Register(&edgenode.NodeEntry{NodeID: rec.ID, LifecycleState: edgenode.LifecycleConnected})
	}

	svc := New(reg, nil)
	svc.SetNodeStore(store)
	svc.SetModelCatalog(catalog)

	storeSnap, catalogSnap, _ := svc.runtimeConfigSnapshot()
	candidates, _, err := svc.resolveProviderPoolCandidates(
		SubmitRunRequest{ModelGroupKey: "model-prio", ProviderPool: true},
		storeSnap, catalogSnap,
	)
	if err != nil {
		t.Fatalf("resolveProviderPoolCandidates: %v", err)
	}
	if len(candidates) != 3 {
		t.Fatalf("expected 3 candidates, got %d", len(candidates))
	}

	byID := map[string]*candidateNode{}
	for i := range candidates {
		byID[candidates[i].providerID] = &candidates[i]
	}

	// prov-high-prio: fixture Priority=3 → candidate.priority must be 3.
	hp := byID["prov-high-prio"]
	if hp == nil {
		t.Fatal("prov-high-prio not in candidates")
	}
	if hp.priority != 3 {
		t.Errorf("prov-high-prio: got priority %d, want 3", hp.priority)
	}

	// prov-low-prio: fixture Priority=10 → candidate.priority must be 10.
	lp := byID["prov-low-prio"]
	if lp == nil {
		t.Fatal("prov-low-prio not in candidates")
	}
	if lp.priority != 10 {
		t.Errorf("prov-low-prio: got priority %d, want 10", lp.priority)
	}

	// prov-no-priority: Priority omitted → defaults to 0.
	np := byID["prov-no-priority"]
	if np == nil {
		t.Fatal("prov-no-priority not in candidates")
	}
	if np.priority != 0 {
		t.Errorf("prov-no-priority: got priority %d, want 0", np.priority)
	}
}

// TestGetSnapshotForNodeLegacyAdapterFallback verifies that a node with no
// providers[] catalog returns adapter snapshots (CLI, OllamaInstances, etc.)
// via the legacy adapter fallback path.
func TestGetSnapshotForNodeLegacyAdapterFallback(t *testing.T) {
	rec := &edgenode.NodeRecord{
		ID: "node-legacy",
		Adapters: config.AdaptersConf{
			CLI: config.CLIConf{Enabled: true},
			OllamaInstances: []config.OllamaInstanceConf{
				{Name: "ollama-local", Enabled: true, Capacity: 3},
			},
		},
		Runtime: config.RuntimeConf{Concurrency: 2},
	}
	store := edgenode.NewNodeStore()
	store.Add(rec)

	m := newModelQueueManager(store)

	snaps := m.getSnapshotForNode("node-legacy", rec)

	// Must return adapter snapshots: CLI + ollama-local = 2 entries.
	if len(snaps) != 2 {
		t.Fatalf("expected 2 adapter snapshots (cli + ollama-local), got %d: %+v", len(snaps), snaps)
	}
	adapters := map[string]bool{}
	for _, s := range snaps {
		adapters[s.Adapter] = true
	}
	if !adapters["cli"] {
		t.Error("expected cli adapter snapshot in legacy fallback")
	}
	if !adapters["ollama-local"] {
		t.Error("expected ollama-local adapter snapshot in legacy fallback")
	}
}

func TestResolveProviderPoolCandidatesClassifiesExecutionPath(t *testing.T) {
	catalog := []config.ModelCatalogEntry{
		{
			ID: "test-model",
			Providers: map[string]string{
				"prov-openai-compat":     "served-model",
				"prov-openai-api":        "served-model",
				"prov-vllm":              "served-model",
				"prov-vllm-mlx":          "served-model",
				"prov-lemonade":          "served-model",
				"prov-sglang":            "served-model",
				"prov-seulgivibe-claude": "served-model",
				"prov-seulgivibe-openai": "served-model",
				"prov-ollama":            "served-model",
				"prov-cli":               "served-model",
				"prov-unknown":           "served-model",
			},
		},
	}

	store := edgenode.NewNodeStore()

	// OpenAI-compatible providers (all should be tunnel path).
	store.Add(&edgenode.NodeRecord{
		ID:      "node-openai",
		Runtime: config.RuntimeConf{Concurrency: 4},
		Providers: []config.NodeProviderConf{
			{ID: "prov-openai-compat", Type: "openai_compat", Category: "api", Health: "available", Capacity: 2, Models: []string{"served-model"}},
			{ID: "prov-openai-api", Type: "openai_api", Category: "api", Health: "available", Capacity: 2, Models: []string{"served-model"}},
			{ID: "prov-vllm", Type: "vllm", Category: "local_inference", Health: "available", Capacity: 2, Models: []string{"served-model"}},
			{ID: "prov-vllm-mlx", Type: "vllm-mlx", Category: "local_inference", Health: "available", Capacity: 2, Models: []string{"served-model"}},
			{ID: "prov-lemonade", Type: "lemonade", Category: "local_inference", Health: "available", Capacity: 2, Models: []string{"served-model"}},
			{ID: "prov-sglang", Type: "sglang", Category: "local_inference", Health: "available", Capacity: 2, Models: []string{"served-model"}},
			{ID: "prov-seulgivibe-claude", Type: "seulgivibe_claude", Category: "api", Health: "available", Capacity: 2, Models: []string{"served-model"}},
			{ID: "prov-seulgivibe-openai", Type: "seulgivibe_openai", Category: "api", Health: "available", Capacity: 2, Models: []string{"served-model"}},
		},
	})

	// Ollama/CLI/unknown providers (all should be normalized path).
	store.Add(&edgenode.NodeRecord{
		ID:      "node-normalized",
		Runtime: config.RuntimeConf{Concurrency: 4},
		Adapters: config.AdaptersConf{
			OllamaInstances: []config.OllamaInstanceConf{{Name: "ollama", Enabled: true}},
			CLI:             config.CLIConf{Enabled: true, Profiles: map[string]config.CLIProfileConf{"cli-profile": {}}},
		},
		Providers: []config.NodeProviderConf{
			{ID: "prov-ollama", Type: "ollama", Category: "local_inference", Adapter: "ollama", Health: "available", Capacity: 2, Models: []string{"served-model"}},
			{ID: "prov-cli", Type: "cli", Category: "cli", Adapter: "cli", Health: "available", Capacity: 2, Models: []string{"served-model"}},
			{ID: "prov-unknown", Type: "unknown_type", Category: "api", Health: "available", Capacity: 2, Models: []string{"served-model"}},
		},
	})

	// Build a fake registry with all nodes.
	reg := edgenode.NewRegistry()
	allRecs := store.All()
	for _, rec := range allRecs {
		entry := &edgenode.NodeEntry{
			NodeID:         rec.ID,
			LifecycleState: edgenode.LifecycleConnected,
		}
		reg.Register(entry)
	}

	// Create Service with catalog and node store.
	svc := New(reg, nil)
	svc.SetNodeStore(store)
	svc.SetModelCatalog(catalog)

	req := SubmitRunRequest{ModelGroupKey: "test-model", ProviderPool: true}
	candidates, _, err := svc.resolveProviderPoolCandidates(req, store, catalog)
	if err != nil {
		t.Fatalf("resolveProviderPoolCandidates: %v", err)
	}

	expectedTunnel := map[string]bool{
		"prov-openai-compat":     true,
		"prov-openai-api":        true,
		"prov-vllm":              true,
		"prov-vllm-mlx":          true,
		"prov-lemonade":          true,
		"prov-sglang":            true,
		"prov-seulgivibe-claude": true,
		"prov-seulgivibe-openai": true,
	}
	expectedNormalized := map[string]bool{
		"prov-ollama":  true,
		"prov-cli":     true,
		"prov-unknown": true,
	}

	// Convert returned candidates to a map.
	got := map[string]providerExecutionPath{}
	for _, c := range candidates {
		got[c.providerID] = c.executionPath
	}

	// Verify tunnel-path providers are present and classified as tunnel.
	for providerID := range expectedTunnel {
		path, ok := got[providerID]
		if !ok {
			t.Errorf("expected tunnel provider %q missing from candidates (got %d total)", providerID, len(candidates))
			continue
		}
		if path != providerExecutionPathTunnel {
			t.Errorf("provider %q: expected tunnel path, got %q", providerID, path)
		}
	}

	// Verify normalized-path providers are present and classified as normalized.
	for providerID := range expectedNormalized {
		path, ok := got[providerID]
		if !ok {
			t.Errorf("expected normalized provider %q missing from candidates (got %d total)", providerID, len(candidates))
			continue
		}
		if path != providerExecutionPathNormalized {
			t.Errorf("provider %q: expected normalized path, got %q", providerID, path)
		}
	}

	// Extra sanity.
	for _, c := range candidates {
		if _, okTunnel := expectedTunnel[c.providerID]; !okTunnel {
			if _, okNorm := expectedNormalized[c.providerID]; !okNorm {
				t.Errorf("unexpected providerID %q", c.providerID)
			}
		}
	}
}

// TestResolveProviderPoolCandidatesKeepsMixedProviderTypes verifies S05.
func TestResolveProviderPoolCandidatesKeepsMixedProviderTypes(t *testing.T) {
	catalog := []config.ModelCatalogEntry{
		{
			ID: "mixed-model",
			Providers: map[string]string{
				"prov-vllm":     "served-vllm",
				"prov-vllm-mlx": "served-vllm-mlx",
				"prov-lemonade": "served-lemonade",
				"prov-ollama":   "served-ollama",
			},
		},
	}

	store := edgenode.NewNodeStore()
	store.Add(&edgenode.NodeRecord{
		ID:      "node-mixed",
		Runtime: config.RuntimeConf{Concurrency: 4},
		Adapters: config.AdaptersConf{
			VllmInstances: []config.VllmInstanceConf{
				{Name: "vllm-gpu", Enabled: true, Endpoint: "http://127.0.0.1:8000/v1"},
				{Name: "vllm-gpu-mlx", Enabled: true, Endpoint: "http://127.0.0.1:8001/v1"},
			},
			OllamaInstances: []config.OllamaInstanceConf{
				{Name: "ollama", Enabled: true, BaseURL: "http://127.0.0.1:11434"},
			},
			OpenAICompatInstances: []config.OpenAICompatInstanceConf{
				{Name: "oaicompat", Enabled: true, Endpoint: "http://127.0.0.1:9000/v1"},
			},
		},
		Providers: []config.NodeProviderConf{
			{
				ID:       "prov-vllm",
				Type:     "vllm",
				Adapter:  "vllm-gpu",
				Models:   []string{"served-vllm"},
				Health:   "available",
				Capacity: 2,
			},
			{
				ID:       "prov-vllm-mlx",
				Type:     "vllm-mlx",
				Adapter:  "vllm-gpu-mlx",
				Models:   []string{"served-vllm-mlx"},
				Health:   "available",
				Capacity: 2,
			},
			{
				ID:       "prov-lemonade",
				Type:     "lemonade",
				Adapter:  "oaicompat",
				Models:   []string{"served-lemonade"},
				Health:   "available",
				Capacity: 2,
			},
			{
				ID:       "prov-ollama",
				Type:     "ollama",
				Adapter:  "ollama",
				Models:   []string{"served-ollama"},
				Health:   "available",
				Capacity: 2,
			},
		},
	})

	reg := edgenode.NewRegistry()
	for _, rec := range store.All() {
		reg.Register(&edgenode.NodeEntry{NodeID: rec.ID, LifecycleState: edgenode.LifecycleConnected})
	}

	svc := New(reg, nil)
	svc.SetNodeStore(store)
	svc.SetModelCatalog(catalog)

	req := SubmitRunRequest{ModelGroupKey: "mixed-model", ProviderPool: true}
	candidates, _, err := svc.resolveProviderPoolCandidates(req, store, catalog)
	if err != nil {
		t.Fatalf("resolveProviderPoolCandidates: %v", err)
	}

	// All 4 providers must be candidates.
	if len(candidates) != 4 {
		ids := make([]string, len(candidates))
		for i, c := range candidates {
			ids[i] = c.providerID
		}
		t.Fatalf("expected 4 candidates (prov-vllm, prov-vllm-mlx, prov-lemonade, prov-ollama), got %d: %v", len(candidates), ids)
	}

	byID := map[string]*candidateNode{}
	for i := range candidates {
		byID[candidates[i].providerID] = &candidates[i]
	}

	// OpenAI-compatible providers (vLLM, vLLM-MLX, lemonade) must be tunnel path.
	for _, provID := range []string{"prov-vllm", "prov-vllm-mlx", "prov-lemonade"} {
		c, ok := byID[provID]
		if !ok {
			t.Errorf("%q not in candidates", provID)
			continue
		}
		if c.executionPath != providerExecutionPathTunnel {
			t.Errorf("%q: expected tunnel path, got %q", provID, c.executionPath)
		}
		// Served target must match the catalog entry.
		expectedServed := catalog[0].Providers[provID]
		if c.servedTarget != expectedServed {
			t.Errorf("%q servedTarget: got %q, want %q", provID, c.servedTarget, expectedServed)
		}
	}

	// Ollama provider must be normalized path.
	ollama, ok := byID["prov-ollama"]
	if !ok {
		t.Fatal("prov-ollama not in candidates")
	}
	if ollama.executionPath != providerExecutionPathNormalized {
		t.Errorf("prov-ollama: expected normalized path, got %q", ollama.executionPath)
	}
	if ollama.servedTarget != "served-ollama" {
		t.Errorf("prov-ollama servedTarget: got %q, want %q", ollama.servedTarget, "served-ollama")
	}
}