package service import ( "context" "fmt" "sync" "time" "google.golang.org/protobuf/types/known/structpb" edgenode "iop/apps/edge/internal/node" eventpkg "iop/packages/go/events" iop "iop/proto/gen/iop" ) type SubmitRunRequest struct { NodeRef string RunID string ModelGroupKey string Adapter string Target string SessionID string Workspace string Prompt string Input map[string]any Background bool TimeoutSec int Metadata map[string]string } // RunDispatch describes a dispatched run in surface-neutral terms. It is the // metadata any caller (console, HTTP, future RPC) needs after submission. type RunDispatch struct { RunID string NodeID string NodeLabel string ModelGroupKey string Adapter string Target string SessionID string Background bool TimeoutSec int } // RunStream carries asynchronous events for a dispatched foreground run. // Background runs leave both channels nil. type RunStream struct { Events <-chan *iop.RunEvent NodeEvents <-chan *iop.EdgeNodeEvent } // RunResult describes the interface for a submitted execution run, allowing // surface-neutral consumption of its dispatch info and event stream. type RunResult interface { Dispatch() RunDispatch Stream() RunStream Close() WaitTimeout() time.Duration } // RunHandle is the legacy combined surface kept for existing callers; it // embeds the surface-neutral DTOs so HTTP/API code can consume RunDispatch // or RunStream directly without depending on this struct. type RunHandle struct { RunDispatch RunStream closeOnce sync.Once close func() } func (h *RunHandle) Close() { if h != nil && h.close != nil { h.closeOnce.Do(h.close) } } func (h *RunHandle) WaitTimeout() time.Duration { if h == nil { return time.Duration(DefaultTimeoutSec+5) * time.Second } return time.Duration(normalizeTimeoutSec(h.TimeoutSec)+5) * time.Second } func (h *RunHandle) Dispatch() RunDispatch { if h == nil { return RunDispatch{} } return h.RunDispatch } func (h *RunHandle) Stream() RunStream { if h == nil { return RunStream{} } return h.RunStream } func (s *Service) SubmitRun(ctx context.Context, req SubmitRunRequest) (RunResult, error) { if req.ModelGroupKey != "" && s.queue != nil { return s.submitRunQueued(ctx, req) } return s.submitRunDirect(req) } func (s *Service) submitRunDirect(req SubmitRunRequest) (RunResult, error) { entry, err := s.ResolveNode(req.NodeRef) if err != nil { return nil, err } return s.dispatchToEntry(entry, req) } func (s *Service) submitRunQueued(ctx context.Context, req SubmitRunRequest) (RunResult, error) { candidates, policy, err := s.resolveQueueCandidates(req) if err != nil { return nil, err } entry, err := s.queue.admit(ctx, req.ModelGroupKey, req.Adapter, req.Target, candidates, policy) if err != nil { return nil, err } runReq, runID, err := BuildRunRequest(req) if err != nil { s.queue.releaseSlot(req.ModelGroupKey, entry.NodeID) return nil, err } // Track inflight before send so the event watcher can release the slot // even if a terminal event arrives before the Send call completes. s.queue.trackInflight(req.ModelGroupKey, runID, entry.NodeID) var runEvents <-chan *iop.RunEvent var unregisterRun func() var nodeEvents <-chan *iop.EdgeNodeEvent var unregisterNode func() if !runReq.GetBackground() { if s.events == nil { s.queue.releaseRun(runID, "no-event-bus") return nil, fmt.Errorf("event bus is not configured") } runEvents, unregisterRun = s.events.SubscribeRun(runID, 4096) nodeEvents, unregisterNode = s.events.SubscribeNode(entry.NodeID, 16) } if err := entry.Client.Send(runReq); err != nil { s.queue.releaseRun(runID, "send-error") if unregisterRun != nil { unregisterRun() } if unregisterNode != nil { unregisterNode() } return nil, err } return &RunHandle{ RunDispatch: RunDispatch{ RunID: runID, NodeID: entry.NodeID, NodeLabel: nodeLabel(entry), ModelGroupKey: req.ModelGroupKey, Adapter: runReq.GetAdapter(), Target: runReq.GetTarget(), SessionID: runReq.GetSessionId(), Background: runReq.GetBackground(), TimeoutSec: int(runReq.GetTimeoutSec()), }, RunStream: RunStream{ Events: runEvents, NodeEvents: nodeEvents, }, close: func() { if unregisterRun != nil { unregisterRun() } if unregisterNode != nil { unregisterNode() } }, }, nil } // resolveQueueCandidates returns candidate nodes filtered by adapter/target capability, // each paired with per-node capacity, plus the group policy derived from adapter config. func (s *Service) resolveQueueCandidates(req SubmitRunRequest) ([]candidateNode, groupPolicy, error) { if req.NodeRef != "" { entry, err := s.ResolveNode(req.NodeRef) if err != nil { return nil, groupPolicy{}, err } cap := defaultNodeCapacity if s.nodeStore != nil { if rec, ok := s.nodeStore.FindByID(entry.NodeID); ok { res := resolveAdapterForNode(rec, req.Adapter, req.Target) if !res.supported { msg := fmt.Sprintf("node %q does not support adapter %q target %q", entry.NodeID, req.Adapter, req.Target) if res.ambiguous { msg = fmt.Sprintf("node %q: adapter %q is ambiguous (multiple enabled instances); use an instance key", entry.NodeID, req.Adapter) } return nil, groupPolicy{}, fmt.Errorf("%s", msg) } cap = res.capacity } } policy := groupPolicyFromStore(s.nodeStore, []*edgenode.NodeEntry{entry}, req.Adapter, req.Target) return []candidateNode{{entry: entry, capacity: cap}}, policy, nil } all := s.registry.All() if len(all) == 0 { return nil, groupPolicy{}, fmt.Errorf("no nodes connected") } var candidates []candidateNode for _, entry := range all { cap := defaultNodeCapacity if s.nodeStore != nil { rec, ok := s.nodeStore.FindByID(entry.NodeID) if ok { res := resolveAdapterForNode(rec, req.Adapter, req.Target) if !res.supported { continue } cap = res.capacity } } candidates = append(candidates, candidateNode{entry: entry, capacity: cap}) } if len(candidates) == 0 { return nil, groupPolicy{}, fmt.Errorf("no nodes support adapter %q target %q", req.Adapter, req.Target) } entries := make([]*edgenode.NodeEntry, len(candidates)) for i, c := range candidates { entries[i] = c.entry } policy := groupPolicyFromStore(s.nodeStore, entries, req.Adapter, req.Target) return candidates, policy, nil } // adapterResolution holds the resolved capacity and queue policy for a single // node/adapter combination. ambiguous is true when a type-name lookup (e.g., // "ollama") matches 2+ enabled instances on that node, mirroring the Node // router's exact-instance-key/ambiguity contract. type adapterResolution struct { supported bool ambiguous bool capacity int maxQueue int queueTimeoutMS int } func positiveOr(v, fallback int) int { if v > 0 { return v } return fallback } // resolveAdapterForNode determines whether a node can handle adapterType/target // and computes the per-node capacity and queue policy fields. // // Resolution order: // 1. Exact instance Name match across OllamaInstances / VllmInstances / // OpenAICompatInstances (instance-key route, matching Node router priority). // 2. Type-name route (e.g. "ollama"): supported only when exactly 1 enabled // instance of that type exists. 2+ enabled instances → ambiguous (fail, // same semantics as Node router ambiguity error). 0 instances → fail-open // for legacy/unconfigured nodes. // 3. "cli": capability gated by CLI.Enabled and profile name in target. // 4. Default (unknown adapter type): fail-open. func resolveAdapterForNode(rec *edgenode.NodeRecord, adapterType, target string) adapterResolution { if rec == nil { return adapterResolution{supported: true, capacity: defaultNodeCapacity} } concurrencyFallback := positiveOr(rec.Runtime.Concurrency, defaultNodeCapacity) // Exact instance Name match (highest priority). for _, inst := range rec.Adapters.OllamaInstances { if inst.Name == adapterType { return adapterResolution{ supported: inst.Enabled, capacity: positiveOr(inst.Capacity, concurrencyFallback), maxQueue: inst.MaxQueue, queueTimeoutMS: inst.QueueTimeoutMS, } } } for _, inst := range rec.Adapters.VllmInstances { if inst.Name == adapterType { return adapterResolution{ supported: inst.Enabled, capacity: positiveOr(inst.Capacity, concurrencyFallback), maxQueue: inst.MaxQueue, queueTimeoutMS: inst.QueueTimeoutMS, } } } for _, inst := range rec.Adapters.OpenAICompatInstances { if inst.Name == adapterType { return adapterResolution{ supported: inst.Enabled, capacity: positiveOr(inst.Capacity, concurrencyFallback), maxQueue: inst.MaxQueue, queueTimeoutMS: inst.QueueTimeoutMS, } } } // Type-name route. switch adapterType { case "ollama": return resolveTypeRoute(rec, ollamaEnabledInstances(rec), concurrencyFallback) case "vllm": return resolveTypeRoute(rec, vllmEnabledInstances(rec), concurrencyFallback) case "openai_compat": return resolveTypeRoute(rec, openAICompatEnabledInstances(rec), concurrencyFallback) case "cli": // CLI has no named multi-instance model; capability is gated by profile. if !rec.Adapters.CLI.Enabled && len(rec.Adapters.CLI.Profiles) == 0 { // No CLI config at all → fail-open for legacy/unconfigured nodes. return adapterResolution{supported: true, capacity: concurrencyFallback} } if !rec.Adapters.CLI.Enabled { return adapterResolution{supported: false} } if target == "" { return adapterResolution{supported: len(rec.Adapters.CLI.Profiles) > 0, capacity: concurrencyFallback} } _, ok := rec.Adapters.CLI.Profiles[target] return adapterResolution{supported: ok, capacity: concurrencyFallback} default: return adapterResolution{supported: true, capacity: concurrencyFallback} } } // instanceFields holds the capacity/policy fields extracted from a single adapter instance. type instanceFields struct { capacity, maxQueue, queueTimeoutMS int } func ollamaEnabledInstances(rec *edgenode.NodeRecord) []instanceFields { var out []instanceFields for _, inst := range rec.Adapters.OllamaInstances { if inst.Enabled { out = append(out, instanceFields{inst.Capacity, inst.MaxQueue, inst.QueueTimeoutMS}) } } return out } func vllmEnabledInstances(rec *edgenode.NodeRecord) []instanceFields { var out []instanceFields for _, inst := range rec.Adapters.VllmInstances { if inst.Enabled { out = append(out, instanceFields{inst.Capacity, inst.MaxQueue, inst.QueueTimeoutMS}) } } return out } func openAICompatEnabledInstances(rec *edgenode.NodeRecord) []instanceFields { var out []instanceFields for _, inst := range rec.Adapters.OpenAICompatInstances { if inst.Enabled { out = append(out, instanceFields{inst.Capacity, inst.MaxQueue, inst.QueueTimeoutMS}) } } return out } // resolveTypeRoute applies Node-router-compatible type-name resolution: // - 0 enabled instances → fail-open (legacy / unconfigured) // - 1 enabled instance → use its capacity and policy // - 2+ enabled instances → ambiguous (reject) func resolveTypeRoute(rec *edgenode.NodeRecord, enabled []instanceFields, concurrencyFallback int) adapterResolution { switch len(enabled) { case 0: return adapterResolution{supported: true, capacity: concurrencyFallback} case 1: f := enabled[0] return adapterResolution{ supported: true, capacity: positiveOr(f.capacity, concurrencyFallback), maxQueue: f.maxQueue, queueTimeoutMS: f.queueTimeoutMS, } default: return adapterResolution{supported: false, ambiguous: true} } } // groupPolicyFromStore derives queue policy from the first resolved candidate node. func groupPolicyFromStore(store *edgenode.NodeStore, entries []*edgenode.NodeEntry, adapterType, target string) groupPolicy { if store != nil { for _, e := range entries { rec, ok := store.FindByID(e.NodeID) if !ok { continue } res := resolveAdapterForNode(rec, adapterType, target) if !res.supported { continue } if res.maxQueue > 0 || res.queueTimeoutMS > 0 { p := groupPolicy{ maxQueue: positiveOr(res.maxQueue, defaultGroupMaxQueue), queueTimeout: time.Duration(positiveOr(res.queueTimeoutMS, 0)) * time.Millisecond, } if p.queueTimeout <= 0 { p.queueTimeout = defaultQueueTimeout } return p } } } return groupPolicy{maxQueue: defaultGroupMaxQueue, queueTimeout: defaultQueueTimeout} } func (s *Service) dispatchToEntry(entry *edgenode.NodeEntry, req SubmitRunRequest) (RunResult, error) { runReq, runID, err := BuildRunRequest(req) if err != nil { return nil, err } var runEvents <-chan *iop.RunEvent var unregisterRun func() var nodeEvents <-chan *iop.EdgeNodeEvent var unregisterNode func() if !runReq.GetBackground() { if s.events == nil { return nil, fmt.Errorf("event bus is not configured") } runEvents, unregisterRun = s.events.SubscribeRun(runID, 4096) nodeEvents, unregisterNode = s.events.SubscribeNode(entry.NodeID, 16) } if err := entry.Client.Send(runReq); err != nil { if unregisterRun != nil { unregisterRun() } if unregisterNode != nil { unregisterNode() } return nil, err } return &RunHandle{ RunDispatch: RunDispatch{ RunID: runID, NodeID: entry.NodeID, NodeLabel: nodeLabel(entry), ModelGroupKey: req.ModelGroupKey, Adapter: runReq.GetAdapter(), Target: runReq.GetTarget(), SessionID: runReq.GetSessionId(), Background: runReq.GetBackground(), TimeoutSec: int(runReq.GetTimeoutSec()), }, RunStream: RunStream{ Events: runEvents, NodeEvents: nodeEvents, }, close: func() { if unregisterRun != nil { unregisterRun() } if unregisterNode != nil { unregisterNode() } }, }, nil } type CancelRunRequest struct { NodeRef string RunID string Adapter string Target string SessionID string } func BuildCancelRunRequest(req CancelRunRequest) *iop.CancelRequest { return &iop.CancelRequest{ RunId: req.RunID, Adapter: req.Adapter, Target: req.Target, SessionId: NormalizeSessionID(req.SessionID), Action: iop.CancelAction_CANCEL_ACTION_CANCEL_RUN, } } func (s *Service) CancelRun(_ context.Context, req CancelRunRequest) (CommandResult, error) { entry, err := s.ResolveNode(req.NodeRef) if err != nil { return CommandResult{}, err } cancelReq := BuildCancelRunRequest(req) if err := entry.Client.Send(cancelReq); err != nil { return CommandResult{}, err } return CommandResult{ NodeID: entry.NodeID, NodeLabel: nodeLabel(entry), SessionID: cancelReq.GetSessionId(), }, nil } type TerminateSessionRequest struct { NodeRef string Adapter string Target string SessionID string } // CommandResult is the surface-neutral acknowledgement for one-shot node // commands (terminate-session, future control RPCs). type CommandResult struct { NodeID string NodeLabel string SessionID string } // TerminateSessionResult is retained as an alias for backward compatibility. type TerminateSessionResult = CommandResult func (s *Service) TerminateSession(_ context.Context, req TerminateSessionRequest) (TerminateSessionResult, error) { entry, err := s.ResolveNode(req.NodeRef) if err != nil { return TerminateSessionResult{}, err } sessionID := NormalizeSessionID(req.SessionID) cancelReq := &iop.CancelRequest{ Adapter: req.Adapter, Target: req.Target, SessionId: sessionID, Action: iop.CancelAction_CANCEL_ACTION_TERMINATE_SESSION, } if err := entry.Client.Send(cancelReq); err != nil { return TerminateSessionResult{}, err } return TerminateSessionResult{ NodeID: entry.NodeID, NodeLabel: nodeLabel(entry), SessionID: sessionID, }, nil } func NewRunID() string { return fmt.Sprintf("manual-%d", time.Now().UnixNano()) } func IsNodeDisconnected(event *iop.EdgeNodeEvent) bool { return event.GetType() == eventpkg.TypeNodeDisconnected } func BuildRunRequest(req SubmitRunRequest) (*iop.RunRequest, string, error) { inputMap := make(map[string]any, len(req.Input)+1) for k, v := range req.Input { inputMap[k] = v } if req.Prompt != "" { if _, ok := inputMap["prompt"]; !ok { inputMap["prompt"] = req.Prompt } } input, err := structpb.NewStruct(inputMap) if err != nil { return nil, "", err } runID := req.RunID if runID == "" { runID = NewRunID() } metadata := make(map[string]string, len(req.Metadata)) for k, v := range req.Metadata { metadata[k] = v } return &iop.RunRequest{ RunId: runID, Adapter: req.Adapter, Target: req.Target, Workspace: req.Workspace, SessionId: NormalizeSessionID(req.SessionID), SessionMode: iop.RunSessionMode_RUN_SESSION_MODE_CREATE_IF_MISSING, Background: req.Background, Input: input, TimeoutSec: int32(normalizeTimeoutSec(req.TimeoutSec)), Metadata: metadata, }, runID, nil }