// Package openai_compat provides an Adapter for OpenAI-compatible inference // servers such as Lemonade Server via the /v1/models and /v1/chat/completions // endpoints. Unlike the vllm adapter it carries an explicit provider label, // per-request auth/header injection and top-level option passthrough so a // provider boundary and its config contract stay distinct. package openai_compat import ( "bufio" "bytes" "context" "encoding/json" "fmt" "io" "net/http" "net/url" "strings" "time" "go.uber.org/zap" "iop/apps/node/internal/runtime" "iop/packages/go/config" ) const Name = "openai_compat" type Adapter struct { instanceName string provider string endpoint string headers map[string]string capacity int maxQueue int queueTimeoutMS int requestTimeoutMS int client *http.Client logger *zap.Logger } // New creates an OpenAI-compatible adapter. The optional instanceName is the // registry instance key used to disambiguate multiple instances on a node. func New(cfg config.OpenAICompatConf, logger *zap.Logger, instanceName ...string) *Adapter { endpoint := strings.TrimRight(cfg.Endpoint, "/") name := Name if len(instanceName) > 0 && instanceName[0] != "" { name = instanceName[0] } var headers map[string]string if len(cfg.Headers) > 0 { headers = make(map[string]string, len(cfg.Headers)) for k, v := range cfg.Headers { headers[k] = v } } return &Adapter{ instanceName: name, provider: cfg.Provider, endpoint: endpoint, headers: headers, capacity: cfg.Capacity, maxQueue: cfg.MaxQueue, queueTimeoutMS: cfg.QueueTimeoutMS, requestTimeoutMS: cfg.RequestTimeoutMS, client: &http.Client{}, logger: logger, } } func (a *Adapter) Name() string { return Name } func (a *Adapter) Capabilities(ctx context.Context) (runtime.Capabilities, error) { probeCtx, cancel := context.WithTimeout(ctx, 2*time.Second) defer cancel() targets, err := a.fetchTargets(probeCtx) status := runtime.ProviderStatusAvailable if err != nil { status = runtime.ProviderStatusUnavailable } return runtime.Capabilities{ AdapterName: Name, InstanceKey: a.instanceName, Targets: targets, MaxConcurrency: effectiveCapacity(a.capacity, 8), MaxQueue: a.maxQueue, QueueTimeoutMS: a.queueTimeoutMS, RequestTimeoutMS: a.requestTimeoutMS, ProviderStatus: runtime.NormalizeProviderStatus(status), }, nil } func effectiveCapacity(capacity, defaultVal int) int { if capacity <= 0 { return defaultVal } return capacity } func (a *Adapter) Execute(ctx context.Context, spec runtime.ExecutionSpec, sink runtime.EventSink) error { if a.endpoint == "" { return fmt.Errorf("openai_compat adapter: endpoint is required") } model := strings.TrimSpace(spec.Target) if model == "" { model = stringInput(spec.Input, "model") } if model == "" { return fmt.Errorf("openai_compat adapter: target/model is required") } messages := messagesFromInput(spec.Input) if len(messages) == 0 { return fmt.Errorf("openai_compat adapter: messages are required") } if err := sink.Emit(ctx, runtime.RuntimeEvent{ RunID: spec.RunID, Type: runtime.EventTypeStart, Timestamp: time.Now(), }); err != nil { return err } reqBody := buildRequestBody(model, messages, spec.Input) body, err := json.Marshal(reqBody) if err != nil { return fmt.Errorf("openai_compat adapter: marshal request: %w", err) } req, err := http.NewRequestWithContext(ctx, http.MethodPost, joinOpenAIPath(a.endpoint, "/v1/chat/completions"), bytes.NewReader(body)) if err != nil { return fmt.Errorf("openai_compat adapter: build request: %w", err) } a.applyHeaders(req, true) a.logger.Info("openai_compat adapter executing", zap.String("run_id", spec.RunID), zap.String("provider", a.provider), zap.String("target", model), zap.String("endpoint", a.endpoint), ) resp, err := a.client.Do(req) if err != nil { _ = emitError(ctx, sink, spec.RunID, fmt.Sprintf("openai_compat request failed: %v", err)) return fmt.Errorf("openai_compat adapter: request: %w", err) } defer resp.Body.Close() if resp.StatusCode < 200 || resp.StatusCode >= 300 { msg := readLimited(resp.Body, 4096) if msg == "" { msg = resp.Status } _ = emitError(ctx, sink, spec.RunID, fmt.Sprintf("openai_compat returned %s: %s", resp.Status, msg)) return fmt.Errorf("openai_compat adapter: non-2xx response: %s", resp.Status) } scanner := bufio.NewScanner(resp.Body) outputTokens := 0 finishReason := "" var usage *runtime.UsageStats for scanner.Scan() { line := scanner.Text() if !strings.HasPrefix(line, "data: ") { continue } payload := strings.TrimPrefix(line, "data: ") if payload == "[DONE]" { return sink.Emit(ctx, completeEvent(spec.RunID, finishReason, usage, outputTokens)) } var chunk chatChunk if err := json.Unmarshal([]byte(payload), &chunk); err != nil { continue } if chunk.Usage != nil { usage = &runtime.UsageStats{ InputTokens: chunk.Usage.PromptTokens, OutputTokens: chunk.Usage.CompletionTokens, } } if len(chunk.Choices) == 0 { continue } choice := chunk.Choices[0] if choice.FinishReason != nil && *choice.FinishReason != "" { finishReason = *choice.FinishReason } if reasoning := choice.Delta.ReasoningContent; reasoning != "" { if err := sink.Emit(ctx, runtime.RuntimeEvent{ RunID: spec.RunID, Type: runtime.EventTypeReasoningDelta, Delta: reasoning, Timestamp: time.Now(), }); err != nil { return err } } if content := choice.Delta.Content; content != "" { outputTokens += len(strings.Fields(content)) if err := sink.Emit(ctx, runtime.RuntimeEvent{ RunID: spec.RunID, Type: runtime.EventTypeDelta, Delta: content, Timestamp: time.Now(), }); err != nil { return err } } } if err := scanner.Err(); err != nil { _ = emitError(ctx, sink, spec.RunID, fmt.Sprintf("openai_compat stream scan failed: %v", err)) return fmt.Errorf("openai_compat adapter: scan stream: %w", err) } _ = emitError(ctx, sink, spec.RunID, "openai_compat stream ended without [DONE]") return fmt.Errorf("openai_compat adapter: stream ended without [DONE]") } // ProbeProvider checks endpoint availability and target presence using the // OpenAI-compatible /v1/models endpoint, returning only the public baseline // status values. func (a *Adapter) ProbeProvider(ctx context.Context, target string) (runtime.ProviderProbeResult, error) { probeCtx, cancel := context.WithTimeout(ctx, 2*time.Second) defer cancel() targets, err := a.fetchTargets(probeCtx) result := runtime.ProviderProbeResult{ AdapterName: Name, InstanceKey: a.instanceName, Target: target, } if a.provider != "" { result.Metadata = map[string]string{"provider": a.provider} } if err != nil { result.Status = runtime.NormalizeProviderStatus(runtime.ProviderStatusUnavailable) result.Detail = err.Error() return result, nil } result.Targets = targets if target == "" { result.Status = runtime.NormalizeProviderStatus(runtime.ProviderStatusAvailable) return result, nil } found := false for _, t := range targets { if t == target { found = true break } } if found { result.Status = runtime.NormalizeProviderStatus(runtime.ProviderStatusAvailable) } else { result.Status = runtime.NormalizeProviderStatus(runtime.ProviderStatusUnavailable) result.Detail = fmt.Sprintf("target model %q not found in provider models", target) } return result, nil } func (a *Adapter) fetchTargets(ctx context.Context) ([]string, error) { if a.endpoint == "" { return nil, fmt.Errorf("openai_compat adapter: endpoint is required") } req, err := http.NewRequestWithContext(ctx, http.MethodGet, joinOpenAIPath(a.endpoint, "/v1/models"), nil) if err != nil { return nil, fmt.Errorf("build request: %w", err) } a.applyHeaders(req, false) resp, err := a.client.Do(req) if err != nil { return nil, fmt.Errorf("request failed: %w", err) } defer resp.Body.Close() if resp.StatusCode < 200 || resp.StatusCode >= 300 { return nil, fmt.Errorf("status code %d", resp.StatusCode) } var models modelsResponse if err := json.NewDecoder(resp.Body).Decode(&models); err != nil { return nil, fmt.Errorf("decode response: %w", err) } out := make([]string, 0, len(models.Data)) for _, model := range models.Data { if model.ID != "" { out = append(out, model.ID) } } return out, nil } // applyHeaders injects configured auth/secret headers. The adapter always owns // Content-Type for JSON POSTs, so a user-provided Content-Type is ignored to // keep the request body contract intact. func (a *Adapter) applyHeaders(req *http.Request, jsonBody bool) { for k, v := range a.headers { if jsonBody && http.CanonicalHeaderKey(k) == "Content-Type" { continue } req.Header.Set(k, v) } if jsonBody { req.Header.Set("Content-Type", "application/json") } } func buildRequestBody(model string, messages []chatMessage, input map[string]any) map[string]any { body := make(map[string]any) // Copy caller options as top-level OpenAI-compatible request fields first so // adapter-owned fields below always win over them. if opts, ok := input["options"].(map[string]any); ok { for k, v := range opts { body[k] = v } } // Pass through optional OpenAI-compatible fields when present in the input. for _, key := range []string{"tools", "format", "think", "keep_alive"} { if v, ok := input[key]; ok { body[key] = v } } body["model"] = model body["messages"] = messages body["stream"] = true return body } func completeEvent(runID, finishReason string, usage *runtime.UsageStats, outputTokens int) runtime.RuntimeEvent { if usage == nil { usage = &runtime.UsageStats{OutputTokens: outputTokens} } var metadata map[string]string if finishReason != "" { metadata = map[string]string{"finish_reason": finishReason} } return runtime.RuntimeEvent{ RunID: runID, Type: runtime.EventTypeComplete, Message: "openai_compat chat complete", Usage: usage, Metadata: metadata, Timestamp: time.Now(), } } func messagesFromInput(input map[string]any) []chatMessage { if raw, ok := input["messages"].([]any); ok { out := make([]chatMessage, 0, len(raw)) for _, item := range raw { m, ok := item.(map[string]any) if !ok { continue } role, _ := m["role"].(string) content, _ := m["content"].(string) role = strings.TrimSpace(role) content = strings.TrimSpace(content) if role == "" || content == "" { continue } out = append(out, chatMessage{Role: role, Content: content}) } if len(out) > 0 { return out } } if prompt := strings.TrimSpace(stringInput(input, "prompt")); prompt != "" { return []chatMessage{{Role: "user", Content: prompt}} } return nil } func emitError(ctx context.Context, sink runtime.EventSink, runID, msg string) error { return sink.Emit(ctx, runtime.RuntimeEvent{ RunID: runID, Type: runtime.EventTypeError, Error: msg, Timestamp: time.Now(), }) } func stringInput(input map[string]any, key string) string { if input == nil { return "" } if v, ok := input[key].(string); ok { return v } return "" } // joinOpenAIPath appends an OpenAI-compatible path to the endpoint. When the // endpoint already targets the /v1 root it strips the leading /v1 from the path // so a configured ".../v1" endpoint does not produce ".../v1/v1/...". func joinOpenAIPath(baseURL, path string) string { u, err := url.Parse(baseURL) if err != nil { return strings.TrimRight(baseURL, "/") + path } basePath := strings.TrimRight(u.Path, "/") if strings.HasSuffix(basePath, "/v1") && strings.HasPrefix(path, "/v1/") { path = strings.TrimPrefix(path, "/v1") } u.Path = basePath + path return u.String() } func readLimited(r io.Reader, limit int64) string { b, _ := io.ReadAll(io.LimitReader(r, limit)) return strings.TrimSpace(string(b)) } type chatMessage struct { Role string `json:"role"` Content string `json:"content"` } type chatChunk struct { Choices []struct { Delta struct { Content string `json:"content"` ReasoningContent string `json:"reasoning_content"` } `json:"delta"` FinishReason *string `json:"finish_reason"` } `json:"choices"` Usage *struct { PromptTokens int `json:"prompt_tokens"` CompletionTokens int `json:"completion_tokens"` } `json:"usage"` } type modelsResponse struct { Data []struct { ID string `json:"id"` } `json:"data"` }