// Package openai_compat provides an Adapter for OpenAI-compatible inference // servers such as Lemonade Server via the /v1/models and /v1/chat/completions // endpoints. Unlike the vllm adapter it carries an explicit provider label, // per-request auth/header injection and top-level option passthrough so a // provider boundary and its config contract stay distinct. package openai_compat import ( "bufio" "bytes" "context" "encoding/json" "fmt" "io" "net/http" "net/url" "os" "strings" "time" "go.uber.org/zap" "iop/apps/node/internal/runtime" "iop/packages/go/config" ) const Name = "openai_compat" const ( runtimeMetadataOpenAIToolCalls = "openai_tool_calls" runtimeMetadataOpenAITextToolFallback = "openai_text_tool_fallback" streamTraceMetadataKey = "iop_trace_stream" streamTraceEnvKey = "IOP_OPENAI_COMPAT_TRACE_STREAM" ) type Adapter struct { instanceName string provider string endpoint string headers map[string]string capacity int maxQueue int queueTimeoutMS int requestTimeoutMS int client *http.Client logger *zap.Logger } // New creates an OpenAI-compatible adapter. The optional instanceName is the // registry instance key used to disambiguate multiple instances on a node. func New(cfg config.OpenAICompatConf, logger *zap.Logger, instanceName ...string) *Adapter { endpoint := strings.TrimRight(cfg.Endpoint, "/") name := Name if len(instanceName) > 0 && instanceName[0] != "" { name = instanceName[0] } var headers map[string]string if len(cfg.Headers) > 0 { headers = make(map[string]string, len(cfg.Headers)) for k, v := range cfg.Headers { headers[k] = v } } return &Adapter{ instanceName: name, provider: cfg.Provider, endpoint: endpoint, headers: headers, capacity: cfg.Capacity, maxQueue: cfg.MaxQueue, queueTimeoutMS: cfg.QueueTimeoutMS, requestTimeoutMS: cfg.RequestTimeoutMS, client: &http.Client{}, logger: logger, } } func (a *Adapter) Name() string { return Name } func (a *Adapter) TunnelProvider(ctx context.Context, req runtime.ProviderTunnelRequest, sink runtime.ProviderTunnelSink) error { var seq int64 = 0 if a.endpoint == "" { err := fmt.Errorf("openai_compat adapter: endpoint is required") _ = emitTunnelError(ctx, sink, req, seq, err) return err } urlStr := joinOpenAIPath(a.endpoint, req.Path) httpReq, err := http.NewRequestWithContext(ctx, req.Method, urlStr, bytes.NewReader(req.Body)) if err != nil { err = fmt.Errorf("build request: %w", err) _ = emitTunnelError(ctx, sink, req, seq, err) return err } isJSON := req.Method == http.MethodPost a.applyHeaders(httpReq, isJSON) for k, v := range req.Headers { httpReq.Header.Set(k, v) } resp, err := a.client.Do(httpReq) if err != nil { err = fmt.Errorf("request failed: %w", err) _ = emitTunnelError(ctx, sink, req, seq, err) return err } defer resp.Body.Close() respHeaders := make(map[string]string) for k, values := range resp.Header { if len(values) > 0 { respHeaders[k] = values[0] } } err = sink.EmitTunnelFrame(ctx, runtime.ProviderTunnelFrame{ RunID: req.RunID, TunnelID: req.TunnelID, Sequence: seq, Kind: runtime.ProviderTunnelFrameKindResponseStart, StatusCode: resp.StatusCode, Headers: respHeaders, Timestamp: time.Now(), }) if err != nil { return err } seq++ buf := make([]byte, 4096) for { n, readErr := resp.Body.Read(buf) if n > 0 { err = sink.EmitTunnelFrame(ctx, runtime.ProviderTunnelFrame{ RunID: req.RunID, TunnelID: req.TunnelID, Sequence: seq, Kind: runtime.ProviderTunnelFrameKindBody, Body: append([]byte(nil), buf[:n]...), Timestamp: time.Now(), }) if err != nil { return err } seq++ } if readErr != nil { if readErr == io.EOF { break } if ctx.Err() != nil { _ = emitTunnelError(ctx, sink, req, seq, ctx.Err()) return ctx.Err() } err = fmt.Errorf("read response body: %w", readErr) _ = emitTunnelError(ctx, sink, req, seq, err) return err } } err = sink.EmitTunnelFrame(ctx, runtime.ProviderTunnelFrame{ RunID: req.RunID, TunnelID: req.TunnelID, Sequence: seq, Kind: runtime.ProviderTunnelFrameKindEnd, End: true, Timestamp: time.Now(), }) return err } func emitTunnelError(ctx context.Context, sink runtime.ProviderTunnelSink, req runtime.ProviderTunnelRequest, seq int64, err error) error { return sink.EmitTunnelFrame(ctx, runtime.ProviderTunnelFrame{ RunID: req.RunID, TunnelID: req.TunnelID, Sequence: seq, Kind: runtime.ProviderTunnelFrameKindError, Error: err.Error(), Timestamp: time.Now(), }) } func (a *Adapter) Capabilities(ctx context.Context) (runtime.Capabilities, error) { probeCtx, cancel := context.WithTimeout(ctx, 2*time.Second) defer cancel() targets, err := a.fetchTargets(probeCtx) status := runtime.ProviderStatusAvailable if err != nil { status = runtime.ProviderStatusUnavailable } return runtime.Capabilities{ AdapterName: Name, InstanceKey: a.instanceName, Targets: targets, MaxConcurrency: effectiveCapacity(a.capacity, 8), MaxQueue: a.maxQueue, QueueTimeoutMS: a.queueTimeoutMS, RequestTimeoutMS: a.requestTimeoutMS, ProviderStatus: runtime.NormalizeProviderStatus(status), }, nil } func effectiveCapacity(capacity, defaultVal int) int { if capacity <= 0 { return defaultVal } return capacity } func (a *Adapter) Execute(ctx context.Context, spec runtime.ExecutionSpec, sink runtime.EventSink) error { if a.endpoint == "" { return fmt.Errorf("openai_compat adapter: endpoint is required") } model := strings.TrimSpace(spec.Target) if model == "" { model = stringInput(spec.Input, "model") } if model == "" { return fmt.Errorf("openai_compat adapter: target/model is required") } messages := messagesFromInput(spec.Input) if len(messages) == 0 { return fmt.Errorf("openai_compat adapter: messages are required") } if err := sink.Emit(ctx, runtime.RuntimeEvent{ RunID: spec.RunID, Type: runtime.EventTypeStart, Timestamp: time.Now(), }); err != nil { return err } reqBody, err := a.buildRequestBody(model, messages, spec.Input) if err != nil { _ = emitError(ctx, sink, spec.RunID, fmt.Sprintf("openai_compat build request body failed: %v", err)) return fmt.Errorf("openai_compat adapter: build request body: %w", err) } body, err := json.Marshal(reqBody) if err != nil { return fmt.Errorf("openai_compat adapter: marshal request: %w", err) } textToolFallback := false a.logger.Info("openai_compat adapter executing", zap.String("run_id", spec.RunID), zap.String("provider", a.provider), zap.String("target", model), zap.String("endpoint", a.endpoint), ) resp, err := a.doChatCompletion(ctx, body) if err != nil { _ = emitError(ctx, sink, spec.RunID, fmt.Sprintf("openai_compat request failed: %v", err)) return fmt.Errorf("openai_compat adapter: request: %w", err) } if resp.StatusCode < 200 || resp.StatusCode >= 300 { msg := readLimited(resp.Body, 4096) _ = resp.Body.Close() if retryBody, ok := retryBodyWithForcedSingleTool(reqBody, msg); ok { body, err = json.Marshal(retryBody) if err != nil { _ = emitError(ctx, sink, spec.RunID, fmt.Sprintf("openai_compat retry marshal failed: %v", err)) return fmt.Errorf("openai_compat adapter: retry marshal: %w", err) } resp, err = a.doChatCompletion(ctx, body) if err != nil { _ = emitError(ctx, sink, spec.RunID, fmt.Sprintf("openai_compat retry request failed: %v", err)) return fmt.Errorf("openai_compat adapter: retry request: %w", err) } if resp.StatusCode >= 200 && resp.StatusCode < 300 { goto streamResponse } msg = readLimited(resp.Body, 4096) _ = resp.Body.Close() } if retryBody, ok := retryBodyWithTextToolFallback(reqBody, msg); ok { body, err = json.Marshal(retryBody) if err != nil { _ = emitError(ctx, sink, spec.RunID, fmt.Sprintf("openai_compat text tool fallback marshal failed: %v", err)) return fmt.Errorf("openai_compat adapter: text tool fallback marshal: %w", err) } resp, err = a.doChatCompletion(ctx, body) if err != nil { _ = emitError(ctx, sink, spec.RunID, fmt.Sprintf("openai_compat text tool fallback request failed: %v", err)) return fmt.Errorf("openai_compat adapter: text tool fallback request: %w", err) } if resp.StatusCode >= 200 && resp.StatusCode < 300 { textToolFallback = true goto streamResponse } msg = readLimited(resp.Body, 4096) _ = resp.Body.Close() } if msg == "" { msg = resp.Status } _ = emitError(ctx, sink, spec.RunID, fmt.Sprintf("openai_compat returned %s: %s", resp.Status, msg)) return fmt.Errorf("openai_compat adapter: non-2xx response: %s", resp.Status) } streamResponse: defer resp.Body.Close() scanner := bufio.NewScanner(resp.Body) outputTokens := 0 finishReason := "" var usage *runtime.UsageStats var toolCalls openAIToolCallAccumulator traceStream := openAICompatTraceStreamEnabled(spec.Metadata) traceSeq := 0 for scanner.Scan() { line := scanner.Text() if !strings.HasPrefix(line, "data: ") { continue } payload := strings.TrimPrefix(line, "data: ") if payload == "[DONE]" { if traceStream { a.logger.Info("openai_compat provider stream done", zap.String("run_id", spec.RunID), zap.Int("seq", traceSeq+1), ) } return sink.Emit(ctx, completeEvent(spec.RunID, finishReason, usage, outputTokens, toolCalls.ToolCalls(), textToolFallback)) } traceSeq++ if traceStream { a.logger.Info("openai_compat provider stream raw chunk", zap.String("run_id", spec.RunID), zap.Int("seq", traceSeq), zap.Int("raw_len", len(payload)), zap.Bool("raw_has_open_think", hasOpenThinkTag(payload)), zap.Bool("raw_has_close_think", hasCloseThinkTag(payload)), zap.String("raw_preview", tracePreview(payload, 4000)), ) } var chunk chatChunk if err := json.Unmarshal([]byte(payload), &chunk); err != nil { if traceStream { a.logger.Warn("openai_compat provider stream raw chunk ignored", zap.String("run_id", spec.RunID), zap.Int("seq", traceSeq), zap.Error(err), ) } continue } if chunk.Usage != nil { usage = &runtime.UsageStats{ InputTokens: chunk.Usage.PromptTokens, OutputTokens: chunk.Usage.CompletionTokens, } if d := chunk.Usage.PromptTokensDetails; d != nil { usage.CachedInputTokens = d.CachedTokens } if d := chunk.Usage.CompletionTokensDetails; d != nil { usage.ReasoningTokens = d.ReasoningTokens } } if len(chunk.Choices) == 0 { continue } choice := chunk.Choices[0] if choice.FinishReason != nil && *choice.FinishReason != "" { finishReason = *choice.FinishReason } if len(choice.Delta.ToolCalls) > 0 { toolCalls.AddDelta(choice.Delta.ToolCalls) } reasoning := choice.Delta.ReasoningText() if traceStream { a.logger.Info("openai_compat provider stream parsed chunk", zap.String("run_id", spec.RunID), zap.Int("seq", traceSeq), zap.String("finish_reason", finishReason), zap.Int("tool_call_delta_count", len(choice.Delta.ToolCalls)), zap.Int("content_len", len(choice.Delta.Content)), zap.Bool("content_has_open_think", hasOpenThinkTag(choice.Delta.Content)), zap.Bool("content_has_close_think", hasCloseThinkTag(choice.Delta.Content)), zap.String("content_preview", tracePreview(choice.Delta.Content, 2000)), zap.Int("reasoning_len", len(reasoning)), zap.Bool("reasoning_has_open_think", hasOpenThinkTag(reasoning)), zap.Bool("reasoning_has_close_think", hasCloseThinkTag(reasoning)), zap.String("reasoning_preview", tracePreview(reasoning, 2000)), ) } if reasoning != "" { if err := sink.Emit(ctx, runtime.RuntimeEvent{ RunID: spec.RunID, Type: runtime.EventTypeReasoningDelta, Delta: reasoning, Timestamp: time.Now(), }); err != nil { return err } } if content := choice.Delta.Content; content != "" { outputTokens += len(strings.Fields(content)) if err := sink.Emit(ctx, runtime.RuntimeEvent{ RunID: spec.RunID, Type: runtime.EventTypeDelta, Delta: content, Timestamp: time.Now(), }); err != nil { return err } } } if err := scanner.Err(); err != nil { _ = emitError(ctx, sink, spec.RunID, fmt.Sprintf("openai_compat stream scan failed: %v", err)) return fmt.Errorf("openai_compat adapter: scan stream: %w", err) } _ = emitError(ctx, sink, spec.RunID, "openai_compat stream ended without [DONE]") return fmt.Errorf("openai_compat adapter: stream ended without [DONE]") } func openAICompatTraceStreamEnabled(metadata map[string]string) bool { if traceBool(os.Getenv(streamTraceEnvKey)) { return true } if metadata == nil { return false } return traceBool(metadata[streamTraceMetadataKey]) } func traceBool(v string) bool { switch strings.ToLower(strings.TrimSpace(v)) { case "1", "true", "yes", "on": return true default: return false } } func tracePreview(s string, max int) string { if max <= 0 || len(s) <= max { return s } return s[:max] + "...[truncated]" } func hasOpenThinkTag(s string) bool { return strings.Contains(strings.ToLower(s), "= 300 { return nil, fmt.Errorf("status code %d", resp.StatusCode) } var models modelsResponse if err := json.NewDecoder(resp.Body).Decode(&models); err != nil { return nil, fmt.Errorf("decode response: %w", err) } out := make([]string, 0, len(models.Data)) for _, model := range models.Data { if model.ID != "" { out = append(out, model.ID) } } return out, nil } // applyHeaders injects configured auth/secret headers. The adapter always owns // Content-Type for JSON POSTs, so a user-provided Content-Type is ignored to // keep the request body contract intact. func (a *Adapter) applyHeaders(req *http.Request, jsonBody bool) { for k, v := range a.headers { if jsonBody && http.CanonicalHeaderKey(k) == "Content-Type" { continue } req.Header.Set(k, v) } if jsonBody { req.Header.Set("Content-Type", "application/json") } } func (a *Adapter) doChatCompletion(ctx context.Context, body []byte) (*http.Response, error) { req, err := http.NewRequestWithContext(ctx, http.MethodPost, joinOpenAIPath(a.endpoint, "/v1/chat/completions"), bytes.NewReader(body)) if err != nil { return nil, fmt.Errorf("build request: %w", err) } a.applyHeaders(req, true) return a.client.Do(req) } func (a *Adapter) buildRequestBody(model string, messages []chatMessage, input map[string]any) (map[string]any, error) { body := make(map[string]any) // Copy caller options as top-level OpenAI-compatible request fields first so // adapter-owned fields below always win over them. if opts, ok := input["options"].(map[string]any); ok { for k, v := range opts { body[k] = v } } // Pass through optional OpenAI-compatible fields when present in the input (except think). for _, key := range []string{"tools", "tool_choice", "format", "keep_alive"} { if v, ok := input[key]; ok { body[key] = v } } body["model"] = model body["messages"] = messages body["stream"] = true // Extract think-control fields hasThink := false var thinkVal bool if v, ok := input["think"]; ok { if bv, ok := v.(bool); ok { hasThink = true thinkVal = bv } } hasReasoningEffort := false var reasoningEffortVal string if v, ok := input["reasoning_effort"]; ok { if sv, ok := v.(string); ok { hasReasoningEffort = true reasoningEffortVal = sv } } hasBudget := false var budgetVal any if v, ok := input["thinking_token_budget"]; ok { hasBudget = true budgetVal = v } // vLLM, vLLM-MLX and Lemonade all serve Qwen3-family templates that honor // chat_template_kwargs.enable_thinking / thinking_token_budget. The Lemonade // build ignores the top-level `think` field, so all three providers map // think-control into chat_template_kwargs rather than top-level fields. switch a.provider { case "vllm", "vllm-mlx", "lemonade": thinkDisableRequested := (hasThink && !thinkVal) || (hasReasoningEffort && reasoningEffortVal == "none") // The vLLM-MLX runtime labels its entire streamed output as // reasoning_content and keeps generating reasoning even with // enable_thinking=false, so a think-disable request can never yield a // reasoning-free stream. Fail with a clear compatibility error instead // of silently returning a reasoning stream for think=false. if a.provider == "vllm-mlx" && thinkDisableRequested { return nil, fmt.Errorf("unsupported think control: provider %q cannot disable reasoning generation for streaming responses (think=false / reasoning_effort=none)", a.provider) } if hasReasoningEffort && reasoningEffortVal != "" && reasoningEffortVal != "none" { return nil, fmt.Errorf("unsupported think control: reasoning_effort %q is not supported by %s", reasoningEffortVal, a.provider) } if hasBudget { var budgetInt int64 valid := false switch bv := budgetVal.(type) { case int: budgetInt = int64(bv) valid = true case int64: budgetInt = bv valid = true case float64: if bv == float64(int64(bv)) { budgetInt = int64(bv) valid = true } } if !valid || budgetInt < 0 { return nil, fmt.Errorf("unsupported think control: invalid thinking_token_budget %v", budgetVal) } } if hasThink || (hasReasoningEffort && reasoningEffortVal == "none") || hasBudget { var chatTemplateKwargs map[string]any if ctk, ok := body["chat_template_kwargs"].(map[string]any); ok { chatTemplateKwargs = make(map[string]any) for k, v := range ctk { chatTemplateKwargs[k] = v } } else { chatTemplateKwargs = make(map[string]any) } if (hasThink && !thinkVal) || (hasReasoningEffort && reasoningEffortVal == "none") { chatTemplateKwargs["enable_thinking"] = false } else if (hasThink && thinkVal) || hasBudget { chatTemplateKwargs["enable_thinking"] = true } if hasBudget { chatTemplateKwargs["thinking_token_budget"] = budgetVal } body["chat_template_kwargs"] = chatTemplateKwargs } default: // empty/unknown provider: pass requested fields through as-is. if hasThink { body["think"] = thinkVal } if hasReasoningEffort { body["reasoning_effort"] = reasoningEffortVal } if hasBudget { body["thinking_token_budget"] = budgetVal } } return body, nil } func retryBodyWithForcedSingleTool(body map[string]any, errorBody string) (map[string]any, bool) { if !isAutoToolChoiceUnsupportedError(errorBody) { return nil, false } forced, ok := forcedToolChoiceForSingleTool(body["tools"]) if !ok { return nil, false } next := make(map[string]any, len(body)+1) for key, value := range body { next[key] = value } next["tool_choice"] = forced return next, true } func isAutoToolChoiceUnsupportedError(errorBody string) bool { errorBody = strings.ToLower(errorBody) return strings.Contains(errorBody, "auto") && strings.Contains(errorBody, "tool choice") && isToolCallingUnsupportedError(errorBody) } func isToolCallingUnsupportedError(errorBody string) bool { errorBody = strings.ToLower(errorBody) return strings.Contains(errorBody, "enable-auto-tool-choice") || strings.Contains(errorBody, "tool-call-parser") } func forcedToolChoiceForSingleTool(tools any) (map[string]any, bool) { items, ok := tools.([]any) if !ok || len(items) != 1 { return nil, false } tool, ok := items[0].(map[string]any) if !ok { return nil, false } name := "" if fn, ok := tool["function"].(map[string]any); ok { name, _ = fn["name"].(string) } if name == "" { name, _ = tool["name"].(string) } name = strings.TrimSpace(name) if name == "" { return nil, false } return map[string]any{ "type": "function", "function": map[string]any{ "name": name, }, }, true } func retryBodyWithTextToolFallback(body map[string]any, errorBody string) (map[string]any, bool) { if !isToolCallingUnsupportedError(errorBody) { return nil, false } instruction, ok := textToolFallbackInstruction(body["tools"]) if !ok { return nil, false } messages, ok := prependTextToolFallbackInstruction(body["messages"], instruction) if !ok { return nil, false } next := make(map[string]any, len(body)+1) for key, value := range body { next[key] = value } delete(next, "tools") delete(next, "tool_choice") next["messages"] = messages return next, true } func prependTextToolFallbackInstruction(value any, instruction string) (any, bool) { switch messages := value.(type) { case []chatMessage: system := chatMessage{Role: "system", Content: instruction} out := make([]chatMessage, 0, len(messages)+1) for _, msg := range messages { if strings.EqualFold(strings.TrimSpace(msg.Role), "system") { system.Content = joinTextToolFallbackSystemContent(system.Content, msg.Content) continue } out = append(out, msg) } return append([]chatMessage{system}, out...), true case []any: systemContent := instruction out := make([]any, 0, len(messages)+1) for _, item := range messages { msg, ok := item.(map[string]any) if !ok { out = append(out, item) continue } role, _ := msg["role"].(string) if strings.EqualFold(strings.TrimSpace(role), "system") { systemContent = joinTextToolFallbackSystemContent(systemContent, stringFromAny(msg["content"])) continue } out = append(out, item) } return append([]any{map[string]any{"role": "system", "content": systemContent}}, out...), true default: return nil, false } } func joinTextToolFallbackSystemContent(first, next string) string { first = strings.TrimSpace(first) next = strings.TrimSpace(next) switch { case first == "": return next case next == "": return first default: return first + "\n\n" + next } } func stringFromAny(value any) string { switch v := value.(type) { case nil: return "" case string: return v default: encoded, _ := json.Marshal(v) return string(encoded) } } func textToolFallbackInstruction(tools any) (string, bool) { items, ok := anyItems(tools) if !ok || len(items) == 0 { return "", false } encoded, err := json.Marshal(items) if err != nil { return "", false } return "Tool calls must be emitted as plain text because this backend does not support native OpenAI tool calling. When a tool is needed, respond with exactly one tool call and no markdown:\n" + "\n\nJSON_VALUE\n\n\n" + "run_commands executes from the client workspace root. Do not prepend cd to an absolute workspace path unless the user explicitly asks to operate in a different directory; prefer current-workspace commands such as git status.\n" + "Use valid JSON for each parameter value and follow the supplied parameter schema. Available tools JSON: " + string(encoded), true } func anyItems(value any) ([]any, bool) { switch v := value.(type) { case []any: return v, true default: encoded, err := json.Marshal(v) if err != nil { return nil, false } var out []any if err := json.Unmarshal(encoded, &out); err != nil { return nil, false } return out, true } } func completeEvent(runID, finishReason string, usage *runtime.UsageStats, outputTokens int, toolCalls []any, textToolFallback bool) runtime.RuntimeEvent { if usage == nil { usage = &runtime.UsageStats{OutputTokens: outputTokens} } var metadata map[string]string if finishReason != "" { metadata = map[string]string{"finish_reason": finishReason} } if textToolFallback { if metadata == nil { metadata = make(map[string]string, 2) } metadata[runtimeMetadataOpenAITextToolFallback] = "true" } if len(toolCalls) > 0 { if metadata == nil { metadata = make(map[string]string, 2) } if finishReason == "" { metadata["finish_reason"] = "tool_calls" } if encoded, err := json.Marshal(toolCalls); err == nil { metadata[runtimeMetadataOpenAIToolCalls] = string(encoded) } } return runtime.RuntimeEvent{ RunID: runID, Type: runtime.EventTypeComplete, Message: "openai_compat chat complete", Usage: usage, Metadata: metadata, Timestamp: time.Now(), } } func messagesFromInput(input map[string]any) []chatMessage { if raw, ok := input["messages"].([]any); ok { out := make([]chatMessage, 0, len(raw)) for _, item := range raw { m, ok := item.(map[string]any) if !ok { continue } role, _ := m["role"].(string) content, _ := m["content"].(string) toolCallID, _ := m["tool_call_id"].(string) toolCalls := anySlice(m["tool_calls"]) role = strings.TrimSpace(role) content = strings.TrimSpace(content) toolCallID = strings.TrimSpace(toolCallID) if role == "" || (content == "" && len(toolCalls) == 0) { continue } out = append(out, chatMessage{ Role: role, Content: content, ToolCalls: toolCalls, ToolCallID: toolCallID, }) } if len(out) > 0 { return out } } if prompt := strings.TrimSpace(stringInput(input, "prompt")); prompt != "" { return []chatMessage{{Role: "user", Content: prompt}} } return nil } func emitError(ctx context.Context, sink runtime.EventSink, runID, msg string) error { return sink.Emit(ctx, runtime.RuntimeEvent{ RunID: runID, Type: runtime.EventTypeError, Error: msg, Timestamp: time.Now(), }) } func stringInput(input map[string]any, key string) string { if input == nil { return "" } if v, ok := input[key].(string); ok { return v } return "" } func anySlice(v any) []any { if items, ok := v.([]any); ok { return items } return nil } // joinOpenAIPath appends an OpenAI-compatible path to the endpoint. When the // endpoint already targets the /v1 root it strips the leading /v1 from the path // so a configured ".../v1" endpoint does not produce ".../v1/v1/...". func joinOpenAIPath(baseURL, path string) string { u, err := url.Parse(baseURL) if err != nil { return strings.TrimRight(baseURL, "/") + path } basePath := strings.TrimRight(u.Path, "/") if strings.HasSuffix(basePath, "/v1") && strings.HasPrefix(path, "/v1/") { path = strings.TrimPrefix(path, "/v1") } u.Path = basePath + path return u.String() } func readLimited(r io.Reader, limit int64) string { b, _ := io.ReadAll(io.LimitReader(r, limit)) return strings.TrimSpace(string(b)) } type chatMessage struct { Role string `json:"role"` Content string `json:"content"` ToolCalls []any `json:"tool_calls,omitempty"` ToolCallID string `json:"tool_call_id,omitempty"` } type chatChunk struct { Choices []struct { Delta chatChunkDelta `json:"delta"` FinishReason *string `json:"finish_reason"` } `json:"choices"` Usage *struct { PromptTokens int `json:"prompt_tokens"` CompletionTokens int `json:"completion_tokens"` PromptTokensDetails *struct { CachedTokens int `json:"cached_tokens"` } `json:"prompt_tokens_details"` CompletionTokensDetails *struct { ReasoningTokens int `json:"reasoning_tokens"` } `json:"completion_tokens_details"` } `json:"usage"` } type chatChunkDelta struct { Content string `json:"content"` ReasoningContent string `json:"reasoning_content"` Reasoning string `json:"reasoning"` ToolCalls []any `json:"tool_calls"` } func (d chatChunkDelta) ReasoningText() string { if d.ReasoningContent != "" { return d.ReasoningContent } return d.Reasoning } type openAIToolCallAccumulator struct { calls map[int]map[string]any order []int } func (a *openAIToolCallAccumulator) AddDelta(raw []any) { for fallbackIndex, item := range raw { call, ok := item.(map[string]any) if !ok { continue } index := fallbackIndex if parsed, ok := numericIndex(call["index"]); ok { index = parsed } dst := a.ensure(index) for key, value := range call { switch key { case "index": continue case "function": mergeToolCallFunction(dst, value) default: if !emptyToolCallValue(value) { dst[key] = value } } } } } func (a *openAIToolCallAccumulator) ToolCalls() []any { if len(a.order) == 0 { return nil } out := make([]any, 0, len(a.order)) for _, index := range a.order { call := a.calls[index] if len(call) == 0 { continue } copyCall := make(map[string]any, len(call)) for key, value := range call { copyCall[key] = value } out = append(out, copyCall) } return out } func (a *openAIToolCallAccumulator) ensure(index int) map[string]any { if a.calls == nil { a.calls = make(map[int]map[string]any) } if call, ok := a.calls[index]; ok { return call } call := make(map[string]any) a.calls[index] = call a.order = append(a.order, index) return call } func mergeToolCallFunction(call map[string]any, value any) { fn, ok := value.(map[string]any) if !ok { return } current, _ := call["function"].(map[string]any) if current == nil { current = make(map[string]any) call["function"] = current } for key, item := range fn { if key == "arguments" { if part, ok := item.(string); ok { current["arguments"] = currentString(current["arguments"]) + part continue } } if !emptyToolCallValue(item) { current[key] = item } } } func numericIndex(value any) (int, bool) { switch v := value.(type) { case int: return v, true case float64: return int(v), true default: return 0, false } } func emptyToolCallValue(value any) bool { if value == nil { return true } if text, ok := value.(string); ok { return text == "" } return false } func currentString(value any) string { if text, ok := value.(string); ok { return text } return "" } type modelsResponse struct { Data []struct { ID string `json:"id"` } `json:"data"` }