1113 lines
31 KiB
Go
1113 lines
31 KiB
Go
// Package openai_compat provides an Adapter for OpenAI-compatible inference
|
|
// servers such as Lemonade Server via the /v1/models and /v1/chat/completions
|
|
// endpoints. Unlike the vllm adapter it carries an explicit provider label,
|
|
// per-request auth/header injection and top-level option passthrough so a
|
|
// provider boundary and its config contract stay distinct.
|
|
package openai_compat
|
|
|
|
import (
|
|
"bufio"
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"os"
|
|
"strings"
|
|
"time"
|
|
|
|
"go.uber.org/zap"
|
|
|
|
"iop/apps/node/internal/runtime"
|
|
"iop/packages/go/config"
|
|
)
|
|
|
|
const Name = "openai_compat"
|
|
|
|
const (
|
|
runtimeMetadataOpenAIToolCalls = "openai_tool_calls"
|
|
runtimeMetadataOpenAITextToolFallback = "openai_text_tool_fallback"
|
|
streamTraceMetadataKey = "iop_trace_stream"
|
|
streamTraceEnvKey = "IOP_OPENAI_COMPAT_TRACE_STREAM"
|
|
)
|
|
|
|
type Adapter struct {
|
|
instanceName string
|
|
provider string
|
|
endpoint string
|
|
headers map[string]string
|
|
capacity int
|
|
maxQueue int
|
|
queueTimeoutMS int
|
|
requestTimeoutMS int
|
|
client *http.Client
|
|
logger *zap.Logger
|
|
}
|
|
|
|
// New creates an OpenAI-compatible adapter. The optional instanceName is the
|
|
// registry instance key used to disambiguate multiple instances on a node.
|
|
func New(cfg config.OpenAICompatConf, logger *zap.Logger, instanceName ...string) *Adapter {
|
|
endpoint := strings.TrimRight(cfg.Endpoint, "/")
|
|
name := Name
|
|
if len(instanceName) > 0 && instanceName[0] != "" {
|
|
name = instanceName[0]
|
|
}
|
|
var headers map[string]string
|
|
if len(cfg.Headers) > 0 {
|
|
headers = make(map[string]string, len(cfg.Headers))
|
|
for k, v := range cfg.Headers {
|
|
headers[k] = v
|
|
}
|
|
}
|
|
return &Adapter{
|
|
instanceName: name,
|
|
provider: cfg.Provider,
|
|
endpoint: endpoint,
|
|
headers: headers,
|
|
capacity: cfg.Capacity,
|
|
maxQueue: cfg.MaxQueue,
|
|
queueTimeoutMS: cfg.QueueTimeoutMS,
|
|
requestTimeoutMS: cfg.RequestTimeoutMS,
|
|
client: &http.Client{},
|
|
logger: logger,
|
|
}
|
|
}
|
|
|
|
func (a *Adapter) Name() string { return Name }
|
|
|
|
func (a *Adapter) TunnelProvider(ctx context.Context, req runtime.ProviderTunnelRequest, sink runtime.ProviderTunnelSink) error {
|
|
var seq int64 = 0
|
|
|
|
if a.endpoint == "" {
|
|
err := fmt.Errorf("openai_compat adapter: endpoint is required")
|
|
_ = emitTunnelError(ctx, sink, req, seq, err)
|
|
return err
|
|
}
|
|
|
|
urlStr := joinOpenAIPath(a.endpoint, req.Path)
|
|
httpReq, err := http.NewRequestWithContext(ctx, req.Method, urlStr, bytes.NewReader(req.Body))
|
|
if err != nil {
|
|
err = fmt.Errorf("build request: %w", err)
|
|
_ = emitTunnelError(ctx, sink, req, seq, err)
|
|
return err
|
|
}
|
|
|
|
isJSON := req.Method == http.MethodPost
|
|
a.applyHeaders(httpReq, isJSON)
|
|
for k, v := range req.Headers {
|
|
httpReq.Header.Set(k, v)
|
|
}
|
|
|
|
resp, err := a.client.Do(httpReq)
|
|
if err != nil {
|
|
err = fmt.Errorf("request failed: %w", err)
|
|
_ = emitTunnelError(ctx, sink, req, seq, err)
|
|
return err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
respHeaders := make(map[string]string)
|
|
for k, values := range resp.Header {
|
|
if len(values) > 0 {
|
|
respHeaders[k] = values[0]
|
|
}
|
|
}
|
|
|
|
err = sink.EmitTunnelFrame(ctx, runtime.ProviderTunnelFrame{
|
|
RunID: req.RunID,
|
|
TunnelID: req.TunnelID,
|
|
Sequence: seq,
|
|
Kind: runtime.ProviderTunnelFrameKindResponseStart,
|
|
StatusCode: resp.StatusCode,
|
|
Headers: respHeaders,
|
|
Timestamp: time.Now(),
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
seq++
|
|
|
|
buf := make([]byte, 4096)
|
|
for {
|
|
n, readErr := resp.Body.Read(buf)
|
|
if n > 0 {
|
|
err = sink.EmitTunnelFrame(ctx, runtime.ProviderTunnelFrame{
|
|
RunID: req.RunID,
|
|
TunnelID: req.TunnelID,
|
|
Sequence: seq,
|
|
Kind: runtime.ProviderTunnelFrameKindBody,
|
|
Body: append([]byte(nil), buf[:n]...),
|
|
Timestamp: time.Now(),
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
seq++
|
|
}
|
|
if readErr != nil {
|
|
if readErr == io.EOF {
|
|
break
|
|
}
|
|
if ctx.Err() != nil {
|
|
_ = emitTunnelError(ctx, sink, req, seq, ctx.Err())
|
|
return ctx.Err()
|
|
}
|
|
err = fmt.Errorf("read response body: %w", readErr)
|
|
_ = emitTunnelError(ctx, sink, req, seq, err)
|
|
return err
|
|
}
|
|
}
|
|
|
|
err = sink.EmitTunnelFrame(ctx, runtime.ProviderTunnelFrame{
|
|
RunID: req.RunID,
|
|
TunnelID: req.TunnelID,
|
|
Sequence: seq,
|
|
Kind: runtime.ProviderTunnelFrameKindEnd,
|
|
End: true,
|
|
Timestamp: time.Now(),
|
|
})
|
|
return err
|
|
}
|
|
|
|
func emitTunnelError(ctx context.Context, sink runtime.ProviderTunnelSink, req runtime.ProviderTunnelRequest, seq int64, err error) error {
|
|
return sink.EmitTunnelFrame(ctx, runtime.ProviderTunnelFrame{
|
|
RunID: req.RunID,
|
|
TunnelID: req.TunnelID,
|
|
Sequence: seq,
|
|
Kind: runtime.ProviderTunnelFrameKindError,
|
|
Error: err.Error(),
|
|
Timestamp: time.Now(),
|
|
})
|
|
}
|
|
|
|
func (a *Adapter) Capabilities(ctx context.Context) (runtime.Capabilities, error) {
|
|
probeCtx, cancel := context.WithTimeout(ctx, 2*time.Second)
|
|
defer cancel()
|
|
targets, err := a.fetchTargets(probeCtx)
|
|
status := runtime.ProviderStatusAvailable
|
|
if err != nil {
|
|
status = runtime.ProviderStatusUnavailable
|
|
}
|
|
return runtime.Capabilities{
|
|
AdapterName: Name,
|
|
InstanceKey: a.instanceName,
|
|
Targets: targets,
|
|
MaxConcurrency: effectiveCapacity(a.capacity, 8),
|
|
MaxQueue: a.maxQueue,
|
|
QueueTimeoutMS: a.queueTimeoutMS,
|
|
RequestTimeoutMS: a.requestTimeoutMS,
|
|
ProviderStatus: runtime.NormalizeProviderStatus(status),
|
|
}, nil
|
|
}
|
|
|
|
func effectiveCapacity(capacity, defaultVal int) int {
|
|
if capacity <= 0 {
|
|
return defaultVal
|
|
}
|
|
return capacity
|
|
}
|
|
|
|
func (a *Adapter) Execute(ctx context.Context, spec runtime.ExecutionSpec, sink runtime.EventSink) error {
|
|
if a.endpoint == "" {
|
|
return fmt.Errorf("openai_compat adapter: endpoint is required")
|
|
}
|
|
model := strings.TrimSpace(spec.Target)
|
|
if model == "" {
|
|
model = stringInput(spec.Input, "model")
|
|
}
|
|
if model == "" {
|
|
return fmt.Errorf("openai_compat adapter: target/model is required")
|
|
}
|
|
messages := messagesFromInput(spec.Input)
|
|
if len(messages) == 0 {
|
|
return fmt.Errorf("openai_compat adapter: messages are required")
|
|
}
|
|
|
|
if err := sink.Emit(ctx, runtime.RuntimeEvent{
|
|
RunID: spec.RunID,
|
|
Type: runtime.EventTypeStart,
|
|
Timestamp: time.Now(),
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
|
|
reqBody, err := a.buildRequestBody(model, messages, spec.Input)
|
|
if err != nil {
|
|
_ = emitError(ctx, sink, spec.RunID, fmt.Sprintf("openai_compat build request body failed: %v", err))
|
|
return fmt.Errorf("openai_compat adapter: build request body: %w", err)
|
|
}
|
|
body, err := json.Marshal(reqBody)
|
|
if err != nil {
|
|
return fmt.Errorf("openai_compat adapter: marshal request: %w", err)
|
|
}
|
|
textToolFallback := false
|
|
|
|
a.logger.Info("openai_compat adapter executing",
|
|
zap.String("run_id", spec.RunID),
|
|
zap.String("provider", a.provider),
|
|
zap.String("target", model),
|
|
zap.String("endpoint", a.endpoint),
|
|
)
|
|
resp, err := a.doChatCompletion(ctx, body)
|
|
if err != nil {
|
|
_ = emitError(ctx, sink, spec.RunID, fmt.Sprintf("openai_compat request failed: %v", err))
|
|
return fmt.Errorf("openai_compat adapter: request: %w", err)
|
|
}
|
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
|
msg := readLimited(resp.Body, 4096)
|
|
_ = resp.Body.Close()
|
|
if retryBody, ok := retryBodyWithForcedSingleTool(reqBody, msg); ok {
|
|
body, err = json.Marshal(retryBody)
|
|
if err != nil {
|
|
_ = emitError(ctx, sink, spec.RunID, fmt.Sprintf("openai_compat retry marshal failed: %v", err))
|
|
return fmt.Errorf("openai_compat adapter: retry marshal: %w", err)
|
|
}
|
|
resp, err = a.doChatCompletion(ctx, body)
|
|
if err != nil {
|
|
_ = emitError(ctx, sink, spec.RunID, fmt.Sprintf("openai_compat retry request failed: %v", err))
|
|
return fmt.Errorf("openai_compat adapter: retry request: %w", err)
|
|
}
|
|
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
|
|
goto streamResponse
|
|
}
|
|
msg = readLimited(resp.Body, 4096)
|
|
_ = resp.Body.Close()
|
|
}
|
|
if retryBody, ok := retryBodyWithTextToolFallback(reqBody, msg); ok {
|
|
body, err = json.Marshal(retryBody)
|
|
if err != nil {
|
|
_ = emitError(ctx, sink, spec.RunID, fmt.Sprintf("openai_compat text tool fallback marshal failed: %v", err))
|
|
return fmt.Errorf("openai_compat adapter: text tool fallback marshal: %w", err)
|
|
}
|
|
resp, err = a.doChatCompletion(ctx, body)
|
|
if err != nil {
|
|
_ = emitError(ctx, sink, spec.RunID, fmt.Sprintf("openai_compat text tool fallback request failed: %v", err))
|
|
return fmt.Errorf("openai_compat adapter: text tool fallback request: %w", err)
|
|
}
|
|
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
|
|
textToolFallback = true
|
|
goto streamResponse
|
|
}
|
|
msg = readLimited(resp.Body, 4096)
|
|
_ = resp.Body.Close()
|
|
}
|
|
if msg == "" {
|
|
msg = resp.Status
|
|
}
|
|
_ = emitError(ctx, sink, spec.RunID, fmt.Sprintf("openai_compat returned %s: %s", resp.Status, msg))
|
|
return fmt.Errorf("openai_compat adapter: non-2xx response: %s", resp.Status)
|
|
}
|
|
|
|
streamResponse:
|
|
defer resp.Body.Close()
|
|
scanner := bufio.NewScanner(resp.Body)
|
|
outputTokens := 0
|
|
finishReason := ""
|
|
var usage *runtime.UsageStats
|
|
var toolCalls openAIToolCallAccumulator
|
|
traceStream := openAICompatTraceStreamEnabled(spec.Metadata)
|
|
traceSeq := 0
|
|
for scanner.Scan() {
|
|
line := scanner.Text()
|
|
if !strings.HasPrefix(line, "data: ") {
|
|
continue
|
|
}
|
|
payload := strings.TrimPrefix(line, "data: ")
|
|
if payload == "[DONE]" {
|
|
if traceStream {
|
|
a.logger.Info("openai_compat provider stream done",
|
|
zap.String("run_id", spec.RunID),
|
|
zap.Int("seq", traceSeq+1),
|
|
)
|
|
}
|
|
return sink.Emit(ctx, completeEvent(spec.RunID, finishReason, usage, outputTokens, toolCalls.ToolCalls(), textToolFallback))
|
|
}
|
|
traceSeq++
|
|
if traceStream {
|
|
a.logger.Info("openai_compat provider stream raw chunk",
|
|
zap.String("run_id", spec.RunID),
|
|
zap.Int("seq", traceSeq),
|
|
zap.Int("raw_len", len(payload)),
|
|
zap.Bool("raw_has_open_think", hasOpenThinkTag(payload)),
|
|
zap.Bool("raw_has_close_think", hasCloseThinkTag(payload)),
|
|
zap.String("raw_preview", tracePreview(payload, 4000)),
|
|
)
|
|
}
|
|
var chunk chatChunk
|
|
if err := json.Unmarshal([]byte(payload), &chunk); err != nil {
|
|
if traceStream {
|
|
a.logger.Warn("openai_compat provider stream raw chunk ignored",
|
|
zap.String("run_id", spec.RunID),
|
|
zap.Int("seq", traceSeq),
|
|
zap.Error(err),
|
|
)
|
|
}
|
|
continue
|
|
}
|
|
if chunk.Usage != nil {
|
|
usage = &runtime.UsageStats{
|
|
InputTokens: chunk.Usage.PromptTokens,
|
|
OutputTokens: chunk.Usage.CompletionTokens,
|
|
}
|
|
}
|
|
if len(chunk.Choices) == 0 {
|
|
continue
|
|
}
|
|
choice := chunk.Choices[0]
|
|
if choice.FinishReason != nil && *choice.FinishReason != "" {
|
|
finishReason = *choice.FinishReason
|
|
}
|
|
if len(choice.Delta.ToolCalls) > 0 {
|
|
toolCalls.AddDelta(choice.Delta.ToolCalls)
|
|
}
|
|
reasoning := choice.Delta.ReasoningText()
|
|
if traceStream {
|
|
a.logger.Info("openai_compat provider stream parsed chunk",
|
|
zap.String("run_id", spec.RunID),
|
|
zap.Int("seq", traceSeq),
|
|
zap.String("finish_reason", finishReason),
|
|
zap.Int("tool_call_delta_count", len(choice.Delta.ToolCalls)),
|
|
zap.Int("content_len", len(choice.Delta.Content)),
|
|
zap.Bool("content_has_open_think", hasOpenThinkTag(choice.Delta.Content)),
|
|
zap.Bool("content_has_close_think", hasCloseThinkTag(choice.Delta.Content)),
|
|
zap.String("content_preview", tracePreview(choice.Delta.Content, 2000)),
|
|
zap.Int("reasoning_len", len(reasoning)),
|
|
zap.Bool("reasoning_has_open_think", hasOpenThinkTag(reasoning)),
|
|
zap.Bool("reasoning_has_close_think", hasCloseThinkTag(reasoning)),
|
|
zap.String("reasoning_preview", tracePreview(reasoning, 2000)),
|
|
)
|
|
}
|
|
if reasoning != "" {
|
|
if err := sink.Emit(ctx, runtime.RuntimeEvent{
|
|
RunID: spec.RunID,
|
|
Type: runtime.EventTypeReasoningDelta,
|
|
Delta: reasoning,
|
|
Timestamp: time.Now(),
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if content := choice.Delta.Content; content != "" {
|
|
outputTokens += len(strings.Fields(content))
|
|
if err := sink.Emit(ctx, runtime.RuntimeEvent{
|
|
RunID: spec.RunID,
|
|
Type: runtime.EventTypeDelta,
|
|
Delta: content,
|
|
Timestamp: time.Now(),
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
if err := scanner.Err(); err != nil {
|
|
_ = emitError(ctx, sink, spec.RunID, fmt.Sprintf("openai_compat stream scan failed: %v", err))
|
|
return fmt.Errorf("openai_compat adapter: scan stream: %w", err)
|
|
}
|
|
_ = emitError(ctx, sink, spec.RunID, "openai_compat stream ended without [DONE]")
|
|
return fmt.Errorf("openai_compat adapter: stream ended without [DONE]")
|
|
}
|
|
|
|
func openAICompatTraceStreamEnabled(metadata map[string]string) bool {
|
|
if traceBool(os.Getenv(streamTraceEnvKey)) {
|
|
return true
|
|
}
|
|
if metadata == nil {
|
|
return false
|
|
}
|
|
return traceBool(metadata[streamTraceMetadataKey])
|
|
}
|
|
|
|
func traceBool(v string) bool {
|
|
switch strings.ToLower(strings.TrimSpace(v)) {
|
|
case "1", "true", "yes", "on":
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func tracePreview(s string, max int) string {
|
|
if max <= 0 || len(s) <= max {
|
|
return s
|
|
}
|
|
return s[:max] + "...[truncated]"
|
|
}
|
|
|
|
func hasOpenThinkTag(s string) bool {
|
|
return strings.Contains(strings.ToLower(s), "<think")
|
|
}
|
|
|
|
func hasCloseThinkTag(s string) bool {
|
|
return strings.Contains(strings.ToLower(s), "</think")
|
|
}
|
|
|
|
// ProbeProvider checks endpoint availability and target presence using the
|
|
// OpenAI-compatible /v1/models endpoint, returning only the public baseline
|
|
// status values.
|
|
func (a *Adapter) ProbeProvider(ctx context.Context, target string) (runtime.ProviderProbeResult, error) {
|
|
probeCtx, cancel := context.WithTimeout(ctx, 2*time.Second)
|
|
defer cancel()
|
|
|
|
targets, err := a.fetchTargets(probeCtx)
|
|
|
|
result := runtime.ProviderProbeResult{
|
|
AdapterName: Name,
|
|
InstanceKey: a.instanceName,
|
|
Target: target,
|
|
}
|
|
if a.provider != "" {
|
|
result.Metadata = map[string]string{"provider": a.provider}
|
|
}
|
|
|
|
if err != nil {
|
|
result.Status = runtime.NormalizeProviderStatus(runtime.ProviderStatusUnavailable)
|
|
result.Detail = err.Error()
|
|
return result, nil
|
|
}
|
|
|
|
result.Targets = targets
|
|
|
|
if target == "" {
|
|
result.Status = runtime.NormalizeProviderStatus(runtime.ProviderStatusAvailable)
|
|
return result, nil
|
|
}
|
|
|
|
found := false
|
|
for _, t := range targets {
|
|
if t == target {
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
|
|
if found {
|
|
result.Status = runtime.NormalizeProviderStatus(runtime.ProviderStatusAvailable)
|
|
} else {
|
|
result.Status = runtime.NormalizeProviderStatus(runtime.ProviderStatusUnavailable)
|
|
result.Detail = fmt.Sprintf("target model %q not found in provider models", target)
|
|
}
|
|
|
|
return result, nil
|
|
}
|
|
|
|
func (a *Adapter) fetchTargets(ctx context.Context) ([]string, error) {
|
|
if a.endpoint == "" {
|
|
return nil, fmt.Errorf("openai_compat adapter: endpoint is required")
|
|
}
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, joinOpenAIPath(a.endpoint, "/v1/models"), nil)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("build request: %w", err)
|
|
}
|
|
a.applyHeaders(req, false)
|
|
resp, err := a.client.Do(req)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("request failed: %w", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
|
return nil, fmt.Errorf("status code %d", resp.StatusCode)
|
|
}
|
|
var models modelsResponse
|
|
if err := json.NewDecoder(resp.Body).Decode(&models); err != nil {
|
|
return nil, fmt.Errorf("decode response: %w", err)
|
|
}
|
|
out := make([]string, 0, len(models.Data))
|
|
for _, model := range models.Data {
|
|
if model.ID != "" {
|
|
out = append(out, model.ID)
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// applyHeaders injects configured auth/secret headers. The adapter always owns
|
|
// Content-Type for JSON POSTs, so a user-provided Content-Type is ignored to
|
|
// keep the request body contract intact.
|
|
func (a *Adapter) applyHeaders(req *http.Request, jsonBody bool) {
|
|
for k, v := range a.headers {
|
|
if jsonBody && http.CanonicalHeaderKey(k) == "Content-Type" {
|
|
continue
|
|
}
|
|
req.Header.Set(k, v)
|
|
}
|
|
if jsonBody {
|
|
req.Header.Set("Content-Type", "application/json")
|
|
}
|
|
}
|
|
|
|
func (a *Adapter) doChatCompletion(ctx context.Context, body []byte) (*http.Response, error) {
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, joinOpenAIPath(a.endpoint, "/v1/chat/completions"), bytes.NewReader(body))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("build request: %w", err)
|
|
}
|
|
a.applyHeaders(req, true)
|
|
return a.client.Do(req)
|
|
}
|
|
|
|
func (a *Adapter) buildRequestBody(model string, messages []chatMessage, input map[string]any) (map[string]any, error) {
|
|
body := make(map[string]any)
|
|
// Copy caller options as top-level OpenAI-compatible request fields first so
|
|
// adapter-owned fields below always win over them.
|
|
if opts, ok := input["options"].(map[string]any); ok {
|
|
for k, v := range opts {
|
|
body[k] = v
|
|
}
|
|
}
|
|
// Pass through optional OpenAI-compatible fields when present in the input (except think).
|
|
for _, key := range []string{"tools", "tool_choice", "format", "keep_alive"} {
|
|
if v, ok := input[key]; ok {
|
|
body[key] = v
|
|
}
|
|
}
|
|
body["model"] = model
|
|
body["messages"] = messages
|
|
body["stream"] = true
|
|
|
|
// Extract think-control fields
|
|
hasThink := false
|
|
var thinkVal bool
|
|
if v, ok := input["think"]; ok {
|
|
if bv, ok := v.(bool); ok {
|
|
hasThink = true
|
|
thinkVal = bv
|
|
}
|
|
}
|
|
|
|
hasReasoningEffort := false
|
|
var reasoningEffortVal string
|
|
if v, ok := input["reasoning_effort"]; ok {
|
|
if sv, ok := v.(string); ok {
|
|
hasReasoningEffort = true
|
|
reasoningEffortVal = sv
|
|
}
|
|
}
|
|
|
|
hasBudget := false
|
|
var budgetVal any
|
|
if v, ok := input["thinking_token_budget"]; ok {
|
|
hasBudget = true
|
|
budgetVal = v
|
|
}
|
|
|
|
// vLLM, vLLM-MLX and Lemonade all serve Qwen3-family templates that honor
|
|
// chat_template_kwargs.enable_thinking / thinking_token_budget. The Lemonade
|
|
// build ignores the top-level `think` field, so all three providers map
|
|
// think-control into chat_template_kwargs rather than top-level fields.
|
|
switch a.provider {
|
|
case "vllm", "vllm-mlx", "lemonade":
|
|
thinkDisableRequested := (hasThink && !thinkVal) || (hasReasoningEffort && reasoningEffortVal == "none")
|
|
|
|
// The vLLM-MLX runtime labels its entire streamed output as
|
|
// reasoning_content and keeps generating reasoning even with
|
|
// enable_thinking=false, so a think-disable request can never yield a
|
|
// reasoning-free stream. Fail with a clear compatibility error instead
|
|
// of silently returning a reasoning stream for think=false.
|
|
if a.provider == "vllm-mlx" && thinkDisableRequested {
|
|
return nil, fmt.Errorf("unsupported think control: provider %q cannot disable reasoning generation for streaming responses (think=false / reasoning_effort=none)", a.provider)
|
|
}
|
|
|
|
if hasReasoningEffort && reasoningEffortVal != "" && reasoningEffortVal != "none" {
|
|
return nil, fmt.Errorf("unsupported think control: reasoning_effort %q is not supported by %s", reasoningEffortVal, a.provider)
|
|
}
|
|
|
|
if hasBudget {
|
|
var budgetInt int64
|
|
valid := false
|
|
switch bv := budgetVal.(type) {
|
|
case int:
|
|
budgetInt = int64(bv)
|
|
valid = true
|
|
case int64:
|
|
budgetInt = bv
|
|
valid = true
|
|
case float64:
|
|
if bv == float64(int64(bv)) {
|
|
budgetInt = int64(bv)
|
|
valid = true
|
|
}
|
|
}
|
|
if !valid || budgetInt < 0 {
|
|
return nil, fmt.Errorf("unsupported think control: invalid thinking_token_budget %v", budgetVal)
|
|
}
|
|
}
|
|
|
|
if hasThink || (hasReasoningEffort && reasoningEffortVal == "none") || hasBudget {
|
|
var chatTemplateKwargs map[string]any
|
|
if ctk, ok := body["chat_template_kwargs"].(map[string]any); ok {
|
|
chatTemplateKwargs = make(map[string]any)
|
|
for k, v := range ctk {
|
|
chatTemplateKwargs[k] = v
|
|
}
|
|
} else {
|
|
chatTemplateKwargs = make(map[string]any)
|
|
}
|
|
|
|
if (hasThink && !thinkVal) || (hasReasoningEffort && reasoningEffortVal == "none") {
|
|
chatTemplateKwargs["enable_thinking"] = false
|
|
} else if (hasThink && thinkVal) || hasBudget {
|
|
chatTemplateKwargs["enable_thinking"] = true
|
|
}
|
|
|
|
if hasBudget {
|
|
chatTemplateKwargs["thinking_token_budget"] = budgetVal
|
|
}
|
|
|
|
body["chat_template_kwargs"] = chatTemplateKwargs
|
|
}
|
|
default:
|
|
// empty/unknown provider: pass requested fields through as-is.
|
|
if hasThink {
|
|
body["think"] = thinkVal
|
|
}
|
|
if hasReasoningEffort {
|
|
body["reasoning_effort"] = reasoningEffortVal
|
|
}
|
|
if hasBudget {
|
|
body["thinking_token_budget"] = budgetVal
|
|
}
|
|
}
|
|
|
|
return body, nil
|
|
}
|
|
|
|
func retryBodyWithForcedSingleTool(body map[string]any, errorBody string) (map[string]any, bool) {
|
|
if !isAutoToolChoiceUnsupportedError(errorBody) {
|
|
return nil, false
|
|
}
|
|
forced, ok := forcedToolChoiceForSingleTool(body["tools"])
|
|
if !ok {
|
|
return nil, false
|
|
}
|
|
next := make(map[string]any, len(body)+1)
|
|
for key, value := range body {
|
|
next[key] = value
|
|
}
|
|
next["tool_choice"] = forced
|
|
return next, true
|
|
}
|
|
|
|
func isAutoToolChoiceUnsupportedError(errorBody string) bool {
|
|
errorBody = strings.ToLower(errorBody)
|
|
return strings.Contains(errorBody, "auto") &&
|
|
strings.Contains(errorBody, "tool choice") &&
|
|
isToolCallingUnsupportedError(errorBody)
|
|
}
|
|
|
|
func isToolCallingUnsupportedError(errorBody string) bool {
|
|
errorBody = strings.ToLower(errorBody)
|
|
return strings.Contains(errorBody, "enable-auto-tool-choice") ||
|
|
strings.Contains(errorBody, "tool-call-parser")
|
|
}
|
|
|
|
func forcedToolChoiceForSingleTool(tools any) (map[string]any, bool) {
|
|
items, ok := tools.([]any)
|
|
if !ok || len(items) != 1 {
|
|
return nil, false
|
|
}
|
|
tool, ok := items[0].(map[string]any)
|
|
if !ok {
|
|
return nil, false
|
|
}
|
|
name := ""
|
|
if fn, ok := tool["function"].(map[string]any); ok {
|
|
name, _ = fn["name"].(string)
|
|
}
|
|
if name == "" {
|
|
name, _ = tool["name"].(string)
|
|
}
|
|
name = strings.TrimSpace(name)
|
|
if name == "" {
|
|
return nil, false
|
|
}
|
|
return map[string]any{
|
|
"type": "function",
|
|
"function": map[string]any{
|
|
"name": name,
|
|
},
|
|
}, true
|
|
}
|
|
|
|
func retryBodyWithTextToolFallback(body map[string]any, errorBody string) (map[string]any, bool) {
|
|
if !isToolCallingUnsupportedError(errorBody) {
|
|
return nil, false
|
|
}
|
|
instruction, ok := textToolFallbackInstruction(body["tools"])
|
|
if !ok {
|
|
return nil, false
|
|
}
|
|
messages, ok := prependTextToolFallbackInstruction(body["messages"], instruction)
|
|
if !ok {
|
|
return nil, false
|
|
}
|
|
next := make(map[string]any, len(body)+1)
|
|
for key, value := range body {
|
|
next[key] = value
|
|
}
|
|
delete(next, "tools")
|
|
delete(next, "tool_choice")
|
|
next["messages"] = messages
|
|
return next, true
|
|
}
|
|
|
|
func prependTextToolFallbackInstruction(value any, instruction string) (any, bool) {
|
|
switch messages := value.(type) {
|
|
case []chatMessage:
|
|
system := chatMessage{Role: "system", Content: instruction}
|
|
out := make([]chatMessage, 0, len(messages)+1)
|
|
for _, msg := range messages {
|
|
if strings.EqualFold(strings.TrimSpace(msg.Role), "system") {
|
|
system.Content = joinTextToolFallbackSystemContent(system.Content, msg.Content)
|
|
continue
|
|
}
|
|
out = append(out, msg)
|
|
}
|
|
return append([]chatMessage{system}, out...), true
|
|
case []any:
|
|
systemContent := instruction
|
|
out := make([]any, 0, len(messages)+1)
|
|
for _, item := range messages {
|
|
msg, ok := item.(map[string]any)
|
|
if !ok {
|
|
out = append(out, item)
|
|
continue
|
|
}
|
|
role, _ := msg["role"].(string)
|
|
if strings.EqualFold(strings.TrimSpace(role), "system") {
|
|
systemContent = joinTextToolFallbackSystemContent(systemContent, stringFromAny(msg["content"]))
|
|
continue
|
|
}
|
|
out = append(out, item)
|
|
}
|
|
return append([]any{map[string]any{"role": "system", "content": systemContent}}, out...), true
|
|
default:
|
|
return nil, false
|
|
}
|
|
}
|
|
|
|
func joinTextToolFallbackSystemContent(first, next string) string {
|
|
first = strings.TrimSpace(first)
|
|
next = strings.TrimSpace(next)
|
|
switch {
|
|
case first == "":
|
|
return next
|
|
case next == "":
|
|
return first
|
|
default:
|
|
return first + "\n\n" + next
|
|
}
|
|
}
|
|
|
|
func stringFromAny(value any) string {
|
|
switch v := value.(type) {
|
|
case nil:
|
|
return ""
|
|
case string:
|
|
return v
|
|
default:
|
|
encoded, _ := json.Marshal(v)
|
|
return string(encoded)
|
|
}
|
|
}
|
|
|
|
func textToolFallbackInstruction(tools any) (string, bool) {
|
|
items, ok := anyItems(tools)
|
|
if !ok || len(items) == 0 {
|
|
return "", false
|
|
}
|
|
encoded, err := json.Marshal(items)
|
|
if err != nil {
|
|
return "", false
|
|
}
|
|
return "Tool calls must be emitted as plain text because this backend does not support native OpenAI tool calling. When a tool is needed, respond with exactly one tool call and no markdown:\n" +
|
|
"<tool_call>\n<function=TOOL_NAME>\n<parameter=PARAMETER_NAME>JSON_VALUE</parameter>\n</function>\n</tool_call>\n" +
|
|
"run_commands executes from the client workspace root. Do not prepend cd to an absolute workspace path unless the user explicitly asks to operate in a different directory; prefer current-workspace commands such as git status.\n" +
|
|
"Use valid JSON for each parameter value and follow the supplied parameter schema. Available tools JSON: " + string(encoded), true
|
|
}
|
|
|
|
func anyItems(value any) ([]any, bool) {
|
|
switch v := value.(type) {
|
|
case []any:
|
|
return v, true
|
|
default:
|
|
encoded, err := json.Marshal(v)
|
|
if err != nil {
|
|
return nil, false
|
|
}
|
|
var out []any
|
|
if err := json.Unmarshal(encoded, &out); err != nil {
|
|
return nil, false
|
|
}
|
|
return out, true
|
|
}
|
|
}
|
|
|
|
func completeEvent(runID, finishReason string, usage *runtime.UsageStats, outputTokens int, toolCalls []any, textToolFallback bool) runtime.RuntimeEvent {
|
|
if usage == nil {
|
|
usage = &runtime.UsageStats{OutputTokens: outputTokens}
|
|
}
|
|
var metadata map[string]string
|
|
if finishReason != "" {
|
|
metadata = map[string]string{"finish_reason": finishReason}
|
|
}
|
|
if textToolFallback {
|
|
if metadata == nil {
|
|
metadata = make(map[string]string, 2)
|
|
}
|
|
metadata[runtimeMetadataOpenAITextToolFallback] = "true"
|
|
}
|
|
if len(toolCalls) > 0 {
|
|
if metadata == nil {
|
|
metadata = make(map[string]string, 2)
|
|
}
|
|
if finishReason == "" {
|
|
metadata["finish_reason"] = "tool_calls"
|
|
}
|
|
if encoded, err := json.Marshal(toolCalls); err == nil {
|
|
metadata[runtimeMetadataOpenAIToolCalls] = string(encoded)
|
|
}
|
|
}
|
|
return runtime.RuntimeEvent{
|
|
RunID: runID,
|
|
Type: runtime.EventTypeComplete,
|
|
Message: "openai_compat chat complete",
|
|
Usage: usage,
|
|
Metadata: metadata,
|
|
Timestamp: time.Now(),
|
|
}
|
|
}
|
|
|
|
func messagesFromInput(input map[string]any) []chatMessage {
|
|
if raw, ok := input["messages"].([]any); ok {
|
|
out := make([]chatMessage, 0, len(raw))
|
|
for _, item := range raw {
|
|
m, ok := item.(map[string]any)
|
|
if !ok {
|
|
continue
|
|
}
|
|
role, _ := m["role"].(string)
|
|
content, _ := m["content"].(string)
|
|
toolCallID, _ := m["tool_call_id"].(string)
|
|
toolCalls := anySlice(m["tool_calls"])
|
|
role = strings.TrimSpace(role)
|
|
content = strings.TrimSpace(content)
|
|
toolCallID = strings.TrimSpace(toolCallID)
|
|
if role == "" || (content == "" && len(toolCalls) == 0) {
|
|
continue
|
|
}
|
|
out = append(out, chatMessage{
|
|
Role: role,
|
|
Content: content,
|
|
ToolCalls: toolCalls,
|
|
ToolCallID: toolCallID,
|
|
})
|
|
}
|
|
if len(out) > 0 {
|
|
return out
|
|
}
|
|
}
|
|
if prompt := strings.TrimSpace(stringInput(input, "prompt")); prompt != "" {
|
|
return []chatMessage{{Role: "user", Content: prompt}}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func emitError(ctx context.Context, sink runtime.EventSink, runID, msg string) error {
|
|
return sink.Emit(ctx, runtime.RuntimeEvent{
|
|
RunID: runID,
|
|
Type: runtime.EventTypeError,
|
|
Error: msg,
|
|
Timestamp: time.Now(),
|
|
})
|
|
}
|
|
|
|
func stringInput(input map[string]any, key string) string {
|
|
if input == nil {
|
|
return ""
|
|
}
|
|
if v, ok := input[key].(string); ok {
|
|
return v
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func anySlice(v any) []any {
|
|
if items, ok := v.([]any); ok {
|
|
return items
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// joinOpenAIPath appends an OpenAI-compatible path to the endpoint. When the
|
|
// endpoint already targets the /v1 root it strips the leading /v1 from the path
|
|
// so a configured ".../v1" endpoint does not produce ".../v1/v1/...".
|
|
func joinOpenAIPath(baseURL, path string) string {
|
|
u, err := url.Parse(baseURL)
|
|
if err != nil {
|
|
return strings.TrimRight(baseURL, "/") + path
|
|
}
|
|
basePath := strings.TrimRight(u.Path, "/")
|
|
if strings.HasSuffix(basePath, "/v1") && strings.HasPrefix(path, "/v1/") {
|
|
path = strings.TrimPrefix(path, "/v1")
|
|
}
|
|
u.Path = basePath + path
|
|
return u.String()
|
|
}
|
|
|
|
func readLimited(r io.Reader, limit int64) string {
|
|
b, _ := io.ReadAll(io.LimitReader(r, limit))
|
|
return strings.TrimSpace(string(b))
|
|
}
|
|
|
|
type chatMessage struct {
|
|
Role string `json:"role"`
|
|
Content string `json:"content"`
|
|
ToolCalls []any `json:"tool_calls,omitempty"`
|
|
ToolCallID string `json:"tool_call_id,omitempty"`
|
|
}
|
|
|
|
type chatChunk struct {
|
|
Choices []struct {
|
|
Delta chatChunkDelta `json:"delta"`
|
|
FinishReason *string `json:"finish_reason"`
|
|
} `json:"choices"`
|
|
Usage *struct {
|
|
PromptTokens int `json:"prompt_tokens"`
|
|
CompletionTokens int `json:"completion_tokens"`
|
|
} `json:"usage"`
|
|
}
|
|
|
|
type chatChunkDelta struct {
|
|
Content string `json:"content"`
|
|
ReasoningContent string `json:"reasoning_content"`
|
|
Reasoning string `json:"reasoning"`
|
|
ToolCalls []any `json:"tool_calls"`
|
|
}
|
|
|
|
func (d chatChunkDelta) ReasoningText() string {
|
|
if d.ReasoningContent != "" {
|
|
return d.ReasoningContent
|
|
}
|
|
return d.Reasoning
|
|
}
|
|
|
|
type openAIToolCallAccumulator struct {
|
|
calls map[int]map[string]any
|
|
order []int
|
|
}
|
|
|
|
func (a *openAIToolCallAccumulator) AddDelta(raw []any) {
|
|
for fallbackIndex, item := range raw {
|
|
call, ok := item.(map[string]any)
|
|
if !ok {
|
|
continue
|
|
}
|
|
index := fallbackIndex
|
|
if parsed, ok := numericIndex(call["index"]); ok {
|
|
index = parsed
|
|
}
|
|
dst := a.ensure(index)
|
|
for key, value := range call {
|
|
switch key {
|
|
case "index":
|
|
continue
|
|
case "function":
|
|
mergeToolCallFunction(dst, value)
|
|
default:
|
|
if !emptyToolCallValue(value) {
|
|
dst[key] = value
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (a *openAIToolCallAccumulator) ToolCalls() []any {
|
|
if len(a.order) == 0 {
|
|
return nil
|
|
}
|
|
out := make([]any, 0, len(a.order))
|
|
for _, index := range a.order {
|
|
call := a.calls[index]
|
|
if len(call) == 0 {
|
|
continue
|
|
}
|
|
copyCall := make(map[string]any, len(call))
|
|
for key, value := range call {
|
|
copyCall[key] = value
|
|
}
|
|
out = append(out, copyCall)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func (a *openAIToolCallAccumulator) ensure(index int) map[string]any {
|
|
if a.calls == nil {
|
|
a.calls = make(map[int]map[string]any)
|
|
}
|
|
if call, ok := a.calls[index]; ok {
|
|
return call
|
|
}
|
|
call := make(map[string]any)
|
|
a.calls[index] = call
|
|
a.order = append(a.order, index)
|
|
return call
|
|
}
|
|
|
|
func mergeToolCallFunction(call map[string]any, value any) {
|
|
fn, ok := value.(map[string]any)
|
|
if !ok {
|
|
return
|
|
}
|
|
current, _ := call["function"].(map[string]any)
|
|
if current == nil {
|
|
current = make(map[string]any)
|
|
call["function"] = current
|
|
}
|
|
for key, item := range fn {
|
|
if key == "arguments" {
|
|
if part, ok := item.(string); ok {
|
|
current["arguments"] = currentString(current["arguments"]) + part
|
|
continue
|
|
}
|
|
}
|
|
if !emptyToolCallValue(item) {
|
|
current[key] = item
|
|
}
|
|
}
|
|
}
|
|
|
|
func numericIndex(value any) (int, bool) {
|
|
switch v := value.(type) {
|
|
case int:
|
|
return v, true
|
|
case float64:
|
|
return int(v), true
|
|
default:
|
|
return 0, false
|
|
}
|
|
}
|
|
|
|
func emptyToolCallValue(value any) bool {
|
|
if value == nil {
|
|
return true
|
|
}
|
|
if text, ok := value.(string); ok {
|
|
return text == ""
|
|
}
|
|
return false
|
|
}
|
|
|
|
func currentString(value any) string {
|
|
if text, ok := value.(string); ok {
|
|
return text
|
|
}
|
|
return ""
|
|
}
|
|
|
|
type modelsResponse struct {
|
|
Data []struct {
|
|
ID string `json:"id"`
|
|
} `json:"data"`
|
|
}
|