428 lines
12 KiB
Go
428 lines
12 KiB
Go
package openai_compat
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"strings"
|
|
"time"
|
|
|
|
"iop/apps/node/internal/runtime"
|
|
)
|
|
|
|
func (a *Adapter) applyHeaders(req *http.Request, jsonBody bool) {
|
|
for k, v := range a.headers {
|
|
if jsonBody && http.CanonicalHeaderKey(k) == "Content-Type" {
|
|
continue
|
|
}
|
|
req.Header.Set(k, v)
|
|
}
|
|
if jsonBody {
|
|
req.Header.Set("Content-Type", "application/json")
|
|
}
|
|
}
|
|
|
|
func (a *Adapter) doChatCompletion(ctx context.Context, body []byte) (*http.Response, error) {
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, joinOpenAIPath(a.endpoint, "/v1/chat/completions"), bytes.NewReader(body))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("build request: %w", err)
|
|
}
|
|
a.applyHeaders(req, true)
|
|
return a.client.Do(req)
|
|
}
|
|
|
|
func (a *Adapter) buildRequestBody(model string, messages []chatMessage, input map[string]any) (map[string]any, error) {
|
|
body := make(map[string]any)
|
|
// Copy caller options as top-level OpenAI-compatible request fields first so
|
|
// adapter-owned fields below always win over them.
|
|
if opts, ok := input["options"].(map[string]any); ok {
|
|
for k, v := range opts {
|
|
body[k] = v
|
|
}
|
|
}
|
|
// Pass through optional OpenAI-compatible fields when present in the input (except think).
|
|
for _, key := range []string{"tools", "tool_choice", "format", "keep_alive"} {
|
|
if v, ok := input[key]; ok {
|
|
body[key] = v
|
|
}
|
|
}
|
|
body["model"] = model
|
|
body["messages"] = messages
|
|
body["stream"] = true
|
|
|
|
// Extract think-control fields
|
|
hasThink := false
|
|
var thinkVal bool
|
|
if v, ok := input["think"]; ok {
|
|
if bv, ok := v.(bool); ok {
|
|
hasThink = true
|
|
thinkVal = bv
|
|
}
|
|
}
|
|
|
|
hasReasoningEffort := false
|
|
var reasoningEffortVal string
|
|
if v, ok := input["reasoning_effort"]; ok {
|
|
if sv, ok := v.(string); ok {
|
|
hasReasoningEffort = true
|
|
reasoningEffortVal = sv
|
|
}
|
|
}
|
|
|
|
hasBudget := false
|
|
var budgetVal any
|
|
if v, ok := input["thinking_token_budget"]; ok {
|
|
hasBudget = true
|
|
budgetVal = v
|
|
}
|
|
|
|
// vLLM, vLLM-MLX and Lemonade all serve Qwen3-family templates that honor
|
|
// chat_template_kwargs.enable_thinking / thinking_token_budget. The Lemonade
|
|
// build ignores the top-level `think` field, so all three providers map
|
|
// think-control into chat_template_kwargs rather than top-level fields.
|
|
switch a.provider {
|
|
case "vllm", "vllm-mlx", "lemonade":
|
|
thinkDisableRequested := (hasThink && !thinkVal) || (hasReasoningEffort && reasoningEffortVal == "none")
|
|
|
|
// The vLLM-MLX runtime labels its entire streamed output as
|
|
// reasoning_content and keeps generating reasoning even with
|
|
// enable_thinking=false, so a think-disable request can never yield a
|
|
// reasoning-free stream. Fail with a clear compatibility error instead
|
|
// of silently returning a reasoning stream for think=false.
|
|
if a.provider == "vllm-mlx" && thinkDisableRequested {
|
|
return nil, fmt.Errorf("unsupported think control: provider %q cannot disable reasoning generation for streaming responses (think=false / reasoning_effort=none)", a.provider)
|
|
}
|
|
|
|
if hasReasoningEffort && reasoningEffortVal != "" && reasoningEffortVal != "none" {
|
|
return nil, fmt.Errorf("unsupported think control: reasoning_effort %q is not supported by %s", reasoningEffortVal, a.provider)
|
|
}
|
|
|
|
if hasBudget {
|
|
var budgetInt int64
|
|
valid := false
|
|
switch bv := budgetVal.(type) {
|
|
case int:
|
|
budgetInt = int64(bv)
|
|
valid = true
|
|
case int64:
|
|
budgetInt = bv
|
|
valid = true
|
|
case float64:
|
|
if bv == float64(int64(bv)) {
|
|
budgetInt = int64(bv)
|
|
valid = true
|
|
}
|
|
}
|
|
if !valid || budgetInt < 0 {
|
|
return nil, fmt.Errorf("unsupported think control: invalid thinking_token_budget %v", budgetVal)
|
|
}
|
|
}
|
|
|
|
if hasThink || (hasReasoningEffort && reasoningEffortVal == "none") || hasBudget {
|
|
var chatTemplateKwargs map[string]any
|
|
if ctk, ok := body["chat_template_kwargs"].(map[string]any); ok {
|
|
chatTemplateKwargs = make(map[string]any)
|
|
for k, v := range ctk {
|
|
chatTemplateKwargs[k] = v
|
|
}
|
|
} else {
|
|
chatTemplateKwargs = make(map[string]any)
|
|
}
|
|
|
|
if (hasThink && !thinkVal) || (hasReasoningEffort && reasoningEffortVal == "none") {
|
|
chatTemplateKwargs["enable_thinking"] = false
|
|
} else if (hasThink && thinkVal) || hasBudget {
|
|
chatTemplateKwargs["enable_thinking"] = true
|
|
}
|
|
|
|
if hasBudget {
|
|
chatTemplateKwargs["thinking_token_budget"] = budgetVal
|
|
}
|
|
|
|
body["chat_template_kwargs"] = chatTemplateKwargs
|
|
}
|
|
default:
|
|
// empty/unknown provider: pass requested fields through as-is.
|
|
if hasThink {
|
|
body["think"] = thinkVal
|
|
}
|
|
if hasReasoningEffort {
|
|
body["reasoning_effort"] = reasoningEffortVal
|
|
}
|
|
if hasBudget {
|
|
body["thinking_token_budget"] = budgetVal
|
|
}
|
|
}
|
|
|
|
return body, nil
|
|
}
|
|
|
|
func retryBodyWithForcedSingleTool(body map[string]any, errorBody string) (map[string]any, bool) {
|
|
if !isAutoToolChoiceUnsupportedError(errorBody) {
|
|
return nil, false
|
|
}
|
|
forced, ok := forcedToolChoiceForSingleTool(body["tools"])
|
|
if !ok {
|
|
return nil, false
|
|
}
|
|
next := make(map[string]any, len(body)+1)
|
|
for key, value := range body {
|
|
next[key] = value
|
|
}
|
|
next["tool_choice"] = forced
|
|
return next, true
|
|
}
|
|
|
|
func isAutoToolChoiceUnsupportedError(errorBody string) bool {
|
|
errorBody = strings.ToLower(errorBody)
|
|
return strings.Contains(errorBody, "auto") &&
|
|
strings.Contains(errorBody, "tool choice") &&
|
|
isToolCallingUnsupportedError(errorBody)
|
|
}
|
|
|
|
func isToolCallingUnsupportedError(errorBody string) bool {
|
|
errorBody = strings.ToLower(errorBody)
|
|
return strings.Contains(errorBody, "enable-auto-tool-choice") ||
|
|
strings.Contains(errorBody, "tool-call-parser")
|
|
}
|
|
|
|
func forcedToolChoiceForSingleTool(tools any) (map[string]any, bool) {
|
|
items, ok := tools.([]any)
|
|
if !ok || len(items) != 1 {
|
|
return nil, false
|
|
}
|
|
tool, ok := items[0].(map[string]any)
|
|
if !ok {
|
|
return nil, false
|
|
}
|
|
name := ""
|
|
if fn, ok := tool["function"].(map[string]any); ok {
|
|
name, _ = fn["name"].(string)
|
|
}
|
|
if name == "" {
|
|
name, _ = tool["name"].(string)
|
|
}
|
|
name = strings.TrimSpace(name)
|
|
if name == "" {
|
|
return nil, false
|
|
}
|
|
return map[string]any{
|
|
"type": "function",
|
|
"function": map[string]any{
|
|
"name": name,
|
|
},
|
|
}, true
|
|
}
|
|
|
|
func retryBodyWithTextToolFallback(body map[string]any, errorBody string) (map[string]any, bool) {
|
|
if !isToolCallingUnsupportedError(errorBody) {
|
|
return nil, false
|
|
}
|
|
instruction, ok := textToolFallbackInstruction(body["tools"])
|
|
if !ok {
|
|
return nil, false
|
|
}
|
|
messages, ok := prependTextToolFallbackInstruction(body["messages"], instruction)
|
|
if !ok {
|
|
return nil, false
|
|
}
|
|
next := make(map[string]any, len(body)+1)
|
|
for key, value := range body {
|
|
next[key] = value
|
|
}
|
|
delete(next, "tools")
|
|
delete(next, "tool_choice")
|
|
next["messages"] = messages
|
|
return next, true
|
|
}
|
|
|
|
func prependTextToolFallbackInstruction(value any, instruction string) (any, bool) {
|
|
switch messages := value.(type) {
|
|
case []chatMessage:
|
|
system := chatMessage{Role: "system", Content: instruction}
|
|
out := make([]chatMessage, 0, len(messages)+1)
|
|
for _, msg := range messages {
|
|
if strings.EqualFold(strings.TrimSpace(msg.Role), "system") {
|
|
system.Content = joinTextToolFallbackSystemContent(system.Content, msg.Content)
|
|
continue
|
|
}
|
|
out = append(out, msg)
|
|
}
|
|
return append([]chatMessage{system}, out...), true
|
|
case []any:
|
|
systemContent := instruction
|
|
out := make([]any, 0, len(messages)+1)
|
|
for _, item := range messages {
|
|
msg, ok := item.(map[string]any)
|
|
if !ok {
|
|
out = append(out, item)
|
|
continue
|
|
}
|
|
role, _ := msg["role"].(string)
|
|
if strings.EqualFold(strings.TrimSpace(role), "system") {
|
|
systemContent = joinTextToolFallbackSystemContent(systemContent, stringFromAny(msg["content"]))
|
|
continue
|
|
}
|
|
out = append(out, item)
|
|
}
|
|
return append([]any{map[string]any{"role": "system", "content": systemContent}}, out...), true
|
|
default:
|
|
return nil, false
|
|
}
|
|
}
|
|
|
|
func joinTextToolFallbackSystemContent(first, next string) string {
|
|
first = strings.TrimSpace(first)
|
|
next = strings.TrimSpace(next)
|
|
switch {
|
|
case first == "":
|
|
return next
|
|
case next == "":
|
|
return first
|
|
default:
|
|
return first + "\n\n" + next
|
|
}
|
|
}
|
|
|
|
func stringFromAny(value any) string {
|
|
switch v := value.(type) {
|
|
case nil:
|
|
return ""
|
|
case string:
|
|
return v
|
|
default:
|
|
encoded, _ := json.Marshal(v)
|
|
return string(encoded)
|
|
}
|
|
}
|
|
|
|
func textToolFallbackInstruction(tools any) (string, bool) {
|
|
items, ok := anyItems(tools)
|
|
if !ok || len(items) == 0 {
|
|
return "", false
|
|
}
|
|
encoded, err := json.Marshal(items)
|
|
if err != nil {
|
|
return "", false
|
|
}
|
|
return "Tool calls must be emitted as plain text because this backend does not support native OpenAI tool calling. When a tool is needed, respond with exactly one tool call and no markdown:\n" +
|
|
"<tool_call>\n<function=TOOL_NAME>\n<parameter=PARAMETER_NAME>JSON_VALUE</parameter>\n</function>\n</tool_call>\n" +
|
|
"run_commands executes from the client workspace root. Do not prepend cd to an absolute workspace path unless the user explicitly asks to operate in a different directory; prefer current-workspace commands such as git status.\n" +
|
|
"Use valid JSON for each parameter value and follow the supplied parameter schema. Available tools JSON: " + string(encoded), true
|
|
}
|
|
|
|
func anyItems(value any) ([]any, bool) {
|
|
switch v := value.(type) {
|
|
case []any:
|
|
return v, true
|
|
default:
|
|
encoded, err := json.Marshal(v)
|
|
if err != nil {
|
|
return nil, false
|
|
}
|
|
var out []any
|
|
if err := json.Unmarshal(encoded, &out); err != nil {
|
|
return nil, false
|
|
}
|
|
return out, true
|
|
}
|
|
}
|
|
|
|
func completeEvent(runID, finishReason string, usage *runtime.UsageStats, outputTokens int, toolCalls []any, textToolFallback bool) runtime.RuntimeEvent {
|
|
if usage == nil {
|
|
usage = &runtime.UsageStats{OutputTokens: outputTokens}
|
|
}
|
|
var metadata map[string]string
|
|
if finishReason != "" {
|
|
metadata = map[string]string{"finish_reason": finishReason}
|
|
}
|
|
if textToolFallback {
|
|
if metadata == nil {
|
|
metadata = make(map[string]string, 2)
|
|
}
|
|
metadata[runtimeMetadataOpenAITextToolFallback] = "true"
|
|
}
|
|
if len(toolCalls) > 0 {
|
|
if metadata == nil {
|
|
metadata = make(map[string]string, 2)
|
|
}
|
|
if finishReason == "" {
|
|
metadata["finish_reason"] = "tool_calls"
|
|
}
|
|
if encoded, err := json.Marshal(toolCalls); err == nil {
|
|
metadata[runtimeMetadataOpenAIToolCalls] = string(encoded)
|
|
}
|
|
}
|
|
return runtime.RuntimeEvent{
|
|
RunID: runID,
|
|
Type: runtime.EventTypeComplete,
|
|
Message: "openai_compat chat complete",
|
|
Usage: usage,
|
|
Metadata: metadata,
|
|
Timestamp: time.Now(),
|
|
}
|
|
}
|
|
|
|
// messagesFromInput extracts chat messages from the execution input.
|
|
func messagesFromInput(input map[string]any) []chatMessage {
|
|
if raw, ok := input["messages"].([]any); ok {
|
|
out := make([]chatMessage, 0, len(raw))
|
|
for _, item := range raw {
|
|
m, ok := item.(map[string]any)
|
|
if !ok {
|
|
continue
|
|
}
|
|
role, _ := m["role"].(string)
|
|
content, _ := m["content"].(string)
|
|
toolCallID, _ := m["tool_call_id"].(string)
|
|
toolCalls := anySlice(m["tool_calls"])
|
|
role = strings.TrimSpace(role)
|
|
content = strings.TrimSpace(content)
|
|
toolCallID = strings.TrimSpace(toolCallID)
|
|
if role == "" || (content == "" && len(toolCalls) == 0) {
|
|
continue
|
|
}
|
|
out = append(out, chatMessage{
|
|
Role: role,
|
|
Content: content,
|
|
ToolCalls: toolCalls,
|
|
ToolCallID: toolCallID,
|
|
})
|
|
}
|
|
if len(out) > 0 {
|
|
return out
|
|
}
|
|
}
|
|
if prompt := strings.TrimSpace(stringInput(input, "prompt")); prompt != "" {
|
|
return []chatMessage{{Role: "user", Content: prompt}}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func emitError(ctx context.Context, sink runtime.EventSink, runID, msg string) error {
|
|
return sink.Emit(ctx, runtime.RuntimeEvent{
|
|
RunID: runID,
|
|
Type: runtime.EventTypeError,
|
|
Error: msg,
|
|
Timestamp: time.Now(),
|
|
})
|
|
}
|
|
|
|
func stringInput(input map[string]any, key string) string {
|
|
if input == nil {
|
|
return ""
|
|
}
|
|
if v, ok := input[key].(string); ok {
|
|
return v
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func anySlice(v any) []any {
|
|
if items, ok := v.([]any); ok {
|
|
return items
|
|
}
|
|
return nil
|
|
}
|