diff --git a/agent-contract/inner/edge-node-runtime-wire.md b/agent-contract/inner/edge-node-runtime-wire.md index ae0c78f..f4a2500 100644 --- a/agent-contract/inner/edge-node-runtime-wire.md +++ b/agent-contract/inner/edge-node-runtime-wire.md @@ -45,6 +45,7 @@ Edge는 Node 연결을 수락하고, Node는 연결 직후 등록 요청을 보 - `RunRequest.metadata`: caller-defined 실행 metadata다. workspace 자체는 별도 `workspace` 필드로 전달한다. - `RunEvent.type`: `start`, `delta`, `complete`, `error`, `cancelled` 같은 실행 이벤트 종류다. - `RunEvent.metadata["openai_tool_calls"]`: OpenAI-compatible provider adapter가 native `tool_calls`를 반환했을 때 완료 이벤트에 싣는 JSON 배열이다. Edge OpenAI-compatible 표면은 이 값을 `message.tool_calls` 또는 stream `delta.tool_calls`로 복원한다. provider assistant content 텍스트를 이 값으로 파싱/합성하지 않는다. +- `RunEvent.metadata["openai_text_tool_fallback"]`: OpenAI-compatible provider adapter가 backend native tool API 거부 후 `tools`/`tool_choice`를 제거하고 text tool-call instruction으로 재시도했을 때 `"true"`를 싣는다. Edge는 이 표시가 있는 실행에서만 assistant content의 text tool-call을 OpenAI-compatible `tool_calls`로 복원할 수 있다. - `NodeCommandRequest.type`: 실행이 아닌 조회/제어성 명령이다. adapter execution 요청과 섞지 않는다. - `NodeConfigPayload.adapters`: Edge가 Node에 내려주는 adapter instance 설정이다. - `AdapterConfig.name`: node 내부 stable adapter instance identity다. 비어 있으면 legacy single-instance type 이름과 동등하다. diff --git a/agent-contract/outer/openai-compatible-api.md b/agent-contract/outer/openai-compatible-api.md index a8718dc..047dff9 100644 --- a/agent-contract/outer/openai-compatible-api.md +++ b/agent-contract/outer/openai-compatible-api.md @@ -165,10 +165,10 @@ Workspace-bound route는 workspace가 없거나 상대 경로이면 OpenAI-compa - `parallel_tool_calls` - `stream_options` -`tools`가 있는 Chat Completions 요청에서 provider route(`openai_compat`, `vllm`, `ollama`, provider pool)는 forced tool 선택 객체와 `"none"` 같은 명시적 `tool_choice`를 backend에 전달한다. 단, `"auto"`는 OpenAI-compatible 기본값과 같으므로 provider request에서는 생략한다. 일부 vLLM 계열 backend는 explicit/default `"auto"`를 `--enable-auto-tool-choice`/`--tool-call-parser` 없이 400으로 거부한다. 이 400이 발생하고 요청 tool이 정확히 1개이면 Node adapter는 해당 tool에 대한 forced `tool_choice`로 1회 재시도한다. 여러 tool이면 임의 선택하지 않는다. +`tools`가 있는 Chat Completions 요청에서 provider route(`openai_compat`, `vllm`, `ollama`, provider pool)는 forced tool 선택 객체와 `"none"` 같은 명시적 `tool_choice`를 backend에 전달한다. 단, `"auto"`는 OpenAI-compatible 기본값과 같으므로 provider request에서는 생략한다. 일부 vLLM 계열 backend는 explicit/default `"auto"`를 `--enable-auto-tool-choice`/`--tool-call-parser` 없이 400으로 거부한다. 이 400이 발생하고 요청 tool이 정확히 1개이면 Node adapter는 해당 tool에 대한 forced `tool_choice`로 1회 재시도한다. forced tool도 `--tool-call-parser` 요구로 거부되거나 여러 tool이라 forced를 고를 수 없으면, Node adapter는 `tools`/`tool_choice`를 제거하고 text tool-call system instruction을 붙여 1회 재시도하며 완료 metadata에 `openai_text_tool_fallback: "true"`를 싣는다. provider가 native OpenAI-compatible `tool_calls`를 반환하면 Node는 내부 `RunEvent.metadata["openai_tool_calls"]` JSON으로 보존하고, Edge는 이를 OpenAI-compatible `message.tool_calls` 또는 stream `delta.tool_calls`로 반환하며 `finish_reason: "tool_calls"`를 사용한다. provider native `tool_calls[].function.arguments`는 OpenAI 계약에 맞는 JSON string으로 반환한다. 단, provider가 요청 `tools[].function.parameters` schema상 배열/객체여야 하는 값을 JSON 문자열로 이중 인코딩한 경우 Edge는 해당 `arguments` JSON만 schema 기준으로 복원해 다시 JSON string으로 직렬화한다. -provider route에서 assistant content에 ``, `{{function(...)}}`, `[Calling tool: ...]` 같은 텍스트가 들어와도 Edge는 이를 파싱하거나 OpenAI `tool_calls`로 합성하지 않는다. 해당 텍스트는 backend가 반환한 content로 그대로 둔다. +provider route에서 assistant content에 ``, `{{function(...)}}`, `[Calling tool: ...]` 같은 텍스트가 들어와도 Edge는 기본적으로 이를 파싱하거나 OpenAI `tool_calls`로 합성하지 않는다. 해당 텍스트는 backend가 반환한 content로 그대로 둔다. 예외적으로 Node adapter가 native tool API unsupported fallback을 수행해 `openai_text_tool_fallback: "true"` metadata를 붙인 실행에 한해서만 Edge가 text tool-call을 OpenAI `tool_calls`로 복원한다. CLI route(`adapter: "cli"`)는 native backend tool calling이 없으므로, 호환 fallback으로만 assistant content의 Cline-style 텍스트 tool call 블록을 OpenAI-compatible `tool_calls`로 합성할 수 있다. 이 fallback이 지원하는 텍스트 블록의 최소 형태는 `>>JSON-or-text` 또는 `{{function_name(key=Python/JSON-like-literal)}}`이다. CLI fallback에서 텍스트 tool call을 구조화할 때만 Edge는 요청의 `tools[].function.parameters` schema를 기준으로 arguments를 정규화한다. 예를 들어 tool schema가 `commands: string[]`만 허용하면 command 객체 입력도 shell string 배열로 접고, `commands: {command,args}[]`를 허용하면 shell 문법이 없는 명령을 structured argv로 만든다. CLI fallback에서 schema에 없는 UI 설명용 `description`이나 실행 위치 힌트용 `runInTerminal`은 command 객체와 최상위 args에서 제거한다. 단, `cd`, `command -v`, `&&`, pipe, redirect, quote 등 shell 해석이 필요한 명령은 schema가 허용할 때 `commands: ["cd /work && git status"]` 같은 shell string으로 유지한다. diff --git a/apps/edge/internal/openai/chat_handler.go b/apps/edge/internal/openai/chat_handler.go index 20b717c..290d262 100644 --- a/apps/edge/internal/openai/chat_handler.go +++ b/apps/edge/internal/openai/chat_handler.go @@ -154,12 +154,12 @@ func routeSupportsNativeToolCalls(dispatch routeDispatch) bool { } } -func shouldSynthesizeTextToolCalls(dispatch edgeservice.RunDispatch) bool { - return strings.TrimSpace(dispatch.Adapter) == "cli" +func shouldSynthesizeTextToolCalls(dispatch edgeservice.RunDispatch, textToolFallback bool) bool { + return textToolFallback || strings.TrimSpace(dispatch.Adapter) == "cli" } func (s *Server) completeChatCompletion(w http.ResponseWriter, r *http.Request, req chatCompletionRequest, handle edgeservice.RunResult, outputPolicy strictOutputPolicy) { - text, reasoning, finishReason, nativeToolCalls, usage, err := collectRunResult(r.Context(), handle.Stream(), handle.WaitTimeout()) + text, reasoning, finishReason, nativeToolCalls, usage, textToolFallback, err := collectRunResult(r.Context(), handle.Stream(), handle.WaitTimeout()) if err != nil { writeError(w, httpStatusForRunError(err), "run_error", err.Error()) return @@ -170,7 +170,7 @@ func (s *Server) completeChatCompletion(w http.ResponseWriter, r *http.Request, nativeToolCalls = normalizeNativeToolCallsForResponse(nativeToolCalls, req.Tools) message.ToolCalls = nativeToolCalls finishReason = "tool_calls" - } else if shouldSynthesizeTextToolCalls(handle.Dispatch()) { + } else if shouldSynthesizeTextToolCalls(handle.Dispatch(), textToolFallback) { cleaned, toolCalls := synthesizeToolCallsFromText(text, req.Tools, handle.Dispatch().RunID) if len(toolCalls) > 0 { message.Content = cleaned diff --git a/apps/edge/internal/openai/responses_handler.go b/apps/edge/internal/openai/responses_handler.go index 470550f..2d3151b 100644 --- a/apps/edge/internal/openai/responses_handler.go +++ b/apps/edge/internal/openai/responses_handler.go @@ -140,7 +140,7 @@ func decodeResponsesRequest(dec *json.Decoder, req *responsesRequest) error { } func (s *Server) completeResponse(w http.ResponseWriter, r *http.Request, req responsesRequest, handle edgeservice.RunResult, outputPolicy strictOutputPolicy) { - text, reasoning, _, _, usage, err := collectRunResult(r.Context(), handle.Stream(), handle.WaitTimeout()) + text, reasoning, _, _, usage, _, err := collectRunResult(r.Context(), handle.Stream(), handle.WaitTimeout()) if err != nil { writeError(w, httpStatusForRunError(err), "run_error", err.Error()) return diff --git a/apps/edge/internal/openai/run_result.go b/apps/edge/internal/openai/run_result.go index 2faeff2..0cf866f 100644 --- a/apps/edge/internal/openai/run_result.go +++ b/apps/edge/internal/openai/run_result.go @@ -10,11 +10,14 @@ import ( edgeservice "iop/apps/edge/internal/service" ) -const runtimeMetadataOpenAIToolCalls = "openai_tool_calls" +const ( + runtimeMetadataOpenAIToolCalls = "openai_tool_calls" + runtimeMetadataOpenAITextToolFallback = "openai_text_tool_fallback" +) -func collectRunResult(ctx context.Context, stream edgeservice.RunStream, timeout time.Duration) (string, string, string, []any, *openAIUsage, error) { +func collectRunResult(ctx context.Context, stream edgeservice.RunStream, timeout time.Duration) (string, string, string, []any, *openAIUsage, bool, error) { if stream.Events == nil { - return "", "", "", nil, nil, fmt.Errorf("run stream unavailable") + return "", "", "", nil, nil, false, fmt.Errorf("run stream unavailable") } var contentBuilder strings.Builder var reasoningBuilder strings.Builder @@ -25,20 +28,20 @@ func collectRunResult(ctx context.Context, stream edgeservice.RunStream, timeout for { select { case <-ctx.Done(): - return "", "", "", nil, nil, ctx.Err() + return "", "", "", nil, nil, false, ctx.Err() case <-timer.C: - return "", "", "", nil, nil, fmt.Errorf("run timed out") + return "", "", "", nil, nil, false, fmt.Errorf("run timed out") case nodeEvent, ok := <-stream.NodeEvents: if !ok { stream.NodeEvents = nil continue } if edgeservice.IsNodeDisconnected(nodeEvent) { - return "", "", "", nil, nil, fmt.Errorf("node disconnected") + return "", "", "", nil, nil, false, fmt.Errorf("node disconnected") } case event, ok := <-stream.Events: if !ok { - return "", "", "", nil, nil, fmt.Errorf("run stream closed") + return "", "", "", nil, nil, false, fmt.Errorf("run stream closed") } if event == nil { continue @@ -63,7 +66,7 @@ func collectRunResult(ctx context.Context, stream edgeservice.RunStream, timeout TotalTokens: int(u.GetInputTokens() + u.GetOutputTokens()), } } - return contentBuilder.String(), reasoningBuilder.String(), finishReason, toolCalls, usage, nil + return contentBuilder.String(), reasoningBuilder.String(), finishReason, toolCalls, usage, isTextToolFallback(event.GetMetadata()), nil case "error", "cancelled": msg := event.GetError() if msg == "" { @@ -72,7 +75,7 @@ func collectRunResult(ctx context.Context, stream edgeservice.RunStream, timeout if msg == "" { msg = "run failed" } - return "", "", "", nil, nil, fmt.Errorf("%s", msg) + return "", "", "", nil, nil, false, fmt.Errorf("%s", msg) } } } @@ -92,3 +95,7 @@ func toolCallsFromRunMetadata(metadata map[string]string) []any { } return toolCalls } + +func isTextToolFallback(metadata map[string]string) bool { + return strings.EqualFold(strings.TrimSpace(metadata[runtimeMetadataOpenAITextToolFallback]), "true") +} diff --git a/apps/edge/internal/openai/server_test.go b/apps/edge/internal/openai/server_test.go index 3844ab6..f87eaab 100644 --- a/apps/edge/internal/openai/server_test.go +++ b/apps/edge/internal/openai/server_test.go @@ -464,6 +464,55 @@ func TestChatCompletionsDoesNotSynthesizeTextToolCallsForProviderRoute(t *testin } } +func TestChatCompletionsSynthesizesProviderTextToolCallsWhenFallbackMarked(t *testing.T) { + fake := &fakeRunService{events: make(chan *iop.RunEvent, 2)} + fake.events <- &iop.RunEvent{Type: "delta", Delta: "먼저 현재 git 상태를 확인하겠습니다.\n\n\n\n\n[\"cd /config/workspace/iop && git status\"]\n\n\n"} + fake.events <- &iop.RunEvent{ + Type: "complete", + Metadata: map[string]string{ + runtimeMetadataOpenAITextToolFallback: "true", + }, + } + + srv := NewServer(config.EdgeOpenAIConf{Adapter: "openai_compat"}, fake, nil) + req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ + "model":"qwen3.6:35b", + "messages":[{"role":"user","content":"status"}], + "tools":[{"type":"function","function":{"name":"run_commands","parameters":{"type":"object","properties":{"commands":{"type":"array","items":{"type":"string"}}},"required":["commands"]}}}], + "tool_choice":"auto" + }`)) + w := httptest.NewRecorder() + + srv.handleChatCompletions(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status: got %d body=%s", w.Code, w.Body.String()) + } + var resp chatCompletionResponse + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatalf("decode response: %v", err) + } + choice := resp.Choices[0] + if choice.FinishReason != "tool_calls" { + t.Fatalf("finish_reason: got %q, want tool_calls", choice.FinishReason) + } + if strings.Contains(choice.Message.Content, "") { + t.Fatalf("content still contains raw tool_call block: %q", choice.Message.Content) + } + if len(choice.Message.ToolCalls) != 1 { + t.Fatalf("tool_calls len: got %d", len(choice.Message.ToolCalls)) + } + call := choice.Message.ToolCalls[0].(map[string]any) + fn := call["function"].(map[string]any) + var args map[string][]string + if err := json.Unmarshal([]byte(fn["arguments"].(string)), &args); err != nil { + t.Fatalf("arguments JSON: %v", err) + } + if len(args["commands"]) != 1 || args["commands"][0] != "cd /config/workspace/iop && git status" { + t.Fatalf("commands args: %+v", args["commands"]) + } +} + func TestChatCompletionsPassesThroughNativeToolCallsFromProviderRoute(t *testing.T) { fake := &fakeRunService{events: make(chan *iop.RunEvent, 1)} fake.events <- &iop.RunEvent{ @@ -1869,7 +1918,7 @@ func TestCollectRunResultTimesOut(t *testing.T) { } ctx, cancel := context.WithTimeout(context.Background(), 1500*time.Millisecond) defer cancel() - _, _, _, _, _, err := collectRunResult(ctx, handle.Stream(), handle.WaitTimeout()) + _, _, _, _, _, _, err := collectRunResult(ctx, handle.Stream(), handle.WaitTimeout()) if err == nil { t.Fatal("expected timeout error") } @@ -2158,7 +2207,7 @@ func TestCollectRunResultFailsWhenEventStreamCloses(t *testing.T) { }, } - _, _, _, _, _, err := collectRunResult(context.Background(), handle.Stream(), handle.WaitTimeout()) + _, _, _, _, _, _, err := collectRunResult(context.Background(), handle.Stream(), handle.WaitTimeout()) if err == nil { t.Fatal("expected closed stream error") } diff --git a/apps/edge/internal/openai/stream.go b/apps/edge/internal/openai/stream.go index 61b1882..39f6389 100644 --- a/apps/edge/internal/openai/stream.go +++ b/apps/edge/internal/openai/stream.go @@ -153,7 +153,7 @@ func (s *Server) streamChatCompletion(w http.ResponseWriter, r *http.Request, re } func (s *Server) streamBufferedChatCompletion(w http.ResponseWriter, r *http.Request, handle edgeservice.RunResult, id string, created int64, model string, flusher http.Flusher, outputPolicy strictOutputPolicy, tools []any) { - text, reasoning, finishReason, nativeToolCalls, _, err := collectRunResult(r.Context(), handle.Stream(), handle.WaitTimeout()) + text, reasoning, finishReason, nativeToolCalls, _, textToolFallback, err := collectRunResult(r.Context(), handle.Stream(), handle.WaitTimeout()) if err != nil { writeSSEError(w, flusher, err.Error()) return @@ -195,7 +195,7 @@ func (s *Server) streamBufferedChatCompletion(w http.ResponseWriter, r *http.Req }}, }) finishReason = "tool_calls" - } else if shouldSynthesizeTextToolCalls(handle.Dispatch()) { + } else if shouldSynthesizeTextToolCalls(handle.Dispatch(), textToolFallback) { cleaned, toolCalls := synthesizeToolCallsFromText(text, tools, handle.Dispatch().RunID) if len(toolCalls) > 0 { if cleaned != "" { diff --git a/apps/node/internal/adapters/openai_compat/openai_compat.go b/apps/node/internal/adapters/openai_compat/openai_compat.go index b651e12..34452f0 100644 --- a/apps/node/internal/adapters/openai_compat/openai_compat.go +++ b/apps/node/internal/adapters/openai_compat/openai_compat.go @@ -25,7 +25,10 @@ import ( const Name = "openai_compat" -const runtimeMetadataOpenAIToolCalls = "openai_tool_calls" +const ( + runtimeMetadataOpenAIToolCalls = "openai_tool_calls" + runtimeMetadataOpenAITextToolFallback = "openai_text_tool_fallback" +) type Adapter struct { instanceName string @@ -127,6 +130,7 @@ func (a *Adapter) Execute(ctx context.Context, spec runtime.ExecutionSpec, sink if err != nil { return fmt.Errorf("openai_compat adapter: marshal request: %w", err) } + textToolFallback := false a.logger.Info("openai_compat adapter executing", zap.String("run_id", spec.RunID), @@ -159,6 +163,24 @@ func (a *Adapter) Execute(ctx context.Context, spec runtime.ExecutionSpec, sink msg = readLimited(resp.Body, 4096) _ = resp.Body.Close() } + if retryBody, ok := retryBodyWithTextToolFallback(reqBody, msg); ok { + body, err = json.Marshal(retryBody) + if err != nil { + _ = emitError(ctx, sink, spec.RunID, fmt.Sprintf("openai_compat text tool fallback marshal failed: %v", err)) + return fmt.Errorf("openai_compat adapter: text tool fallback marshal: %w", err) + } + resp, err = a.doChatCompletion(ctx, body) + if err != nil { + _ = emitError(ctx, sink, spec.RunID, fmt.Sprintf("openai_compat text tool fallback request failed: %v", err)) + return fmt.Errorf("openai_compat adapter: text tool fallback request: %w", err) + } + if resp.StatusCode >= 200 && resp.StatusCode < 300 { + textToolFallback = true + goto streamResponse + } + msg = readLimited(resp.Body, 4096) + _ = resp.Body.Close() + } if msg == "" { msg = resp.Status } @@ -180,7 +202,7 @@ streamResponse: } payload := strings.TrimPrefix(line, "data: ") if payload == "[DONE]" { - return sink.Emit(ctx, completeEvent(spec.RunID, finishReason, usage, outputTokens, toolCalls.ToolCalls())) + return sink.Emit(ctx, completeEvent(spec.RunID, finishReason, usage, outputTokens, toolCalls.ToolCalls(), textToolFallback)) } var chunk chatChunk if err := json.Unmarshal([]byte(payload), &chunk); err != nil { @@ -376,7 +398,13 @@ func isAutoToolChoiceUnsupportedError(errorBody string) bool { errorBody = strings.ToLower(errorBody) return strings.Contains(errorBody, "auto") && strings.Contains(errorBody, "tool choice") && - (strings.Contains(errorBody, "enable-auto-tool-choice") || strings.Contains(errorBody, "tool-call-parser")) + isToolCallingUnsupportedError(errorBody) +} + +func isToolCallingUnsupportedError(errorBody string) bool { + errorBody = strings.ToLower(errorBody) + return strings.Contains(errorBody, "enable-auto-tool-choice") || + strings.Contains(errorBody, "tool-call-parser") } func forcedToolChoiceForSingleTool(tools any) (map[string]any, bool) { @@ -407,7 +435,77 @@ func forcedToolChoiceForSingleTool(tools any) (map[string]any, bool) { }, true } -func completeEvent(runID, finishReason string, usage *runtime.UsageStats, outputTokens int, toolCalls []any) runtime.RuntimeEvent { +func retryBodyWithTextToolFallback(body map[string]any, errorBody string) (map[string]any, bool) { + if !isToolCallingUnsupportedError(errorBody) { + return nil, false + } + instruction, ok := textToolFallbackInstruction(body["tools"]) + if !ok { + return nil, false + } + messages, ok := prependTextToolFallbackInstruction(body["messages"], instruction) + if !ok { + return nil, false + } + next := make(map[string]any, len(body)+1) + for key, value := range body { + next[key] = value + } + delete(next, "tools") + delete(next, "tool_choice") + next["messages"] = messages + return next, true +} + +func prependTextToolFallbackInstruction(value any, instruction string) (any, bool) { + switch messages := value.(type) { + case []chatMessage: + out := make([]chatMessage, 0, len(messages)+1) + out = append(out, chatMessage{Role: "system", Content: instruction}) + out = append(out, messages...) + return out, true + case []any: + out := make([]any, 0, len(messages)+1) + out = append(out, map[string]any{"role": "system", "content": instruction}) + out = append(out, messages...) + return out, true + default: + return nil, false + } +} + +func textToolFallbackInstruction(tools any) (string, bool) { + items, ok := anyItems(tools) + if !ok || len(items) == 0 { + return "", false + } + encoded, err := json.Marshal(items) + if err != nil { + return "", false + } + return "Tool calls must be emitted as plain text because this backend does not support native OpenAI tool calling. When a tool is needed, respond with exactly one tool call and no markdown:\n" + + "\n\nJSON_VALUE\n\n\n" + + "Use valid JSON for each parameter value and follow the supplied parameter schema. Available tools JSON: " + string(encoded), true +} + +func anyItems(value any) ([]any, bool) { + switch v := value.(type) { + case []any: + return v, true + default: + encoded, err := json.Marshal(v) + if err != nil { + return nil, false + } + var out []any + if err := json.Unmarshal(encoded, &out); err != nil { + return nil, false + } + return out, true + } +} + +func completeEvent(runID, finishReason string, usage *runtime.UsageStats, outputTokens int, toolCalls []any, textToolFallback bool) runtime.RuntimeEvent { if usage == nil { usage = &runtime.UsageStats{OutputTokens: outputTokens} } @@ -415,6 +513,12 @@ func completeEvent(runID, finishReason string, usage *runtime.UsageStats, output if finishReason != "" { metadata = map[string]string{"finish_reason": finishReason} } + if textToolFallback { + if metadata == nil { + metadata = make(map[string]string, 2) + } + metadata[runtimeMetadataOpenAITextToolFallback] = "true" + } if len(toolCalls) > 0 { if metadata == nil { metadata = make(map[string]string, 2) diff --git a/apps/node/internal/adapters/openai_compat/openai_compat_test.go b/apps/node/internal/adapters/openai_compat/openai_compat_test.go index f9a534f..7289689 100644 --- a/apps/node/internal/adapters/openai_compat/openai_compat_test.go +++ b/apps/node/internal/adapters/openai_compat/openai_compat_test.go @@ -370,6 +370,74 @@ func TestOpenAICompatExecuteRetriesSingleToolWhenAutoUnsupported(t *testing.T) { } } +func TestOpenAICompatExecuteFallsBackToTextToolsWhenNativeToolsUnsupported(t *testing.T) { + attempts := 0 + var fallbackBody map[string]any + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + attempts++ + switch attempts { + case 1: + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"error":{"message":"\"auto\" tool choice requires --enable-auto-tool-choice and --tool-call-parser to be set"}}`)) + return + case 2: + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"error":{"message":"tool_choice=\"function=ChatCompletionNamedFunction(name='run_commands') type='function'\" requires --tool-call-parser to be set"}}`)) + return + } + if err := json.NewDecoder(r.Body).Decode(&fallbackBody); err != nil { + t.Fatalf("decode fallback: %v", err) + } + w.Header().Set("Content-Type", "text/event-stream") + _, _ = fmt.Fprintf(w, "data: %s\n\n", `{"choices":[{"delta":{"content":"\n\n[\"git status\"]\n\n"}}]}`) + _, _ = fmt.Fprintf(w, "data: [DONE]\n\n") + })) + defer server.Close() + + adapter := New(config.OpenAICompatConf{Endpoint: server.URL}, zap.NewNop()) + sink := &fakeSink{} + if err := adapter.Execute(context.Background(), runtime.ExecutionSpec{ + RunID: "run-text-fallback", + Target: "qwen3.6:35b", + Input: map[string]any{ + "prompt": "status", + "tools": []any{map[string]any{ + "type": "function", + "function": map[string]any{ + "name": "run_commands", + "parameters": map[string]any{ + "type": "object", + "properties": map[string]any{ + "commands": map[string]any{"type": "array", "items": map[string]any{"type": "string"}}, + }, + }, + }, + }}, + }, + }, sink); err != nil { + t.Fatalf("Execute failed: %v", err) + } + if attempts != 3 { + t.Fatalf("attempts: got %d, want 3", attempts) + } + if _, ok := fallbackBody["tools"]; ok { + t.Fatalf("fallback must omit tools: %+v", fallbackBody["tools"]) + } + if _, ok := fallbackBody["tool_choice"]; ok { + t.Fatalf("fallback must omit tool_choice: %+v", fallbackBody["tool_choice"]) + } + messages := fallbackBody["messages"].([]any) + first := messages[0].(map[string]any) + if first["role"] != "system" || !strings.Contains(first["content"].(string), "") || !strings.Contains(first["content"].(string), "run_commands") { + t.Fatalf("fallback system instruction missing tool format/name: %+v", first) + } + events := sink.all() + complete := events[len(events)-1] + if complete.Metadata[runtimeMetadataOpenAITextToolFallback] != "true" { + t.Fatalf("fallback metadata missing: %+v", complete.Metadata) + } +} + func TestOpenAICompatExecutePreservesToolCallMessages(t *testing.T) { var body map[string]any server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/apps/node/internal/adapters/vllm/vllm.go b/apps/node/internal/adapters/vllm/vllm.go index 76671bf..2d46f36 100644 --- a/apps/node/internal/adapters/vllm/vllm.go +++ b/apps/node/internal/adapters/vllm/vllm.go @@ -22,7 +22,10 @@ import ( const Name = "vllm" -const runtimeMetadataOpenAIToolCalls = "openai_tool_calls" +const ( + runtimeMetadataOpenAIToolCalls = "openai_tool_calls" + runtimeMetadataOpenAITextToolFallback = "openai_text_tool_fallback" +) type Vllm struct { instanceName string @@ -123,6 +126,7 @@ func (v *Vllm) Execute(ctx context.Context, spec runtime.ExecutionSpec, sink run if err != nil { return fmt.Errorf("vllm adapter: marshal request: %w", err) } + textToolFallback := false v.logger.Info("vllm adapter executing", zap.String("run_id", spec.RunID), @@ -157,6 +161,24 @@ func (v *Vllm) Execute(ctx context.Context, spec runtime.ExecutionSpec, sink run _ = resp.Body.Close() } } + if fallbackReq, ok := textToolFallbackChatRequest(chatReq, msg); ok { + body, err = json.Marshal(fallbackReq) + if err != nil { + _ = emitError(ctx, sink, spec.RunID, fmt.Sprintf("vllm text tool fallback marshal failed: %v", err)) + return fmt.Errorf("vllm adapter: text tool fallback marshal: %w", err) + } + resp, err = v.doChatCompletion(ctx, body) + if err != nil { + _ = emitError(ctx, sink, spec.RunID, fmt.Sprintf("vllm text tool fallback request failed: %v", err)) + return fmt.Errorf("vllm adapter: text tool fallback request: %w", err) + } + if resp.StatusCode >= 200 && resp.StatusCode < 300 { + textToolFallback = true + goto streamResponse + } + msg = readLimited(resp.Body, 4096) + _ = resp.Body.Close() + } if msg == "" { msg = resp.Status } @@ -178,7 +200,7 @@ streamResponse: } payload := strings.TrimPrefix(line, "data: ") if payload == "[DONE]" { - return sink.Emit(ctx, completeEvent(spec.RunID, finishReason, usage, outputTokens, toolCalls.ToolCalls())) + return sink.Emit(ctx, completeEvent(spec.RunID, finishReason, usage, outputTokens, toolCalls.ToolCalls(), textToolFallback)) } var chunk vllmChatChunk if err := json.Unmarshal([]byte(payload), &chunk); err != nil { @@ -352,7 +374,13 @@ func isAutoToolChoiceUnsupportedError(errorBody string) bool { errorBody = strings.ToLower(errorBody) return strings.Contains(errorBody, "auto") && strings.Contains(errorBody, "tool choice") && - (strings.Contains(errorBody, "enable-auto-tool-choice") || strings.Contains(errorBody, "tool-call-parser")) + isToolCallingUnsupportedError(errorBody) +} + +func isToolCallingUnsupportedError(errorBody string) bool { + errorBody = strings.ToLower(errorBody) + return strings.Contains(errorBody, "enable-auto-tool-choice") || + strings.Contains(errorBody, "tool-call-parser") } func forcedToolChoiceForSingleTool(tools any) (map[string]any, bool) { @@ -383,7 +411,55 @@ func forcedToolChoiceForSingleTool(tools any) (map[string]any, bool) { }, true } -func completeEvent(runID, finishReason string, usage *runtime.UsageStats, outputTokens int, toolCalls []any) runtime.RuntimeEvent { +func textToolFallbackChatRequest(req vllmChatRequest, errorBody string) (vllmChatRequest, bool) { + if !isToolCallingUnsupportedError(errorBody) { + return vllmChatRequest{}, false + } + instruction, ok := textToolFallbackInstruction(req.Tools) + if !ok { + return vllmChatRequest{}, false + } + next := req + next.Tools = nil + next.ToolChoice = nil + next.Messages = make([]vllmMessage, 0, len(req.Messages)+1) + next.Messages = append(next.Messages, vllmMessage{Role: "system", Content: instruction}) + next.Messages = append(next.Messages, req.Messages...) + return next, true +} + +func textToolFallbackInstruction(tools any) (string, bool) { + items, ok := anyItems(tools) + if !ok || len(items) == 0 { + return "", false + } + encoded, err := json.Marshal(items) + if err != nil { + return "", false + } + return "Tool calls must be emitted as plain text because this backend does not support native OpenAI tool calling. When a tool is needed, respond with exactly one tool call and no markdown:\n" + + "\n\nJSON_VALUE\n\n\n" + + "Use valid JSON for each parameter value and follow the supplied parameter schema. Available tools JSON: " + string(encoded), true +} + +func anyItems(value any) ([]any, bool) { + switch v := value.(type) { + case []any: + return v, true + default: + encoded, err := json.Marshal(v) + if err != nil { + return nil, false + } + var out []any + if err := json.Unmarshal(encoded, &out); err != nil { + return nil, false + } + return out, true + } +} + +func completeEvent(runID, finishReason string, usage *runtime.UsageStats, outputTokens int, toolCalls []any, textToolFallback bool) runtime.RuntimeEvent { if usage == nil { usage = &runtime.UsageStats{OutputTokens: outputTokens} } @@ -391,6 +467,12 @@ func completeEvent(runID, finishReason string, usage *runtime.UsageStats, output if finishReason != "" { metadata = map[string]string{"finish_reason": finishReason} } + if textToolFallback { + if metadata == nil { + metadata = make(map[string]string, 2) + } + metadata[runtimeMetadataOpenAITextToolFallback] = "true" + } if len(toolCalls) > 0 { if metadata == nil { metadata = make(map[string]string, 2) diff --git a/apps/node/internal/adapters/vllm/vllm_test.go b/apps/node/internal/adapters/vllm/vllm_test.go index b95e985..0988cd4 100644 --- a/apps/node/internal/adapters/vllm/vllm_test.go +++ b/apps/node/internal/adapters/vllm/vllm_test.go @@ -241,6 +241,75 @@ func TestVllmExecuteRetriesSingleToolWhenAutoUnsupported(t *testing.T) { } } +func TestVllmExecuteFallsBackToTextToolsWhenNativeToolsUnsupported(t *testing.T) { + attempts := 0 + var fallbackBody map[string]any + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + attempts++ + switch attempts { + case 1: + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"error":{"message":"\"auto\" tool choice requires --enable-auto-tool-choice and --tool-call-parser to be set"}}`)) + return + case 2: + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"error":{"message":"tool_choice=\"function=ChatCompletionNamedFunction(name='run_commands') type='function'\" requires --tool-call-parser to be set"}}`)) + return + } + if err := json.NewDecoder(r.Body).Decode(&fallbackBody); err != nil { + t.Fatalf("decode fallback: %v", err) + } + w.Header().Set("Content-Type", "text/event-stream") + _, _ = fmt.Fprintf(w, "data: %s\n\n", `{"choices":[{"delta":{"content":"\n\n[\"git status\"]\n\n"}}]}`) + _, _ = fmt.Fprintf(w, "data: [DONE]\n\n") + })) + defer server.Close() + + adapter := New(config.VllmConf{Endpoint: server.URL}, zap.NewNop()) + sink := &fakeSink{} + err := adapter.Execute(context.Background(), runtime.ExecutionSpec{ + RunID: "run-text-fallback", + Target: "llama-3", + Input: map[string]any{ + "prompt": "status", + "tools": []any{map[string]any{ + "type": "function", + "function": map[string]any{ + "name": "run_commands", + "parameters": map[string]any{ + "type": "object", + "properties": map[string]any{ + "commands": map[string]any{"type": "array", "items": map[string]any{"type": "string"}}, + }, + }, + }, + }}, + }, + }, sink) + if err != nil { + t.Fatalf("Execute failed: %v", err) + } + if attempts != 3 { + t.Fatalf("attempts: got %d, want 3", attempts) + } + if _, ok := fallbackBody["tools"]; ok { + t.Fatalf("fallback must omit tools: %+v", fallbackBody["tools"]) + } + if _, ok := fallbackBody["tool_choice"]; ok { + t.Fatalf("fallback must omit tool_choice: %+v", fallbackBody["tool_choice"]) + } + messages := fallbackBody["messages"].([]any) + first := messages[0].(map[string]any) + if first["role"] != "system" || !strings.Contains(first["content"].(string), "") || !strings.Contains(first["content"].(string), "run_commands") { + t.Fatalf("fallback system instruction missing tool format/name: %+v", first) + } + events := sink.all() + complete := events[len(events)-1] + if complete.Metadata[runtimeMetadataOpenAITextToolFallback] != "true" { + t.Fatalf("fallback metadata missing: %+v", complete.Metadata) + } +} + func TestVllmExecuteUsesMessagesInput(t *testing.T) { var gotMessages []map[string]any server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {