From 643a3a93cab3d2b087b91fa8624faaaa6065c9f7 Mon Sep 17 00:00:00 2001 From: toki Date: Sat, 27 Jun 2026 22:14:43 +0900 Subject: [PATCH] =?UTF-8?q?fix(openai):=20auto=20tool=20choice=20fallback?= =?UTF-8?q?=20=EB=B3=B4=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- agent-contract/outer/openai-compatible-api.md | 2 +- .../adapters/openai_compat/openai_compat.go | 88 +++++++++++++++++-- .../openai_compat/openai_compat_test.go | 49 +++++++++++ apps/node/internal/adapters/vllm/vllm.go | 75 ++++++++++++++-- apps/node/internal/adapters/vllm/vllm_test.go | 50 +++++++++++ 5 files changed, 249 insertions(+), 15 deletions(-) diff --git a/agent-contract/outer/openai-compatible-api.md b/agent-contract/outer/openai-compatible-api.md index c5e9fc8..a8718dc 100644 --- a/agent-contract/outer/openai-compatible-api.md +++ b/agent-contract/outer/openai-compatible-api.md @@ -165,7 +165,7 @@ Workspace-bound route는 workspace가 없거나 상대 경로이면 OpenAI-compa - `parallel_tool_calls` - `stream_options` -`tools`가 있는 Chat Completions 요청에서 provider route(`openai_compat`, `vllm`, `ollama`, provider pool)는 forced tool 선택 객체와 `"none"` 같은 명시적 `tool_choice`를 backend에 전달한다. 단, `"auto"`는 OpenAI-compatible 기본값과 같으므로 provider request에서는 생략한다. 일부 vLLM 계열 backend는 explicit `"auto"`를 `--enable-auto-tool-choice`/`--tool-call-parser` 없이 400으로 거부하기 때문이다. +`tools`가 있는 Chat Completions 요청에서 provider route(`openai_compat`, `vllm`, `ollama`, provider pool)는 forced tool 선택 객체와 `"none"` 같은 명시적 `tool_choice`를 backend에 전달한다. 단, `"auto"`는 OpenAI-compatible 기본값과 같으므로 provider request에서는 생략한다. 일부 vLLM 계열 backend는 explicit/default `"auto"`를 `--enable-auto-tool-choice`/`--tool-call-parser` 없이 400으로 거부한다. 이 400이 발생하고 요청 tool이 정확히 1개이면 Node adapter는 해당 tool에 대한 forced `tool_choice`로 1회 재시도한다. 여러 tool이면 임의 선택하지 않는다. provider가 native OpenAI-compatible `tool_calls`를 반환하면 Node는 내부 `RunEvent.metadata["openai_tool_calls"]` JSON으로 보존하고, Edge는 이를 OpenAI-compatible `message.tool_calls` 또는 stream `delta.tool_calls`로 반환하며 `finish_reason: "tool_calls"`를 사용한다. provider native `tool_calls[].function.arguments`는 OpenAI 계약에 맞는 JSON string으로 반환한다. 단, provider가 요청 `tools[].function.parameters` schema상 배열/객체여야 하는 값을 JSON 문자열로 이중 인코딩한 경우 Edge는 해당 `arguments` JSON만 schema 기준으로 복원해 다시 JSON string으로 직렬화한다. provider route에서 assistant content에 ``, `{{function(...)}}`, `[Calling tool: ...]` 같은 텍스트가 들어와도 Edge는 이를 파싱하거나 OpenAI `tool_calls`로 합성하지 않는다. 해당 텍스트는 backend가 반환한 content로 그대로 둔다. diff --git a/apps/node/internal/adapters/openai_compat/openai_compat.go b/apps/node/internal/adapters/openai_compat/openai_compat.go index c29363a..b651e12 100644 --- a/apps/node/internal/adapters/openai_compat/openai_compat.go +++ b/apps/node/internal/adapters/openai_compat/openai_compat.go @@ -127,11 +127,6 @@ func (a *Adapter) Execute(ctx context.Context, spec runtime.ExecutionSpec, sink if err != nil { return fmt.Errorf("openai_compat adapter: marshal request: %w", err) } - req, err := http.NewRequestWithContext(ctx, http.MethodPost, joinOpenAIPath(a.endpoint, "/v1/chat/completions"), bytes.NewReader(body)) - if err != nil { - return fmt.Errorf("openai_compat adapter: build request: %w", err) - } - a.applyHeaders(req, true) a.logger.Info("openai_compat adapter executing", zap.String("run_id", spec.RunID), @@ -139,14 +134,31 @@ func (a *Adapter) Execute(ctx context.Context, spec runtime.ExecutionSpec, sink zap.String("target", model), zap.String("endpoint", a.endpoint), ) - resp, err := a.client.Do(req) + resp, err := a.doChatCompletion(ctx, body) if err != nil { _ = emitError(ctx, sink, spec.RunID, fmt.Sprintf("openai_compat request failed: %v", err)) return fmt.Errorf("openai_compat adapter: request: %w", err) } - defer resp.Body.Close() if resp.StatusCode < 200 || resp.StatusCode >= 300 { msg := readLimited(resp.Body, 4096) + _ = resp.Body.Close() + if retryBody, ok := retryBodyWithForcedSingleTool(reqBody, msg); ok { + body, err = json.Marshal(retryBody) + if err != nil { + _ = emitError(ctx, sink, spec.RunID, fmt.Sprintf("openai_compat retry marshal failed: %v", err)) + return fmt.Errorf("openai_compat adapter: retry marshal: %w", err) + } + resp, err = a.doChatCompletion(ctx, body) + if err != nil { + _ = emitError(ctx, sink, spec.RunID, fmt.Sprintf("openai_compat retry request failed: %v", err)) + return fmt.Errorf("openai_compat adapter: retry request: %w", err) + } + if resp.StatusCode >= 200 && resp.StatusCode < 300 { + goto streamResponse + } + msg = readLimited(resp.Body, 4096) + _ = resp.Body.Close() + } if msg == "" { msg = resp.Status } @@ -154,6 +166,8 @@ func (a *Adapter) Execute(ctx context.Context, spec runtime.ExecutionSpec, sink return fmt.Errorf("openai_compat adapter: non-2xx response: %s", resp.Status) } +streamResponse: + defer resp.Body.Close() scanner := bufio.NewScanner(resp.Body) outputTokens := 0 finishReason := "" @@ -312,6 +326,15 @@ func (a *Adapter) applyHeaders(req *http.Request, jsonBody bool) { } } +func (a *Adapter) doChatCompletion(ctx context.Context, body []byte) (*http.Response, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, joinOpenAIPath(a.endpoint, "/v1/chat/completions"), bytes.NewReader(body)) + if err != nil { + return nil, fmt.Errorf("build request: %w", err) + } + a.applyHeaders(req, true) + return a.client.Do(req) +} + func buildRequestBody(model string, messages []chatMessage, input map[string]any) map[string]any { body := make(map[string]any) // Copy caller options as top-level OpenAI-compatible request fields first so @@ -333,6 +356,57 @@ func buildRequestBody(model string, messages []chatMessage, input map[string]any return body } +func retryBodyWithForcedSingleTool(body map[string]any, errorBody string) (map[string]any, bool) { + if !isAutoToolChoiceUnsupportedError(errorBody) { + return nil, false + } + forced, ok := forcedToolChoiceForSingleTool(body["tools"]) + if !ok { + return nil, false + } + next := make(map[string]any, len(body)+1) + for key, value := range body { + next[key] = value + } + next["tool_choice"] = forced + return next, true +} + +func isAutoToolChoiceUnsupportedError(errorBody string) bool { + errorBody = strings.ToLower(errorBody) + return strings.Contains(errorBody, "auto") && + strings.Contains(errorBody, "tool choice") && + (strings.Contains(errorBody, "enable-auto-tool-choice") || strings.Contains(errorBody, "tool-call-parser")) +} + +func forcedToolChoiceForSingleTool(tools any) (map[string]any, bool) { + items, ok := tools.([]any) + if !ok || len(items) != 1 { + return nil, false + } + tool, ok := items[0].(map[string]any) + if !ok { + return nil, false + } + name := "" + if fn, ok := tool["function"].(map[string]any); ok { + name, _ = fn["name"].(string) + } + if name == "" { + name, _ = tool["name"].(string) + } + name = strings.TrimSpace(name) + if name == "" { + return nil, false + } + return map[string]any{ + "type": "function", + "function": map[string]any{ + "name": name, + }, + }, true +} + func completeEvent(runID, finishReason string, usage *runtime.UsageStats, outputTokens int, toolCalls []any) runtime.RuntimeEvent { if usage == nil { usage = &runtime.UsageStats{OutputTokens: outputTokens} diff --git a/apps/node/internal/adapters/openai_compat/openai_compat_test.go b/apps/node/internal/adapters/openai_compat/openai_compat_test.go index 22ebef7..f9a534f 100644 --- a/apps/node/internal/adapters/openai_compat/openai_compat_test.go +++ b/apps/node/internal/adapters/openai_compat/openai_compat_test.go @@ -321,6 +321,55 @@ func TestOpenAICompatExecutePassesToolsAndToolChoice(t *testing.T) { } } +func TestOpenAICompatExecuteRetriesSingleToolWhenAutoUnsupported(t *testing.T) { + attempts := 0 + var retryBody map[string]any + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + attempts++ + if attempts == 1 { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"error":{"message":"\"auto\" tool choice requires --enable-auto-tool-choice and --tool-call-parser to be set"}}`)) + return + } + if err := json.NewDecoder(r.Body).Decode(&retryBody); err != nil { + t.Fatalf("decode retry: %v", err) + } + w.Header().Set("Content-Type", "text/event-stream") + _, _ = fmt.Fprintf(w, "data: %s\n\n", `{"choices":[{"delta":{"content":"ok"}}]}`) + _, _ = fmt.Fprintf(w, "data: [DONE]\n\n") + })) + defer server.Close() + + adapter := New(config.OpenAICompatConf{Endpoint: server.URL}, zap.NewNop()) + sink := &fakeSink{} + if err := adapter.Execute(context.Background(), runtime.ExecutionSpec{ + RunID: "run-retry", + Target: "qwen3.6:35b", + Input: map[string]any{ + "prompt": "status", + "tools": []any{map[string]any{ + "type": "function", + "function": map[string]any{ + "name": "run_commands", + }, + }}, + }, + }, sink); err != nil { + t.Fatalf("Execute failed: %v", err) + } + if attempts != 2 { + t.Fatalf("attempts: got %d, want 2", attempts) + } + choice, ok := retryBody["tool_choice"].(map[string]any) + if !ok { + t.Fatalf("retry tool_choice missing: %+v", retryBody) + } + fn := choice["function"].(map[string]any) + if choice["type"] != "function" || fn["name"] != "run_commands" { + t.Fatalf("retry tool_choice: %+v", choice) + } +} + func TestOpenAICompatExecutePreservesToolCallMessages(t *testing.T) { var body map[string]any server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/apps/node/internal/adapters/vllm/vllm.go b/apps/node/internal/adapters/vllm/vllm.go index 5e144a1..76671bf 100644 --- a/apps/node/internal/adapters/vllm/vllm.go +++ b/apps/node/internal/adapters/vllm/vllm.go @@ -123,25 +123,40 @@ func (v *Vllm) Execute(ctx context.Context, spec runtime.ExecutionSpec, sink run if err != nil { return fmt.Errorf("vllm adapter: marshal request: %w", err) } - req, err := http.NewRequestWithContext(ctx, http.MethodPost, joinURL(v.endpoint, "/v1/chat/completions"), bytes.NewReader(body)) - if err != nil { - return fmt.Errorf("vllm adapter: build request: %w", err) - } - req.Header.Set("Content-Type", "application/json") v.logger.Info("vllm adapter executing", zap.String("run_id", spec.RunID), zap.String("target", model), zap.String("endpoint", v.endpoint), ) - resp, err := v.client.Do(req) + resp, err := v.doChatCompletion(ctx, body) if err != nil { _ = emitError(ctx, sink, spec.RunID, fmt.Sprintf("vllm request failed: %v", err)) return fmt.Errorf("vllm adapter: request: %w", err) } - defer resp.Body.Close() if resp.StatusCode < 200 || resp.StatusCode >= 300 { msg := readLimited(resp.Body, 4096) + _ = resp.Body.Close() + if isAutoToolChoiceUnsupportedError(msg) { + if forced, ok := forcedToolChoiceForSingleTool(chatReq.Tools); ok { + chatReq.ToolChoice = forced + body, err = json.Marshal(chatReq) + if err != nil { + _ = emitError(ctx, sink, spec.RunID, fmt.Sprintf("vllm retry marshal failed: %v", err)) + return fmt.Errorf("vllm adapter: retry marshal: %w", err) + } + resp, err = v.doChatCompletion(ctx, body) + if err != nil { + _ = emitError(ctx, sink, spec.RunID, fmt.Sprintf("vllm retry request failed: %v", err)) + return fmt.Errorf("vllm adapter: retry request: %w", err) + } + if resp.StatusCode >= 200 && resp.StatusCode < 300 { + goto streamResponse + } + msg = readLimited(resp.Body, 4096) + _ = resp.Body.Close() + } + } if msg == "" { msg = resp.Status } @@ -149,6 +164,8 @@ func (v *Vllm) Execute(ctx context.Context, spec runtime.ExecutionSpec, sink run return fmt.Errorf("vllm adapter: non-2xx response: %s", resp.Status) } +streamResponse: + defer resp.Body.Close() scanner := bufio.NewScanner(resp.Body) outputTokens := 0 finishReason := "" @@ -322,6 +339,50 @@ func emitError(ctx context.Context, sink runtime.EventSink, runID, msg string) e }) } +func (v *Vllm) doChatCompletion(ctx context.Context, body []byte) (*http.Response, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, joinURL(v.endpoint, "/v1/chat/completions"), bytes.NewReader(body)) + if err != nil { + return nil, fmt.Errorf("build request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + return v.client.Do(req) +} + +func isAutoToolChoiceUnsupportedError(errorBody string) bool { + errorBody = strings.ToLower(errorBody) + return strings.Contains(errorBody, "auto") && + strings.Contains(errorBody, "tool choice") && + (strings.Contains(errorBody, "enable-auto-tool-choice") || strings.Contains(errorBody, "tool-call-parser")) +} + +func forcedToolChoiceForSingleTool(tools any) (map[string]any, bool) { + items, ok := tools.([]any) + if !ok || len(items) != 1 { + return nil, false + } + tool, ok := items[0].(map[string]any) + if !ok { + return nil, false + } + name := "" + if fn, ok := tool["function"].(map[string]any); ok { + name, _ = fn["name"].(string) + } + if name == "" { + name, _ = tool["name"].(string) + } + name = strings.TrimSpace(name) + if name == "" { + return nil, false + } + return map[string]any{ + "type": "function", + "function": map[string]any{ + "name": name, + }, + }, true +} + func completeEvent(runID, finishReason string, usage *runtime.UsageStats, outputTokens int, toolCalls []any) runtime.RuntimeEvent { if usage == nil { usage = &runtime.UsageStats{OutputTokens: outputTokens} diff --git a/apps/node/internal/adapters/vllm/vllm_test.go b/apps/node/internal/adapters/vllm/vllm_test.go index f6e0aef..b95e985 100644 --- a/apps/node/internal/adapters/vllm/vllm_test.go +++ b/apps/node/internal/adapters/vllm/vllm_test.go @@ -191,6 +191,56 @@ func TestVllmExecutePassesToolsAndPreservesNativeToolCalls(t *testing.T) { } } +func TestVllmExecuteRetriesSingleToolWhenAutoUnsupported(t *testing.T) { + attempts := 0 + var retryBody map[string]any + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + attempts++ + if attempts == 1 { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"error":{"message":"\"auto\" tool choice requires --enable-auto-tool-choice and --tool-call-parser to be set"}}`)) + return + } + if err := json.NewDecoder(r.Body).Decode(&retryBody); err != nil { + t.Fatalf("decode retry: %v", err) + } + w.Header().Set("Content-Type", "text/event-stream") + _, _ = fmt.Fprintf(w, "data: %s\n\n", `{"choices":[{"delta":{"content":"ok"}}]}`) + _, _ = fmt.Fprintf(w, "data: [DONE]\n\n") + })) + defer server.Close() + + adapter := New(config.VllmConf{Endpoint: server.URL}, zap.NewNop()) + sink := &fakeSink{} + err := adapter.Execute(context.Background(), runtime.ExecutionSpec{ + RunID: "run-retry", + Target: "llama-3", + Input: map[string]any{ + "prompt": "status", + "tools": []any{map[string]any{ + "type": "function", + "function": map[string]any{ + "name": "run_commands", + }, + }}, + }, + }, sink) + if err != nil { + t.Fatalf("Execute failed: %v", err) + } + if attempts != 2 { + t.Fatalf("attempts: got %d, want 2", attempts) + } + choice, ok := retryBody["tool_choice"].(map[string]any) + if !ok { + t.Fatalf("retry tool_choice missing: %+v", retryBody) + } + fn := choice["function"].(map[string]any) + if choice["type"] != "function" || fn["name"] != "run_commands" { + t.Fatalf("retry tool_choice: %+v", choice) + } +} + func TestVllmExecuteUsesMessagesInput(t *testing.T) { var gotMessages []map[string]any server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {