toki/iop
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(openai): auto tool choice fallback 보강

Browse source
This commit is contained in:
toki 2026-06-27 22:14:43 +09:00
parent a32264e908
commit 643a3a93ca
5 changed files with 249 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
agent-contract/outer/openai-compatible-api.md
View file

@ -165,7 +165,7 @@ Workspace-bound route는 workspace가 없거나 상대 경로이면 OpenAI-compa
- `parallel_tool_calls`
- `stream_options`
`tools`가 있는 Chat Completions 요청에서 provider route(`openai_compat`, `vllm`, `ollama`, provider pool)는 forced tool 선택 객체와 `"none"` 같은 명시적 `tool_choice`를 backend에 전달한다. 단, `"auto"`는 OpenAI-compatible 기본값과 같으므로 provider request에서는 생략한다. 일부 vLLM 계열 backend는 explicit `"auto"`를 `--enable-auto-tool-choice`/`--tool-call-parser` 없이 400으로 거부하기 때문이다.
`tools`가 있는 Chat Completions 요청에서 provider route(`openai_compat`, `vllm`, `ollama`, provider pool)는 forced tool 선택 객체와 `"none"` 같은 명시적 `tool_choice`를 backend에 전달한다. 단, `"auto"`는 OpenAI-compatible 기본값과 같으므로 provider request에서는 생략한다. 일부 vLLM 계열 backend는 explicit/default `"auto"`를 `--enable-auto-tool-choice`/`--tool-call-parser` 없이 400으로 거부한다. 이 400이 발생하고 요청 tool이 정확히 1개이면 Node adapter는 해당 tool에 대한 forced `tool_choice`로 1회 재시도한다. 여러 tool이면 임의 선택하지 않는다.
provider가 native OpenAI-compatible `tool_calls`를 반환하면 Node는 내부 `RunEvent.metadata["openai_tool_calls"]` JSON으로 보존하고, Edge는 이를 OpenAI-compatible `message.tool_calls` 또는 stream `delta.tool_calls`로 반환하며 `finish_reason: "tool_calls"`를 사용한다.
provider native `tool_calls[].function.arguments`는 OpenAI 계약에 맞는 JSON string으로 반환한다. 단, provider가 요청 `tools[].function.parameters` schema상 배열/객체여야 하는 값을 JSON 문자열로 이중 인코딩한 경우 Edge는 해당 `arguments` JSON만 schema 기준으로 복원해 다시 JSON string으로 직렬화한다.
provider route에서 assistant content에 `<tool_call>`, `{{function(...)}}`, `[Calling tool: ...]` 같은 텍스트가 들어와도 Edge는 이를 파싱하거나 OpenAI `tool_calls`로 합성하지 않는다. 해당 텍스트는 backend가 반환한 content로 그대로 둔다.

88
apps/node/internal/adapters/openai_compat/openai_compat.go
View file

@ -127,11 +127,6 @@ func (a *Adapter) Execute(ctx context.Context, spec runtime.ExecutionSpec, sink
if err != nil {
return fmt.Errorf("openai_compat adapter: marshal request: %w", err)
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, joinOpenAIPath(a.endpoint, "/v1/chat/completions"), bytes.NewReader(body))
if err != nil {
return fmt.Errorf("openai_compat adapter: build request: %w", err)
}
a.applyHeaders(req, true)
a.logger.Info("openai_compat adapter executing",
zap.String("run_id", spec.RunID),
@ -139,14 +134,31 @@ func (a *Adapter) Execute(ctx context.Context, spec runtime.ExecutionSpec, sink
zap.String("target", model),
zap.String("endpoint", a.endpoint),
)
resp, err := a.client.Do(req)
resp, err := a.doChatCompletion(ctx, body)
if err != nil {
_ = emitError(ctx, sink, spec.RunID, fmt.Sprintf("openai_compat request failed: %v", err))
return fmt.Errorf("openai_compat adapter: request: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
msg := readLimited(resp.Body, 4096)
_ = resp.Body.Close()
if retryBody, ok := retryBodyWithForcedSingleTool(reqBody, msg); ok {
body, err = json.Marshal(retryBody)
if err != nil {
_ = emitError(ctx, sink, spec.RunID, fmt.Sprintf("openai_compat retry marshal failed: %v", err))
return fmt.Errorf("openai_compat adapter: retry marshal: %w", err)
}
resp, err = a.doChatCompletion(ctx, body)
if err != nil {
_ = emitError(ctx, sink, spec.RunID, fmt.Sprintf("openai_compat retry request failed: %v", err))
return fmt.Errorf("openai_compat adapter: retry request: %w", err)
}
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
goto streamResponse
}
msg = readLimited(resp.Body, 4096)
_ = resp.Body.Close()
}
if msg == "" {
msg = resp.Status
}
@ -154,6 +166,8 @@ func (a *Adapter) Execute(ctx context.Context, spec runtime.ExecutionSpec, sink
return fmt.Errorf("openai_compat adapter: non-2xx response: %s", resp.Status)
}
streamResponse:
defer resp.Body.Close()
scanner := bufio.NewScanner(resp.Body)
outputTokens := 0
finishReason := ""
@ -312,6 +326,15 @@ func (a *Adapter) applyHeaders(req *http.Request, jsonBody bool) {
}
}
func (a *Adapter) doChatCompletion(ctx context.Context, body []byte) (*http.Response, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodPost, joinOpenAIPath(a.endpoint, "/v1/chat/completions"), bytes.NewReader(body))
if err != nil {
return nil, fmt.Errorf("build request: %w", err)
}
a.applyHeaders(req, true)
return a.client.Do(req)
}
func buildRequestBody(model string, messages []chatMessage, input map[string]any) map[string]any {
body := make(map[string]any)
// Copy caller options as top-level OpenAI-compatible request fields first so
@ -333,6 +356,57 @@ func buildRequestBody(model string, messages []chatMessage, input map[string]any
return body
}
func retryBodyWithForcedSingleTool(body map[string]any, errorBody string) (map[string]any, bool) {
if !isAutoToolChoiceUnsupportedError(errorBody) {
return nil, false
}
forced, ok := forcedToolChoiceForSingleTool(body["tools"])
if !ok {
return nil, false
}
next := make(map[string]any, len(body)+1)
for key, value := range body {
next[key] = value
}
next["tool_choice"] = forced
return next, true
}
func isAutoToolChoiceUnsupportedError(errorBody string) bool {
errorBody = strings.ToLower(errorBody)
return strings.Contains(errorBody, "auto") &&
strings.Contains(errorBody, "tool choice") &&
(strings.Contains(errorBody, "enable-auto-tool-choice") || strings.Contains(errorBody, "tool-call-parser"))
}
func forcedToolChoiceForSingleTool(tools any) (map[string]any, bool) {
items, ok := tools.([]any)
if !ok || len(items) != 1 {
return nil, false
}
tool, ok := items[0].(map[string]any)
if !ok {
return nil, false
}
name := ""
if fn, ok := tool["function"].(map[string]any); ok {
name, _ = fn["name"].(string)
}
if name == "" {
name, _ = tool["name"].(string)
}
name = strings.TrimSpace(name)
if name == "" {
return nil, false
}
return map[string]any{
"type": "function",
"function": map[string]any{
"name": name,
},
}, true
}
func completeEvent(runID, finishReason string, usage *runtime.UsageStats, outputTokens int, toolCalls []any) runtime.RuntimeEvent {
if usage == nil {
usage = &runtime.UsageStats{OutputTokens: outputTokens}

49
apps/node/internal/adapters/openai_compat/openai_compat_test.go
View file

@ -321,6 +321,55 @@ func TestOpenAICompatExecutePassesToolsAndToolChoice(t *testing.T) {
}
}
func TestOpenAICompatExecuteRetriesSingleToolWhenAutoUnsupported(t *testing.T) {
attempts := 0
var retryBody map[string]any
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
attempts++
if attempts == 1 {
w.WriteHeader(http.StatusBadRequest)
_, _ = w.Write([]byte(`{"error":{"message":"\"auto\" tool choice requires --enable-auto-tool-choice and --tool-call-parser to be set"}}`))
return
}
if err := json.NewDecoder(r.Body).Decode(&retryBody); err != nil {
t.Fatalf("decode retry: %v", err)
}
w.Header().Set("Content-Type", "text/event-stream")
_, _ = fmt.Fprintf(w, "data: %s\n\n", `{"choices":[{"delta":{"content":"ok"}}]}`)
_, _ = fmt.Fprintf(w, "data: [DONE]\n\n")
}))
defer server.Close()
adapter := New(config.OpenAICompatConf{Endpoint: server.URL}, zap.NewNop())
sink := &fakeSink{}
if err := adapter.Execute(context.Background(), runtime.ExecutionSpec{
RunID: "run-retry",
Target: "qwen3.6:35b",
Input: map[string]any{
"prompt": "status",
"tools": []any{map[string]any{
"type": "function",
"function": map[string]any{
"name": "run_commands",
},
}},
},
}, sink); err != nil {
t.Fatalf("Execute failed: %v", err)
}
if attempts != 2 {
t.Fatalf("attempts: got %d, want 2", attempts)
}
choice, ok := retryBody["tool_choice"].(map[string]any)
if !ok {
t.Fatalf("retry tool_choice missing: %+v", retryBody)
}
fn := choice["function"].(map[string]any)
if choice["type"] != "function" || fn["name"] != "run_commands" {
t.Fatalf("retry tool_choice: %+v", choice)
}
}
func TestOpenAICompatExecutePreservesToolCallMessages(t *testing.T) {
var body map[string]any
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

75
apps/node/internal/adapters/vllm/vllm.go
View file

@ -123,25 +123,40 @@ func (v *Vllm) Execute(ctx context.Context, spec runtime.ExecutionSpec, sink run
if err != nil {
return fmt.Errorf("vllm adapter: marshal request: %w", err)
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, joinURL(v.endpoint, "/v1/chat/completions"), bytes.NewReader(body))
if err != nil {
return fmt.Errorf("vllm adapter: build request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
v.logger.Info("vllm adapter executing",
zap.String("run_id", spec.RunID),
zap.String("target", model),
zap.String("endpoint", v.endpoint),
)
resp, err := v.client.Do(req)
resp, err := v.doChatCompletion(ctx, body)
if err != nil {
_ = emitError(ctx, sink, spec.RunID, fmt.Sprintf("vllm request failed: %v", err))
return fmt.Errorf("vllm adapter: request: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
msg := readLimited(resp.Body, 4096)
_ = resp.Body.Close()
if isAutoToolChoiceUnsupportedError(msg) {
if forced, ok := forcedToolChoiceForSingleTool(chatReq.Tools); ok {
chatReq.ToolChoice = forced
body, err = json.Marshal(chatReq)
if err != nil {
_ = emitError(ctx, sink, spec.RunID, fmt.Sprintf("vllm retry marshal failed: %v", err))
return fmt.Errorf("vllm adapter: retry marshal: %w", err)
}
resp, err = v.doChatCompletion(ctx, body)
if err != nil {
_ = emitError(ctx, sink, spec.RunID, fmt.Sprintf("vllm retry request failed: %v", err))
return fmt.Errorf("vllm adapter: retry request: %w", err)
}
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
goto streamResponse
}
msg = readLimited(resp.Body, 4096)
_ = resp.Body.Close()
}
}
if msg == "" {
msg = resp.Status
}
@ -149,6 +164,8 @@ func (v *Vllm) Execute(ctx context.Context, spec runtime.ExecutionSpec, sink run
return fmt.Errorf("vllm adapter: non-2xx response: %s", resp.Status)
}
streamResponse:
defer resp.Body.Close()
scanner := bufio.NewScanner(resp.Body)
outputTokens := 0
finishReason := ""
@ -322,6 +339,50 @@ func emitError(ctx context.Context, sink runtime.EventSink, runID, msg string) e
})
}
func (v *Vllm) doChatCompletion(ctx context.Context, body []byte) (*http.Response, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodPost, joinURL(v.endpoint, "/v1/chat/completions"), bytes.NewReader(body))
if err != nil {
return nil, fmt.Errorf("build request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
return v.client.Do(req)
}
func isAutoToolChoiceUnsupportedError(errorBody string) bool {
errorBody = strings.ToLower(errorBody)
return strings.Contains(errorBody, "auto") &&
strings.Contains(errorBody, "tool choice") &&
(strings.Contains(errorBody, "enable-auto-tool-choice") || strings.Contains(errorBody, "tool-call-parser"))
}
func forcedToolChoiceForSingleTool(tools any) (map[string]any, bool) {
items, ok := tools.([]any)
if !ok || len(items) != 1 {
return nil, false
}
tool, ok := items[0].(map[string]any)
if !ok {
return nil, false
}
name := ""
if fn, ok := tool["function"].(map[string]any); ok {
name, _ = fn["name"].(string)
}
if name == "" {
name, _ = tool["name"].(string)
}
name = strings.TrimSpace(name)
if name == "" {
return nil, false
}
return map[string]any{
"type": "function",
"function": map[string]any{
"name": name,
},
}, true
}
func completeEvent(runID, finishReason string, usage *runtime.UsageStats, outputTokens int, toolCalls []any) runtime.RuntimeEvent {
if usage == nil {
usage = &runtime.UsageStats{OutputTokens: outputTokens}

50
apps/node/internal/adapters/vllm/vllm_test.go
View file

@ -191,6 +191,56 @@ func TestVllmExecutePassesToolsAndPreservesNativeToolCalls(t *testing.T) {
}
}
func TestVllmExecuteRetriesSingleToolWhenAutoUnsupported(t *testing.T) {
attempts := 0
var retryBody map[string]any
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
attempts++
if attempts == 1 {
w.WriteHeader(http.StatusBadRequest)
_, _ = w.Write([]byte(`{"error":{"message":"\"auto\" tool choice requires --enable-auto-tool-choice and --tool-call-parser to be set"}}`))
return
}
if err := json.NewDecoder(r.Body).Decode(&retryBody); err != nil {
t.Fatalf("decode retry: %v", err)
}
w.Header().Set("Content-Type", "text/event-stream")
_, _ = fmt.Fprintf(w, "data: %s\n\n", `{"choices":[{"delta":{"content":"ok"}}]}`)
_, _ = fmt.Fprintf(w, "data: [DONE]\n\n")
}))
defer server.Close()
adapter := New(config.VllmConf{Endpoint: server.URL}, zap.NewNop())
sink := &fakeSink{}
err := adapter.Execute(context.Background(), runtime.ExecutionSpec{
RunID: "run-retry",
Target: "llama-3",
Input: map[string]any{
"prompt": "status",
"tools": []any{map[string]any{
"type": "function",
"function": map[string]any{
"name": "run_commands",
},
}},
},
}, sink)
if err != nil {
t.Fatalf("Execute failed: %v", err)
}
if attempts != 2 {
t.Fatalf("attempts: got %d, want 2", attempts)
}
choice, ok := retryBody["tool_choice"].(map[string]any)
if !ok {
t.Fatalf("retry tool_choice missing: %+v", retryBody)
}
fn := choice["function"].(map[string]any)
if choice["type"] != "function" || fn["name"] != "run_commands" {
t.Fatalf("retry tool_choice: %+v", choice)
}
}
func TestVllmExecuteUsesMessagesInput(t *testing.T) {
var gotMessages []map[string]any
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {