fix(openai): native tool unsupported fallback 추가
This commit is contained in:
parent
643a3a93ca
commit
ea7388cfe9
11 changed files with 408 additions and 28 deletions
|
|
@ -45,6 +45,7 @@ Edge는 Node 연결을 수락하고, Node는 연결 직후 등록 요청을 보
|
|||
- `RunRequest.metadata`: caller-defined 실행 metadata다. workspace 자체는 별도 `workspace` 필드로 전달한다.
|
||||
- `RunEvent.type`: `start`, `delta`, `complete`, `error`, `cancelled` 같은 실행 이벤트 종류다.
|
||||
- `RunEvent.metadata["openai_tool_calls"]`: OpenAI-compatible provider adapter가 native `tool_calls`를 반환했을 때 완료 이벤트에 싣는 JSON 배열이다. Edge OpenAI-compatible 표면은 이 값을 `message.tool_calls` 또는 stream `delta.tool_calls`로 복원한다. provider assistant content 텍스트를 이 값으로 파싱/합성하지 않는다.
|
||||
- `RunEvent.metadata["openai_text_tool_fallback"]`: OpenAI-compatible provider adapter가 backend native tool API 거부 후 `tools`/`tool_choice`를 제거하고 text tool-call instruction으로 재시도했을 때 `"true"`를 싣는다. Edge는 이 표시가 있는 실행에서만 assistant content의 text tool-call을 OpenAI-compatible `tool_calls`로 복원할 수 있다.
|
||||
- `NodeCommandRequest.type`: 실행이 아닌 조회/제어성 명령이다. adapter execution 요청과 섞지 않는다.
|
||||
- `NodeConfigPayload.adapters`: Edge가 Node에 내려주는 adapter instance 설정이다.
|
||||
- `AdapterConfig.name`: node 내부 stable adapter instance identity다. 비어 있으면 legacy single-instance type 이름과 동등하다.
|
||||
|
|
|
|||
|
|
@ -165,10 +165,10 @@ Workspace-bound route는 workspace가 없거나 상대 경로이면 OpenAI-compa
|
|||
- `parallel_tool_calls`
|
||||
- `stream_options`
|
||||
|
||||
`tools`가 있는 Chat Completions 요청에서 provider route(`openai_compat`, `vllm`, `ollama`, provider pool)는 forced tool 선택 객체와 `"none"` 같은 명시적 `tool_choice`를 backend에 전달한다. 단, `"auto"`는 OpenAI-compatible 기본값과 같으므로 provider request에서는 생략한다. 일부 vLLM 계열 backend는 explicit/default `"auto"`를 `--enable-auto-tool-choice`/`--tool-call-parser` 없이 400으로 거부한다. 이 400이 발생하고 요청 tool이 정확히 1개이면 Node adapter는 해당 tool에 대한 forced `tool_choice`로 1회 재시도한다. 여러 tool이면 임의 선택하지 않는다.
|
||||
`tools`가 있는 Chat Completions 요청에서 provider route(`openai_compat`, `vllm`, `ollama`, provider pool)는 forced tool 선택 객체와 `"none"` 같은 명시적 `tool_choice`를 backend에 전달한다. 단, `"auto"`는 OpenAI-compatible 기본값과 같으므로 provider request에서는 생략한다. 일부 vLLM 계열 backend는 explicit/default `"auto"`를 `--enable-auto-tool-choice`/`--tool-call-parser` 없이 400으로 거부한다. 이 400이 발생하고 요청 tool이 정확히 1개이면 Node adapter는 해당 tool에 대한 forced `tool_choice`로 1회 재시도한다. forced tool도 `--tool-call-parser` 요구로 거부되거나 여러 tool이라 forced를 고를 수 없으면, Node adapter는 `tools`/`tool_choice`를 제거하고 text tool-call system instruction을 붙여 1회 재시도하며 완료 metadata에 `openai_text_tool_fallback: "true"`를 싣는다.
|
||||
provider가 native OpenAI-compatible `tool_calls`를 반환하면 Node는 내부 `RunEvent.metadata["openai_tool_calls"]` JSON으로 보존하고, Edge는 이를 OpenAI-compatible `message.tool_calls` 또는 stream `delta.tool_calls`로 반환하며 `finish_reason: "tool_calls"`를 사용한다.
|
||||
provider native `tool_calls[].function.arguments`는 OpenAI 계약에 맞는 JSON string으로 반환한다. 단, provider가 요청 `tools[].function.parameters` schema상 배열/객체여야 하는 값을 JSON 문자열로 이중 인코딩한 경우 Edge는 해당 `arguments` JSON만 schema 기준으로 복원해 다시 JSON string으로 직렬화한다.
|
||||
provider route에서 assistant content에 `<tool_call>`, `{{function(...)}}`, `[Calling tool: ...]` 같은 텍스트가 들어와도 Edge는 이를 파싱하거나 OpenAI `tool_calls`로 합성하지 않는다. 해당 텍스트는 backend가 반환한 content로 그대로 둔다.
|
||||
provider route에서 assistant content에 `<tool_call>`, `{{function(...)}}`, `[Calling tool: ...]` 같은 텍스트가 들어와도 Edge는 기본적으로 이를 파싱하거나 OpenAI `tool_calls`로 합성하지 않는다. 해당 텍스트는 backend가 반환한 content로 그대로 둔다. 예외적으로 Node adapter가 native tool API unsupported fallback을 수행해 `openai_text_tool_fallback: "true"` metadata를 붙인 실행에 한해서만 Edge가 text tool-call을 OpenAI `tool_calls`로 복원한다.
|
||||
CLI route(`adapter: "cli"`)는 native backend tool calling이 없으므로, 호환 fallback으로만 assistant content의 Cline-style 텍스트 tool call 블록을 OpenAI-compatible `tool_calls`로 합성할 수 있다. 이 fallback이 지원하는 텍스트 블록의 최소 형태는 `<tool_call><function=<name>><parameter=<key>>JSON-or-text</parameter></function></tool_call>` 또는 `{{function_name(key=Python/JSON-like-literal)}}`이다.
|
||||
CLI fallback에서 텍스트 tool call을 구조화할 때만 Edge는 요청의 `tools[].function.parameters` schema를 기준으로 arguments를 정규화한다. 예를 들어 tool schema가 `commands: string[]`만 허용하면 command 객체 입력도 shell string 배열로 접고, `commands: {command,args}[]`를 허용하면 shell 문법이 없는 명령을 structured argv로 만든다.
|
||||
CLI fallback에서 schema에 없는 UI 설명용 `description`이나 실행 위치 힌트용 `runInTerminal`은 command 객체와 최상위 args에서 제거한다. 단, `cd`, `command -v`, `&&`, pipe, redirect, quote 등 shell 해석이 필요한 명령은 schema가 허용할 때 `commands: ["cd /work && git status"]` 같은 shell string으로 유지한다.
|
||||
|
|
|
|||
|
|
@ -154,12 +154,12 @@ func routeSupportsNativeToolCalls(dispatch routeDispatch) bool {
|
|||
}
|
||||
}
|
||||
|
||||
func shouldSynthesizeTextToolCalls(dispatch edgeservice.RunDispatch) bool {
|
||||
return strings.TrimSpace(dispatch.Adapter) == "cli"
|
||||
func shouldSynthesizeTextToolCalls(dispatch edgeservice.RunDispatch, textToolFallback bool) bool {
|
||||
return textToolFallback || strings.TrimSpace(dispatch.Adapter) == "cli"
|
||||
}
|
||||
|
||||
func (s *Server) completeChatCompletion(w http.ResponseWriter, r *http.Request, req chatCompletionRequest, handle edgeservice.RunResult, outputPolicy strictOutputPolicy) {
|
||||
text, reasoning, finishReason, nativeToolCalls, usage, err := collectRunResult(r.Context(), handle.Stream(), handle.WaitTimeout())
|
||||
text, reasoning, finishReason, nativeToolCalls, usage, textToolFallback, err := collectRunResult(r.Context(), handle.Stream(), handle.WaitTimeout())
|
||||
if err != nil {
|
||||
writeError(w, httpStatusForRunError(err), "run_error", err.Error())
|
||||
return
|
||||
|
|
@ -170,7 +170,7 @@ func (s *Server) completeChatCompletion(w http.ResponseWriter, r *http.Request,
|
|||
nativeToolCalls = normalizeNativeToolCallsForResponse(nativeToolCalls, req.Tools)
|
||||
message.ToolCalls = nativeToolCalls
|
||||
finishReason = "tool_calls"
|
||||
} else if shouldSynthesizeTextToolCalls(handle.Dispatch()) {
|
||||
} else if shouldSynthesizeTextToolCalls(handle.Dispatch(), textToolFallback) {
|
||||
cleaned, toolCalls := synthesizeToolCallsFromText(text, req.Tools, handle.Dispatch().RunID)
|
||||
if len(toolCalls) > 0 {
|
||||
message.Content = cleaned
|
||||
|
|
|
|||
|
|
@ -140,7 +140,7 @@ func decodeResponsesRequest(dec *json.Decoder, req *responsesRequest) error {
|
|||
}
|
||||
|
||||
func (s *Server) completeResponse(w http.ResponseWriter, r *http.Request, req responsesRequest, handle edgeservice.RunResult, outputPolicy strictOutputPolicy) {
|
||||
text, reasoning, _, _, usage, err := collectRunResult(r.Context(), handle.Stream(), handle.WaitTimeout())
|
||||
text, reasoning, _, _, usage, _, err := collectRunResult(r.Context(), handle.Stream(), handle.WaitTimeout())
|
||||
if err != nil {
|
||||
writeError(w, httpStatusForRunError(err), "run_error", err.Error())
|
||||
return
|
||||
|
|
|
|||
|
|
@ -10,11 +10,14 @@ import (
|
|||
edgeservice "iop/apps/edge/internal/service"
|
||||
)
|
||||
|
||||
const runtimeMetadataOpenAIToolCalls = "openai_tool_calls"
|
||||
const (
|
||||
runtimeMetadataOpenAIToolCalls = "openai_tool_calls"
|
||||
runtimeMetadataOpenAITextToolFallback = "openai_text_tool_fallback"
|
||||
)
|
||||
|
||||
func collectRunResult(ctx context.Context, stream edgeservice.RunStream, timeout time.Duration) (string, string, string, []any, *openAIUsage, error) {
|
||||
func collectRunResult(ctx context.Context, stream edgeservice.RunStream, timeout time.Duration) (string, string, string, []any, *openAIUsage, bool, error) {
|
||||
if stream.Events == nil {
|
||||
return "", "", "", nil, nil, fmt.Errorf("run stream unavailable")
|
||||
return "", "", "", nil, nil, false, fmt.Errorf("run stream unavailable")
|
||||
}
|
||||
var contentBuilder strings.Builder
|
||||
var reasoningBuilder strings.Builder
|
||||
|
|
@ -25,20 +28,20 @@ func collectRunResult(ctx context.Context, stream edgeservice.RunStream, timeout
|
|||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return "", "", "", nil, nil, ctx.Err()
|
||||
return "", "", "", nil, nil, false, ctx.Err()
|
||||
case <-timer.C:
|
||||
return "", "", "", nil, nil, fmt.Errorf("run timed out")
|
||||
return "", "", "", nil, nil, false, fmt.Errorf("run timed out")
|
||||
case nodeEvent, ok := <-stream.NodeEvents:
|
||||
if !ok {
|
||||
stream.NodeEvents = nil
|
||||
continue
|
||||
}
|
||||
if edgeservice.IsNodeDisconnected(nodeEvent) {
|
||||
return "", "", "", nil, nil, fmt.Errorf("node disconnected")
|
||||
return "", "", "", nil, nil, false, fmt.Errorf("node disconnected")
|
||||
}
|
||||
case event, ok := <-stream.Events:
|
||||
if !ok {
|
||||
return "", "", "", nil, nil, fmt.Errorf("run stream closed")
|
||||
return "", "", "", nil, nil, false, fmt.Errorf("run stream closed")
|
||||
}
|
||||
if event == nil {
|
||||
continue
|
||||
|
|
@ -63,7 +66,7 @@ func collectRunResult(ctx context.Context, stream edgeservice.RunStream, timeout
|
|||
TotalTokens: int(u.GetInputTokens() + u.GetOutputTokens()),
|
||||
}
|
||||
}
|
||||
return contentBuilder.String(), reasoningBuilder.String(), finishReason, toolCalls, usage, nil
|
||||
return contentBuilder.String(), reasoningBuilder.String(), finishReason, toolCalls, usage, isTextToolFallback(event.GetMetadata()), nil
|
||||
case "error", "cancelled":
|
||||
msg := event.GetError()
|
||||
if msg == "" {
|
||||
|
|
@ -72,7 +75,7 @@ func collectRunResult(ctx context.Context, stream edgeservice.RunStream, timeout
|
|||
if msg == "" {
|
||||
msg = "run failed"
|
||||
}
|
||||
return "", "", "", nil, nil, fmt.Errorf("%s", msg)
|
||||
return "", "", "", nil, nil, false, fmt.Errorf("%s", msg)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -92,3 +95,7 @@ func toolCallsFromRunMetadata(metadata map[string]string) []any {
|
|||
}
|
||||
return toolCalls
|
||||
}
|
||||
|
||||
func isTextToolFallback(metadata map[string]string) bool {
|
||||
return strings.EqualFold(strings.TrimSpace(metadata[runtimeMetadataOpenAITextToolFallback]), "true")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -464,6 +464,55 @@ func TestChatCompletionsDoesNotSynthesizeTextToolCallsForProviderRoute(t *testin
|
|||
}
|
||||
}
|
||||
|
||||
func TestChatCompletionsSynthesizesProviderTextToolCallsWhenFallbackMarked(t *testing.T) {
|
||||
fake := &fakeRunService{events: make(chan *iop.RunEvent, 2)}
|
||||
fake.events <- &iop.RunEvent{Type: "delta", Delta: "먼저 현재 git 상태를 확인하겠습니다.\n\n<tool_call>\n<function=run_commands>\n<parameter=commands>\n[\"cd /config/workspace/iop && git status\"]\n</parameter>\n</function>\n</tool_call>"}
|
||||
fake.events <- &iop.RunEvent{
|
||||
Type: "complete",
|
||||
Metadata: map[string]string{
|
||||
runtimeMetadataOpenAITextToolFallback: "true",
|
||||
},
|
||||
}
|
||||
|
||||
srv := NewServer(config.EdgeOpenAIConf{Adapter: "openai_compat"}, fake, nil)
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
||||
"model":"qwen3.6:35b",
|
||||
"messages":[{"role":"user","content":"status"}],
|
||||
"tools":[{"type":"function","function":{"name":"run_commands","parameters":{"type":"object","properties":{"commands":{"type":"array","items":{"type":"string"}}},"required":["commands"]}}}],
|
||||
"tool_choice":"auto"
|
||||
}`))
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
srv.handleChatCompletions(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
var resp chatCompletionResponse
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
choice := resp.Choices[0]
|
||||
if choice.FinishReason != "tool_calls" {
|
||||
t.Fatalf("finish_reason: got %q, want tool_calls", choice.FinishReason)
|
||||
}
|
||||
if strings.Contains(choice.Message.Content, "<tool_call>") {
|
||||
t.Fatalf("content still contains raw tool_call block: %q", choice.Message.Content)
|
||||
}
|
||||
if len(choice.Message.ToolCalls) != 1 {
|
||||
t.Fatalf("tool_calls len: got %d", len(choice.Message.ToolCalls))
|
||||
}
|
||||
call := choice.Message.ToolCalls[0].(map[string]any)
|
||||
fn := call["function"].(map[string]any)
|
||||
var args map[string][]string
|
||||
if err := json.Unmarshal([]byte(fn["arguments"].(string)), &args); err != nil {
|
||||
t.Fatalf("arguments JSON: %v", err)
|
||||
}
|
||||
if len(args["commands"]) != 1 || args["commands"][0] != "cd /config/workspace/iop && git status" {
|
||||
t.Fatalf("commands args: %+v", args["commands"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestChatCompletionsPassesThroughNativeToolCallsFromProviderRoute(t *testing.T) {
|
||||
fake := &fakeRunService{events: make(chan *iop.RunEvent, 1)}
|
||||
fake.events <- &iop.RunEvent{
|
||||
|
|
@ -1869,7 +1918,7 @@ func TestCollectRunResultTimesOut(t *testing.T) {
|
|||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 1500*time.Millisecond)
|
||||
defer cancel()
|
||||
_, _, _, _, _, err := collectRunResult(ctx, handle.Stream(), handle.WaitTimeout())
|
||||
_, _, _, _, _, _, err := collectRunResult(ctx, handle.Stream(), handle.WaitTimeout())
|
||||
if err == nil {
|
||||
t.Fatal("expected timeout error")
|
||||
}
|
||||
|
|
@ -2158,7 +2207,7 @@ func TestCollectRunResultFailsWhenEventStreamCloses(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
_, _, _, _, _, err := collectRunResult(context.Background(), handle.Stream(), handle.WaitTimeout())
|
||||
_, _, _, _, _, _, err := collectRunResult(context.Background(), handle.Stream(), handle.WaitTimeout())
|
||||
if err == nil {
|
||||
t.Fatal("expected closed stream error")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -153,7 +153,7 @@ func (s *Server) streamChatCompletion(w http.ResponseWriter, r *http.Request, re
|
|||
}
|
||||
|
||||
func (s *Server) streamBufferedChatCompletion(w http.ResponseWriter, r *http.Request, handle edgeservice.RunResult, id string, created int64, model string, flusher http.Flusher, outputPolicy strictOutputPolicy, tools []any) {
|
||||
text, reasoning, finishReason, nativeToolCalls, _, err := collectRunResult(r.Context(), handle.Stream(), handle.WaitTimeout())
|
||||
text, reasoning, finishReason, nativeToolCalls, _, textToolFallback, err := collectRunResult(r.Context(), handle.Stream(), handle.WaitTimeout())
|
||||
if err != nil {
|
||||
writeSSEError(w, flusher, err.Error())
|
||||
return
|
||||
|
|
@ -195,7 +195,7 @@ func (s *Server) streamBufferedChatCompletion(w http.ResponseWriter, r *http.Req
|
|||
}},
|
||||
})
|
||||
finishReason = "tool_calls"
|
||||
} else if shouldSynthesizeTextToolCalls(handle.Dispatch()) {
|
||||
} else if shouldSynthesizeTextToolCalls(handle.Dispatch(), textToolFallback) {
|
||||
cleaned, toolCalls := synthesizeToolCallsFromText(text, tools, handle.Dispatch().RunID)
|
||||
if len(toolCalls) > 0 {
|
||||
if cleaned != "" {
|
||||
|
|
|
|||
|
|
@ -25,7 +25,10 @@ import (
|
|||
|
||||
const Name = "openai_compat"
|
||||
|
||||
const runtimeMetadataOpenAIToolCalls = "openai_tool_calls"
|
||||
const (
|
||||
runtimeMetadataOpenAIToolCalls = "openai_tool_calls"
|
||||
runtimeMetadataOpenAITextToolFallback = "openai_text_tool_fallback"
|
||||
)
|
||||
|
||||
type Adapter struct {
|
||||
instanceName string
|
||||
|
|
@ -127,6 +130,7 @@ func (a *Adapter) Execute(ctx context.Context, spec runtime.ExecutionSpec, sink
|
|||
if err != nil {
|
||||
return fmt.Errorf("openai_compat adapter: marshal request: %w", err)
|
||||
}
|
||||
textToolFallback := false
|
||||
|
||||
a.logger.Info("openai_compat adapter executing",
|
||||
zap.String("run_id", spec.RunID),
|
||||
|
|
@ -159,6 +163,24 @@ func (a *Adapter) Execute(ctx context.Context, spec runtime.ExecutionSpec, sink
|
|||
msg = readLimited(resp.Body, 4096)
|
||||
_ = resp.Body.Close()
|
||||
}
|
||||
if retryBody, ok := retryBodyWithTextToolFallback(reqBody, msg); ok {
|
||||
body, err = json.Marshal(retryBody)
|
||||
if err != nil {
|
||||
_ = emitError(ctx, sink, spec.RunID, fmt.Sprintf("openai_compat text tool fallback marshal failed: %v", err))
|
||||
return fmt.Errorf("openai_compat adapter: text tool fallback marshal: %w", err)
|
||||
}
|
||||
resp, err = a.doChatCompletion(ctx, body)
|
||||
if err != nil {
|
||||
_ = emitError(ctx, sink, spec.RunID, fmt.Sprintf("openai_compat text tool fallback request failed: %v", err))
|
||||
return fmt.Errorf("openai_compat adapter: text tool fallback request: %w", err)
|
||||
}
|
||||
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
|
||||
textToolFallback = true
|
||||
goto streamResponse
|
||||
}
|
||||
msg = readLimited(resp.Body, 4096)
|
||||
_ = resp.Body.Close()
|
||||
}
|
||||
if msg == "" {
|
||||
msg = resp.Status
|
||||
}
|
||||
|
|
@ -180,7 +202,7 @@ streamResponse:
|
|||
}
|
||||
payload := strings.TrimPrefix(line, "data: ")
|
||||
if payload == "[DONE]" {
|
||||
return sink.Emit(ctx, completeEvent(spec.RunID, finishReason, usage, outputTokens, toolCalls.ToolCalls()))
|
||||
return sink.Emit(ctx, completeEvent(spec.RunID, finishReason, usage, outputTokens, toolCalls.ToolCalls(), textToolFallback))
|
||||
}
|
||||
var chunk chatChunk
|
||||
if err := json.Unmarshal([]byte(payload), &chunk); err != nil {
|
||||
|
|
@ -376,7 +398,13 @@ func isAutoToolChoiceUnsupportedError(errorBody string) bool {
|
|||
errorBody = strings.ToLower(errorBody)
|
||||
return strings.Contains(errorBody, "auto") &&
|
||||
strings.Contains(errorBody, "tool choice") &&
|
||||
(strings.Contains(errorBody, "enable-auto-tool-choice") || strings.Contains(errorBody, "tool-call-parser"))
|
||||
isToolCallingUnsupportedError(errorBody)
|
||||
}
|
||||
|
||||
func isToolCallingUnsupportedError(errorBody string) bool {
|
||||
errorBody = strings.ToLower(errorBody)
|
||||
return strings.Contains(errorBody, "enable-auto-tool-choice") ||
|
||||
strings.Contains(errorBody, "tool-call-parser")
|
||||
}
|
||||
|
||||
func forcedToolChoiceForSingleTool(tools any) (map[string]any, bool) {
|
||||
|
|
@ -407,7 +435,77 @@ func forcedToolChoiceForSingleTool(tools any) (map[string]any, bool) {
|
|||
}, true
|
||||
}
|
||||
|
||||
func completeEvent(runID, finishReason string, usage *runtime.UsageStats, outputTokens int, toolCalls []any) runtime.RuntimeEvent {
|
||||
func retryBodyWithTextToolFallback(body map[string]any, errorBody string) (map[string]any, bool) {
|
||||
if !isToolCallingUnsupportedError(errorBody) {
|
||||
return nil, false
|
||||
}
|
||||
instruction, ok := textToolFallbackInstruction(body["tools"])
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
messages, ok := prependTextToolFallbackInstruction(body["messages"], instruction)
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
next := make(map[string]any, len(body)+1)
|
||||
for key, value := range body {
|
||||
next[key] = value
|
||||
}
|
||||
delete(next, "tools")
|
||||
delete(next, "tool_choice")
|
||||
next["messages"] = messages
|
||||
return next, true
|
||||
}
|
||||
|
||||
func prependTextToolFallbackInstruction(value any, instruction string) (any, bool) {
|
||||
switch messages := value.(type) {
|
||||
case []chatMessage:
|
||||
out := make([]chatMessage, 0, len(messages)+1)
|
||||
out = append(out, chatMessage{Role: "system", Content: instruction})
|
||||
out = append(out, messages...)
|
||||
return out, true
|
||||
case []any:
|
||||
out := make([]any, 0, len(messages)+1)
|
||||
out = append(out, map[string]any{"role": "system", "content": instruction})
|
||||
out = append(out, messages...)
|
||||
return out, true
|
||||
default:
|
||||
return nil, false
|
||||
}
|
||||
}
|
||||
|
||||
func textToolFallbackInstruction(tools any) (string, bool) {
|
||||
items, ok := anyItems(tools)
|
||||
if !ok || len(items) == 0 {
|
||||
return "", false
|
||||
}
|
||||
encoded, err := json.Marshal(items)
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
return "Tool calls must be emitted as plain text because this backend does not support native OpenAI tool calling. When a tool is needed, respond with exactly one tool call and no markdown:\n" +
|
||||
"<tool_call>\n<function=TOOL_NAME>\n<parameter=PARAMETER_NAME>JSON_VALUE</parameter>\n</function>\n</tool_call>\n" +
|
||||
"Use valid JSON for each parameter value and follow the supplied parameter schema. Available tools JSON: " + string(encoded), true
|
||||
}
|
||||
|
||||
func anyItems(value any) ([]any, bool) {
|
||||
switch v := value.(type) {
|
||||
case []any:
|
||||
return v, true
|
||||
default:
|
||||
encoded, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
var out []any
|
||||
if err := json.Unmarshal(encoded, &out); err != nil {
|
||||
return nil, false
|
||||
}
|
||||
return out, true
|
||||
}
|
||||
}
|
||||
|
||||
func completeEvent(runID, finishReason string, usage *runtime.UsageStats, outputTokens int, toolCalls []any, textToolFallback bool) runtime.RuntimeEvent {
|
||||
if usage == nil {
|
||||
usage = &runtime.UsageStats{OutputTokens: outputTokens}
|
||||
}
|
||||
|
|
@ -415,6 +513,12 @@ func completeEvent(runID, finishReason string, usage *runtime.UsageStats, output
|
|||
if finishReason != "" {
|
||||
metadata = map[string]string{"finish_reason": finishReason}
|
||||
}
|
||||
if textToolFallback {
|
||||
if metadata == nil {
|
||||
metadata = make(map[string]string, 2)
|
||||
}
|
||||
metadata[runtimeMetadataOpenAITextToolFallback] = "true"
|
||||
}
|
||||
if len(toolCalls) > 0 {
|
||||
if metadata == nil {
|
||||
metadata = make(map[string]string, 2)
|
||||
|
|
|
|||
|
|
@ -370,6 +370,74 @@ func TestOpenAICompatExecuteRetriesSingleToolWhenAutoUnsupported(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatExecuteFallsBackToTextToolsWhenNativeToolsUnsupported(t *testing.T) {
|
||||
attempts := 0
|
||||
var fallbackBody map[string]any
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
attempts++
|
||||
switch attempts {
|
||||
case 1:
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
_, _ = w.Write([]byte(`{"error":{"message":"\"auto\" tool choice requires --enable-auto-tool-choice and --tool-call-parser to be set"}}`))
|
||||
return
|
||||
case 2:
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
_, _ = w.Write([]byte(`{"error":{"message":"tool_choice=\"function=ChatCompletionNamedFunction(name='run_commands') type='function'\" requires --tool-call-parser to be set"}}`))
|
||||
return
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&fallbackBody); err != nil {
|
||||
t.Fatalf("decode fallback: %v", err)
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
_, _ = fmt.Fprintf(w, "data: %s\n\n", `{"choices":[{"delta":{"content":"<tool_call>\n<function=run_commands>\n<parameter=commands>[\"git status\"]</parameter>\n</function>\n</tool_call>"}}]}`)
|
||||
_, _ = fmt.Fprintf(w, "data: [DONE]\n\n")
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
adapter := New(config.OpenAICompatConf{Endpoint: server.URL}, zap.NewNop())
|
||||
sink := &fakeSink{}
|
||||
if err := adapter.Execute(context.Background(), runtime.ExecutionSpec{
|
||||
RunID: "run-text-fallback",
|
||||
Target: "qwen3.6:35b",
|
||||
Input: map[string]any{
|
||||
"prompt": "status",
|
||||
"tools": []any{map[string]any{
|
||||
"type": "function",
|
||||
"function": map[string]any{
|
||||
"name": "run_commands",
|
||||
"parameters": map[string]any{
|
||||
"type": "object",
|
||||
"properties": map[string]any{
|
||||
"commands": map[string]any{"type": "array", "items": map[string]any{"type": "string"}},
|
||||
},
|
||||
},
|
||||
},
|
||||
}},
|
||||
},
|
||||
}, sink); err != nil {
|
||||
t.Fatalf("Execute failed: %v", err)
|
||||
}
|
||||
if attempts != 3 {
|
||||
t.Fatalf("attempts: got %d, want 3", attempts)
|
||||
}
|
||||
if _, ok := fallbackBody["tools"]; ok {
|
||||
t.Fatalf("fallback must omit tools: %+v", fallbackBody["tools"])
|
||||
}
|
||||
if _, ok := fallbackBody["tool_choice"]; ok {
|
||||
t.Fatalf("fallback must omit tool_choice: %+v", fallbackBody["tool_choice"])
|
||||
}
|
||||
messages := fallbackBody["messages"].([]any)
|
||||
first := messages[0].(map[string]any)
|
||||
if first["role"] != "system" || !strings.Contains(first["content"].(string), "<tool_call>") || !strings.Contains(first["content"].(string), "run_commands") {
|
||||
t.Fatalf("fallback system instruction missing tool format/name: %+v", first)
|
||||
}
|
||||
events := sink.all()
|
||||
complete := events[len(events)-1]
|
||||
if complete.Metadata[runtimeMetadataOpenAITextToolFallback] != "true" {
|
||||
t.Fatalf("fallback metadata missing: %+v", complete.Metadata)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatExecutePreservesToolCallMessages(t *testing.T) {
|
||||
var body map[string]any
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
|
|
|
|||
|
|
@ -22,7 +22,10 @@ import (
|
|||
|
||||
const Name = "vllm"
|
||||
|
||||
const runtimeMetadataOpenAIToolCalls = "openai_tool_calls"
|
||||
const (
|
||||
runtimeMetadataOpenAIToolCalls = "openai_tool_calls"
|
||||
runtimeMetadataOpenAITextToolFallback = "openai_text_tool_fallback"
|
||||
)
|
||||
|
||||
type Vllm struct {
|
||||
instanceName string
|
||||
|
|
@ -123,6 +126,7 @@ func (v *Vllm) Execute(ctx context.Context, spec runtime.ExecutionSpec, sink run
|
|||
if err != nil {
|
||||
return fmt.Errorf("vllm adapter: marshal request: %w", err)
|
||||
}
|
||||
textToolFallback := false
|
||||
|
||||
v.logger.Info("vllm adapter executing",
|
||||
zap.String("run_id", spec.RunID),
|
||||
|
|
@ -157,6 +161,24 @@ func (v *Vllm) Execute(ctx context.Context, spec runtime.ExecutionSpec, sink run
|
|||
_ = resp.Body.Close()
|
||||
}
|
||||
}
|
||||
if fallbackReq, ok := textToolFallbackChatRequest(chatReq, msg); ok {
|
||||
body, err = json.Marshal(fallbackReq)
|
||||
if err != nil {
|
||||
_ = emitError(ctx, sink, spec.RunID, fmt.Sprintf("vllm text tool fallback marshal failed: %v", err))
|
||||
return fmt.Errorf("vllm adapter: text tool fallback marshal: %w", err)
|
||||
}
|
||||
resp, err = v.doChatCompletion(ctx, body)
|
||||
if err != nil {
|
||||
_ = emitError(ctx, sink, spec.RunID, fmt.Sprintf("vllm text tool fallback request failed: %v", err))
|
||||
return fmt.Errorf("vllm adapter: text tool fallback request: %w", err)
|
||||
}
|
||||
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
|
||||
textToolFallback = true
|
||||
goto streamResponse
|
||||
}
|
||||
msg = readLimited(resp.Body, 4096)
|
||||
_ = resp.Body.Close()
|
||||
}
|
||||
if msg == "" {
|
||||
msg = resp.Status
|
||||
}
|
||||
|
|
@ -178,7 +200,7 @@ streamResponse:
|
|||
}
|
||||
payload := strings.TrimPrefix(line, "data: ")
|
||||
if payload == "[DONE]" {
|
||||
return sink.Emit(ctx, completeEvent(spec.RunID, finishReason, usage, outputTokens, toolCalls.ToolCalls()))
|
||||
return sink.Emit(ctx, completeEvent(spec.RunID, finishReason, usage, outputTokens, toolCalls.ToolCalls(), textToolFallback))
|
||||
}
|
||||
var chunk vllmChatChunk
|
||||
if err := json.Unmarshal([]byte(payload), &chunk); err != nil {
|
||||
|
|
@ -352,7 +374,13 @@ func isAutoToolChoiceUnsupportedError(errorBody string) bool {
|
|||
errorBody = strings.ToLower(errorBody)
|
||||
return strings.Contains(errorBody, "auto") &&
|
||||
strings.Contains(errorBody, "tool choice") &&
|
||||
(strings.Contains(errorBody, "enable-auto-tool-choice") || strings.Contains(errorBody, "tool-call-parser"))
|
||||
isToolCallingUnsupportedError(errorBody)
|
||||
}
|
||||
|
||||
func isToolCallingUnsupportedError(errorBody string) bool {
|
||||
errorBody = strings.ToLower(errorBody)
|
||||
return strings.Contains(errorBody, "enable-auto-tool-choice") ||
|
||||
strings.Contains(errorBody, "tool-call-parser")
|
||||
}
|
||||
|
||||
func forcedToolChoiceForSingleTool(tools any) (map[string]any, bool) {
|
||||
|
|
@ -383,7 +411,55 @@ func forcedToolChoiceForSingleTool(tools any) (map[string]any, bool) {
|
|||
}, true
|
||||
}
|
||||
|
||||
func completeEvent(runID, finishReason string, usage *runtime.UsageStats, outputTokens int, toolCalls []any) runtime.RuntimeEvent {
|
||||
func textToolFallbackChatRequest(req vllmChatRequest, errorBody string) (vllmChatRequest, bool) {
|
||||
if !isToolCallingUnsupportedError(errorBody) {
|
||||
return vllmChatRequest{}, false
|
||||
}
|
||||
instruction, ok := textToolFallbackInstruction(req.Tools)
|
||||
if !ok {
|
||||
return vllmChatRequest{}, false
|
||||
}
|
||||
next := req
|
||||
next.Tools = nil
|
||||
next.ToolChoice = nil
|
||||
next.Messages = make([]vllmMessage, 0, len(req.Messages)+1)
|
||||
next.Messages = append(next.Messages, vllmMessage{Role: "system", Content: instruction})
|
||||
next.Messages = append(next.Messages, req.Messages...)
|
||||
return next, true
|
||||
}
|
||||
|
||||
func textToolFallbackInstruction(tools any) (string, bool) {
|
||||
items, ok := anyItems(tools)
|
||||
if !ok || len(items) == 0 {
|
||||
return "", false
|
||||
}
|
||||
encoded, err := json.Marshal(items)
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
return "Tool calls must be emitted as plain text because this backend does not support native OpenAI tool calling. When a tool is needed, respond with exactly one tool call and no markdown:\n" +
|
||||
"<tool_call>\n<function=TOOL_NAME>\n<parameter=PARAMETER_NAME>JSON_VALUE</parameter>\n</function>\n</tool_call>\n" +
|
||||
"Use valid JSON for each parameter value and follow the supplied parameter schema. Available tools JSON: " + string(encoded), true
|
||||
}
|
||||
|
||||
func anyItems(value any) ([]any, bool) {
|
||||
switch v := value.(type) {
|
||||
case []any:
|
||||
return v, true
|
||||
default:
|
||||
encoded, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
var out []any
|
||||
if err := json.Unmarshal(encoded, &out); err != nil {
|
||||
return nil, false
|
||||
}
|
||||
return out, true
|
||||
}
|
||||
}
|
||||
|
||||
func completeEvent(runID, finishReason string, usage *runtime.UsageStats, outputTokens int, toolCalls []any, textToolFallback bool) runtime.RuntimeEvent {
|
||||
if usage == nil {
|
||||
usage = &runtime.UsageStats{OutputTokens: outputTokens}
|
||||
}
|
||||
|
|
@ -391,6 +467,12 @@ func completeEvent(runID, finishReason string, usage *runtime.UsageStats, output
|
|||
if finishReason != "" {
|
||||
metadata = map[string]string{"finish_reason": finishReason}
|
||||
}
|
||||
if textToolFallback {
|
||||
if metadata == nil {
|
||||
metadata = make(map[string]string, 2)
|
||||
}
|
||||
metadata[runtimeMetadataOpenAITextToolFallback] = "true"
|
||||
}
|
||||
if len(toolCalls) > 0 {
|
||||
if metadata == nil {
|
||||
metadata = make(map[string]string, 2)
|
||||
|
|
|
|||
|
|
@ -241,6 +241,75 @@ func TestVllmExecuteRetriesSingleToolWhenAutoUnsupported(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestVllmExecuteFallsBackToTextToolsWhenNativeToolsUnsupported(t *testing.T) {
|
||||
attempts := 0
|
||||
var fallbackBody map[string]any
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
attempts++
|
||||
switch attempts {
|
||||
case 1:
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
_, _ = w.Write([]byte(`{"error":{"message":"\"auto\" tool choice requires --enable-auto-tool-choice and --tool-call-parser to be set"}}`))
|
||||
return
|
||||
case 2:
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
_, _ = w.Write([]byte(`{"error":{"message":"tool_choice=\"function=ChatCompletionNamedFunction(name='run_commands') type='function'\" requires --tool-call-parser to be set"}}`))
|
||||
return
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&fallbackBody); err != nil {
|
||||
t.Fatalf("decode fallback: %v", err)
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
_, _ = fmt.Fprintf(w, "data: %s\n\n", `{"choices":[{"delta":{"content":"<tool_call>\n<function=run_commands>\n<parameter=commands>[\"git status\"]</parameter>\n</function>\n</tool_call>"}}]}`)
|
||||
_, _ = fmt.Fprintf(w, "data: [DONE]\n\n")
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
adapter := New(config.VllmConf{Endpoint: server.URL}, zap.NewNop())
|
||||
sink := &fakeSink{}
|
||||
err := adapter.Execute(context.Background(), runtime.ExecutionSpec{
|
||||
RunID: "run-text-fallback",
|
||||
Target: "llama-3",
|
||||
Input: map[string]any{
|
||||
"prompt": "status",
|
||||
"tools": []any{map[string]any{
|
||||
"type": "function",
|
||||
"function": map[string]any{
|
||||
"name": "run_commands",
|
||||
"parameters": map[string]any{
|
||||
"type": "object",
|
||||
"properties": map[string]any{
|
||||
"commands": map[string]any{"type": "array", "items": map[string]any{"type": "string"}},
|
||||
},
|
||||
},
|
||||
},
|
||||
}},
|
||||
},
|
||||
}, sink)
|
||||
if err != nil {
|
||||
t.Fatalf("Execute failed: %v", err)
|
||||
}
|
||||
if attempts != 3 {
|
||||
t.Fatalf("attempts: got %d, want 3", attempts)
|
||||
}
|
||||
if _, ok := fallbackBody["tools"]; ok {
|
||||
t.Fatalf("fallback must omit tools: %+v", fallbackBody["tools"])
|
||||
}
|
||||
if _, ok := fallbackBody["tool_choice"]; ok {
|
||||
t.Fatalf("fallback must omit tool_choice: %+v", fallbackBody["tool_choice"])
|
||||
}
|
||||
messages := fallbackBody["messages"].([]any)
|
||||
first := messages[0].(map[string]any)
|
||||
if first["role"] != "system" || !strings.Contains(first["content"].(string), "<tool_call>") || !strings.Contains(first["content"].(string), "run_commands") {
|
||||
t.Fatalf("fallback system instruction missing tool format/name: %+v", first)
|
||||
}
|
||||
events := sink.all()
|
||||
complete := events[len(events)-1]
|
||||
if complete.Metadata[runtimeMetadataOpenAITextToolFallback] != "true" {
|
||||
t.Fatalf("fallback metadata missing: %+v", complete.Metadata)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVllmExecuteUsesMessagesInput(t *testing.T) {
|
||||
var gotMessages []map[string]any
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
|
|
|
|||
Loading…
Reference in a new issue