feat(openai): explicit reasoning 보존 로직 추가 및 관련 핸들러 개선
Strict output 모드에서 reasoning이 명시적으로 요청된 경우 이를 보존할 수 있도록 normalizeCompletionOutput 함수와 관련 핸들러들의 로직을 업데이트한다.
This commit is contained in:
parent
5aece17895
commit
b6363f7dd2
6 changed files with 133 additions and 6 deletions
|
|
@ -65,6 +65,9 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) {
|
|||
return
|
||||
}
|
||||
outputPolicy := s.resolveOutputPolicy(basePrompt)
|
||||
if responseMode == responseModePassthrough && !responseModeWasExplicit(runMeta) && outputPolicy.Strict {
|
||||
responseMode = responseModeTransformed
|
||||
}
|
||||
if catalogEntry := s.findProviderPoolEntry(req.Model); catalogEntry != nil {
|
||||
applyModelCatalogGenerationPolicyToChat(&req, *catalogEntry, outputPolicy.Strict)
|
||||
}
|
||||
|
|
@ -312,6 +315,14 @@ func parseResponseMode(runMeta map[string]string) (string, error) {
|
|||
}
|
||||
}
|
||||
|
||||
func responseModeWasExplicit(runMeta map[string]string) bool {
|
||||
if runMeta == nil {
|
||||
return false
|
||||
}
|
||||
mode, ok := runMeta[responseModeMetadataKey]
|
||||
return ok && strings.TrimSpace(mode) != ""
|
||||
}
|
||||
|
||||
// routeUsesProviderTunnel reports whether the resolved dispatch targets an
|
||||
// OpenAI-compatible provider that serves raw tunnel passthrough. Provider-pool
|
||||
// catalog routes and openai_compat/vllm type routes qualify; CLI and other
|
||||
|
|
@ -499,7 +510,7 @@ func collectChatCompletionOutput(ctx context.Context, req chatCompletionRequest,
|
|||
if err != nil {
|
||||
return chatCompletionOutput{}, err
|
||||
}
|
||||
text, reasoning, normalized := normalizeCompletionOutput(outputPolicy, text, reasoning)
|
||||
text, reasoning, normalized := normalizeCompletionOutput(outputPolicy, text, reasoning, req.explicitlyIncludesReasoning())
|
||||
message := chatMessage{Role: "assistant", Content: text, ReasoningContent: reasoning}
|
||||
includeReasoning := req.includeReasoning()
|
||||
if !includeReasoning {
|
||||
|
|
|
|||
|
|
@ -174,7 +174,7 @@ func (s *Server) completeResponse(w http.ResponseWriter, r *http.Request, req re
|
|||
writeError(w, httpStatusForRunError(err), "run_error", err.Error())
|
||||
return
|
||||
}
|
||||
text, reasoning, normalized := normalizeCompletionOutput(outputPolicy, text, reasoning)
|
||||
text, reasoning, normalized := normalizeCompletionOutput(outputPolicy, text, reasoning, false)
|
||||
s.logger.Info("openai responses output",
|
||||
zap.String("run_id", handle.Dispatch().RunID),
|
||||
zap.Bool("strict_output", outputPolicy.Strict),
|
||||
|
|
|
|||
|
|
@ -2364,6 +2364,34 @@ func TestChatCompletionsStreamsReasoningSSE(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestChatCompletionsStrictOutputStreamsExplicitReasoningSSE(t *testing.T) {
|
||||
fake := &fakeRunService{events: make(chan *iop.RunEvent, 4)}
|
||||
fake.events <- &iop.RunEvent{Type: "reasoning_delta", Delta: "think"}
|
||||
fake.events <- &iop.RunEvent{Type: "delta", Delta: "<attempt_completion><result>hi</result></attempt_completion>"}
|
||||
fake.events <- &iop.RunEvent{Type: "complete"}
|
||||
|
||||
srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama", StrictOutput: true}, fake, nil)
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
||||
"model":"stream-model",
|
||||
"stream":true,
|
||||
"messages":[{"role":"user","content":"hi"}],
|
||||
"include_reasoning": true
|
||||
}`))
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
srv.handleChatCompletions(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
body := w.Body.String()
|
||||
if !strings.Contains(body, `"reasoning_content":"think"`) ||
|
||||
!strings.Contains(body, `"content":"\u003cattempt_completion\u003e\u003cresult\u003ehi\u003c/result\u003e\u003c/attempt_completion\u003e"`) ||
|
||||
!strings.Contains(body, "data: [DONE]") {
|
||||
t.Fatalf("unexpected SSE body:\n%s", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChatCompletionsStreamFallsBackToReasoningWhenContentEmpty(t *testing.T) {
|
||||
fake := &fakeRunService{events: make(chan *iop.RunEvent, 3)}
|
||||
fake.events <- &iop.RunEvent{Type: "reasoning_delta", Delta: "think only"}
|
||||
|
|
@ -2517,6 +2545,37 @@ func TestChatCompletionsStrictOutputNormalizesXMLStyleAgentResponse(t *testing.T
|
|||
}
|
||||
}
|
||||
|
||||
func TestChatCompletionsStrictOutputPreservesExplicitReasoningContent(t *testing.T) {
|
||||
fake := &fakeRunService{events: make(chan *iop.RunEvent, 4)}
|
||||
fake.events <- &iop.RunEvent{Type: "reasoning_delta", Delta: "hidden thought"}
|
||||
fake.events <- &iop.RunEvent{Type: "delta", Delta: "<attempt_completion>\n<result>ok</result>\n</attempt_completion>"}
|
||||
fake.events <- &iop.RunEvent{Type: "complete"}
|
||||
|
||||
srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama", StrictOutput: true}, fake, nil)
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
||||
"model":"agent-model",
|
||||
"messages":[{"role":"user","content":"finish"}],
|
||||
"include_reasoning": true
|
||||
}`))
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
srv.handleChatCompletions(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
var resp chatCompletionResponse
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if resp.Choices[0].Message.Content != "<attempt_completion>\n<result>ok</result>\n</attempt_completion>" {
|
||||
t.Fatalf("content: %q", resp.Choices[0].Message.Content)
|
||||
}
|
||||
if resp.Choices[0].Message.ReasoningContent != "hidden thought" {
|
||||
t.Fatalf("reasoning_content: got %q, want hidden thought", resp.Choices[0].Message.ReasoningContent)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChatCompletionsStrictOutputWrapsPlainTextWhenPromptDeclaresCompletionTool(t *testing.T) {
|
||||
fake := &fakeRunService{events: make(chan *iop.RunEvent, 3)}
|
||||
fake.events <- &iop.RunEvent{Type: "delta", Delta: "plain answer"}
|
||||
|
|
@ -4084,6 +4143,54 @@ func TestChatCompletionsProviderPoolDispatch(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestChatCompletionsStrictOutputProviderPoolDefaultUsesTransformedPath(t *testing.T) {
|
||||
fake := &fakeRunService{events: make(chan *iop.RunEvent, 4)}
|
||||
fake.events <- &iop.RunEvent{Type: "reasoning_delta", Delta: "think"}
|
||||
fake.events <- &iop.RunEvent{Type: "delta", Delta: "ok"}
|
||||
fake.events <- &iop.RunEvent{Type: "complete"}
|
||||
|
||||
catalog := []config.ModelCatalogEntry{
|
||||
{ID: "qwen3.6:35b", Providers: map[string]string{"prov-vllm": "Qwen3-35B-A22B"}},
|
||||
}
|
||||
srv := NewServer(config.EdgeOpenAIConf{StrictOutput: true}, fake, nil)
|
||||
srv.SetModelCatalog(catalog)
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
||||
"model":"qwen3.6:35b",
|
||||
"messages":[{"role":"user","content":"hello"}],
|
||||
"include_reasoning": true
|
||||
}`))
|
||||
w := httptest.NewRecorder()
|
||||
srv.handleChatCompletions(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
if len(fake.tunnelReqsSnapshot()) != 0 {
|
||||
t.Fatal("strict output default must keep normalized path unless passthrough is explicit")
|
||||
}
|
||||
reqs := fake.reqsSnapshot()
|
||||
if len(reqs) != 1 {
|
||||
t.Fatalf("expected 1 SubmitRun call, got %d", len(reqs))
|
||||
}
|
||||
if !reqs[0].ProviderPool {
|
||||
t.Fatal("provider-pool dispatch should be preserved on transformed path")
|
||||
}
|
||||
if got := reqs[0].Metadata[responseModeMetadataKey]; got != responseModeTransformed {
|
||||
t.Fatalf("response mode metadata: got %q, want %q", got, responseModeTransformed)
|
||||
}
|
||||
if got := w.Header().Get(responseModeHeaderName); got != responseModeTransformed {
|
||||
t.Fatalf("response mode header: got %q, want %q", got, responseModeTransformed)
|
||||
}
|
||||
var resp chatCompletionResponse
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if resp.Choices[0].Message.ReasoningContent != "think" {
|
||||
t.Fatalf("reasoning_content: got %q, want think", resp.Choices[0].Message.ReasoningContent)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChatCompletionsResponseModeRouting(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
|
|
@ -4625,7 +4732,8 @@ func TestChatCompletionsPassthroughProviderPoolGenerationPolicy(t *testing.T) {
|
|||
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
||||
"model":"pool-model",
|
||||
"messages":[{"role":"user","content":"hello"}],
|
||||
"max_completion_tokens":20
|
||||
"max_completion_tokens":20,
|
||||
"metadata":{"iop_response_mode":"passthrough"}
|
||||
}`))
|
||||
w := httptest.NewRecorder()
|
||||
srv.handleChatCompletions(w, req)
|
||||
|
|
|
|||
|
|
@ -50,6 +50,7 @@ func (s *Server) streamChatCompletion(w http.ResponseWriter, r *http.Request, re
|
|||
model := responseModel(req.Model, handle.Dispatch().Target)
|
||||
traceStream := openAICompatTraceStreamEnabled(submitReq.Metadata)
|
||||
traceSeq := 0
|
||||
shouldExposeReasoning := req.includeReasoning() && (!outputPolicy.Strict || req.explicitlyIncludesReasoning())
|
||||
writeTracedContentDelta := func(content, source string, filtered bool) {
|
||||
if content == "" {
|
||||
return
|
||||
|
|
@ -181,7 +182,7 @@ func (s *Server) streamChatCompletion(w http.ResponseWriter, r *http.Request, re
|
|||
continue
|
||||
}
|
||||
reasoningBuilder.WriteString(reasoning)
|
||||
if outputPolicy.Strict || !req.includeReasoning() {
|
||||
if !shouldExposeReasoning {
|
||||
continue
|
||||
}
|
||||
writeTracedReasoningDelta(reasoning)
|
||||
|
|
@ -189,7 +190,7 @@ func (s *Server) streamChatCompletion(w http.ResponseWriter, r *http.Request, re
|
|||
processContentDelta(contentSentinelFilter.Flush())
|
||||
if reasoning := reasoningSentinelFilter.Flush(); reasoning != "" {
|
||||
reasoningBuilder.WriteString(reasoning)
|
||||
if !outputPolicy.Strict && req.includeReasoning() {
|
||||
if shouldExposeReasoning {
|
||||
writeTracedReasoningDelta(reasoning)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ func prependSystemMessage(messages []chatMessage, content string) []chatMessage
|
|||
return out
|
||||
}
|
||||
|
||||
func normalizeCompletionOutput(policy strictOutputPolicy, content, reasoning string) (string, string, bool) {
|
||||
func normalizeCompletionOutput(policy strictOutputPolicy, content, reasoning string, preserveReasoning bool) (string, string, bool) {
|
||||
var sanitized bool
|
||||
content, sanitized = sanitizeKnownChatTemplateSentinels(content)
|
||||
reasoning, reasoningSanitized := sanitizeKnownChatTemplateSentinels(reasoning)
|
||||
|
|
@ -43,6 +43,9 @@ func normalizeCompletionOutput(policy strictOutputPolicy, content, reasoning str
|
|||
if policy.XMLCompletionTool != "" && !startsWithXMLRoot(normalized) {
|
||||
normalized = wrapXMLCompletion(policy, normalized)
|
||||
}
|
||||
if preserveReasoning {
|
||||
return normalized, reasoning, sanitized || normalized != content
|
||||
}
|
||||
return normalized, "", sanitized || normalized != content || reasoning != ""
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -130,6 +130,10 @@ func (req chatCompletionRequest) includeReasoning() bool {
|
|||
return true
|
||||
}
|
||||
|
||||
func (req chatCompletionRequest) explicitlyIncludesReasoning() bool {
|
||||
return req.IncludeReasoning != nil && *req.IncludeReasoning
|
||||
}
|
||||
|
||||
func shouldForwardProviderToolChoice(toolChoice any) bool {
|
||||
if toolChoice == nil {
|
||||
return false
|
||||
|
|
|
|||
Loading…
Reference in a new issue