682 lines
21 KiB
Go
682 lines
21 KiB
Go
package openai
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"net/http"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"go.uber.org/zap"
|
|
|
|
edgeservice "iop/apps/edge/internal/service"
|
|
"iop/packages/go/config"
|
|
)
|
|
|
|
func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodPost {
|
|
writeError(w, http.StatusMethodNotAllowed, "method_not_allowed", "method not allowed")
|
|
return
|
|
}
|
|
defer r.Body.Close()
|
|
|
|
var req chatCompletionRequest
|
|
if err := decodeChatCompletionRequest(json.NewDecoder(r.Body), &req); err != nil {
|
|
writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error())
|
|
return
|
|
}
|
|
runMeta, workspace, err := parseOpenAIMetadata(req.Metadata)
|
|
if err != nil {
|
|
writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error())
|
|
return
|
|
}
|
|
|
|
dispatch, ok := s.resolveRouteDispatch(req.Model)
|
|
if !ok {
|
|
writeError(w, http.StatusBadRequest, "invalid_request_error", "model is required")
|
|
return
|
|
}
|
|
if err := validateWorkspaceForRoute(dispatch, workspace); err != nil {
|
|
writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error())
|
|
return
|
|
}
|
|
basePrompt := promptFromMessages(req.Messages)
|
|
if strings.TrimSpace(basePrompt) == "" {
|
|
writeError(w, http.StatusBadRequest, "invalid_request_error", "messages are required")
|
|
return
|
|
}
|
|
outputPolicy := s.resolveOutputPolicy(basePrompt)
|
|
if catalogEntry := s.findProviderPoolEntry(req.Model); catalogEntry != nil {
|
|
applyModelCatalogGenerationPolicyToChat(&req, *catalogEntry, outputPolicy.Strict)
|
|
}
|
|
messages := req.Messages
|
|
if instruction := strictOutputContractInstruction(outputPolicy); instruction != "" {
|
|
messages = prependSystemMessage(messages, instruction)
|
|
}
|
|
prompt := promptFromMessages(messages)
|
|
input := req.runInput(prompt, messages, outputPolicy.Strict, routeSupportsNativeToolCalls(dispatch))
|
|
s.logger.Info("openai chat completion input",
|
|
zap.String("model", req.Model),
|
|
zap.String("target", dispatch.Target),
|
|
zap.String("adapter", dispatch.Adapter),
|
|
zap.Bool("strict_output", outputPolicy.Strict),
|
|
zap.Bool("strict_stream_buffer", outputPolicy.StreamBuffer),
|
|
zap.String("xml_completion_tool", outputPolicy.XMLCompletionTool),
|
|
zap.Bool("contract_instruction", outputPolicy.ContractInstruction),
|
|
zap.Bool("stream", req.Stream),
|
|
zap.Int("message_count", len(req.Messages)),
|
|
zap.Int("prompt_len", len(prompt)),
|
|
zap.Any("input_keys", mapKeys(input)),
|
|
)
|
|
|
|
estimate := s.estimateChatInputTokens(prompt, runMeta, req.Tools, req.ToolChoice)
|
|
contextClass := classifyContext(estimate, s.longContextThreshold())
|
|
|
|
validation := toolValidationContractFromRequest(req)
|
|
metadata := chatRunMetadata(runMeta, req, outputPolicy)
|
|
metadata["estimated_input_tokens"] = strconv.Itoa(estimate)
|
|
metadata["context_class"] = contextClass
|
|
if validation.enabled {
|
|
metadata = toolValidationAttemptMetadata(metadata, 1, "", "")
|
|
}
|
|
submitReq := chatSubmitRunRequest(dispatch, req, workspace, prompt, input, metadata)
|
|
submitReq.EstimatedInputTokens = estimate
|
|
submitReq.ContextClass = contextClass
|
|
handle, err := s.service.SubmitRun(r.Context(), submitReq)
|
|
if err != nil {
|
|
writeError(w, http.StatusBadGateway, "node_dispatch_error", err.Error())
|
|
return
|
|
}
|
|
s.logger.Info("openai chat completion dispatch",
|
|
zap.String("run_id", handle.Dispatch().RunID),
|
|
zap.String("node_id", handle.Dispatch().NodeID),
|
|
zap.String("model_group", handle.Dispatch().ModelGroupKey),
|
|
zap.String("adapter", handle.Dispatch().Adapter),
|
|
zap.String("target", handle.Dispatch().Target),
|
|
zap.Int("estimated_input_tokens", handle.Dispatch().EstimatedInputTokens),
|
|
zap.String("context_class", handle.Dispatch().ContextClass),
|
|
zap.String("queue_reason", handle.Dispatch().QueueReason),
|
|
)
|
|
|
|
if req.Stream {
|
|
s.streamChatCompletion(w, r, req, submitReq, handle, outputPolicy, validation)
|
|
return
|
|
}
|
|
s.completeChatCompletion(w, r, req, submitReq, handle, outputPolicy, validation)
|
|
}
|
|
|
|
func decodeChatCompletionRequest(dec *json.Decoder, req *chatCompletionRequest) error {
|
|
var raw map[string]json.RawMessage
|
|
if err := dec.Decode(&raw); err != nil {
|
|
return fmt.Errorf("invalid JSON request")
|
|
}
|
|
for key := range raw {
|
|
switch key {
|
|
case "model", "messages", "stream", "metadata", "max_tokens", "max_completion_tokens", "temperature", "top_p", "presence_penalty", "frequency_penalty", "seed", "stop", "response_format", "tools", "tool_choice", "parallel_tool_calls", "stream_options", "store", "think", "reasoning_effort", "thinking_token_budget", "include_reasoning":
|
|
default:
|
|
return fmt.Errorf("%s is not supported for /v1/chat/completions", key)
|
|
}
|
|
}
|
|
normalized, err := json.Marshal(raw)
|
|
if err != nil {
|
|
return fmt.Errorf("invalid JSON request")
|
|
}
|
|
if err := json.Unmarshal(normalized, req); err != nil {
|
|
return fmt.Errorf("invalid /v1/chat/completions request format")
|
|
}
|
|
if req.MaxTokens != nil && *req.MaxTokens <= 0 {
|
|
return fmt.Errorf("max_tokens must be greater than zero")
|
|
}
|
|
if req.MaxCompletionTokens != nil && *req.MaxCompletionTokens <= 0 {
|
|
return fmt.Errorf("max_completion_tokens must be greater than zero")
|
|
}
|
|
if req.Temperature != nil && (*req.Temperature < 0 || *req.Temperature > 2) {
|
|
return fmt.Errorf("temperature must be between 0 and 2")
|
|
}
|
|
if req.TopP != nil && (*req.TopP < 0 || *req.TopP > 1) {
|
|
return fmt.Errorf("top_p must be between 0 and 1")
|
|
}
|
|
if err := validateThinkControl(req); err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validateThinkControl(req *chatCompletionRequest) error {
|
|
if req.ThinkingTokenBudget != nil && *req.ThinkingTokenBudget < 0 {
|
|
return fmt.Errorf("thinking_token_budget must be non-negative")
|
|
}
|
|
if req.ReasoningEffort != nil {
|
|
eff := *req.ReasoningEffort
|
|
if eff == "" {
|
|
return fmt.Errorf("reasoning_effort cannot be empty when present")
|
|
}
|
|
if eff != "none" && eff != "low" && eff != "medium" && eff != "high" {
|
|
return fmt.Errorf("reasoning_effort must be one of none, low, medium, or high")
|
|
}
|
|
}
|
|
if req.Think != nil && !*req.Think {
|
|
if req.ReasoningEffort != nil && *req.ReasoningEffort != "none" {
|
|
return fmt.Errorf("think=false conflicts with reasoning_effort=%s", *req.ReasoningEffort)
|
|
}
|
|
}
|
|
if req.ThinkingTokenBudget != nil {
|
|
if req.Think != nil && !*req.Think {
|
|
return fmt.Errorf("thinking_token_budget cannot be set when think=false")
|
|
}
|
|
if req.ReasoningEffort != nil && *req.ReasoningEffort == "none" {
|
|
return fmt.Errorf("thinking_token_budget cannot be set when reasoning_effort=none")
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func applyModelCatalogGenerationPolicyToChat(req *chatCompletionRequest, entry config.ModelCatalogEntry, strictOutput bool) {
|
|
if req == nil {
|
|
return
|
|
}
|
|
applyOutputTokenPolicy(&req.MaxTokens, req.MaxCompletionTokens, entry.DefaultMaxTokens, entry.MinMaxTokens)
|
|
if entry.DefaultThinkingTokenBudget > 0 && chatRequestAllowsDefaultThinkingBudget(*req) {
|
|
if req.ThinkingTokenBudget == nil {
|
|
req.ThinkingTokenBudget = intPtr(entry.DefaultThinkingTokenBudget)
|
|
}
|
|
if strictOutput && req.Think == nil {
|
|
req.Think = boolPtr(true)
|
|
}
|
|
}
|
|
}
|
|
|
|
func applyModelCatalogGenerationPolicyToResponses(req *responsesRequest, entry config.ModelCatalogEntry) {
|
|
if req == nil {
|
|
return
|
|
}
|
|
applyOutputTokenPolicy(&req.MaxOutputTokens, nil, entry.DefaultMaxTokens, entry.MinMaxTokens)
|
|
}
|
|
|
|
func applyOutputTokenPolicy(primary **int, fallback *int, defaultTokens, minTokens int) {
|
|
if primary == nil {
|
|
return
|
|
}
|
|
current := 0
|
|
if *primary != nil {
|
|
current = **primary
|
|
} else if fallback != nil {
|
|
current = *fallback
|
|
}
|
|
|
|
next := current
|
|
if next == 0 && defaultTokens > 0 {
|
|
next = defaultTokens
|
|
}
|
|
if minTokens > 0 && (next == 0 || next < minTokens) {
|
|
next = minTokens
|
|
}
|
|
if next > 0 && next != current {
|
|
*primary = intPtr(next)
|
|
}
|
|
}
|
|
|
|
func chatRequestAllowsDefaultThinkingBudget(req chatCompletionRequest) bool {
|
|
if req.Think != nil && !*req.Think {
|
|
return false
|
|
}
|
|
if req.ReasoningEffort != nil && *req.ReasoningEffort == "none" {
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
func intPtr(v int) *int {
|
|
return &v
|
|
}
|
|
|
|
func boolPtr(v bool) *bool {
|
|
return &v
|
|
}
|
|
|
|
func chatRunMetadata(runMeta map[string]string, req chatCompletionRequest, outputPolicy strictOutputPolicy) map[string]string {
|
|
if runMeta == nil {
|
|
runMeta = make(map[string]string)
|
|
}
|
|
runMeta["openai_model"] = req.Model
|
|
runMeta["openai_stream"] = fmt.Sprintf("%t", req.Stream)
|
|
runMeta["strict_output"] = fmt.Sprintf("%t", outputPolicy.Strict)
|
|
return runMeta
|
|
}
|
|
|
|
func routeSupportsNativeToolCalls(dispatch routeDispatch) bool {
|
|
if dispatch.ProviderPool {
|
|
return true
|
|
}
|
|
switch strings.TrimSpace(dispatch.Adapter) {
|
|
case "openai_compat", "vllm", "ollama":
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func shouldSynthesizeTextToolCalls(dispatch edgeservice.RunDispatch, textToolFallback bool) bool {
|
|
return textToolFallback || strings.TrimSpace(dispatch.Adapter) == "cli"
|
|
}
|
|
|
|
func chatSubmitRunRequest(dispatch routeDispatch, req chatCompletionRequest, workspace, prompt string, input map[string]any, metadata map[string]string) edgeservice.SubmitRunRequest {
|
|
return edgeservice.SubmitRunRequest{
|
|
NodeRef: dispatch.NodeRef,
|
|
ModelGroupKey: strings.TrimSpace(req.Model),
|
|
Adapter: dispatch.Adapter,
|
|
Target: dispatch.Target,
|
|
SessionID: dispatch.SessionID,
|
|
Workspace: workspace,
|
|
Prompt: prompt,
|
|
Input: input,
|
|
TimeoutSec: dispatch.TimeoutSec,
|
|
MaxQueue: dispatch.MaxQueue,
|
|
QueueTimeoutMS: dispatch.QueueTimeoutMS,
|
|
Metadata: metadata,
|
|
ProviderPool: dispatch.ProviderPool,
|
|
}
|
|
}
|
|
|
|
func (s *Server) completeChatCompletion(w http.ResponseWriter, r *http.Request, req chatCompletionRequest, submitReq edgeservice.SubmitRunRequest, handle edgeservice.RunResult, outputPolicy strictOutputPolicy, validation toolValidationContract) {
|
|
attempt := 1
|
|
for {
|
|
result, err := collectChatCompletionOutput(r.Context(), req, handle, outputPolicy)
|
|
if err != nil {
|
|
s.cancelRunOnHTTPGiveUp(handle.Dispatch(), err)
|
|
handle.Close()
|
|
writeError(w, httpStatusForRunError(err), "run_error", err.Error())
|
|
return
|
|
}
|
|
valErr := result.toolValidationErr
|
|
if valErr == nil {
|
|
valErr = validateToolCallResponse(validation, result.toolCalls, result.toolCallOrigin)
|
|
}
|
|
if valErr != nil {
|
|
failedRunID := handle.Dispatch().RunID
|
|
if attempt < maxToolValidationAttempts {
|
|
handle.Close()
|
|
attempt++
|
|
retryReq := submitReq
|
|
retryReq.Metadata = toolValidationAttemptMetadata(submitReq.Metadata, attempt, failedRunID, valErr.Error())
|
|
s.logger.Warn("openai chat completion tool validation retry",
|
|
zap.String("run_id", failedRunID),
|
|
zap.Int("attempt", attempt),
|
|
zap.String("reason", valErr.Error()),
|
|
)
|
|
next, submitErr := s.service.SubmitRun(r.Context(), retryReq)
|
|
if submitErr != nil {
|
|
writeError(w, http.StatusBadGateway, "tool_validation_retry_error", submitErr.Error())
|
|
return
|
|
}
|
|
handle = next
|
|
continue
|
|
}
|
|
handle.Close()
|
|
s.logger.Warn("openai chat completion tool validation failed",
|
|
zap.String("run_id", failedRunID),
|
|
zap.Int("attempt", attempt),
|
|
zap.String("reason", valErr.Error()),
|
|
)
|
|
writeError(w, http.StatusBadGateway, "tool_validation_error", valErr.Error())
|
|
return
|
|
}
|
|
handle.Close()
|
|
s.logger.Info("openai chat completion output",
|
|
zap.String("run_id", handle.Dispatch().RunID),
|
|
zap.Bool("strict_output", outputPolicy.Strict),
|
|
zap.Bool("strict_stream_buffer", outputPolicy.StreamBuffer),
|
|
zap.String("xml_completion_tool", outputPolicy.XMLCompletionTool),
|
|
zap.Bool("normalized", result.normalized),
|
|
zap.Int("content_len", result.contentLen),
|
|
zap.Int("reasoning_len", result.reasoningLen),
|
|
)
|
|
created := time.Now().Unix()
|
|
writeJSON(w, http.StatusOK, chatCompletionResponse{
|
|
ID: "chatcmpl-" + handle.Dispatch().RunID,
|
|
Object: "chat.completion",
|
|
Created: created,
|
|
Model: responseModel(req.Model, handle.Dispatch().Target),
|
|
Choices: []chatCompletionChoice{{
|
|
Index: 0,
|
|
Message: result.message,
|
|
FinishReason: result.finishReason,
|
|
}},
|
|
Usage: result.usage,
|
|
})
|
|
return
|
|
}
|
|
}
|
|
|
|
type chatCompletionOutput struct {
|
|
message chatMessage
|
|
finishReason string
|
|
usage *openAIUsage
|
|
normalized bool
|
|
contentLen int
|
|
reasoningLen int
|
|
toolCalls []any
|
|
toolCallOrigin string
|
|
toolValidationErr error
|
|
}
|
|
|
|
func collectChatCompletionOutput(ctx context.Context, req chatCompletionRequest, handle edgeservice.RunResult, outputPolicy strictOutputPolicy) (chatCompletionOutput, error) {
|
|
text, reasoning, finishReason, nativeToolCalls, usage, _, err := collectRunResult(ctx, handle.Stream(), handle.WaitTimeout())
|
|
if err != nil {
|
|
return chatCompletionOutput{}, err
|
|
}
|
|
text, reasoning, normalized := normalizeCompletionOutput(outputPolicy, text, reasoning)
|
|
message := chatMessage{Role: "assistant", Content: text, ReasoningContent: reasoning}
|
|
includeReasoning := req.includeReasoning()
|
|
if !includeReasoning {
|
|
message.ReasoningContent = ""
|
|
}
|
|
var toolCalls []any
|
|
toolCallOrigin := ""
|
|
var toolValidationErr error
|
|
|
|
if len(nativeToolCalls) > 0 {
|
|
if len(req.Tools) > 0 && strings.TrimSpace(message.Content) != "" {
|
|
filter := &streamToolTextFilter{}
|
|
message.Content = filter.Append(message.Content) + filter.FlushNonCandidate()
|
|
}
|
|
nativeToolCalls = normalizeNativeToolCallsForResponse(nativeToolCalls, req.Tools)
|
|
message.ToolCalls = nativeToolCalls
|
|
toolCalls = nativeToolCalls
|
|
toolCallOrigin = toolCallOriginNative
|
|
finishReason = "tool_calls"
|
|
} else if len(req.Tools) > 0 {
|
|
res := synthesizeToolCallsFromTextResult(text, req.Tools, handle.Dispatch().RunID)
|
|
if res.validationErr != nil {
|
|
toolValidationErr = res.validationErr
|
|
} else if len(res.toolCalls) > 0 {
|
|
message.Content = res.cleaned
|
|
message.ReasoningContent = ""
|
|
message.ToolCalls = res.toolCalls
|
|
toolCalls = res.toolCalls
|
|
toolCallOrigin = toolCallOriginText
|
|
finishReason = "tool_calls"
|
|
}
|
|
}
|
|
if len(message.ToolCalls) == 0 && strings.TrimSpace(message.Content) == "" && strings.TrimSpace(reasoning) != "" {
|
|
if includeReasoning {
|
|
message.Content = reasoningOnlyFallbackContent(reasoning, finishReason)
|
|
} else {
|
|
message.Content = hiddenReasoningFallbackContent(finishReason)
|
|
}
|
|
}
|
|
return chatCompletionOutput{
|
|
message: message,
|
|
finishReason: finishReason,
|
|
usage: usage,
|
|
normalized: normalized,
|
|
contentLen: len(message.Content),
|
|
reasoningLen: len(reasoning),
|
|
toolCalls: toolCalls,
|
|
toolCallOrigin: toolCallOrigin,
|
|
toolValidationErr: toolValidationErr,
|
|
}, nil
|
|
}
|
|
|
|
func reasoningOnlyFallbackContent(reasoning, finishReason string) string {
|
|
content := strings.TrimSpace(reasoning)
|
|
if content == "" {
|
|
return ""
|
|
}
|
|
reason := strings.TrimSpace(finishReason)
|
|
if reason == "" || reason == "stop" {
|
|
return content
|
|
}
|
|
return content + "\n\n[IOP notice: model finished before producing final assistant content; finish_reason=" + reason + "]"
|
|
}
|
|
|
|
func hiddenReasoningFallbackContent(finishReason string) string {
|
|
reason := strings.TrimSpace(finishReason)
|
|
if reason == "" {
|
|
reason = "unknown"
|
|
}
|
|
return "[IOP notice: model produced reasoning but no final assistant content; reasoning was hidden by include_reasoning=false; finish_reason=" + reason + "]"
|
|
}
|
|
|
|
func (s *Server) resolveAdapter() string {
|
|
if s.cfg.Adapter != "" {
|
|
return s.cfg.Adapter
|
|
}
|
|
return "ollama"
|
|
}
|
|
|
|
func (s *Server) resolveTarget(model string) string {
|
|
if s.cfg.Target != "" {
|
|
return s.cfg.Target
|
|
}
|
|
return strings.TrimSpace(model)
|
|
}
|
|
|
|
// routeDispatch holds fully-resolved dispatch parameters for a single request.
|
|
type routeDispatch struct {
|
|
NodeRef string
|
|
Adapter string
|
|
Target string
|
|
SessionID string
|
|
TimeoutSec int
|
|
MaxQueue int
|
|
QueueTimeoutMS int
|
|
WorkspaceRequired bool
|
|
// ProviderPool is true when the request model matched a provider-pool
|
|
// catalog entry. Adapter and Target are empty; the service layer resolves
|
|
// them per-candidate and rewrites Target after admission.
|
|
ProviderPool bool
|
|
}
|
|
|
|
// resolveRoute returns the first catalog entry whose Model matches model.
|
|
// Entries with an empty Target are skipped.
|
|
func (s *Server) resolveRoute(model string) *config.OpenAIRouteEntry {
|
|
model = strings.TrimSpace(model)
|
|
if model == "" {
|
|
return nil
|
|
}
|
|
for i := range s.cfg.ModelRoutes {
|
|
r := &s.cfg.ModelRoutes[i]
|
|
if strings.TrimSpace(r.Model) == model && r.Target != "" {
|
|
return r
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// findProviderPoolEntry returns the catalog entry matching model, or nil.
|
|
func (s *Server) findProviderPoolEntry(model string) *config.ModelCatalogEntry {
|
|
model = strings.TrimSpace(model)
|
|
if model == "" {
|
|
return nil
|
|
}
|
|
modelCatalog := s.modelCatalogSnapshot()
|
|
for i := range modelCatalog {
|
|
if modelCatalog[i].ID == model {
|
|
entry := modelCatalog[i]
|
|
return &entry
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// resolveRouteDispatch returns fully-resolved dispatch params for model.
|
|
// Priority: provider-pool catalog → legacy model_routes → single-target fallback.
|
|
// Returns (dispatch, true) on success; (zero, false) when no target can be resolved.
|
|
func (s *Server) resolveRouteDispatch(model string) (routeDispatch, bool) {
|
|
// Provider-pool catalog takes highest priority.
|
|
if s.findProviderPoolEntry(model) != nil {
|
|
return routeDispatch{
|
|
SessionID: s.resolveSessionID(),
|
|
TimeoutSec: s.resolveTimeoutSec(),
|
|
ProviderPool: true,
|
|
}, true
|
|
}
|
|
|
|
if route := s.resolveRoute(model); route != nil {
|
|
adapter := route.Adapter
|
|
if adapter == "" {
|
|
adapter = s.resolveAdapter()
|
|
}
|
|
nodeRef := route.NodeRef
|
|
if nodeRef == "" {
|
|
nodeRef = s.cfg.NodeRef
|
|
}
|
|
sessionID := route.SessionID
|
|
if sessionID == "" {
|
|
sessionID = s.resolveSessionID()
|
|
}
|
|
timeoutSec := route.TimeoutSec
|
|
if timeoutSec <= 0 {
|
|
timeoutSec = s.resolveTimeoutSec()
|
|
}
|
|
return routeDispatch{
|
|
NodeRef: nodeRef,
|
|
Adapter: adapter,
|
|
Target: route.Target,
|
|
SessionID: sessionID,
|
|
TimeoutSec: timeoutSec,
|
|
MaxQueue: route.MaxQueue,
|
|
QueueTimeoutMS: route.QueueTimeoutMS,
|
|
WorkspaceRequired: route.WorkspaceRequired,
|
|
}, true
|
|
}
|
|
target := s.resolveTarget(model)
|
|
if target == "" {
|
|
return routeDispatch{}, false
|
|
}
|
|
return routeDispatch{
|
|
NodeRef: s.cfg.NodeRef,
|
|
Adapter: s.resolveAdapter(),
|
|
Target: target,
|
|
SessionID: s.resolveSessionID(),
|
|
TimeoutSec: s.resolveTimeoutSec(),
|
|
}, true
|
|
}
|
|
|
|
func (s *Server) resolveSessionID() string {
|
|
if s.cfg.SessionID != "" {
|
|
return s.cfg.SessionID
|
|
}
|
|
return edgeservice.DefaultSessionID
|
|
}
|
|
|
|
func (s *Server) resolveTimeoutSec() int {
|
|
if s.cfg.TimeoutSec > 0 {
|
|
return s.cfg.TimeoutSec
|
|
}
|
|
return edgeservice.DefaultTimeoutSec
|
|
}
|
|
|
|
func (s *Server) resolveStrictOutput() bool {
|
|
return s.cfg.StrictOutput
|
|
}
|
|
|
|
func (s *Server) resolveStrictStreamBuffer() bool {
|
|
return s.cfg.StrictStreamBuffer
|
|
}
|
|
|
|
func (s *Server) resolveOutputPolicy(prompt string) strictOutputPolicy {
|
|
policy := strictOutputPolicy{
|
|
Strict: s.resolveStrictOutput(),
|
|
StreamBuffer: s.resolveStrictStreamBuffer(),
|
|
}
|
|
if !policy.Strict {
|
|
return policy
|
|
}
|
|
policy.XMLCompletionTool, policy.XMLResultTag = inferXMLCompletionContract(prompt)
|
|
policy.ContractInstruction = policy.XMLCompletionTool != ""
|
|
return policy
|
|
}
|
|
|
|
func promptFromMessages(messages []chatMessage) string {
|
|
var b strings.Builder
|
|
for _, msg := range messages {
|
|
content := strings.TrimSpace(msg.Content)
|
|
if content == "" {
|
|
continue
|
|
}
|
|
role := strings.TrimSpace(msg.Role)
|
|
if role == "" {
|
|
role = "user"
|
|
}
|
|
if b.Len() > 0 {
|
|
b.WriteString("\n")
|
|
}
|
|
b.WriteString(role)
|
|
b.WriteString(": ")
|
|
b.WriteString(content)
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
func responseModel(requestModel, target string) string {
|
|
if requestModel != "" {
|
|
return requestModel
|
|
}
|
|
return target
|
|
}
|
|
|
|
func httpStatusForRunError(err error) int {
|
|
if errors.Is(err, context.Canceled) {
|
|
return http.StatusRequestTimeout
|
|
}
|
|
return http.StatusBadGateway
|
|
}
|
|
|
|
func validateWorkspaceForRoute(d routeDispatch, workspace string) error {
|
|
if !d.WorkspaceRequired {
|
|
return nil
|
|
}
|
|
if strings.TrimSpace(workspace) == "" {
|
|
return fmt.Errorf("workspace is required for this model route")
|
|
}
|
|
if !filepath.IsAbs(workspace) {
|
|
return fmt.Errorf("workspace must be an absolute path")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func writeJSON(w http.ResponseWriter, status int, v any) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(status)
|
|
_ = json.NewEncoder(w).Encode(v)
|
|
}
|
|
|
|
func writeError(w http.ResponseWriter, status int, code, message string) {
|
|
writeJSON(w, status, errorResponse{Error: errorBody{Type: code, Message: message}})
|
|
}
|
|
|
|
// estimateChatInputTokens computes a conservative token-count approximation for
|
|
// a chat completion request. The prompt argument is the already-computed
|
|
// payload (including any strict-output contract instruction) so the estimate
|
|
// does not double-count the contract text that is also sent to the node.
|
|
func (s *Server) estimateChatInputTokens(prompt string, runMeta map[string]string, tools []any, toolChoice any) int {
|
|
textParts := []string{prompt}
|
|
// Metadata keys+values.
|
|
for k, v := range runMeta {
|
|
textParts = append(textParts, k, v)
|
|
}
|
|
// Tool schemas.
|
|
for _, t := range tools {
|
|
textParts = append(textParts, toJSON(t))
|
|
}
|
|
if toolChoice != nil {
|
|
textParts = append(textParts, toJSON(toolChoice))
|
|
}
|
|
input := strings.Join(textParts, "\n")
|
|
return estimateInputTokens(input, runMeta, tools, toolChoice)
|
|
}
|
|
|
|
// toJSON is a lightweight serialiser for any json.RawMessage-compatible value.
|
|
func toJSON(v any) string {
|
|
b, err := json.Marshal(v)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
return string(b)
|
|
}
|