// Package vllm provides an Adapter for OpenAI-compatible inference engines // such as vLLM and SGLang via the /v1/chat/completions SSE endpoint. package vllm type vllmMessage struct { Role string `json:"role"` Content string `json:"content"` ToolCalls []any `json:"tool_calls,omitempty"` ToolCallID string `json:"tool_call_id,omitempty"` } type vllmChatRequest struct { Model string `json:"model"` Messages []vllmMessage `json:"messages"` Stream bool `json:"stream"` Tools any `json:"tools,omitempty"` ToolChoice any `json:"tool_choice,omitempty"` } type vllmChatChunk struct { Choices []struct { Delta vllmChatChunkDelta `json:"delta"` FinishReason *string `json:"finish_reason"` } `json:"choices"` Usage *struct { PromptTokens int `json:"prompt_tokens"` CompletionTokens int `json:"completion_tokens"` PromptTokensDetails *struct { CachedTokens int `json:"cached_tokens"` } `json:"prompt_tokens_details"` CompletionTokensDetails *struct { ReasoningTokens int `json:"reasoning_tokens"` } `json:"completion_tokens_details"` } `json:"usage"` } type vllmChatChunkDelta struct { Content string `json:"content"` ReasoningContent string `json:"reasoning_content"` Reasoning string `json:"reasoning"` ToolCalls []any `json:"tool_calls"` } func (d vllmChatChunkDelta) ReasoningText() string { if d.ReasoningContent != "" { return d.ReasoningContent } return d.Reasoning } type openAIToolCallAccumulator struct { calls map[int]map[string]any order []int } func (a *openAIToolCallAccumulator) AddDelta(raw []any) { for fallbackIndex, item := range raw { call, ok := item.(map[string]any) if !ok { continue } index := fallbackIndex if parsed, ok := numericIndex(call["index"]); ok { index = parsed } dst := a.ensure(index) for key, value := range call { switch key { case "index": continue case "function": mergeToolCallFunction(dst, value) default: if !emptyToolCallValue(value) { dst[key] = value } } } } } func (a *openAIToolCallAccumulator) ToolCalls() []any { if len(a.order) == 0 { return nil } out := make([]any, 0, len(a.order)) for _, index := range a.order { call := a.calls[index] if len(call) == 0 { continue } copyCall := make(map[string]any, len(call)) for key, value := range call { copyCall[key] = value } out = append(out, copyCall) } return out } func (a *openAIToolCallAccumulator) ensure(index int) map[string]any { if a.calls == nil { a.calls = make(map[int]map[string]any) } if call, ok := a.calls[index]; ok { return call } call := make(map[string]any) a.calls[index] = call a.order = append(a.order, index) return call } func mergeToolCallFunction(call map[string]any, value any) { fn, ok := value.(map[string]any) if !ok { return } current, _ := call["function"].(map[string]any) if current == nil { current = make(map[string]any) call["function"] = current } for key, item := range fn { if key == "arguments" { if part, ok := item.(string); ok { current["arguments"] = currentString(current["arguments"]) + part continue } } if !emptyToolCallValue(item) { current[key] = item } } } func numericIndex(value any) (int, bool) { switch v := value.(type) { case int: return v, true case float64: return int(v), true default: return 0, false } } func emptyToolCallValue(value any) bool { if value == nil { return true } if text, ok := value.(string); ok { return text == "" } return false } func currentString(value any) string { if text, ok := value.(string); ok { return text } return "" } type modelsResponse struct { Data []struct { ID string `json:"id"` } `json:"data"` }