iop/apps/node/internal/adapters/vllm/vllm.go

// Package vllm provides an Adapter for OpenAI-compatible inference engines
// such as vLLM and SGLang via the /v1/chat/completions SSE endpoint.
package vllm

type vllmMessage struct {
	Role       string `json:"role"`
	Content    string `json:"content"`
	ToolCalls  []any  `json:"tool_calls,omitempty"`
	ToolCallID string `json:"tool_call_id,omitempty"`
}

type vllmChatRequest struct {
	Model      string        `json:"model"`
	Messages   []vllmMessage `json:"messages"`
	Stream     bool          `json:"stream"`
	Tools      any           `json:"tools,omitempty"`
	ToolChoice any           `json:"tool_choice,omitempty"`
}

type vllmChatChunk struct {
	Choices []struct {
		Delta        vllmChatChunkDelta `json:"delta"`
		FinishReason *string            `json:"finish_reason"`
	} `json:"choices"`
	Usage *struct {
		PromptTokens        int `json:"prompt_tokens"`
		CompletionTokens    int `json:"completion_tokens"`
		PromptTokensDetails *struct {
			CachedTokens int `json:"cached_tokens"`
		} `json:"prompt_tokens_details"`
		CompletionTokensDetails *struct {
			ReasoningTokens int `json:"reasoning_tokens"`
		} `json:"completion_tokens_details"`
	} `json:"usage"`
}

type vllmChatChunkDelta struct {
	Content          string `json:"content"`
	ReasoningContent string `json:"reasoning_content"`
	Reasoning        string `json:"reasoning"`
	ToolCalls        []any  `json:"tool_calls"`
}

func (d vllmChatChunkDelta) ReasoningText() string {
	if d.ReasoningContent != "" {
		return d.ReasoningContent
	}
	return d.Reasoning
}

type openAIToolCallAccumulator struct {
	calls map[int]map[string]any
	order []int
}

func (a *openAIToolCallAccumulator) AddDelta(raw []any) {
	for fallbackIndex, item := range raw {
		call, ok := item.(map[string]any)
		if !ok {
			continue
		}
		index := fallbackIndex
		if parsed, ok := numericIndex(call["index"]); ok {
			index = parsed
		}
		dst := a.ensure(index)
		for key, value := range call {
			switch key {
			case "index":
				continue
			case "function":
				mergeToolCallFunction(dst, value)
			default:
				if !emptyToolCallValue(value) {
					dst[key] = value
				}
			}
		}
	}
}

func (a *openAIToolCallAccumulator) ToolCalls() []any {
	if len(a.order) == 0 {
		return nil
	}
	out := make([]any, 0, len(a.order))
	for _, index := range a.order {
		call := a.calls[index]
		if len(call) == 0 {
			continue
		}
		copyCall := make(map[string]any, len(call))
		for key, value := range call {
			copyCall[key] = value
		}
		out = append(out, copyCall)
	}
	return out
}

func (a *openAIToolCallAccumulator) ensure(index int) map[string]any {
	if a.calls == nil {
		a.calls = make(map[int]map[string]any)
	}
	if call, ok := a.calls[index]; ok {
		return call
	}
	call := make(map[string]any)
	a.calls[index] = call
	a.order = append(a.order, index)
	return call
}

func mergeToolCallFunction(call map[string]any, value any) {
	fn, ok := value.(map[string]any)
	if !ok {
		return
	}
	current, _ := call["function"].(map[string]any)
	if current == nil {
		current = make(map[string]any)
		call["function"] = current
	}
	for key, item := range fn {
		if key == "arguments" {
			if part, ok := item.(string); ok {
				current["arguments"] = currentString(current["arguments"]) + part
				continue
			}
		}
		if !emptyToolCallValue(item) {
			current[key] = item
		}
	}
}

func numericIndex(value any) (int, bool) {
	switch v := value.(type) {
	case int:
		return v, true
	case float64:
		return int(v), true
	default:
		return 0, false
	}
}

func emptyToolCallValue(value any) bool {
	if value == nil {
		return true
	}
	if text, ok := value.(string); ok {
		return text == ""
	}
	return false
}

func currentString(value any) string {
	if text, ok := value.(string); ok {
		return text
	}
	return ""
}

type modelsResponse struct {
	Data []struct {
		ID string `json:"id"`
	} `json:"data"`
}