// Package vllm provides an Adapter that calls a vLLM server via its // OpenAI-compatible /v1/chat/completions endpoint. package vllm import ( "context" "encoding/json" "fmt" "net/http" "net/url" "strings" "time" "go.uber.org/zap" "iop/apps/node/internal/runtime" "iop/packages/go/config" ) const Name = "vllm" type Vllm struct { endpoint string client *http.Client logger *zap.Logger } func New(cfg config.VllmConf, logger *zap.Logger) *Vllm { endpoint := strings.TrimRight(cfg.Endpoint, "/") return &Vllm{ endpoint: endpoint, client: &http.Client{}, logger: logger, } } func (v *Vllm) Name() string { return Name } func (v *Vllm) Capabilities(_ context.Context) (runtime.Capabilities, error) { return runtime.Capabilities{ AdapterName: Name, Targets: v.fetchTargets(), MaxConcurrency: 8, }, nil } func (v *Vllm) Execute(ctx context.Context, spec runtime.ExecutionSpec, sink runtime.EventSink) error { // TODO: implement streaming POST to v.endpoint/v1/chat/completions // with "stream": true and SSE response parsing. v.logger.Info("vllm adapter called", zap.String("run_id", spec.RunID), zap.String("target", spec.Target), zap.String("endpoint", v.endpoint), ) return fmt.Errorf("vllm adapter: not yet implemented") } func (v *Vllm) fetchTargets() []string { if v.endpoint == "" { return nil } ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() req, err := http.NewRequestWithContext(ctx, http.MethodGet, joinURL(v.endpoint, "/v1/models"), nil) if err != nil { return nil } resp, err := v.client.Do(req) if err != nil { return nil } defer resp.Body.Close() if resp.StatusCode < 200 || resp.StatusCode >= 300 { return nil } var models vllmModelsResponse if err := json.NewDecoder(resp.Body).Decode(&models); err != nil { return nil } out := make([]string, 0, len(models.Data)) for _, model := range models.Data { if model.ID != "" { out = append(out, model.ID) } } return out } func joinURL(baseURL, path string) string { u, err := url.Parse(baseURL) if err != nil { return strings.TrimRight(baseURL, "/") + path } u.Path = strings.TrimRight(u.Path, "/") + path return u.String() } type vllmModelsResponse struct { Data []struct { ID string `json:"id"` } `json:"data"` }