48 lines
1.3 KiB
Go
48 lines
1.3 KiB
Go
package vllm
|
|
|
|
import (
|
|
"net/http"
|
|
"strings"
|
|
|
|
"go.uber.org/zap"
|
|
|
|
"iop/packages/go/config"
|
|
)
|
|
|
|
const Name = "vllm"
|
|
|
|
// Vllm is the vLLM/SGLang adapter. It targets OpenAI-compatible endpoints
|
|
// without the per-request auth/header injection that the openai_compat adapter
|
|
// carries, so a provider boundary and its config contract stay distinct.
|
|
type Vllm struct {
|
|
instanceName string
|
|
endpoint string
|
|
capacity int
|
|
maxQueue int
|
|
queueTimeoutMS int
|
|
requestTimeoutMS int
|
|
client *http.Client
|
|
logger *zap.Logger
|
|
}
|
|
|
|
// New creates a vLLM adapter. The optional instanceName is the registry
|
|
// instance key used to disambiguate multiple vLLM instances on the same node.
|
|
func New(cfg config.VllmConf, logger *zap.Logger, instanceName ...string) *Vllm {
|
|
endpoint := strings.TrimRight(cfg.Endpoint, "/")
|
|
name := Name
|
|
if len(instanceName) > 0 && instanceName[0] != "" {
|
|
name = instanceName[0]
|
|
}
|
|
return &Vllm{
|
|
instanceName: name,
|
|
endpoint: endpoint,
|
|
capacity: cfg.Capacity,
|
|
maxQueue: cfg.MaxQueue,
|
|
queueTimeoutMS: cfg.QueueTimeoutMS,
|
|
requestTimeoutMS: cfg.RequestTimeoutMS,
|
|
client: &http.Client{},
|
|
logger: logger,
|
|
}
|
|
}
|
|
|
|
func (v *Vllm) Name() string { return Name }
|