toki
a52ecb8efe
필드 테스트를 위해 edge OpenAI-compatible 경로와 node adapter 설정을 확장하고, Jenkins 바이너리 빌드 및 control-plane/web compose 배포 구성을 함께 정리한다.
102 lines
2.3 KiB
Go
102 lines
2.3 KiB
Go
// Package vllm provides an Adapter that calls a vLLM server via its
|
|
// OpenAI-compatible /v1/chat/completions endpoint.
|
|
package vllm
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"net/url"
|
|
"strings"
|
|
"time"
|
|
|
|
"go.uber.org/zap"
|
|
|
|
"iop/apps/node/internal/runtime"
|
|
"iop/packages/config"
|
|
)
|
|
|
|
const Name = "vllm"
|
|
|
|
type Vllm struct {
|
|
endpoint string
|
|
client *http.Client
|
|
logger *zap.Logger
|
|
}
|
|
|
|
func New(cfg config.VllmConf, logger *zap.Logger) *Vllm {
|
|
endpoint := strings.TrimRight(cfg.Endpoint, "/")
|
|
return &Vllm{
|
|
endpoint: endpoint,
|
|
client: &http.Client{},
|
|
logger: logger,
|
|
}
|
|
}
|
|
|
|
func (v *Vllm) Name() string { return Name }
|
|
|
|
func (v *Vllm) Capabilities(_ context.Context) (runtime.Capabilities, error) {
|
|
return runtime.Capabilities{
|
|
AdapterName: Name,
|
|
Targets: v.fetchTargets(),
|
|
MaxConcurrency: 8,
|
|
}, nil
|
|
}
|
|
|
|
func (v *Vllm) Execute(ctx context.Context, spec runtime.ExecutionSpec, sink runtime.EventSink) error {
|
|
// TODO: implement streaming POST to v.endpoint/v1/chat/completions
|
|
// with "stream": true and SSE response parsing.
|
|
v.logger.Info("vllm adapter called",
|
|
zap.String("run_id", spec.RunID),
|
|
zap.String("target", spec.Target),
|
|
zap.String("endpoint", v.endpoint),
|
|
)
|
|
return fmt.Errorf("vllm adapter: not yet implemented")
|
|
}
|
|
|
|
func (v *Vllm) fetchTargets() []string {
|
|
if v.endpoint == "" {
|
|
return nil
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
|
defer cancel()
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, joinURL(v.endpoint, "/v1/models"), nil)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
resp, err := v.client.Do(req)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
|
return nil
|
|
}
|
|
var models vllmModelsResponse
|
|
if err := json.NewDecoder(resp.Body).Decode(&models); err != nil {
|
|
return nil
|
|
}
|
|
out := make([]string, 0, len(models.Data))
|
|
for _, model := range models.Data {
|
|
if model.ID != "" {
|
|
out = append(out, model.ID)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func joinURL(baseURL, path string) string {
|
|
u, err := url.Parse(baseURL)
|
|
if err != nil {
|
|
return strings.TrimRight(baseURL, "/") + path
|
|
}
|
|
u.Path = strings.TrimRight(u.Path, "/") + path
|
|
return u.String()
|
|
}
|
|
|
|
type vllmModelsResponse struct {
|
|
Data []struct {
|
|
ID string `json:"id"`
|
|
} `json:"data"`
|
|
}
|