toki
f01c7e5ecd
터미널 세션 core와 CLI mode executor 경계를 분리해 후속 remote terminal bridge가 재사용할 내부 실행 기반을 만든다. adapter 기본값과 vLLM experimental surface도 명시해 production 실행 경로가 mock fallback이나 미구현 실행으로 숨지 않게 한다.
100 lines
2.2 KiB
Go
100 lines
2.2 KiB
Go
// Package vllm contains the experimental vLLM adapter surface. It can query
|
|
// models, but execution remains disabled until streaming support is implemented.
|
|
package vllm
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"net/url"
|
|
"strings"
|
|
"time"
|
|
|
|
"go.uber.org/zap"
|
|
|
|
"iop/apps/node/internal/runtime"
|
|
"iop/packages/go/config"
|
|
)
|
|
|
|
const Name = "vllm"
|
|
|
|
type Vllm struct {
|
|
endpoint string
|
|
client *http.Client
|
|
logger *zap.Logger
|
|
}
|
|
|
|
func New(cfg config.VllmConf, logger *zap.Logger) *Vllm {
|
|
endpoint := strings.TrimRight(cfg.Endpoint, "/")
|
|
return &Vllm{
|
|
endpoint: endpoint,
|
|
client: &http.Client{},
|
|
logger: logger,
|
|
}
|
|
}
|
|
|
|
func (v *Vllm) Name() string { return Name }
|
|
|
|
func (v *Vllm) Capabilities(_ context.Context) (runtime.Capabilities, error) {
|
|
return runtime.Capabilities{
|
|
AdapterName: Name,
|
|
Targets: v.fetchTargets(),
|
|
MaxConcurrency: 8,
|
|
}, nil
|
|
}
|
|
|
|
func (v *Vllm) Execute(ctx context.Context, spec runtime.ExecutionSpec, sink runtime.EventSink) error {
|
|
v.logger.Info("vllm adapter called",
|
|
zap.String("run_id", spec.RunID),
|
|
zap.String("target", spec.Target),
|
|
zap.String("endpoint", v.endpoint),
|
|
)
|
|
return fmt.Errorf("vllm adapter: experimental adapter disabled until streaming execution is implemented")
|
|
}
|
|
|
|
func (v *Vllm) fetchTargets() []string {
|
|
if v.endpoint == "" {
|
|
return nil
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
|
defer cancel()
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, joinURL(v.endpoint, "/v1/models"), nil)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
resp, err := v.client.Do(req)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
|
return nil
|
|
}
|
|
var models vllmModelsResponse
|
|
if err := json.NewDecoder(resp.Body).Decode(&models); err != nil {
|
|
return nil
|
|
}
|
|
out := make([]string, 0, len(models.Data))
|
|
for _, model := range models.Data {
|
|
if model.ID != "" {
|
|
out = append(out, model.ID)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func joinURL(baseURL, path string) string {
|
|
u, err := url.Parse(baseURL)
|
|
if err != nil {
|
|
return strings.TrimRight(baseURL, "/") + path
|
|
}
|
|
u.Path = strings.TrimRight(u.Path, "/") + path
|
|
return u.String()
|
|
}
|
|
|
|
type vllmModelsResponse struct {
|
|
Data []struct {
|
|
ID string `json:"id"`
|
|
} `json:"data"`
|
|
}
|