toki
d764adb390
- edge node store 및 console 업데이트 - node adapters(cli, ollama, vllm, mock) 리팩토링 - node router, run_manager, store 개선 - proto 파일(control.proto, runtime.proto) 및 생성 코드 업데이트 - config 업데이트
49 lines
1.1 KiB
Go
49 lines
1.1 KiB
Go
// Package vllm provides an Adapter that calls a vLLM server via its
|
|
// OpenAI-compatible /v1/chat/completions endpoint.
|
|
package vllm
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
|
|
"go.uber.org/zap"
|
|
|
|
"iop/apps/node/internal/runtime"
|
|
"iop/packages/config"
|
|
)
|
|
|
|
const Name = "vllm"
|
|
|
|
type Vllm struct {
|
|
endpoint string
|
|
logger *zap.Logger
|
|
}
|
|
|
|
func New(cfg config.VllmConf, logger *zap.Logger) *Vllm {
|
|
return &Vllm{
|
|
endpoint: cfg.Endpoint,
|
|
logger: logger,
|
|
}
|
|
}
|
|
|
|
func (v *Vllm) Name() string { return Name }
|
|
|
|
func (v *Vllm) Capabilities(_ context.Context) (runtime.Capabilities, error) {
|
|
// TODO: query /v1/models from vLLM endpoint.
|
|
return runtime.Capabilities{
|
|
AdapterName: Name,
|
|
Targets: []string{},
|
|
MaxConcurrency: 8,
|
|
}, nil
|
|
}
|
|
|
|
func (v *Vllm) Execute(ctx context.Context, spec runtime.ExecutionSpec, sink runtime.EventSink) error {
|
|
// TODO: implement streaming POST to v.endpoint/v1/chat/completions
|
|
// with "stream": true and SSE response parsing.
|
|
v.logger.Info("vllm adapter called",
|
|
zap.String("run_id", spec.RunID),
|
|
zap.String("target", spec.Target),
|
|
zap.String("endpoint", v.endpoint),
|
|
)
|
|
return fmt.Errorf("vllm adapter: not yet implemented")
|
|
}
|