toki
2d6fde0876
apps/node 중심 구현 — TCP+JSON transport, Hexagonal Architecture, mock/cli adapter, fx DI, SQLite 실행 이력 저장. edge/control-plane/worker는 cobra placeholder. 유닛 테스트 및 통합 테스트 클라이언트 계획서 추가.
49 lines
1.1 KiB
Go
49 lines
1.1 KiB
Go
// Package vllm provides an Adapter that calls a vLLM server via its
|
|
// OpenAI-compatible /v1/chat/completions endpoint.
|
|
package vllm
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
|
|
"go.uber.org/zap"
|
|
|
|
"iop/apps/node/internal/runtime"
|
|
"iop/packages/config"
|
|
)
|
|
|
|
const Name = "vllm"
|
|
|
|
type Vllm struct {
|
|
endpoint string
|
|
logger *zap.Logger
|
|
}
|
|
|
|
func New(cfg config.VllmConf, logger *zap.Logger) *Vllm {
|
|
return &Vllm{
|
|
endpoint: cfg.Endpoint,
|
|
logger: logger,
|
|
}
|
|
}
|
|
|
|
func (v *Vllm) Name() string { return Name }
|
|
|
|
func (v *Vllm) Capabilities(_ context.Context) (runtime.Capabilities, error) {
|
|
// TODO: query /v1/models from vLLM endpoint.
|
|
return runtime.Capabilities{
|
|
AdapterName: Name,
|
|
Models: []string{},
|
|
MaxConcurrency: 8,
|
|
}, nil
|
|
}
|
|
|
|
func (v *Vllm) Execute(ctx context.Context, spec runtime.ExecutionSpec, sink runtime.EventSink) error {
|
|
// TODO: implement streaming POST to v.endpoint/v1/chat/completions
|
|
// with "stream": true and SSE response parsing.
|
|
v.logger.Info("vllm adapter called",
|
|
zap.String("run_id", spec.RunID),
|
|
zap.String("model", spec.Model),
|
|
zap.String("endpoint", v.endpoint),
|
|
)
|
|
return fmt.Errorf("vllm adapter: not yet implemented")
|
|
}
|