toki
445716cc3e
- Add configrefresh package for config classification, request handling - Add bootstrap refresh_admin.go for admin-level config refresh - Add edgevalidate package for config validation - Update edge bootstrap runtime with refresh capabilities - Update edge service layer with config refresh integration - Update openai layer routes/chat_handler/server for config refresh - Update edgecmd config handling with refresh support - Update input manager, transport server, status provider - Add edge.yaml config updates and go/config package changes - Add hostsetup templates and test updates - Update roadmap and phase documentation for runtime-reconnect-config-refresh
114 lines
2.8 KiB
Go
114 lines
2.8 KiB
Go
package openai
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"net"
|
|
"net/http"
|
|
"sync"
|
|
"time"
|
|
|
|
"go.uber.org/zap"
|
|
|
|
edgeservice "iop/apps/edge/internal/service"
|
|
"iop/packages/go/config"
|
|
)
|
|
|
|
type runService interface {
|
|
SubmitRun(context.Context, edgeservice.SubmitRunRequest) (edgeservice.RunResult, error)
|
|
OllamaAPI(context.Context, edgeservice.OllamaAPIRequest) (edgeservice.OllamaAPIView, error)
|
|
}
|
|
|
|
type Server struct {
|
|
mu sync.RWMutex
|
|
cfg config.EdgeOpenAIConf
|
|
modelCatalog []config.ModelCatalogEntry
|
|
service runService
|
|
logger *zap.Logger
|
|
server *http.Server
|
|
}
|
|
|
|
func NewServer(cfg config.EdgeOpenAIConf, svc runService, logger *zap.Logger) *Server {
|
|
if logger == nil {
|
|
logger = zap.NewNop()
|
|
}
|
|
return &Server{cfg: cfg, service: svc, logger: logger}
|
|
}
|
|
|
|
// SetModelCatalog provides the provider-pool model catalog to the OpenAI server.
|
|
// When set, /v1/models lists catalog IDs and requests matching catalog entries
|
|
// are dispatched via the provider pool instead of the legacy model_routes path.
|
|
func (s *Server) SetModelCatalog(catalog []config.ModelCatalogEntry) {
|
|
s.mu.Lock()
|
|
s.modelCatalog = cloneModelCatalog(catalog)
|
|
s.mu.Unlock()
|
|
}
|
|
|
|
func (s *Server) modelCatalogSnapshot() []config.ModelCatalogEntry {
|
|
s.mu.RLock()
|
|
defer s.mu.RUnlock()
|
|
return cloneModelCatalog(s.modelCatalog)
|
|
}
|
|
|
|
func cloneModelCatalog(catalog []config.ModelCatalogEntry) []config.ModelCatalogEntry {
|
|
if len(catalog) == 0 {
|
|
return nil
|
|
}
|
|
out := make([]config.ModelCatalogEntry, len(catalog))
|
|
for i, entry := range catalog {
|
|
out[i] = entry
|
|
if entry.Providers != nil {
|
|
out[i].Providers = make(map[string]string, len(entry.Providers))
|
|
for k, v := range entry.Providers {
|
|
out[i].Providers[k] = v
|
|
}
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func (s *Server) Enabled() bool {
|
|
return s != nil && s.cfg.Enabled
|
|
}
|
|
|
|
func (s *Server) Start(ctx context.Context) error {
|
|
if !s.Enabled() {
|
|
return nil
|
|
}
|
|
if s.cfg.Listen == "" {
|
|
s.cfg.Listen = "0.0.0.0:18081"
|
|
}
|
|
|
|
mux := s.routes()
|
|
|
|
s.server = &http.Server{
|
|
Addr: s.cfg.Listen,
|
|
Handler: mux,
|
|
ReadHeaderTimeout: 5 * time.Second,
|
|
}
|
|
ln, err := net.Listen("tcp", s.cfg.Listen)
|
|
if err != nil {
|
|
return fmt.Errorf("openai server listen %s: %w", s.cfg.Listen, err)
|
|
}
|
|
go func() {
|
|
<-ctx.Done()
|
|
shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
_ = s.server.Shutdown(shutdownCtx)
|
|
}()
|
|
go func() {
|
|
s.logger.Info("openai-compatible server listening", zap.String("addr", s.cfg.Listen))
|
|
if err := s.server.Serve(ln); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
|
s.logger.Warn("openai-compatible server exited", zap.Error(err))
|
|
}
|
|
}()
|
|
return nil
|
|
}
|
|
|
|
func (s *Server) Stop(ctx context.Context) error {
|
|
if s == nil || s.server == nil {
|
|
return nil
|
|
}
|
|
return s.server.Shutdown(ctx)
|
|
}
|