OpenAI-compatible passthrough에서 caller가 보낸 provider-native thinking field가 catalog 기본 thinking budget으로 덮이지 않아야 한다. provider가 거부한 extension field도 Edge가 선판단하지 않고 provider status/body로 relay되는지 테스트 근거를 남긴다.
1647 lines
54 KiB
Go
1647 lines
54 KiB
Go
package openai_compat
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"go.uber.org/zap"
|
|
|
|
"iop/apps/node/internal/runtime"
|
|
"iop/packages/go/config"
|
|
)
|
|
|
|
type fakeSink struct {
|
|
mu sync.Mutex
|
|
events []runtime.RuntimeEvent
|
|
}
|
|
|
|
func (s *fakeSink) Emit(_ context.Context, event runtime.RuntimeEvent) error {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
s.events = append(s.events, event)
|
|
return nil
|
|
}
|
|
|
|
func (s *fakeSink) all() []runtime.RuntimeEvent {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
return append([]runtime.RuntimeEvent(nil), s.events...)
|
|
}
|
|
|
|
func TestOpenAICompatCapabilitiesQueryModels(t *testing.T) {
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if r.URL.Path != "/v1/models" {
|
|
t.Fatalf("unexpected path %s", r.URL.Path)
|
|
}
|
|
_, _ = w.Write([]byte(`{"object":"list","data":[{"id":"model-a"},{"id":"model-b"}]}`))
|
|
}))
|
|
defer server.Close()
|
|
|
|
adapter := New(config.OpenAICompatConf{
|
|
Provider: "lemonade",
|
|
Endpoint: server.URL,
|
|
Capacity: 10,
|
|
MaxQueue: 20,
|
|
QueueTimeoutMS: 5000,
|
|
RequestTimeoutMS: 10000,
|
|
}, zap.NewNop())
|
|
caps, err := adapter.Capabilities(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("Capabilities failed: %v", err)
|
|
}
|
|
if caps.AdapterName != Name {
|
|
t.Fatalf("AdapterName: got %q want %q", caps.AdapterName, Name)
|
|
}
|
|
if got := strings.Join(caps.Targets, ","); got != "model-a,model-b" {
|
|
t.Fatalf("targets: got %q", got)
|
|
}
|
|
if caps.MaxConcurrency != 10 {
|
|
t.Fatalf("expected MaxConcurrency 10, got %d", caps.MaxConcurrency)
|
|
}
|
|
if caps.MaxQueue != 20 || caps.QueueTimeoutMS != 5000 || caps.RequestTimeoutMS != 10000 {
|
|
t.Fatalf("queue config: got %+v", caps)
|
|
}
|
|
if caps.ProviderStatus != runtime.ProviderStatusAvailable {
|
|
t.Fatalf("expected available, got %s", caps.ProviderStatus)
|
|
}
|
|
|
|
adapterDefault := New(config.OpenAICompatConf{Endpoint: server.URL}, zap.NewNop())
|
|
capsDefault, err := adapterDefault.Capabilities(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("Capabilities failed: %v", err)
|
|
}
|
|
if capsDefault.MaxConcurrency != 8 {
|
|
t.Fatalf("expected default MaxConcurrency 8, got %d", capsDefault.MaxConcurrency)
|
|
}
|
|
}
|
|
|
|
func TestOpenAICompatProbeProviderAvailability(t *testing.T) {
|
|
t.Run("200_ok_and_target_hit", func(t *testing.T) {
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if r.URL.Path != "/v1/models" {
|
|
t.Fatalf("unexpected path %s", r.URL.Path)
|
|
}
|
|
_, _ = w.Write([]byte(`{"object":"list","data":[{"id":"model-a"},{"id":"model-b"}]}`))
|
|
}))
|
|
defer server.Close()
|
|
|
|
adapter := New(config.OpenAICompatConf{Provider: "lemonade", Endpoint: server.URL}, zap.NewNop())
|
|
res, err := adapter.ProbeProvider(context.Background(), "model-a")
|
|
if err != nil {
|
|
t.Fatalf("ProbeProvider failed: %v", err)
|
|
}
|
|
if res.Status != runtime.ProviderStatusAvailable {
|
|
t.Errorf("expected available, got %s", res.Status)
|
|
}
|
|
if len(res.Targets) != 2 || res.Targets[0] != "model-a" {
|
|
t.Errorf("unexpected Targets: %+v", res.Targets)
|
|
}
|
|
if res.Metadata["provider"] != "lemonade" {
|
|
t.Errorf("expected provider metadata, got %+v", res.Metadata)
|
|
}
|
|
})
|
|
|
|
t.Run("200_ok_and_target_miss", func(t *testing.T) {
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
_, _ = w.Write([]byte(`{"object":"list","data":[{"id":"model-a"}]}`))
|
|
}))
|
|
defer server.Close()
|
|
|
|
adapter := New(config.OpenAICompatConf{Endpoint: server.URL}, zap.NewNop())
|
|
res, err := adapter.ProbeProvider(context.Background(), "model-b")
|
|
if err != nil {
|
|
t.Fatalf("ProbeProvider failed: %v", err)
|
|
}
|
|
if res.Status != runtime.ProviderStatusUnavailable {
|
|
t.Errorf("expected unavailable, got %s", res.Status)
|
|
}
|
|
if !strings.Contains(res.Detail, "not found") {
|
|
t.Errorf("expected 'not found' in detail, got %s", res.Detail)
|
|
}
|
|
})
|
|
|
|
t.Run("500_internal_error", func(t *testing.T) {
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
w.WriteHeader(http.StatusInternalServerError)
|
|
}))
|
|
defer server.Close()
|
|
|
|
adapter := New(config.OpenAICompatConf{Endpoint: server.URL}, zap.NewNop())
|
|
res, err := adapter.ProbeProvider(context.Background(), "model-a")
|
|
if err != nil {
|
|
t.Fatalf("ProbeProvider failed: %v", err)
|
|
}
|
|
if res.Status != runtime.ProviderStatusUnavailable {
|
|
t.Errorf("expected unavailable, got %s", res.Status)
|
|
}
|
|
})
|
|
|
|
t.Run("empty_target", func(t *testing.T) {
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
_, _ = w.Write([]byte(`{"object":"list","data":[{"id":"model-a"}]}`))
|
|
}))
|
|
defer server.Close()
|
|
|
|
adapter := New(config.OpenAICompatConf{Endpoint: server.URL}, zap.NewNop())
|
|
res, err := adapter.ProbeProvider(context.Background(), "")
|
|
if err != nil {
|
|
t.Fatalf("ProbeProvider failed: %v", err)
|
|
}
|
|
if res.Status != runtime.ProviderStatusAvailable {
|
|
t.Errorf("expected available, got %s", res.Status)
|
|
}
|
|
})
|
|
}
|
|
func TestOpenAICompatExecuteStreamsDeltasAndFinishReason(t *testing.T) {
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if r.URL.Path != "/v1/chat/completions" {
|
|
t.Fatalf("unexpected path %s", r.URL.Path)
|
|
}
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
_, _ = fmt.Fprintf(w, "data: %s\n\n", `{"choices":[{"delta":{"reasoning_content":"think "}}]}`)
|
|
_, _ = fmt.Fprintf(w, "data: %s\n\n", `{"choices":[{"delta":{"content":"hello "}}]}`)
|
|
_, _ = fmt.Fprintf(w, "data: %s\n\n", `{"choices":[{"delta":{"content":"world"},"finish_reason":"stop"}],"usage":{"prompt_tokens":5,"completion_tokens":2}}`)
|
|
_, _ = fmt.Fprintf(w, "data: [DONE]\n\n")
|
|
}))
|
|
defer server.Close()
|
|
|
|
adapter := New(config.OpenAICompatConf{Endpoint: server.URL}, zap.NewNop())
|
|
sink := &fakeSink{}
|
|
err := adapter.Execute(context.Background(), runtime.ExecutionSpec{
|
|
RunID: "run-1",
|
|
Target: "lemonade-model",
|
|
Input: map[string]any{"prompt": "say hello"},
|
|
}, sink)
|
|
if err != nil {
|
|
t.Fatalf("Execute failed: %v", err)
|
|
}
|
|
|
|
events := sink.all()
|
|
// start + reasoning_delta + delta + delta + complete
|
|
if len(events) != 5 {
|
|
t.Fatalf("expected 5 events, got %d: %+v", len(events), events)
|
|
}
|
|
if events[0].Type != runtime.EventTypeStart {
|
|
t.Fatalf("expected start, got %+v", events[0])
|
|
}
|
|
if events[1].Type != runtime.EventTypeReasoningDelta || events[1].Delta != "think " {
|
|
t.Fatalf("expected reasoning delta, got %+v", events[1])
|
|
}
|
|
if events[2].Delta+events[3].Delta != "hello world" {
|
|
t.Fatalf("unexpected deltas: %q + %q", events[2].Delta, events[3].Delta)
|
|
}
|
|
complete := events[4]
|
|
if complete.Type != runtime.EventTypeComplete {
|
|
t.Fatalf("expected complete, got %+v", complete)
|
|
}
|
|
if complete.Metadata["finish_reason"] != "stop" {
|
|
t.Fatalf("expected finish_reason stop, got %+v", complete.Metadata)
|
|
}
|
|
if complete.Usage == nil || complete.Usage.OutputTokens != 2 || complete.Usage.InputTokens != 5 {
|
|
t.Fatalf("expected usage from chunk, got %+v", complete.Usage)
|
|
}
|
|
}
|
|
|
|
func TestOpenAICompatExecuteParsesReasoningAndCachedInputTokens(t *testing.T) {
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
_, _ = fmt.Fprintf(w, "data: %s\n\n", `{"choices":[{"delta":{"content":"hi"},"finish_reason":"stop"}],"usage":{"prompt_tokens":12,"completion_tokens":7,"prompt_tokens_details":{"cached_tokens":4},"completion_tokens_details":{"reasoning_tokens":3}}}`)
|
|
_, _ = fmt.Fprintf(w, "data: [DONE]\n\n")
|
|
}))
|
|
defer server.Close()
|
|
|
|
adapter := New(config.OpenAICompatConf{Endpoint: server.URL}, zap.NewNop())
|
|
sink := &fakeSink{}
|
|
if err := adapter.Execute(context.Background(), runtime.ExecutionSpec{
|
|
RunID: "run-1",
|
|
Target: "qwen-model",
|
|
Input: map[string]any{"prompt": "hi"},
|
|
}, sink); err != nil {
|
|
t.Fatalf("Execute failed: %v", err)
|
|
}
|
|
|
|
events := sink.all()
|
|
complete := events[len(events)-1]
|
|
if complete.Type != runtime.EventTypeComplete || complete.Usage == nil {
|
|
t.Fatalf("expected complete event with usage, got %+v", complete)
|
|
}
|
|
u := complete.Usage
|
|
if u.InputTokens != 12 || u.OutputTokens != 7 {
|
|
t.Fatalf("input/output tokens: got %d/%d, want 12/7", u.InputTokens, u.OutputTokens)
|
|
}
|
|
if u.CachedInputTokens != 4 {
|
|
t.Fatalf("cached_input_tokens: got %d, want 4", u.CachedInputTokens)
|
|
}
|
|
if u.ReasoningTokens != 3 {
|
|
t.Fatalf("reasoning_tokens: got %d, want 3", u.ReasoningTokens)
|
|
}
|
|
}
|
|
|
|
func TestOpenAICompatExecuteStreamsReasoningAlias(t *testing.T) {
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if r.URL.Path != "/v1/chat/completions" {
|
|
t.Fatalf("unexpected path %s", r.URL.Path)
|
|
}
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
_, _ = fmt.Fprintf(w, "data: %s\n\n", `{"choices":[{"delta":{"reasoning":"think alias "}}]}`)
|
|
_, _ = fmt.Fprintf(w, "data: %s\n\n", `{"choices":[{"delta":{"content":"answer"}}]}`)
|
|
_, _ = fmt.Fprintf(w, "data: [DONE]\n\n")
|
|
}))
|
|
defer server.Close()
|
|
|
|
adapter := New(config.OpenAICompatConf{Endpoint: server.URL}, zap.NewNop())
|
|
sink := &fakeSink{}
|
|
err := adapter.Execute(context.Background(), runtime.ExecutionSpec{
|
|
RunID: "run-1",
|
|
Target: "qwen-model",
|
|
Input: map[string]any{"prompt": "say hello"},
|
|
}, sink)
|
|
if err != nil {
|
|
t.Fatalf("Execute failed: %v", err)
|
|
}
|
|
|
|
events := sink.all()
|
|
if len(events) != 4 {
|
|
t.Fatalf("expected 4 events, got %d: %+v", len(events), events)
|
|
}
|
|
if events[1].Type != runtime.EventTypeReasoningDelta || events[1].Delta != "think alias " {
|
|
t.Fatalf("expected reasoning alias delta, got %+v", events[1])
|
|
}
|
|
if events[2].Type != runtime.EventTypeDelta || events[2].Delta != "answer" {
|
|
t.Fatalf("expected answer delta, got %+v", events[2])
|
|
}
|
|
}
|
|
|
|
func TestOpenAICompatExecuteSendsHeaders(t *testing.T) {
|
|
var gotAuth, gotContentType string
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
gotAuth = r.Header.Get("Authorization")
|
|
gotContentType = r.Header.Get("Content-Type")
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
_, _ = fmt.Fprintf(w, "data: %s\n\n", `{"choices":[{"delta":{"content":"ok"}}]}`)
|
|
_, _ = fmt.Fprintf(w, "data: [DONE]\n\n")
|
|
}))
|
|
defer server.Close()
|
|
|
|
adapter := New(config.OpenAICompatConf{
|
|
Endpoint: server.URL,
|
|
Headers: map[string]string{
|
|
"Authorization": "Bearer secret-token",
|
|
// A user-provided Content-Type must not override the adapter-owned one.
|
|
"Content-Type": "text/plain",
|
|
},
|
|
}, zap.NewNop())
|
|
sink := &fakeSink{}
|
|
if err := adapter.Execute(context.Background(), runtime.ExecutionSpec{
|
|
RunID: "run-h",
|
|
Target: "lemonade-model",
|
|
Input: map[string]any{"prompt": "hi"},
|
|
}, sink); err != nil {
|
|
t.Fatalf("Execute failed: %v", err)
|
|
}
|
|
if gotAuth != "Bearer secret-token" {
|
|
t.Fatalf("expected Authorization header, got %q", gotAuth)
|
|
}
|
|
if gotContentType != "application/json" {
|
|
t.Fatalf("expected adapter-owned content type, got %q", gotContentType)
|
|
}
|
|
}
|
|
|
|
func TestOpenAICompatExecutePassesOptionsAsTopLevelFields(t *testing.T) {
|
|
var body map[string]any
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
|
t.Fatalf("decode: %v", err)
|
|
}
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
_, _ = fmt.Fprintf(w, "data: %s\n\n", `{"choices":[{"delta":{"content":"ok"}}]}`)
|
|
_, _ = fmt.Fprintf(w, "data: [DONE]\n\n")
|
|
}))
|
|
defer server.Close()
|
|
|
|
adapter := New(config.OpenAICompatConf{Endpoint: server.URL}, zap.NewNop())
|
|
sink := &fakeSink{}
|
|
if err := adapter.Execute(context.Background(), runtime.ExecutionSpec{
|
|
RunID: "run-o",
|
|
Target: "lemonade-model",
|
|
Input: map[string]any{
|
|
"prompt": "hi",
|
|
"options": map[string]any{
|
|
"temperature": 0.2,
|
|
"max_tokens": float64(8),
|
|
// adapter-owned fields must not be overridden by options
|
|
"model": "evil",
|
|
"stream": false,
|
|
},
|
|
},
|
|
}, sink); err != nil {
|
|
t.Fatalf("Execute failed: %v", err)
|
|
}
|
|
if body["temperature"] != 0.2 {
|
|
t.Fatalf("expected temperature top-level, got %v", body["temperature"])
|
|
}
|
|
if body["max_tokens"] != float64(8) {
|
|
t.Fatalf("expected max_tokens top-level, got %v", body["max_tokens"])
|
|
}
|
|
if body["model"] != "lemonade-model" {
|
|
t.Fatalf("options must not override model, got %v", body["model"])
|
|
}
|
|
if body["stream"] != true {
|
|
t.Fatalf("options must not override stream, got %v", body["stream"])
|
|
}
|
|
}
|
|
|
|
func TestOpenAICompatExecutePassesToolsAndToolChoice(t *testing.T) {
|
|
var body map[string]any
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
|
t.Fatalf("decode: %v", err)
|
|
}
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
_, _ = fmt.Fprintf(w, "data: %s\n\n", `{"choices":[{"delta":{"content":"ok"}}]}`)
|
|
_, _ = fmt.Fprintf(w, "data: [DONE]\n\n")
|
|
}))
|
|
defer server.Close()
|
|
|
|
adapter := New(config.OpenAICompatConf{Endpoint: server.URL}, zap.NewNop())
|
|
sink := &fakeSink{}
|
|
if err := adapter.Execute(context.Background(), runtime.ExecutionSpec{
|
|
RunID: "run-t",
|
|
Target: "qwen3.6:35b",
|
|
Input: map[string]any{
|
|
"prompt": "hi",
|
|
"tools": []any{map[string]any{"type": "function"}},
|
|
"tool_choice": "none",
|
|
},
|
|
}, sink); err != nil {
|
|
t.Fatalf("Execute failed: %v", err)
|
|
}
|
|
if tools, ok := body["tools"].([]any); !ok || len(tools) != 1 {
|
|
t.Fatalf("tools not passed: %+v", body["tools"])
|
|
}
|
|
if body["tool_choice"] != "none" {
|
|
t.Fatalf("tool_choice not passed: %+v", body["tool_choice"])
|
|
}
|
|
}
|
|
|
|
func TestOpenAICompatExecuteRetriesSingleToolWhenAutoUnsupported(t *testing.T) {
|
|
attempts := 0
|
|
var retryBody map[string]any
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
attempts++
|
|
if attempts == 1 {
|
|
w.WriteHeader(http.StatusBadRequest)
|
|
_, _ = w.Write([]byte(`{"error":{"message":"\"auto\" tool choice requires --enable-auto-tool-choice and --tool-call-parser to be set"}}`))
|
|
return
|
|
}
|
|
if err := json.NewDecoder(r.Body).Decode(&retryBody); err != nil {
|
|
t.Fatalf("decode retry: %v", err)
|
|
}
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
_, _ = fmt.Fprintf(w, "data: %s\n\n", `{"choices":[{"delta":{"content":"ok"}}]}`)
|
|
_, _ = fmt.Fprintf(w, "data: [DONE]\n\n")
|
|
}))
|
|
defer server.Close()
|
|
|
|
adapter := New(config.OpenAICompatConf{Endpoint: server.URL}, zap.NewNop())
|
|
sink := &fakeSink{}
|
|
if err := adapter.Execute(context.Background(), runtime.ExecutionSpec{
|
|
RunID: "run-retry",
|
|
Target: "qwen3.6:35b",
|
|
Input: map[string]any{
|
|
"prompt": "status",
|
|
"tools": []any{map[string]any{
|
|
"type": "function",
|
|
"function": map[string]any{
|
|
"name": "run_commands",
|
|
},
|
|
}},
|
|
},
|
|
}, sink); err != nil {
|
|
t.Fatalf("Execute failed: %v", err)
|
|
}
|
|
if attempts != 2 {
|
|
t.Fatalf("attempts: got %d, want 2", attempts)
|
|
}
|
|
choice, ok := retryBody["tool_choice"].(map[string]any)
|
|
if !ok {
|
|
t.Fatalf("retry tool_choice missing: %+v", retryBody)
|
|
}
|
|
fn := choice["function"].(map[string]any)
|
|
if choice["type"] != "function" || fn["name"] != "run_commands" {
|
|
t.Fatalf("retry tool_choice: %+v", choice)
|
|
}
|
|
}
|
|
|
|
func TestOpenAICompatExecuteFallsBackToTextToolsWhenNativeToolsUnsupported(t *testing.T) {
|
|
attempts := 0
|
|
var fallbackBody map[string]any
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
attempts++
|
|
switch attempts {
|
|
case 1:
|
|
w.WriteHeader(http.StatusBadRequest)
|
|
_, _ = w.Write([]byte(`{"error":{"message":"\"auto\" tool choice requires --enable-auto-tool-choice and --tool-call-parser to be set"}}`))
|
|
return
|
|
case 2:
|
|
w.WriteHeader(http.StatusBadRequest)
|
|
_, _ = w.Write([]byte(`{"error":{"message":"tool_choice=\"function=ChatCompletionNamedFunction(name='run_commands') type='function'\" requires --tool-call-parser to be set"}}`))
|
|
return
|
|
}
|
|
if err := json.NewDecoder(r.Body).Decode(&fallbackBody); err != nil {
|
|
t.Fatalf("decode fallback: %v", err)
|
|
}
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
_, _ = fmt.Fprintf(w, "data: %s\n\n", `{"choices":[{"delta":{"content":"<tool_call>\n<function=run_commands>\n<parameter=commands>[\"git status\"]</parameter>\n</function>\n</tool_call>"}}]}`)
|
|
_, _ = fmt.Fprintf(w, "data: [DONE]\n\n")
|
|
}))
|
|
defer server.Close()
|
|
|
|
adapter := New(config.OpenAICompatConf{Endpoint: server.URL}, zap.NewNop())
|
|
sink := &fakeSink{}
|
|
if err := adapter.Execute(context.Background(), runtime.ExecutionSpec{
|
|
RunID: "run-text-fallback",
|
|
Target: "qwen3.6:35b",
|
|
Input: map[string]any{
|
|
"messages": []any{
|
|
map[string]any{"role": "system", "content": "Existing Cline system prompt."},
|
|
map[string]any{"role": "user", "content": "status"},
|
|
},
|
|
"tools": []any{map[string]any{
|
|
"type": "function",
|
|
"function": map[string]any{
|
|
"name": "run_commands",
|
|
"parameters": map[string]any{
|
|
"type": "object",
|
|
"properties": map[string]any{
|
|
"commands": map[string]any{"type": "array", "items": map[string]any{"type": "string"}},
|
|
},
|
|
},
|
|
},
|
|
}},
|
|
},
|
|
}, sink); err != nil {
|
|
t.Fatalf("Execute failed: %v", err)
|
|
}
|
|
if attempts != 3 {
|
|
t.Fatalf("attempts: got %d, want 3", attempts)
|
|
}
|
|
if _, ok := fallbackBody["tools"]; ok {
|
|
t.Fatalf("fallback must omit tools: %+v", fallbackBody["tools"])
|
|
}
|
|
if _, ok := fallbackBody["tool_choice"]; ok {
|
|
t.Fatalf("fallback must omit tool_choice: %+v", fallbackBody["tool_choice"])
|
|
}
|
|
messages := fallbackBody["messages"].([]any)
|
|
first := messages[0].(map[string]any)
|
|
if first["role"] != "system" || !strings.Contains(first["content"].(string), "<tool_call>") || !strings.Contains(first["content"].(string), "run_commands") {
|
|
t.Fatalf("fallback system instruction missing tool format/name: %+v", first)
|
|
}
|
|
if !strings.Contains(first["content"].(string), "client workspace root") || !strings.Contains(first["content"].(string), "Do not prepend cd") {
|
|
t.Fatalf("fallback system instruction missing workspace-root guidance: %+v", first)
|
|
}
|
|
if !strings.Contains(first["content"].(string), "Existing Cline system prompt.") {
|
|
t.Fatalf("fallback system instruction did not preserve existing system content: %+v", first)
|
|
}
|
|
if len(messages) != 2 || messages[1].(map[string]any)["role"] != "user" {
|
|
t.Fatalf("fallback should keep a single leading system message: %+v", messages)
|
|
}
|
|
events := sink.all()
|
|
complete := events[len(events)-1]
|
|
if complete.Metadata[runtimeMetadataOpenAITextToolFallback] != "true" {
|
|
t.Fatalf("fallback metadata missing: %+v", complete.Metadata)
|
|
}
|
|
}
|
|
|
|
func TestOpenAICompatExecutePreservesToolCallMessages(t *testing.T) {
|
|
var body map[string]any
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
|
t.Fatalf("decode: %v", err)
|
|
}
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
_, _ = fmt.Fprintf(w, "data: %s\n\n", `{"choices":[{"delta":{"content":"ok"}}]}`)
|
|
_, _ = fmt.Fprintf(w, "data: [DONE]\n\n")
|
|
}))
|
|
defer server.Close()
|
|
|
|
adapter := New(config.OpenAICompatConf{Endpoint: server.URL}, zap.NewNop())
|
|
sink := &fakeSink{}
|
|
if err := adapter.Execute(context.Background(), runtime.ExecutionSpec{
|
|
RunID: "run-tool-turn",
|
|
Target: "qwen3.6:35b",
|
|
Input: map[string]any{
|
|
"messages": []any{
|
|
map[string]any{
|
|
"role": "assistant",
|
|
"content": "",
|
|
"tool_calls": []any{map[string]any{
|
|
"id": "call_iop_1",
|
|
"type": "function",
|
|
"function": map[string]any{
|
|
"name": "run_commands",
|
|
"arguments": `{"commands":["git status"]}`,
|
|
},
|
|
}},
|
|
},
|
|
map[string]any{
|
|
"role": "tool",
|
|
"content": "clean",
|
|
"tool_call_id": "call_iop_1",
|
|
},
|
|
},
|
|
},
|
|
}, sink); err != nil {
|
|
t.Fatalf("Execute failed: %v", err)
|
|
}
|
|
messages, ok := body["messages"].([]any)
|
|
if !ok || len(messages) != 2 {
|
|
t.Fatalf("messages not preserved: %+v", body["messages"])
|
|
}
|
|
assistantMsg := messages[0].(map[string]any)
|
|
if calls, ok := assistantMsg["tool_calls"].([]any); !ok || len(calls) != 1 {
|
|
t.Fatalf("assistant tool_calls not preserved: %+v", assistantMsg)
|
|
}
|
|
toolMsg := messages[1].(map[string]any)
|
|
if toolMsg["tool_call_id"] != "call_iop_1" {
|
|
t.Fatalf("tool_call_id not preserved: %+v", toolMsg)
|
|
}
|
|
}
|
|
|
|
func TestOpenAICompatExecutePreservesNativeToolCalls(t *testing.T) {
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
_, _ = fmt.Fprintf(w, "data: %s\n\n", `{"choices":[{"delta":{"tool_calls":[{"index":0,"id":"call_1","type":"function","function":{"name":"run_commands","arguments":"{\"commands\":"}}]}}]}`)
|
|
_, _ = fmt.Fprintf(w, "data: %s\n\n", `{"choices":[{"delta":{"tool_calls":[{"index":0,"function":{"arguments":"[\"git status\"]}"}}]},"finish_reason":"tool_calls"}]}`)
|
|
_, _ = fmt.Fprintf(w, "data: [DONE]\n\n")
|
|
}))
|
|
defer server.Close()
|
|
|
|
adapter := New(config.OpenAICompatConf{Endpoint: server.URL}, zap.NewNop())
|
|
sink := &fakeSink{}
|
|
if err := adapter.Execute(context.Background(), runtime.ExecutionSpec{
|
|
RunID: "run-native-tools",
|
|
Target: "qwen3.6:35b",
|
|
Input: map[string]any{"prompt": "status"},
|
|
}, sink); err != nil {
|
|
t.Fatalf("Execute failed: %v", err)
|
|
}
|
|
events := sink.all()
|
|
complete := events[len(events)-1]
|
|
if complete.Metadata["finish_reason"] != "tool_calls" {
|
|
t.Fatalf("finish_reason: %+v", complete.Metadata)
|
|
}
|
|
var toolCalls []map[string]any
|
|
if err := json.Unmarshal([]byte(complete.Metadata[runtimeMetadataOpenAIToolCalls]), &toolCalls); err != nil {
|
|
t.Fatalf("tool_calls metadata JSON: %v", err)
|
|
}
|
|
if len(toolCalls) != 1 || toolCalls[0]["id"] != "call_1" {
|
|
t.Fatalf("tool_calls: %+v", toolCalls)
|
|
}
|
|
fn := toolCalls[0]["function"].(map[string]any)
|
|
if fn["name"] != "run_commands" || fn["arguments"] != `{"commands":["git status"]}` {
|
|
t.Fatalf("function: %+v", fn)
|
|
}
|
|
}
|
|
|
|
func TestOpenAICompatExecuteRejectsEmptyEndpointOrModel(t *testing.T) {
|
|
t.Run("empty_endpoint", func(t *testing.T) {
|
|
adapter := New(config.OpenAICompatConf{}, zap.NewNop())
|
|
err := adapter.Execute(context.Background(), runtime.ExecutionSpec{
|
|
RunID: "r", Target: "m", Input: map[string]any{"prompt": "hi"},
|
|
}, &fakeSink{})
|
|
if err == nil || !strings.Contains(err.Error(), "endpoint is required") {
|
|
t.Fatalf("expected endpoint error, got %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("empty_model", func(t *testing.T) {
|
|
adapter := New(config.OpenAICompatConf{Endpoint: "http://localhost:8000"}, zap.NewNop())
|
|
err := adapter.Execute(context.Background(), runtime.ExecutionSpec{
|
|
RunID: "r", Input: map[string]any{"prompt": "hi"},
|
|
}, &fakeSink{})
|
|
if err == nil || !strings.Contains(err.Error(), "target/model is required") {
|
|
t.Fatalf("expected model error, got %v", err)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestOpenAICompatJoinOpenAIPathNoDuplicateV1(t *testing.T) {
|
|
cases := []struct {
|
|
endpoint string
|
|
path string
|
|
wantTail string
|
|
}{
|
|
{"http://host:8000", "/v1/models", "/v1/models"},
|
|
{"http://host:8000/", "/v1/chat/completions", "/v1/chat/completions"},
|
|
{"http://host:8000/v1", "/v1/models", "/v1/models"},
|
|
{"http://host:8000/v1/", "/v1/chat/completions", "/v1/chat/completions"},
|
|
}
|
|
for _, c := range cases {
|
|
got := joinOpenAIPath(strings.TrimRight(c.endpoint, "/"), c.path)
|
|
if !strings.HasSuffix(got, c.wantTail) || strings.Contains(got, "/v1/v1") {
|
|
t.Errorf("joinOpenAIPath(%q,%q)=%q want suffix %q without /v1/v1", c.endpoint, c.path, got, c.wantTail)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestOpenAICompatExecuteMapsThinkFalseForVLLMChatTemplateKwargs(t *testing.T) {
|
|
var body map[string]any
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
|
t.Fatalf("decode: %v", err)
|
|
}
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
_, _ = fmt.Fprintf(w, "data: %s\n\n", `{"choices":[{"delta":{"content":"ok"}}]}`)
|
|
_, _ = fmt.Fprintf(w, "data: [DONE]\n\n")
|
|
}))
|
|
defer server.Close()
|
|
|
|
adapter := New(config.OpenAICompatConf{
|
|
Provider: "vllm",
|
|
Endpoint: server.URL,
|
|
}, zap.NewNop())
|
|
|
|
sink := &fakeSink{}
|
|
err := adapter.Execute(context.Background(), runtime.ExecutionSpec{
|
|
RunID: "run-vllm",
|
|
Target: "qwen-model",
|
|
Input: map[string]any{
|
|
"prompt": "hello",
|
|
"think": false,
|
|
},
|
|
}, sink)
|
|
if err != nil {
|
|
t.Fatalf("Execute failed: %v", err)
|
|
}
|
|
|
|
ctk, ok := body["chat_template_kwargs"].(map[string]any)
|
|
if !ok {
|
|
t.Fatalf("expected chat_template_kwargs to be a map, got %T", body["chat_template_kwargs"])
|
|
}
|
|
if ctk["enable_thinking"] != false {
|
|
t.Fatalf("expected enable_thinking to be false, got %v", ctk["enable_thinking"])
|
|
}
|
|
}
|
|
|
|
func TestOpenAICompatExecuteVLLMMLXUnsupportedThinkDisable(t *testing.T) {
|
|
// The vLLM-MLX runtime cannot produce a reasoning-free streaming response,
|
|
// so think=false / reasoning_effort=none must fail loudly instead of being
|
|
// silently accepted as a reasoning stream.
|
|
for _, disable := range []map[string]any{
|
|
{"prompt": "hello", "think": false},
|
|
{"prompt": "hello", "reasoning_effort": "none"},
|
|
} {
|
|
adapter := New(config.OpenAICompatConf{
|
|
Provider: "vllm-mlx",
|
|
Endpoint: "http://localhost:8000",
|
|
}, zap.NewNop())
|
|
|
|
sink := &fakeSink{}
|
|
err := adapter.Execute(context.Background(), runtime.ExecutionSpec{
|
|
RunID: "run-vllm-mlx-disable",
|
|
Target: "qwen-model",
|
|
Input: disable,
|
|
}, sink)
|
|
if err == nil {
|
|
t.Fatalf("expected error for vllm-mlx think disable %v, got nil", disable)
|
|
}
|
|
if !strings.Contains(err.Error(), "unsupported think control") {
|
|
t.Fatalf("expected error message to contain 'unsupported think control' for %v, got: %v", disable, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestOpenAICompatExecuteMapsThinkFalseForLemonadeChatTemplateKwargs(t *testing.T) {
|
|
var body map[string]any
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
|
t.Fatalf("decode: %v", err)
|
|
}
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
_, _ = fmt.Fprintf(w, "data: %s\n\n", `{"choices":[{"delta":{"content":"ok"}}]}`)
|
|
_, _ = fmt.Fprintf(w, "data: [DONE]\n\n")
|
|
}))
|
|
defer server.Close()
|
|
|
|
adapter := New(config.OpenAICompatConf{
|
|
Provider: "lemonade",
|
|
Endpoint: server.URL,
|
|
}, zap.NewNop())
|
|
|
|
sink := &fakeSink{}
|
|
err := adapter.Execute(context.Background(), runtime.ExecutionSpec{
|
|
RunID: "run-lemonade",
|
|
Target: "lemonade-model",
|
|
Input: map[string]any{
|
|
"prompt": "hello",
|
|
"think": false,
|
|
},
|
|
}, sink)
|
|
if err != nil {
|
|
t.Fatalf("Execute failed: %v", err)
|
|
}
|
|
|
|
if _, ok := body["think"]; ok {
|
|
t.Fatalf("expected top-level think to be omitted for lemonade, got %v", body["think"])
|
|
}
|
|
ctk, ok := body["chat_template_kwargs"].(map[string]any)
|
|
if !ok {
|
|
t.Fatalf("expected chat_template_kwargs to be a map, got %T", body["chat_template_kwargs"])
|
|
}
|
|
if ctk["enable_thinking"] != false {
|
|
t.Fatalf("expected enable_thinking to be false, got %v", ctk["enable_thinking"])
|
|
}
|
|
}
|
|
|
|
func TestOpenAICompatExecuteOmitsThinkControlWhenRequestOmitsIt(t *testing.T) {
|
|
for _, provider := range []string{"vllm", "vllm-mlx", "lemonade", ""} {
|
|
t.Run(provider, func(t *testing.T) {
|
|
var body map[string]any
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
|
t.Fatalf("decode: %v", err)
|
|
}
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
_, _ = fmt.Fprintf(w, "data: %s\n\n", `{"choices":[{"delta":{"content":"ok"}}]}`)
|
|
_, _ = fmt.Fprintf(w, "data: [DONE]\n\n")
|
|
}))
|
|
defer server.Close()
|
|
|
|
adapter := New(config.OpenAICompatConf{
|
|
Provider: provider,
|
|
Endpoint: server.URL,
|
|
}, zap.NewNop())
|
|
|
|
sink := &fakeSink{}
|
|
err := adapter.Execute(context.Background(), runtime.ExecutionSpec{
|
|
RunID: "run-omit",
|
|
Target: "qwen-model",
|
|
Input: map[string]any{
|
|
"prompt": "hello",
|
|
},
|
|
}, sink)
|
|
if err != nil {
|
|
t.Fatalf("Execute failed: %v", err)
|
|
}
|
|
|
|
for _, key := range []string{"think", "reasoning_effort", "thinking_token_budget", "chat_template_kwargs"} {
|
|
if _, ok := body[key]; ok {
|
|
t.Fatalf("expected key %q to be omitted for provider %q", key, provider)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestOpenAICompatExecuteVLLMUnsupportedReasoningEffort(t *testing.T) {
|
|
adapter := New(config.OpenAICompatConf{
|
|
Provider: "vllm",
|
|
Endpoint: "http://localhost:8000",
|
|
}, zap.NewNop())
|
|
|
|
sink := &fakeSink{}
|
|
err := adapter.Execute(context.Background(), runtime.ExecutionSpec{
|
|
RunID: "run-unsupported",
|
|
Target: "qwen-model",
|
|
Input: map[string]any{
|
|
"prompt": "hello",
|
|
"reasoning_effort": "low",
|
|
},
|
|
}, sink)
|
|
if err == nil {
|
|
t.Fatalf("expected error for unsupported reasoning_effort, got nil")
|
|
}
|
|
if !strings.Contains(err.Error(), "unsupported think control") {
|
|
t.Fatalf("expected error message to contain 'unsupported think control', got: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestOpenAICompatExecuteVLLMUnsupportedNegativeBudget(t *testing.T) {
|
|
adapter := New(config.OpenAICompatConf{
|
|
Provider: "vllm",
|
|
Endpoint: "http://localhost:8000",
|
|
}, zap.NewNop())
|
|
|
|
sink := &fakeSink{}
|
|
err := adapter.Execute(context.Background(), runtime.ExecutionSpec{
|
|
RunID: "run-unsupported",
|
|
Target: "qwen-model",
|
|
Input: map[string]any{
|
|
"prompt": "hello",
|
|
"thinking_token_budget": -10,
|
|
},
|
|
}, sink)
|
|
if err == nil {
|
|
t.Fatalf("expected error for negative budget, got nil")
|
|
}
|
|
if !strings.Contains(err.Error(), "unsupported think control") {
|
|
t.Fatalf("expected error message to contain 'unsupported think control', got: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestOpenAICompatExecuteMapsReasoningEffortNone(t *testing.T) {
|
|
t.Run("vllm", func(t *testing.T) {
|
|
var body map[string]any
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
|
t.Fatalf("decode: %v", err)
|
|
}
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
_, _ = fmt.Fprintf(w, "data: %s\n\n", `{"choices":[{"delta":{"content":"ok"}}]}`)
|
|
_, _ = fmt.Fprintf(w, "data: [DONE]\n\n")
|
|
}))
|
|
defer server.Close()
|
|
|
|
adapter := New(config.OpenAICompatConf{
|
|
Provider: "vllm",
|
|
Endpoint: server.URL,
|
|
}, zap.NewNop())
|
|
|
|
sink := &fakeSink{}
|
|
err := adapter.Execute(context.Background(), runtime.ExecutionSpec{
|
|
RunID: "run-vllm-none",
|
|
Target: "qwen-model",
|
|
Input: map[string]any{
|
|
"prompt": "hello",
|
|
"reasoning_effort": "none",
|
|
},
|
|
}, sink)
|
|
if err != nil {
|
|
t.Fatalf("Execute failed: %v", err)
|
|
}
|
|
|
|
ctk, ok := body["chat_template_kwargs"].(map[string]any)
|
|
if !ok {
|
|
t.Fatalf("expected chat_template_kwargs to be a map, got %T", body["chat_template_kwargs"])
|
|
}
|
|
if ctk["enable_thinking"] != false {
|
|
t.Fatalf("expected enable_thinking to be false, got %v", ctk["enable_thinking"])
|
|
}
|
|
})
|
|
|
|
t.Run("lemonade", func(t *testing.T) {
|
|
var body map[string]any
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
|
t.Fatalf("decode: %v", err)
|
|
}
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
_, _ = fmt.Fprintf(w, "data: %s\n\n", `{"choices":[{"delta":{"content":"ok"}}]}`)
|
|
_, _ = fmt.Fprintf(w, "data: [DONE]\n\n")
|
|
}))
|
|
defer server.Close()
|
|
|
|
adapter := New(config.OpenAICompatConf{
|
|
Provider: "lemonade",
|
|
Endpoint: server.URL,
|
|
}, zap.NewNop())
|
|
|
|
sink := &fakeSink{}
|
|
err := adapter.Execute(context.Background(), runtime.ExecutionSpec{
|
|
RunID: "run-lemonade-none",
|
|
Target: "lemonade-model",
|
|
Input: map[string]any{
|
|
"prompt": "hello",
|
|
"reasoning_effort": "none",
|
|
},
|
|
}, sink)
|
|
if err != nil {
|
|
t.Fatalf("Execute failed: %v", err)
|
|
}
|
|
|
|
if _, ok := body["reasoning_effort"]; ok {
|
|
t.Fatalf("expected reasoning_effort to be omitted, got %v", body["reasoning_effort"])
|
|
}
|
|
ctk, ok := body["chat_template_kwargs"].(map[string]any)
|
|
if !ok {
|
|
t.Fatalf("expected chat_template_kwargs to be a map, got %T", body["chat_template_kwargs"])
|
|
}
|
|
if ctk["enable_thinking"] != false {
|
|
t.Fatalf("expected enable_thinking to be false, got %v", ctk["enable_thinking"])
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestOpenAICompatExecuteMapsThinkTrueForVLLMChatTemplateKwargs(t *testing.T) {
|
|
var body map[string]any
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
|
t.Fatalf("decode: %v", err)
|
|
}
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
_, _ = fmt.Fprintf(w, "data: %s\n\n", `{"choices":[{"delta":{"content":"ok"}}]}`)
|
|
_, _ = fmt.Fprintf(w, "data: [DONE]\n\n")
|
|
}))
|
|
defer server.Close()
|
|
|
|
adapter := New(config.OpenAICompatConf{
|
|
Provider: "vllm",
|
|
Endpoint: server.URL,
|
|
}, zap.NewNop())
|
|
|
|
sink := &fakeSink{}
|
|
err := adapter.Execute(context.Background(), runtime.ExecutionSpec{
|
|
RunID: "run-vllm",
|
|
Target: "qwen-model",
|
|
Input: map[string]any{
|
|
"prompt": "hello",
|
|
"think": true,
|
|
},
|
|
}, sink)
|
|
if err != nil {
|
|
t.Fatalf("Execute failed: %v", err)
|
|
}
|
|
|
|
ctk, ok := body["chat_template_kwargs"].(map[string]any)
|
|
if !ok {
|
|
t.Fatalf("expected chat_template_kwargs to be a map, got %T", body["chat_template_kwargs"])
|
|
}
|
|
if ctk["enable_thinking"] != true {
|
|
t.Fatalf("expected enable_thinking to be true, got %v", ctk["enable_thinking"])
|
|
}
|
|
}
|
|
|
|
func TestOpenAICompatExecuteMapsThinkTrueForVLLMMLXChatTemplateKwargs(t *testing.T) {
|
|
var body map[string]any
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
|
t.Fatalf("decode: %v", err)
|
|
}
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
_, _ = fmt.Fprintf(w, "data: %s\n\n", `{"choices":[{"delta":{"content":"ok"}}]}`)
|
|
_, _ = fmt.Fprintf(w, "data: [DONE]\n\n")
|
|
}))
|
|
defer server.Close()
|
|
|
|
adapter := New(config.OpenAICompatConf{
|
|
Provider: "vllm-mlx",
|
|
Endpoint: server.URL,
|
|
}, zap.NewNop())
|
|
|
|
sink := &fakeSink{}
|
|
err := adapter.Execute(context.Background(), runtime.ExecutionSpec{
|
|
RunID: "run-vllm-mlx",
|
|
Target: "qwen-model",
|
|
Input: map[string]any{
|
|
"prompt": "hello",
|
|
"think": true,
|
|
},
|
|
}, sink)
|
|
if err != nil {
|
|
t.Fatalf("Execute failed: %v", err)
|
|
}
|
|
|
|
ctk, ok := body["chat_template_kwargs"].(map[string]any)
|
|
if !ok {
|
|
t.Fatalf("expected chat_template_kwargs to be a map, got %T", body["chat_template_kwargs"])
|
|
}
|
|
if ctk["enable_thinking"] != true {
|
|
t.Fatalf("expected enable_thinking to be true, got %v", ctk["enable_thinking"])
|
|
}
|
|
}
|
|
|
|
func TestOpenAICompatExecuteMapsThinkTrueForLemonadeChatTemplateKwargs(t *testing.T) {
|
|
var body map[string]any
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
|
t.Fatalf("decode: %v", err)
|
|
}
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
_, _ = fmt.Fprintf(w, "data: %s\n\n", `{"choices":[{"delta":{"content":"ok"}}]}`)
|
|
_, _ = fmt.Fprintf(w, "data: [DONE]\n\n")
|
|
}))
|
|
defer server.Close()
|
|
|
|
adapter := New(config.OpenAICompatConf{
|
|
Provider: "lemonade",
|
|
Endpoint: server.URL,
|
|
}, zap.NewNop())
|
|
|
|
sink := &fakeSink{}
|
|
err := adapter.Execute(context.Background(), runtime.ExecutionSpec{
|
|
RunID: "run-lemonade",
|
|
Target: "lemonade-model",
|
|
Input: map[string]any{
|
|
"prompt": "hello",
|
|
"think": true,
|
|
"thinking_token_budget": 1024,
|
|
},
|
|
}, sink)
|
|
if err != nil {
|
|
t.Fatalf("Execute failed: %v", err)
|
|
}
|
|
|
|
if _, ok := body["think"]; ok {
|
|
t.Fatalf("expected top-level think to be omitted for lemonade, got %v", body["think"])
|
|
}
|
|
if _, ok := body["thinking_token_budget"]; ok {
|
|
t.Fatalf("expected top-level thinking_token_budget to be omitted for lemonade, got %v", body["thinking_token_budget"])
|
|
}
|
|
ctk, ok := body["chat_template_kwargs"].(map[string]any)
|
|
if !ok {
|
|
t.Fatalf("expected chat_template_kwargs to be a map, got %T", body["chat_template_kwargs"])
|
|
}
|
|
if ctk["enable_thinking"] != true {
|
|
t.Fatalf("expected enable_thinking to be true, got %v", ctk["enable_thinking"])
|
|
}
|
|
if ctk["thinking_token_budget"] != float64(1024) {
|
|
t.Fatalf("expected thinking_token_budget to be float64 1024, got %v", ctk["thinking_token_budget"])
|
|
}
|
|
}
|
|
|
|
func TestOpenAICompatExecuteMapsThinkingTokenBudgetForVLLMChatTemplateKwargs(t *testing.T) {
|
|
var body map[string]any
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
|
t.Fatalf("decode: %v", err)
|
|
}
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
_, _ = fmt.Fprintf(w, "data: %s\n\n", `{"choices":[{"delta":{"content":"ok"}}]}`)
|
|
_, _ = fmt.Fprintf(w, "data: [DONE]\n\n")
|
|
}))
|
|
defer server.Close()
|
|
|
|
adapter := New(config.OpenAICompatConf{
|
|
Provider: "vllm",
|
|
Endpoint: server.URL,
|
|
}, zap.NewNop())
|
|
|
|
sink := &fakeSink{}
|
|
err := adapter.Execute(context.Background(), runtime.ExecutionSpec{
|
|
RunID: "run-vllm-budget",
|
|
Target: "qwen-model",
|
|
Input: map[string]any{
|
|
"prompt": "hello",
|
|
"thinking_token_budget": 2048,
|
|
},
|
|
}, sink)
|
|
if err != nil {
|
|
t.Fatalf("Execute failed: %v", err)
|
|
}
|
|
|
|
if _, ok := body["thinking_token_budget"]; ok {
|
|
t.Fatalf("expected top-level thinking_token_budget to be omitted for vllm, got %v", body["thinking_token_budget"])
|
|
}
|
|
|
|
ctk, ok := body["chat_template_kwargs"].(map[string]any)
|
|
if !ok {
|
|
t.Fatalf("expected chat_template_kwargs to be a map, got %T", body["chat_template_kwargs"])
|
|
}
|
|
if ctk["enable_thinking"] != true {
|
|
t.Fatalf("expected enable_thinking to be true for budget-only request, got %v", ctk["enable_thinking"])
|
|
}
|
|
if ctk["thinking_token_budget"] != float64(2048) {
|
|
t.Fatalf("expected thinking_token_budget to be 2048, got %v", ctk["thinking_token_budget"])
|
|
}
|
|
}
|
|
|
|
func TestOpenAICompatExecuteVLLMUnsupportedInvalidBudget(t *testing.T) {
|
|
adapter := New(config.OpenAICompatConf{
|
|
Provider: "vllm",
|
|
Endpoint: "http://localhost:8000",
|
|
}, zap.NewNop())
|
|
|
|
sink := &fakeSink{}
|
|
invalidBudgets := []any{"invalid", -5, 12.34}
|
|
for _, val := range invalidBudgets {
|
|
err := adapter.Execute(context.Background(), runtime.ExecutionSpec{
|
|
RunID: "run-vllm-invalid-budget",
|
|
Target: "qwen-model",
|
|
Input: map[string]any{
|
|
"prompt": "hello",
|
|
"thinking_token_budget": val,
|
|
},
|
|
}, sink)
|
|
if err == nil {
|
|
t.Fatalf("expected error for budget val %v, got nil", val)
|
|
}
|
|
if !strings.Contains(err.Error(), "unsupported think control") {
|
|
t.Fatalf("expected error message to contain 'unsupported think control' for val %v, got: %v", val, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
type fakeTunnelSink struct {
|
|
mu sync.Mutex
|
|
frames []runtime.ProviderTunnelFrame
|
|
}
|
|
|
|
func (s *fakeTunnelSink) EmitTunnelFrame(_ context.Context, frame runtime.ProviderTunnelFrame) error {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
s.frames = append(s.frames, frame)
|
|
return nil
|
|
}
|
|
|
|
func (s *fakeTunnelSink) all() []runtime.ProviderTunnelFrame {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
return append([]runtime.ProviderTunnelFrame(nil), s.frames...)
|
|
}
|
|
|
|
// failingTunnelSink emulates the Edge socket write failing after failAfter
|
|
// successful frames, as when the Edge relay aborts an abandoned tunnel.
|
|
type failingTunnelSink struct {
|
|
mu sync.Mutex
|
|
frames []runtime.ProviderTunnelFrame
|
|
failAfter int
|
|
}
|
|
|
|
func (s *failingTunnelSink) EmitTunnelFrame(_ context.Context, frame runtime.ProviderTunnelFrame) error {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
if len(s.frames) >= s.failAfter {
|
|
return errors.New("edge socket write failed")
|
|
}
|
|
s.frames = append(s.frames, frame)
|
|
return nil
|
|
}
|
|
|
|
func (s *failingTunnelSink) all() []runtime.ProviderTunnelFrame {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
return append([]runtime.ProviderTunnelFrame(nil), s.frames...)
|
|
}
|
|
|
|
// assertOrderedTunnelFrames verifies S09 frame semantics: contiguous sequences
|
|
// starting at 0 (no drop/reorder) and exactly one terminal END/ERROR frame,
|
|
// which must be the last frame emitted.
|
|
func assertOrderedTunnelFrames(t *testing.T, frames []runtime.ProviderTunnelFrame) {
|
|
t.Helper()
|
|
if len(frames) == 0 {
|
|
t.Fatal("no tunnel frames emitted")
|
|
}
|
|
terminals := 0
|
|
for i, f := range frames {
|
|
if f.Sequence != int64(i) {
|
|
t.Errorf("frame %d: sequence %d, want %d (ordered, lossless)", i, f.Sequence, i)
|
|
}
|
|
switch f.Kind {
|
|
case runtime.ProviderTunnelFrameKindEnd, runtime.ProviderTunnelFrameKindError:
|
|
terminals++
|
|
if i != len(frames)-1 {
|
|
t.Errorf("terminal %s frame at index %d must be the last of %d frames", f.Kind, i, len(frames))
|
|
}
|
|
}
|
|
}
|
|
if terminals != 1 {
|
|
t.Errorf("expected exactly one terminal END/ERROR frame, got %d", terminals)
|
|
}
|
|
}
|
|
|
|
func TestOpenAICompatTunnelProvider(t *testing.T) {
|
|
expectedBody := "data: {\"choices\":[{\"delta\":{\"reasoning_content\":\"think Qwen\"}}]}\n\ndata: [DONE]\n\n"
|
|
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodPost {
|
|
t.Errorf("expected POST method, got %s", r.Method)
|
|
}
|
|
if r.URL.Path != "/v1/chat/completions" {
|
|
t.Errorf("expected path /v1/chat/completions, got %s", r.URL.Path)
|
|
}
|
|
if r.Header.Get("Authorization") != "Bearer test-key" {
|
|
t.Errorf("expected Authorization header, got %s", r.Header.Get("Authorization"))
|
|
}
|
|
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
w.Header().Set("X-Custom-Header", "custom-value")
|
|
w.WriteHeader(http.StatusOK)
|
|
_, _ = w.Write([]byte(expectedBody))
|
|
}))
|
|
defer server.Close()
|
|
|
|
adapter := New(config.OpenAICompatConf{
|
|
Endpoint: server.URL,
|
|
}, zap.NewNop())
|
|
|
|
sink := &fakeTunnelSink{}
|
|
req := runtime.ProviderTunnelRequest{
|
|
RunID: "run-1",
|
|
TunnelID: "tunnel-1",
|
|
Method: "POST",
|
|
Path: "/v1/chat/completions",
|
|
Headers: map[string]string{"Authorization": "Bearer test-key"},
|
|
Body: []byte(`{"model":"qwen3.6:35b","prompt":"hi"}`),
|
|
}
|
|
|
|
err := adapter.TunnelProvider(context.Background(), req, sink)
|
|
if err != nil {
|
|
t.Fatalf("TunnelProvider failed: %v", err)
|
|
}
|
|
|
|
frames := sink.all()
|
|
if len(frames) < 3 {
|
|
t.Fatalf("expected at least 3 frames, got %d", len(frames))
|
|
}
|
|
|
|
startFrame := frames[0]
|
|
if startFrame.Kind != runtime.ProviderTunnelFrameKindResponseStart {
|
|
t.Errorf("expected RESPONSE_START, got %s", startFrame.Kind)
|
|
}
|
|
if startFrame.StatusCode != http.StatusOK {
|
|
t.Errorf("expected 200 OK, got %d", startFrame.StatusCode)
|
|
}
|
|
if startFrame.Headers["X-Custom-Header"] != "custom-value" {
|
|
t.Errorf("expected Custom Header, got %v", startFrame.Headers)
|
|
}
|
|
|
|
var bodyBuffer bytes.Buffer
|
|
var lastSeq int64 = 0
|
|
for _, f := range frames[1 : len(frames)-1] {
|
|
if f.Kind != runtime.ProviderTunnelFrameKindBody {
|
|
t.Errorf("expected BODY kind, got %s", f.Kind)
|
|
}
|
|
if f.Sequence != lastSeq+1 {
|
|
t.Errorf("expected seq %d, got %d", lastSeq+1, f.Sequence)
|
|
}
|
|
bodyBuffer.Write(f.Body)
|
|
lastSeq = f.Sequence
|
|
}
|
|
if bodyBuffer.String() != expectedBody {
|
|
t.Errorf("body mismatch: got %q, want %q", bodyBuffer.String(), expectedBody)
|
|
}
|
|
|
|
endFrame := frames[len(frames)-1]
|
|
if endFrame.Kind != runtime.ProviderTunnelFrameKindEnd {
|
|
t.Errorf("expected END, got %s", endFrame.Kind)
|
|
}
|
|
if !endFrame.End {
|
|
t.Errorf("expected End true, got %v", endFrame.End)
|
|
}
|
|
if endFrame.Sequence != lastSeq+1 {
|
|
t.Errorf("expected seq %d, got %d", lastSeq+1, endFrame.Sequence)
|
|
}
|
|
assertOrderedTunnelFrames(t, frames)
|
|
}
|
|
|
|
func TestOpenAICompatTunnelProvider_RelaysProviderHTTPError(t *testing.T) {
|
|
expectedBody := `{"error":{"message":"unsupported field","type":"invalid_request_error","param":"custom_provider_options"}}`
|
|
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodPost {
|
|
t.Errorf("expected POST method, got %s", r.Method)
|
|
}
|
|
if r.URL.Path != "/v1/chat/completions" {
|
|
t.Errorf("expected path /v1/chat/completions, got %s", r.URL.Path)
|
|
}
|
|
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(http.StatusUnprocessableEntity)
|
|
_, _ = w.Write([]byte(expectedBody))
|
|
}))
|
|
defer server.Close()
|
|
|
|
adapter := New(config.OpenAICompatConf{
|
|
Endpoint: server.URL,
|
|
}, zap.NewNop())
|
|
|
|
sink := &fakeTunnelSink{}
|
|
req := runtime.ProviderTunnelRequest{
|
|
RunID: "run-1",
|
|
TunnelID: "tunnel-1",
|
|
Method: "POST",
|
|
Path: "/v1/chat/completions",
|
|
Body: []byte(`{"model":"qwen3.6:35b","custom_provider_options":{"reject":true}}`),
|
|
}
|
|
|
|
err := adapter.TunnelProvider(context.Background(), req, sink)
|
|
if err != nil {
|
|
t.Fatalf("TunnelProvider must relay provider HTTP errors as response frames, got error: %v", err)
|
|
}
|
|
|
|
frames := sink.all()
|
|
if len(frames) < 3 {
|
|
t.Fatalf("expected at least 3 frames, got %d", len(frames))
|
|
}
|
|
if frames[0].Kind != runtime.ProviderTunnelFrameKindResponseStart {
|
|
t.Fatalf("expected RESPONSE_START, got %s", frames[0].Kind)
|
|
}
|
|
if frames[0].StatusCode != http.StatusUnprocessableEntity {
|
|
t.Fatalf("expected provider status 422, got %d", frames[0].StatusCode)
|
|
}
|
|
if got := frames[0].Headers["Content-Type"]; got != "application/json" {
|
|
t.Fatalf("expected provider Content-Type header, got %q", got)
|
|
}
|
|
var bodyBuffer bytes.Buffer
|
|
for _, f := range frames[1 : len(frames)-1] {
|
|
if f.Kind != runtime.ProviderTunnelFrameKindBody {
|
|
t.Errorf("expected BODY kind, got %s", f.Kind)
|
|
}
|
|
bodyBuffer.Write(f.Body)
|
|
}
|
|
if bodyBuffer.String() != expectedBody {
|
|
t.Fatalf("provider error body mismatch:\n got: %q\nwant: %q", bodyBuffer.String(), expectedBody)
|
|
}
|
|
if frames[len(frames)-1].Kind != runtime.ProviderTunnelFrameKindEnd {
|
|
t.Fatalf("expected END, got %s", frames[len(frames)-1].Kind)
|
|
}
|
|
assertOrderedTunnelFrames(t, frames)
|
|
}
|
|
|
|
func TestOpenAICompatTunnelProvider_ResponsesPath(t *testing.T) {
|
|
requestBody := `{"model":"served-model","input":"hi","max_output_tokens":123,"store":false}`
|
|
expectedBody := "data: {\"type\":\"response.output_text.delta\",\"delta\":\"hi\"}\n\ndata: [DONE]\n\n"
|
|
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodPost {
|
|
t.Errorf("expected POST method, got %s", r.Method)
|
|
}
|
|
if r.URL.Path != "/v1/responses" {
|
|
t.Errorf("expected path /v1/responses, got %s", r.URL.Path)
|
|
}
|
|
if r.Header.Get("Authorization") != "Bearer test-key" {
|
|
t.Errorf("expected Authorization header, got %s", r.Header.Get("Authorization"))
|
|
}
|
|
var got bytes.Buffer
|
|
if _, err := got.ReadFrom(r.Body); err != nil {
|
|
t.Fatalf("read request body: %v", err)
|
|
}
|
|
if got.String() != requestBody {
|
|
t.Errorf("request body mismatch: got %q want %q", got.String(), requestBody)
|
|
}
|
|
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
w.WriteHeader(http.StatusOK)
|
|
_, _ = w.Write([]byte(expectedBody))
|
|
}))
|
|
defer server.Close()
|
|
|
|
adapter := New(config.OpenAICompatConf{
|
|
Endpoint: server.URL,
|
|
}, zap.NewNop())
|
|
|
|
sink := &fakeTunnelSink{}
|
|
req := runtime.ProviderTunnelRequest{
|
|
RunID: "run-1",
|
|
TunnelID: "tunnel-1",
|
|
Method: "POST",
|
|
Path: "/v1/responses",
|
|
Headers: map[string]string{"Authorization": "Bearer test-key"},
|
|
Body: []byte(requestBody),
|
|
Stream: true,
|
|
}
|
|
|
|
err := adapter.TunnelProvider(context.Background(), req, sink)
|
|
if err != nil {
|
|
t.Fatalf("TunnelProvider failed: %v", err)
|
|
}
|
|
|
|
frames := sink.all()
|
|
if len(frames) < 3 {
|
|
t.Fatalf("expected at least 3 frames, got %d", len(frames))
|
|
}
|
|
if frames[0].Kind != runtime.ProviderTunnelFrameKindResponseStart {
|
|
t.Errorf("expected RESPONSE_START, got %s", frames[0].Kind)
|
|
}
|
|
if frames[0].StatusCode != http.StatusOK {
|
|
t.Errorf("expected 200 OK, got %d", frames[0].StatusCode)
|
|
}
|
|
var bodyBuffer bytes.Buffer
|
|
for _, f := range frames[1 : len(frames)-1] {
|
|
if f.Kind != runtime.ProviderTunnelFrameKindBody {
|
|
t.Errorf("expected BODY kind, got %s", f.Kind)
|
|
}
|
|
bodyBuffer.Write(f.Body)
|
|
}
|
|
if bodyBuffer.String() != expectedBody {
|
|
t.Errorf("body mismatch: got %q, want %q", bodyBuffer.String(), expectedBody)
|
|
}
|
|
if frames[len(frames)-1].Kind != runtime.ProviderTunnelFrameKindEnd {
|
|
t.Errorf("expected END, got %s", frames[len(frames)-1].Kind)
|
|
}
|
|
assertOrderedTunnelFrames(t, frames)
|
|
}
|
|
|
|
func TestOpenAICompatTunnelProvider_Cancel(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
handlerObservedCancel := make(chan struct{})
|
|
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
w.WriteHeader(http.StatusOK)
|
|
|
|
flusher, ok := w.(http.Flusher)
|
|
for {
|
|
_, err := w.Write([]byte("data: chunk\n\n"))
|
|
if err != nil {
|
|
close(handlerObservedCancel)
|
|
return
|
|
}
|
|
if ok {
|
|
flusher.Flush()
|
|
}
|
|
select {
|
|
case <-r.Context().Done():
|
|
close(handlerObservedCancel)
|
|
return
|
|
case <-time.After(10 * time.Millisecond):
|
|
}
|
|
}
|
|
}))
|
|
defer server.Close()
|
|
|
|
adapter := New(config.OpenAICompatConf{
|
|
Endpoint: server.URL,
|
|
}, zap.NewNop())
|
|
|
|
sink := &fakeTunnelSink{}
|
|
req := runtime.ProviderTunnelRequest{
|
|
RunID: "run-1",
|
|
TunnelID: "tunnel-1",
|
|
Method: "POST",
|
|
Path: "/v1/chat/completions",
|
|
Body: []byte(`{"prompt":"hi"}`),
|
|
}
|
|
|
|
errCh := make(chan error, 1)
|
|
go func() {
|
|
errCh <- adapter.TunnelProvider(ctx, req, sink)
|
|
}()
|
|
|
|
time.Sleep(100 * time.Millisecond)
|
|
cancel()
|
|
|
|
select {
|
|
case err := <-errCh:
|
|
if err == nil {
|
|
t.Error("expected error from TunnelProvider on cancellation, got nil")
|
|
}
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal("timeout waiting for TunnelProvider to return")
|
|
}
|
|
|
|
select {
|
|
case <-handlerObservedCancel:
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal("handler did not observe context cancel")
|
|
}
|
|
|
|
frames := sink.all()
|
|
if len(frames) < 2 {
|
|
t.Fatalf("expected at least 2 frames, got %d: %+v", len(frames), frames)
|
|
}
|
|
assertOrderedTunnelFrames(t, frames)
|
|
|
|
if frames[0].Kind != runtime.ProviderTunnelFrameKindResponseStart {
|
|
t.Errorf("expected first frame RESPONSE_START, got %s", frames[0].Kind)
|
|
}
|
|
for i, f := range frames[1 : len(frames)-1] {
|
|
if f.Kind != runtime.ProviderTunnelFrameKindBody {
|
|
t.Errorf("mid frame %d: expected BODY before the terminal ERROR, got %s", i+1, f.Kind)
|
|
}
|
|
}
|
|
lastFrame := frames[len(frames)-1]
|
|
if lastFrame.Kind != runtime.ProviderTunnelFrameKindError {
|
|
t.Errorf("expected last frame to be ERROR, got %s", lastFrame.Kind)
|
|
}
|
|
if lastFrame.Error == "" {
|
|
t.Error("expected non-empty error message in frame")
|
|
}
|
|
if !strings.Contains(lastFrame.Error, "context canceled") {
|
|
t.Errorf("expected cancellation error in terminal frame, got %q", lastFrame.Error)
|
|
}
|
|
}
|
|
|
|
func TestOpenAICompatTunnelProvider_BodyReadError(t *testing.T) {
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
hj, ok := w.(http.Hijacker)
|
|
if !ok {
|
|
t.Fatal("webserver doesn't support hijacking")
|
|
}
|
|
conn, _, err := hj.Hijack()
|
|
if err != nil {
|
|
t.Fatalf("hijack failed: %v", err)
|
|
}
|
|
defer conn.Close()
|
|
|
|
_, _ = conn.Write([]byte("HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nContent-Length: 100\r\n\r\n"))
|
|
_, _ = conn.Write([]byte("data: chunk1\n\n"))
|
|
}))
|
|
defer server.Close()
|
|
|
|
adapter := New(config.OpenAICompatConf{
|
|
Endpoint: server.URL,
|
|
}, zap.NewNop())
|
|
|
|
sink := &fakeTunnelSink{}
|
|
req := runtime.ProviderTunnelRequest{
|
|
RunID: "run-1",
|
|
TunnelID: "tunnel-1",
|
|
Method: "POST",
|
|
Path: "/v1/chat/completions",
|
|
Body: []byte(`{"prompt":"hi"}`),
|
|
}
|
|
|
|
err := adapter.TunnelProvider(context.Background(), req, sink)
|
|
if err == nil {
|
|
t.Fatal("expected error from body read failure, got nil")
|
|
}
|
|
|
|
frames := sink.all()
|
|
if len(frames) < 2 {
|
|
t.Fatalf("expected at least 2 frames, got %d: %+v", len(frames), frames)
|
|
}
|
|
assertOrderedTunnelFrames(t, frames)
|
|
|
|
lastFrame := frames[len(frames)-1]
|
|
if lastFrame.Kind != runtime.ProviderTunnelFrameKindError {
|
|
t.Errorf("expected last frame to be ERROR, got %s", lastFrame.Kind)
|
|
}
|
|
if !strings.Contains(lastFrame.Error, "read response body") && !strings.Contains(lastFrame.Error, "EOF") {
|
|
t.Errorf("expected body read error msg, got %q", lastFrame.Error)
|
|
}
|
|
}
|
|
|
|
// TestOpenAICompatTunnelProvider_SinkWriteFailureStopsUpstreamRelay covers the
|
|
// Edge-write-failure leg of S09: when EmitTunnelFrame fails mid-stream the
|
|
// adapter stops relaying immediately, returns the emit error without appending
|
|
// further frames, and the upstream provider request is torn down.
|
|
func TestOpenAICompatTunnelProvider_SinkWriteFailureStopsUpstreamRelay(t *testing.T) {
|
|
handlerObservedTeardown := make(chan struct{})
|
|
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
w.WriteHeader(http.StatusOK)
|
|
|
|
flusher, ok := w.(http.Flusher)
|
|
for {
|
|
_, err := w.Write([]byte("data: chunk\n\n"))
|
|
if err != nil {
|
|
close(handlerObservedTeardown)
|
|
return
|
|
}
|
|
if ok {
|
|
flusher.Flush()
|
|
}
|
|
select {
|
|
case <-r.Context().Done():
|
|
close(handlerObservedTeardown)
|
|
return
|
|
case <-time.After(10 * time.Millisecond):
|
|
}
|
|
}
|
|
}))
|
|
defer server.Close()
|
|
|
|
adapter := New(config.OpenAICompatConf{
|
|
Endpoint: server.URL,
|
|
}, zap.NewNop())
|
|
|
|
// RESPONSE_START and the first BODY frame succeed; the next emit fails as
|
|
// if the Edge socket write broke after the caller went away.
|
|
sink := &failingTunnelSink{failAfter: 2}
|
|
req := runtime.ProviderTunnelRequest{
|
|
RunID: "run-1",
|
|
TunnelID: "tunnel-1",
|
|
Method: "POST",
|
|
Path: "/v1/chat/completions",
|
|
Body: []byte(`{"prompt":"hi"}`),
|
|
}
|
|
|
|
errCh := make(chan error, 1)
|
|
go func() {
|
|
errCh <- adapter.TunnelProvider(context.Background(), req, sink)
|
|
}()
|
|
|
|
var tunnelErr error
|
|
select {
|
|
case tunnelErr = <-errCh:
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal("timeout waiting for TunnelProvider to return after sink write failure")
|
|
}
|
|
if tunnelErr == nil || !strings.Contains(tunnelErr.Error(), "edge socket write failed") {
|
|
t.Fatalf("expected the sink emit error to propagate, got %v", tunnelErr)
|
|
}
|
|
|
|
select {
|
|
case <-handlerObservedTeardown:
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal("provider handler did not observe upstream request teardown")
|
|
}
|
|
|
|
frames := sink.all()
|
|
if len(frames) != 2 {
|
|
t.Fatalf("expected relay to stop at 2 frames after emit failure, got %d: %+v", len(frames), frames)
|
|
}
|
|
if frames[0].Kind != runtime.ProviderTunnelFrameKindResponseStart {
|
|
t.Errorf("expected first frame RESPONSE_START, got %s", frames[0].Kind)
|
|
}
|
|
if frames[1].Kind != runtime.ProviderTunnelFrameKindBody {
|
|
t.Errorf("expected second frame BODY, got %s", frames[1].Kind)
|
|
}
|
|
for i, f := range frames {
|
|
if f.Sequence != int64(i) {
|
|
t.Errorf("frame %d: sequence %d, want %d", i, f.Sequence, i)
|
|
}
|
|
}
|
|
}
|