- Add unit tests for OpenAI edge adapter (chat handler, stream reasoning, tool synthesis, auth routes, test support) - Add roadmap: agent-readable-repository-refactor milestone with subtask plans - Update PHASE.md, priority-queue.md, .gitignore - Delete old edge/node/openai.test files (replaced by proper test structure)
4966 lines
188 KiB
Go
4966 lines
188 KiB
Go
package openai
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"go.uber.org/zap"
|
|
"go.uber.org/zap/zaptest/observer"
|
|
|
|
edgeservice "iop/apps/edge/internal/service"
|
|
"iop/packages/go/config"
|
|
iop "iop/proto/gen/iop"
|
|
)
|
|
|
|
func TestResponsesDispatchesNonStreamingRequest(t *testing.T) {
|
|
fake := &fakeRunService{events: make(chan *iop.RunEvent, 3)}
|
|
fake.events <- &iop.RunEvent{Type: "delta", Delta: "hello"}
|
|
fake.events <- &iop.RunEvent{Type: "delta", Delta: " world"}
|
|
fake.events <- &iop.RunEvent{Type: "complete", Usage: &iop.Usage{InputTokens: 1, OutputTokens: 1}}
|
|
|
|
srv := NewServer(config.EdgeOpenAIConf{
|
|
Adapter: "ollama",
|
|
Target: "llama-fixed",
|
|
SessionID: "configured-session",
|
|
TimeoutSec: 42,
|
|
}, fake, nil)
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(`{
|
|
"model":"client-model",
|
|
"input":"say hello",
|
|
"instructions":"Use short answers.",
|
|
"max_output_tokens":16,
|
|
"temperature":0.2,
|
|
"top_p":0.9,
|
|
"background":false
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
|
|
srv.routes().ServeHTTP(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
if fake.req.Target != "llama-fixed" {
|
|
t.Fatalf("target: got %q, want llama-fixed", fake.req.Target)
|
|
}
|
|
if fake.req.Prompt != "Use short answers.\n\nsay hello" {
|
|
t.Fatalf("prompt: got %q", fake.req.Prompt)
|
|
}
|
|
options, ok := fake.req.Input["options"].(map[string]any)
|
|
if !ok {
|
|
t.Fatalf("options not passed: %+v", fake.req.Input)
|
|
}
|
|
if options["max_tokens"].(int) != 16 || options["temperature"].(float64) != 0.2 || options["top_p"].(float64) != 0.9 {
|
|
t.Fatalf("unexpected response options: %+v", options)
|
|
}
|
|
if fake.req.TimeoutSec != 42 {
|
|
t.Fatalf("timeout: got %d, want 42", fake.req.TimeoutSec)
|
|
}
|
|
if fake.req.SessionID != "configured-session" {
|
|
t.Fatalf("session_id: got %q, want configured-session", fake.req.SessionID)
|
|
}
|
|
if fake.req.Background {
|
|
t.Fatal("background should remain false for responses")
|
|
}
|
|
|
|
var resp responsesResponse
|
|
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
|
t.Fatalf("decode response: %v", err)
|
|
}
|
|
if resp.OutputText != "hello world" {
|
|
t.Fatalf("output_text: got %q", resp.OutputText)
|
|
}
|
|
if len(resp.Output) == 0 || len(resp.Output[0].Content) == 0 || resp.Output[0].Content[0].Text != "hello world" {
|
|
t.Fatalf("output[0].content[0].text wrong: %+v", resp.Output)
|
|
}
|
|
}
|
|
|
|
func TestResponsesRejectsUnsupportedRequests(t *testing.T) {
|
|
srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama", Target: "llama"}, &fakeRunService{}, nil)
|
|
|
|
cases := []struct {
|
|
name string
|
|
method string
|
|
body string
|
|
want int
|
|
}{
|
|
{"wrong method", http.MethodGet, "", http.StatusMethodNotAllowed},
|
|
{"stream=true", http.MethodPost, `{"model":"m","input":"hi","stream":true}`, http.StatusBadRequest},
|
|
{"empty input", http.MethodPost, `{"model":"m","input":""}`, http.StatusBadRequest},
|
|
{"non-string input", http.MethodPost, `{"model":"m","input":["a","b"]}`, http.StatusBadRequest},
|
|
{"options wrapper unsupported", http.MethodPost, `{"model":"m","input":"hi","options":{"max_output_tokens":16}}`, http.StatusBadRequest},
|
|
{"background unsupported", http.MethodPost, `{"model":"m","input":"hi","background":true}`, http.StatusBadRequest},
|
|
{"bad max_output_tokens", http.MethodPost, `{"model":"m","input":"hi","max_output_tokens":0}`, http.StatusBadRequest},
|
|
{"bad top_p", http.MethodPost, `{"model":"m","input":"hi","top_p":1.5}`, http.StatusBadRequest},
|
|
}
|
|
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
req := httptest.NewRequest(tc.method, "/v1/responses", strings.NewReader(tc.body))
|
|
w := httptest.NewRecorder()
|
|
srv.routes().ServeHTTP(w, req)
|
|
if w.Code != tc.want {
|
|
t.Fatalf("got %d want %d body=%s", w.Code, tc.want, w.Body.String())
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestResponsesGenericMetadataContract(t *testing.T) {
|
|
fake := &fakeRunService{events: make(chan *iop.RunEvent, 2)}
|
|
fake.events <- &iop.RunEvent{Type: "delta", Delta: "ok"}
|
|
fake.events <- &iop.RunEvent{Type: "complete"}
|
|
|
|
srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama"}, fake, nil)
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(`{
|
|
"model":"client-model",
|
|
"input":"test",
|
|
"metadata":{
|
|
"request_id":"req-001",
|
|
"workspace":"/config/workspace/iop",
|
|
"task_id":"task-123",
|
|
"custom":"value"
|
|
}
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.routes().ServeHTTP(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
if fake.req.Target != "client-model" {
|
|
t.Fatalf("target: got %q, want client-model", fake.req.Target)
|
|
}
|
|
if fake.req.Workspace != "/config/workspace/iop" {
|
|
t.Fatalf("workspace: got %q", fake.req.Workspace)
|
|
}
|
|
if fake.req.Metadata["request_id"] != "req-001" {
|
|
t.Fatalf("request_id: got %q", fake.req.Metadata["request_id"])
|
|
}
|
|
if _, ok := fake.req.Metadata["workspace"]; ok {
|
|
t.Fatal("workspace should not be copied into run metadata")
|
|
}
|
|
if fake.req.Metadata["task_id"] != "task-123" {
|
|
t.Fatalf("task_id: got %q", fake.req.Metadata["task_id"])
|
|
}
|
|
if fake.req.Metadata["custom"] != "value" {
|
|
t.Fatalf("custom: got %q", fake.req.Metadata["custom"])
|
|
}
|
|
if fake.req.Metadata["openai_model"] != "client-model" {
|
|
t.Fatalf("openai_model: got %q", fake.req.Metadata["openai_model"])
|
|
}
|
|
if fake.req.ModelGroupKey != "client-model" {
|
|
t.Fatalf("model group key: got %q, want client-model", fake.req.ModelGroupKey)
|
|
}
|
|
if fake.req.Metadata["openai_stream"] != "false" {
|
|
t.Fatalf("openai_stream: got %q", fake.req.Metadata["openai_stream"])
|
|
}
|
|
if fake.req.Metadata["strict_output"] != "false" {
|
|
t.Fatalf("strict_output: got %q", fake.req.Metadata["strict_output"])
|
|
}
|
|
}
|
|
|
|
func TestChatCompletionsMetadataContractAndWorkspace(t *testing.T) {
|
|
fake := &fakeRunService{events: make(chan *iop.RunEvent, 2)}
|
|
fake.events <- &iop.RunEvent{Type: "delta", Delta: "ok"}
|
|
fake.events <- &iop.RunEvent{Type: "complete"}
|
|
|
|
srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama"}, fake, nil)
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"client-model",
|
|
"messages":[{"role":"user","content":"hi"}],
|
|
"metadata":{
|
|
"request_id":"req-chat-001",
|
|
"workspace":"/config/workspace/iop",
|
|
"task_id":"task-123"
|
|
}
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.routes().ServeHTTP(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
if fake.req.Target != "client-model" {
|
|
t.Fatalf("target: got %q, want client-model", fake.req.Target)
|
|
}
|
|
if fake.req.Workspace != "/config/workspace/iop" {
|
|
t.Fatalf("workspace: got %q", fake.req.Workspace)
|
|
}
|
|
if fake.req.Metadata["request_id"] != "req-chat-001" {
|
|
t.Fatalf("request_id: got %q", fake.req.Metadata["request_id"])
|
|
}
|
|
if fake.req.Metadata["task_id"] != "task-123" {
|
|
t.Fatalf("task_id: got %q", fake.req.Metadata["task_id"])
|
|
}
|
|
if fake.req.ModelGroupKey != "client-model" {
|
|
t.Fatalf("model group key: got %q, want client-model", fake.req.ModelGroupKey)
|
|
}
|
|
if _, ok := fake.req.Metadata["workspace"]; ok {
|
|
t.Fatal("workspace should not be copied into run metadata")
|
|
}
|
|
}
|
|
|
|
func TestChatCompletionsRejectsObjectMetadata(t *testing.T) {
|
|
srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama", Target: "llama"}, &fakeRunService{}, nil)
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"m",
|
|
"messages":[{"role":"user","content":"hi"}],
|
|
"metadata":{"cli":{"flag":"x"}}
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.routes().ServeHTTP(w, req)
|
|
if w.Code != http.StatusBadRequest {
|
|
t.Fatalf("got %d want 400, body=%s", w.Code, w.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestChatCompletionsRejectsUnsupportedFields(t *testing.T) {
|
|
srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama", Target: "llama"}, &fakeRunService{}, nil)
|
|
cases := []struct {
|
|
name string
|
|
body string
|
|
}{
|
|
{"options wrapper", `{"model":"m","messages":[{"role":"user","content":"hi"}],"options":{"num_ctx":8192}}`},
|
|
{"format", `{"model":"m","messages":[{"role":"user","content":"hi"}],"format":"json"}`},
|
|
{"keep_alive", `{"model":"m","messages":[{"role":"user","content":"hi"}],"keep_alive":"10m"}`},
|
|
{"bad max_tokens", `{"model":"m","messages":[{"role":"user","content":"hi"}],"max_tokens":0}`},
|
|
{"bad top_p", `{"model":"m","messages":[{"role":"user","content":"hi"}],"top_p":2}`},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(tc.body))
|
|
w := httptest.NewRecorder()
|
|
srv.routes().ServeHTTP(w, req)
|
|
if w.Code != http.StatusBadRequest {
|
|
t.Fatalf("got %d want 400, body=%s", w.Code, w.Body.String())
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestChatCompletionsAcceptsThinkControlFields(t *testing.T) {
|
|
fake := &fakeRunService{
|
|
events: make(chan *iop.RunEvent, 1),
|
|
}
|
|
fake.events <- &iop.RunEvent{Type: "complete"}
|
|
|
|
srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama"}, fake, nil)
|
|
|
|
// Case 1: All valid think-control fields provided
|
|
reqBody := `{
|
|
"model":"client-model",
|
|
"messages":[{"role":"user","content":"hi"}],
|
|
"think": true,
|
|
"reasoning_effort": "medium",
|
|
"thinking_token_budget": 1024,
|
|
"include_reasoning": false
|
|
}`
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(reqBody))
|
|
w := httptest.NewRecorder()
|
|
srv.routes().ServeHTTP(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
if fake.req.Input["think"] != true {
|
|
t.Fatalf("think: got %v, want true", fake.req.Input["think"])
|
|
}
|
|
if fake.req.Input["reasoning_effort"] != "medium" {
|
|
t.Fatalf("reasoning_effort: got %v, want 'medium'", fake.req.Input["reasoning_effort"])
|
|
}
|
|
|
|
budgetVal := fake.req.Input["thinking_token_budget"]
|
|
switch v := budgetVal.(type) {
|
|
case float64:
|
|
if v != 1024 {
|
|
t.Fatalf("thinking_token_budget: got %f, want 1024", v)
|
|
}
|
|
case int:
|
|
if v != 1024 {
|
|
t.Fatalf("thinking_token_budget: got %d, want 1024", v)
|
|
}
|
|
default:
|
|
t.Fatalf("thinking_token_budget: got %T, want number type", budgetVal)
|
|
}
|
|
|
|
if fake.req.Input["include_reasoning"] != false {
|
|
t.Fatalf("include_reasoning: got %v, want false", fake.req.Input["include_reasoning"])
|
|
}
|
|
|
|
// Case 2: Omitted think-control fields should NOT add keys to runInput
|
|
fake2 := &fakeRunService{
|
|
events: make(chan *iop.RunEvent, 1),
|
|
}
|
|
fake2.events <- &iop.RunEvent{Type: "complete"}
|
|
srv2 := NewServer(config.EdgeOpenAIConf{Adapter: "ollama"}, fake2, nil)
|
|
|
|
reqBody2 := `{
|
|
"model":"client-model",
|
|
"messages":[{"role":"user","content":"hi"}]
|
|
}`
|
|
req2 := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(reqBody2))
|
|
w2 := httptest.NewRecorder()
|
|
srv2.routes().ServeHTTP(w2, req2)
|
|
|
|
if w2.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w2.Code, w2.Body.String())
|
|
}
|
|
for _, key := range []string{"think", "reasoning_effort", "thinking_token_budget", "include_reasoning"} {
|
|
if _, ok := fake2.req.Input[key]; ok {
|
|
t.Fatalf("expected key %q to be omitted from runInput, but it was present", key)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestChatCompletionsRejectsInvalidThinkControlFields(t *testing.T) {
|
|
srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama", Target: "llama"}, &fakeRunService{}, nil)
|
|
cases := []struct {
|
|
name string
|
|
body string
|
|
}{
|
|
{"negative budget", `{"model":"m","messages":[{"role":"user","content":"hi"}],"thinking_token_budget":-1}`},
|
|
{"bad effort", `{"model":"m","messages":[{"role":"user","content":"hi"}],"reasoning_effort":"super_high"}`},
|
|
{"empty effort", `{"model":"m","messages":[{"role":"user","content":"hi"}],"reasoning_effort":""}`},
|
|
{"think=false and reasoning_effort other than none", `{"model":"m","messages":[{"role":"user","content":"hi"}],"think":false,"reasoning_effort":"high"}`},
|
|
{"thinking_token_budget conflicts with think=false", `{"model":"m","messages":[{"role":"user","content":"hi"}],"think":false,"thinking_token_budget":100}`},
|
|
{"thinking_token_budget conflicts with reasoning_effort=none", `{"model":"m","messages":[{"role":"user","content":"hi"}],"reasoning_effort":"none","thinking_token_budget":100}`},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(tc.body))
|
|
w := httptest.NewRecorder()
|
|
srv.routes().ServeHTTP(w, req)
|
|
if w.Code != http.StatusBadRequest {
|
|
t.Fatalf("got %d want 400, body=%s", w.Code, w.Body.String())
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestChatCompletionsRejectsSourceMetadata(t *testing.T) {
|
|
srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama", Target: "llama"}, &fakeRunService{}, nil)
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"m",
|
|
"messages":[{"role":"user","content":"hi"}],
|
|
"metadata":{"source":"manual"}
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.routes().ServeHTTP(w, req)
|
|
if w.Code != http.StatusBadRequest {
|
|
t.Fatalf("got %d want 400, body=%s", w.Code, w.Body.String())
|
|
}
|
|
if !strings.Contains(w.Body.String(), "metadata.source is not supported") {
|
|
t.Fatalf("expected source unsupported error, got %s", w.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestChatCompletionsRejectsNonStringMetadataValue(t *testing.T) {
|
|
srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama", Target: "llama"}, &fakeRunService{}, nil)
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"m",
|
|
"messages":[{"role":"user","content":"hi"}],
|
|
"metadata":{"attempt":2}
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.routes().ServeHTTP(w, req)
|
|
if w.Code != http.StatusBadRequest {
|
|
t.Fatalf("got %d want 400, body=%s", w.Code, w.Body.String())
|
|
}
|
|
if !strings.Contains(w.Body.String(), "metadata.attempt must be a string") {
|
|
t.Fatalf("expected string metadata error, got %s", w.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestChatCompletionsRejectsObjectWorkspaceMetadata(t *testing.T) {
|
|
srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama", Target: "llama"}, &fakeRunService{}, nil)
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"m",
|
|
"messages":[{"role":"user","content":"hi"}],
|
|
"metadata":{
|
|
"workspace": {
|
|
"path": "/home/user/workspace",
|
|
"source_branch": "develop"
|
|
}
|
|
}
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.routes().ServeHTTP(w, req)
|
|
if w.Code != http.StatusBadRequest {
|
|
t.Fatalf("got %d want 400, body=%s", w.Code, w.Body.String())
|
|
}
|
|
if !strings.Contains(w.Body.String(), "metadata.workspace must be a string") {
|
|
t.Fatalf("expected workspace string error, got %s", w.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestResponsesPreservesGenericTaskMetadata(t *testing.T) {
|
|
fake := &fakeRunService{events: make(chan *iop.RunEvent, 2)}
|
|
fake.events <- &iop.RunEvent{Type: "delta", Delta: "ok"}
|
|
fake.events <- &iop.RunEvent{Type: "complete"}
|
|
|
|
srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama"}, fake, nil)
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(`{
|
|
"model":"client-model",
|
|
"input":"test",
|
|
"metadata":{
|
|
"request_id":"req-flat-001",
|
|
"task_id":"task-flat"
|
|
}
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.routes().ServeHTTP(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
if fake.req.Metadata["task_id"] != "task-flat" {
|
|
t.Fatalf("task_id: got %q", fake.req.Metadata["task_id"])
|
|
}
|
|
}
|
|
|
|
func TestResponsesConfiguredTargetWinsOverRequestModel(t *testing.T) {
|
|
fake := &fakeRunService{events: make(chan *iop.RunEvent, 2)}
|
|
fake.events <- &iop.RunEvent{Type: "delta", Delta: "ok"}
|
|
fake.events <- &iop.RunEvent{Type: "complete"}
|
|
|
|
srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama", Target: "config-target"}, fake, nil)
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(`{
|
|
"model":"client-model",
|
|
"input":"test"
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.routes().ServeHTTP(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
if fake.req.Target != "config-target" {
|
|
t.Fatalf("target: got %q, want config-target", fake.req.Target)
|
|
}
|
|
|
|
var resp responsesResponse
|
|
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
|
t.Fatalf("decode: %v", err)
|
|
}
|
|
if resp.Model != "client-model" {
|
|
t.Fatalf("model: got %q, want client-model", resp.Model)
|
|
}
|
|
}
|
|
|
|
func TestResponsesRejectsNonStringMetadata(t *testing.T) {
|
|
srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama", Target: "llama"}, &fakeRunService{}, nil)
|
|
|
|
cases := []struct {
|
|
name string
|
|
body string
|
|
}{
|
|
{"cli only", `{"model":"m","input":"hi","metadata":{"cli":{"flag":"x"}}}`},
|
|
{"inference target", `{"model":"m","input":"hi","metadata":{"inference":{"target":"t"}}}`},
|
|
{"nomadcode metadata", `{"model":"m","input":"hi","metadata":{"nomadcode":{"task_id":"t"}}}`},
|
|
{"source metadata", `{"model":"m","input":"hi","metadata":{"source":"manual"}}`},
|
|
{"number metadata", `{"model":"m","input":"hi","metadata":{"attempt":2}}`},
|
|
{"boolean metadata", `{"model":"m","input":"hi","metadata":{"urgent":true}}`},
|
|
}
|
|
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(tc.body))
|
|
w := httptest.NewRecorder()
|
|
srv.routes().ServeHTTP(w, req)
|
|
if w.Code != http.StatusBadRequest {
|
|
t.Fatalf("got %d want 400, body=%s", w.Code, w.Body.String())
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestResponsesRejectsObjectWorkspaceMetadata(t *testing.T) {
|
|
srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama", Target: "llama"}, &fakeRunService{}, nil)
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(`{
|
|
"model":"m",
|
|
"input":"hi",
|
|
"metadata":{
|
|
"workspace": {
|
|
"path": "/home/user/workspace",
|
|
"source_branch": "develop"
|
|
}
|
|
}
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.routes().ServeHTTP(w, req)
|
|
if w.Code != http.StatusBadRequest {
|
|
t.Fatalf("got %d want 400, body=%s", w.Code, w.Body.String())
|
|
}
|
|
if !strings.Contains(w.Body.String(), "metadata.workspace must be a string") {
|
|
t.Fatalf("expected workspace string error, got %s", w.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestResponsesReturnsOpenAICompatibleShape(t *testing.T) {
|
|
fake := &fakeRunService{events: make(chan *iop.RunEvent, 3)}
|
|
fake.events <- &iop.RunEvent{Type: "delta", Delta: "answer"}
|
|
fake.events <- &iop.RunEvent{Type: "complete", Usage: &iop.Usage{InputTokens: 1, OutputTokens: 1}}
|
|
|
|
srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama", Target: "llama"}, fake, nil)
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(`{
|
|
"model":"client-model",
|
|
"input":"say hello"
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.routes().ServeHTTP(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
|
|
var resp responsesResponse
|
|
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
|
t.Fatalf("decode: %v", err)
|
|
}
|
|
if !strings.HasPrefix(resp.ID, "resp-") {
|
|
t.Fatalf("id: got %q, want resp- prefix", resp.ID)
|
|
}
|
|
if resp.Model != "client-model" {
|
|
t.Fatalf("model: got %q", resp.Model)
|
|
}
|
|
if resp.OutputText != "answer" {
|
|
t.Fatalf("output_text: got %q", resp.OutputText)
|
|
}
|
|
if len(resp.Output) == 0 || len(resp.Output[0].Content) == 0 || resp.Output[0].Content[0].Text != "answer" {
|
|
t.Fatalf("output[0].content[0].text wrong: %+v", resp.Output)
|
|
}
|
|
if resp.Usage.TotalTokens != 2 {
|
|
t.Fatalf("usage.total_tokens: got %d", resp.Usage.TotalTokens)
|
|
}
|
|
}
|
|
|
|
func TestResponsesSanitizesKnownSentinelToken(t *testing.T) {
|
|
fake := &fakeRunService{events: make(chan *iop.RunEvent, 3)}
|
|
fake.events <- &iop.RunEvent{Type: "reasoning_delta", Delta: "hidden <|mask_end|>"}
|
|
fake.events <- &iop.RunEvent{Type: "delta", Delta: "answer <|mask_end|>"}
|
|
fake.events <- &iop.RunEvent{Type: "complete", Usage: &iop.Usage{InputTokens: 1, OutputTokens: 1}}
|
|
|
|
srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama"}, fake, nil)
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(`{
|
|
"model":"client-model",
|
|
"input":"say hello"
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
|
|
srv.routes().ServeHTTP(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
var resp responsesResponse
|
|
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
|
t.Fatalf("decode: %v", err)
|
|
}
|
|
if strings.Contains(resp.OutputText, "<|mask_end|>") || strings.Contains(w.Body.String(), "<|mask_end|>") {
|
|
t.Fatalf("sentinel leaked in responses output: %s", w.Body.String())
|
|
}
|
|
if resp.OutputText != "answer" {
|
|
t.Fatalf("output_text: got %q, want answer", resp.OutputText)
|
|
}
|
|
}
|
|
|
|
func TestResponsesUsageDefaultsToZeroObject(t *testing.T) {
|
|
fake := &fakeRunService{events: make(chan *iop.RunEvent, 2)}
|
|
fake.events <- &iop.RunEvent{Type: "delta", Delta: "ok"}
|
|
fake.events <- &iop.RunEvent{Type: "complete"}
|
|
|
|
srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama", Target: "llama"}, fake, nil)
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(`{
|
|
"model":"m",
|
|
"input":"hi"
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.routes().ServeHTTP(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
|
|
var body map[string]json.RawMessage
|
|
if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {
|
|
t.Fatalf("decode: %v", err)
|
|
}
|
|
if _, ok := body["usage"]; !ok {
|
|
t.Fatal("usage field missing from response")
|
|
}
|
|
var usage openAIUsage
|
|
if err := json.Unmarshal(body["usage"], &usage); err != nil {
|
|
t.Fatalf("usage is not an object: %s", body["usage"])
|
|
}
|
|
}
|
|
|
|
func TestResponsesStrictOutputNormalizesAgentResponse(t *testing.T) {
|
|
fake := &fakeRunService{events: make(chan *iop.RunEvent, 3)}
|
|
fake.events <- &iop.RunEvent{Type: "reasoning_delta", Delta: "hidden"}
|
|
fake.events <- &iop.RunEvent{Type: "delta", Delta: "plain answer"}
|
|
fake.events <- &iop.RunEvent{Type: "complete"}
|
|
|
|
srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama", Target: "llama", StrictOutput: true}, fake, nil)
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(`{
|
|
"model":"client-model",
|
|
"input":"Once you've completed the user's task, you must use the attempt_completion tool to present the result.\n\n<attempt_completion>\n<result>done</result>\n</attempt_completion>\n\nfinish"
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.routes().ServeHTTP(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
if fake.req.Metadata["strict_output"] != "true" {
|
|
t.Fatalf("strict_output metadata: got %q", fake.req.Metadata["strict_output"])
|
|
}
|
|
if fake.req.Input["think"] != false {
|
|
t.Fatalf("strict output should disable thinking by default: %+v", fake.req.Input)
|
|
}
|
|
if !strings.Contains(fake.req.Prompt, "output exactly one <attempt_completion> block") {
|
|
t.Fatalf("strict contract instruction not injected:\n%s", fake.req.Prompt)
|
|
}
|
|
|
|
var resp responsesResponse
|
|
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
|
t.Fatalf("decode: %v", err)
|
|
}
|
|
want := "<attempt_completion>\n<result>plain answer</result>\n</attempt_completion>"
|
|
if resp.OutputText != want {
|
|
t.Fatalf("output_text:\ngot %q\nwant %q", resp.OutputText, want)
|
|
}
|
|
if len(resp.Output) == 0 || len(resp.Output[0].Content) == 0 || resp.Output[0].Content[0].Text != want {
|
|
t.Fatalf("output[0].content[0].text wrong: %+v", resp.Output)
|
|
}
|
|
}
|
|
|
|
func TestCollectRunResultTimesOut(t *testing.T) {
|
|
handle := &edgeservice.RunHandle{
|
|
RunDispatch: edgeservice.RunDispatch{TimeoutSec: 1},
|
|
RunStream: edgeservice.RunStream{
|
|
Events: make(chan *iop.RunEvent),
|
|
NodeEvents: make(chan *iop.EdgeNodeEvent),
|
|
},
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 1500*time.Millisecond)
|
|
defer cancel()
|
|
_, _, _, _, _, _, err := collectRunResult(ctx, handle.Stream(), handle.WaitTimeout())
|
|
if err == nil {
|
|
t.Fatal("expected timeout error")
|
|
}
|
|
}
|
|
|
|
// fakeRunResultWithTimeout implements edgeservice.RunResult with a directly
|
|
// configurable WaitTimeout, so tests can exercise the run-timeout cancel path
|
|
// without waiting out the real DefaultTimeoutSec-derived duration.
|
|
type fakeRunResultWithTimeout struct {
|
|
dispatch edgeservice.RunDispatch
|
|
stream edgeservice.RunStream
|
|
waitTimeout time.Duration
|
|
}
|
|
|
|
func (f *fakeRunResultWithTimeout) Dispatch() edgeservice.RunDispatch { return f.dispatch }
|
|
func (f *fakeRunResultWithTimeout) Stream() edgeservice.RunStream { return f.stream }
|
|
func (f *fakeRunResultWithTimeout) Close() {}
|
|
func (f *fakeRunResultWithTimeout) WaitTimeout() time.Duration { return f.waitTimeout }
|
|
|
|
func TestChatCompletionContextCancelSendsCancelRun(t *testing.T) {
|
|
fake := &fakeRunService{events: make(chan *iop.RunEvent)}
|
|
srv := NewServer(config.EdgeOpenAIConf{
|
|
Adapter: "ollama",
|
|
Target: "llama-fixed",
|
|
SessionID: "cline",
|
|
TimeoutSec: 15,
|
|
}, fake, nil)
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"client-model",
|
|
"messages":[{"role":"user","content":"hi"}]
|
|
}`)).WithContext(ctx)
|
|
w := httptest.NewRecorder()
|
|
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
if w.Code != http.StatusRequestTimeout {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
calls := fake.cancelCallsSnapshot()
|
|
if len(calls) != 1 {
|
|
t.Fatalf("expected 1 CancelRun call, got %d: %+v", len(calls), calls)
|
|
}
|
|
if calls[0].RunID != "run-test" || calls[0].Adapter != "ollama" || calls[0].Target != "llama-fixed" || calls[0].SessionID != "cline" {
|
|
t.Fatalf("unexpected cancel request: %+v", calls[0])
|
|
}
|
|
}
|
|
|
|
func TestResponsesContextCancelSendsCancelRun(t *testing.T) {
|
|
fake := &fakeRunService{events: make(chan *iop.RunEvent)}
|
|
srv := NewServer(config.EdgeOpenAIConf{
|
|
Adapter: "ollama",
|
|
Target: "llama-fixed",
|
|
SessionID: "cline",
|
|
TimeoutSec: 15,
|
|
}, fake, nil)
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(`{
|
|
"model":"client-model",
|
|
"input":"say hello"
|
|
}`)).WithContext(ctx)
|
|
w := httptest.NewRecorder()
|
|
|
|
srv.handleResponses(w, req)
|
|
|
|
if w.Code != http.StatusRequestTimeout {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
calls := fake.cancelCallsSnapshot()
|
|
if len(calls) != 1 {
|
|
t.Fatalf("expected 1 CancelRun call, got %d: %+v", len(calls), calls)
|
|
}
|
|
if calls[0].RunID != "run-test" || calls[0].Adapter != "ollama" || calls[0].Target != "llama-fixed" || calls[0].SessionID != "cline" {
|
|
t.Fatalf("unexpected cancel request: %+v", calls[0])
|
|
}
|
|
}
|
|
|
|
func TestStreamChatCompletionContextCancelSendsCancelRun(t *testing.T) {
|
|
fake := &fakeRunService{events: make(chan *iop.RunEvent)}
|
|
srv := NewServer(config.EdgeOpenAIConf{
|
|
Adapter: "ollama",
|
|
Target: "llama-fixed",
|
|
SessionID: "cline",
|
|
TimeoutSec: 15,
|
|
}, fake, nil)
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"client-model",
|
|
"stream":true,
|
|
"messages":[{"role":"user","content":"hi"}]
|
|
}`)).WithContext(ctx)
|
|
w := httptest.NewRecorder()
|
|
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
calls := fake.cancelCallsSnapshot()
|
|
if len(calls) != 1 {
|
|
t.Fatalf("expected 1 CancelRun call, got %d: %+v", len(calls), calls)
|
|
}
|
|
if calls[0].RunID != "run-test" || calls[0].Adapter != "ollama" || calls[0].Target != "llama-fixed" || calls[0].SessionID != "cline" {
|
|
t.Fatalf("unexpected cancel request: %+v", calls[0])
|
|
}
|
|
}
|
|
|
|
func TestStreamChatCompletionTimeoutSendsCancelRun(t *testing.T) {
|
|
fake := &fakeRunService{}
|
|
srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama"}, fake, nil)
|
|
|
|
handle := &fakeRunResultWithTimeout{
|
|
dispatch: edgeservice.RunDispatch{
|
|
RunID: "run-timeout",
|
|
NodeID: "node-1",
|
|
Adapter: "ollama",
|
|
Target: "llama3",
|
|
SessionID: "cline",
|
|
},
|
|
stream: edgeservice.RunStream{
|
|
Events: make(chan *iop.RunEvent),
|
|
NodeEvents: make(chan *iop.EdgeNodeEvent),
|
|
},
|
|
waitTimeout: 50 * time.Millisecond,
|
|
}
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", nil)
|
|
w := httptest.NewRecorder()
|
|
|
|
srv.streamChatCompletion(w, req, chatCompletionRequest{Model: "llama3"}, edgeservice.SubmitRunRequest{}, handle, strictOutputPolicy{}, toolValidationContract{}, func(ctx context.Context, req edgeservice.SubmitRunRequest) (any, error) {
|
|
return nil, nil
|
|
})
|
|
|
|
calls := fake.cancelCallsSnapshot()
|
|
if len(calls) != 1 {
|
|
t.Fatalf("expected 1 CancelRun call, got %d: %+v", len(calls), calls)
|
|
}
|
|
if calls[0].RunID != "run-timeout" || calls[0].NodeRef != "node-1" {
|
|
t.Fatalf("unexpected cancel request: %+v", calls[0])
|
|
}
|
|
if !strings.Contains(w.Body.String(), "run timed out") {
|
|
t.Fatalf("expected timeout SSE error, got %s", w.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestChatCompletionCompleteDoesNotSendCancelRun(t *testing.T) {
|
|
fake := &fakeRunService{events: make(chan *iop.RunEvent, 2)}
|
|
fake.events <- &iop.RunEvent{Type: "delta", Delta: "hello"}
|
|
fake.events <- &iop.RunEvent{Type: "complete"}
|
|
|
|
srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama", Target: "llama-fixed"}, fake, nil)
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"client-model",
|
|
"messages":[{"role":"user","content":"hi"}]
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
if calls := fake.cancelCallsSnapshot(); len(calls) != 0 {
|
|
t.Fatalf("expected no CancelRun calls for a normal completion, got %d: %+v", len(calls), calls)
|
|
}
|
|
}
|
|
|
|
func TestModelsExposesCatalogRouteModels(t *testing.T) {
|
|
srv := NewServer(config.EdgeOpenAIConf{
|
|
ModelRoutes: []config.OpenAIRouteEntry{
|
|
{Model: "model-a", Adapter: "ollama", Target: "llama3"},
|
|
{Model: "model-b", Adapter: "vllm", Target: "qwen"},
|
|
},
|
|
}, &fakeRunService{}, nil)
|
|
req := httptest.NewRequest(http.MethodGet, "/v1/models", nil)
|
|
w := httptest.NewRecorder()
|
|
srv.handleModels(w, req)
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d", w.Code)
|
|
}
|
|
body := w.Body.String()
|
|
if !strings.Contains(body, `"id":"model-a"`) || !strings.Contains(body, `"id":"model-b"`) {
|
|
t.Fatalf("expected catalog model IDs, got %s", body)
|
|
}
|
|
}
|
|
|
|
func TestModelsSkipsRoutesWithEmptyTarget(t *testing.T) {
|
|
srv := NewServer(config.EdgeOpenAIConf{
|
|
ModelRoutes: []config.OpenAIRouteEntry{
|
|
{Model: "model-with-target", Adapter: "ollama", Target: "llama3"},
|
|
{Model: "model-no-target", Adapter: "ollama", Target: ""},
|
|
},
|
|
}, &fakeRunService{}, nil)
|
|
req := httptest.NewRequest(http.MethodGet, "/v1/models", nil)
|
|
w := httptest.NewRecorder()
|
|
srv.handleModels(w, req)
|
|
body := w.Body.String()
|
|
if !strings.Contains(body, `"id":"model-with-target"`) {
|
|
t.Fatalf("expected model-with-target in response, got %s", body)
|
|
}
|
|
if strings.Contains(body, "model-no-target") {
|
|
t.Fatalf("model-no-target should be skipped (no target), got %s", body)
|
|
}
|
|
}
|
|
|
|
func TestModelsRouteCatalogWinsOverModelsField(t *testing.T) {
|
|
srv := NewServer(config.EdgeOpenAIConf{
|
|
Models: []string{"legacy-model"},
|
|
ModelRoutes: []config.OpenAIRouteEntry{
|
|
{Model: "catalog-model", Adapter: "ollama", Target: "llama3"},
|
|
},
|
|
}, &fakeRunService{}, nil)
|
|
req := httptest.NewRequest(http.MethodGet, "/v1/models", nil)
|
|
w := httptest.NewRecorder()
|
|
srv.handleModels(w, req)
|
|
body := w.Body.String()
|
|
if !strings.Contains(body, `"id":"catalog-model"`) {
|
|
t.Fatalf("expected catalog model in response, got %s", body)
|
|
}
|
|
if strings.Contains(body, "legacy-model") {
|
|
t.Fatalf("legacy model should be hidden when catalog is set, got %s", body)
|
|
}
|
|
}
|
|
|
|
func TestChatCompletionsRouteCatalogDispatches(t *testing.T) {
|
|
fake := &fakeRunService{events: make(chan *iop.RunEvent, 2)}
|
|
fake.events <- &iop.RunEvent{Type: "delta", Delta: "ok"}
|
|
fake.events <- &iop.RunEvent{Type: "complete"}
|
|
|
|
srv := NewServer(config.EdgeOpenAIConf{
|
|
ModelRoutes: []config.OpenAIRouteEntry{
|
|
{Model: "model-a", Adapter: "ollama", Target: "llama3"},
|
|
{Model: "model-b", Adapter: "vllm", Target: "qwen"},
|
|
},
|
|
}, fake, nil)
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"model-a",
|
|
"messages":[{"role":"user","content":"hi"}]
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
if fake.req.Adapter != "ollama" || fake.req.Target != "llama3" {
|
|
t.Fatalf("dispatch mismatch: adapter=%q target=%q", fake.req.Adapter, fake.req.Target)
|
|
}
|
|
}
|
|
|
|
func TestChatCompletionsRouteCatalogDispatchesModelB(t *testing.T) {
|
|
fake := &fakeRunService{events: make(chan *iop.RunEvent, 2)}
|
|
fake.events <- &iop.RunEvent{Type: "delta", Delta: "ok"}
|
|
fake.events <- &iop.RunEvent{Type: "complete"}
|
|
|
|
srv := NewServer(config.EdgeOpenAIConf{
|
|
ModelRoutes: []config.OpenAIRouteEntry{
|
|
{Model: "model-a", Adapter: "ollama", Target: "llama3"},
|
|
{Model: "model-b", Adapter: "ollama", Target: "qwen", MaxQueue: 5, QueueTimeoutMS: 2000},
|
|
},
|
|
}, fake, nil)
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"metadata":{},
|
|
"model":"model-b",
|
|
"messages":[{"role":"user","content":"hi"}]
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
if fake.req.Adapter != "ollama" || fake.req.Target != "qwen" {
|
|
t.Fatalf("dispatch mismatch: adapter=%q target=%q", fake.req.Adapter, fake.req.Target)
|
|
}
|
|
if fake.req.ModelGroupKey != "model-b" {
|
|
t.Fatalf("model group key: got %q, want model-b", fake.req.ModelGroupKey)
|
|
}
|
|
if fake.req.MaxQueue != 5 || fake.req.QueueTimeoutMS != 2000 {
|
|
t.Fatalf("queue policy mismatch: MaxQueue=%d, QueueTimeoutMS=%d", fake.req.MaxQueue, fake.req.QueueTimeoutMS)
|
|
}
|
|
}
|
|
|
|
func TestChatCompletionsCatalogMissFallsToLegacyTarget(t *testing.T) {
|
|
fake := &fakeRunService{events: make(chan *iop.RunEvent, 2)}
|
|
fake.events <- &iop.RunEvent{Type: "delta", Delta: "ok"}
|
|
fake.events <- &iop.RunEvent{Type: "complete"}
|
|
|
|
srv := NewServer(config.EdgeOpenAIConf{
|
|
Adapter: "ollama",
|
|
Target: "legacy-target",
|
|
ModelRoutes: []config.OpenAIRouteEntry{
|
|
{Model: "model-a", Adapter: "ollama", Target: "llama3"},
|
|
},
|
|
}, fake, nil)
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"unknown-model",
|
|
"messages":[{"role":"user","content":"hi"}]
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
if fake.req.Target != "legacy-target" {
|
|
t.Fatalf("expected legacy-target fallback, got %q", fake.req.Target)
|
|
}
|
|
}
|
|
|
|
func TestChatCompletionsEmptyModelReturns400(t *testing.T) {
|
|
srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama"}, &fakeRunService{}, nil)
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"",
|
|
"messages":[{"role":"user","content":"hi"}]
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
if w.Code != http.StatusBadRequest {
|
|
t.Fatalf("expected 400 for empty model, got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestResponsesRouteCatalogDispatchesRoute(t *testing.T) {
|
|
fake := &fakeRunService{events: make(chan *iop.RunEvent, 2)}
|
|
fake.events <- &iop.RunEvent{Type: "delta", Delta: "ok"}
|
|
fake.events <- &iop.RunEvent{Type: "complete"}
|
|
|
|
srv := NewServer(config.EdgeOpenAIConf{
|
|
ModelRoutes: []config.OpenAIRouteEntry{
|
|
{Model: "model-a", Adapter: "ollama", Target: "qwen"},
|
|
},
|
|
}, fake, nil)
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(`{
|
|
"model":"model-a",
|
|
"input":"say hello"
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.routes().ServeHTTP(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
if fake.req.Adapter != "ollama" || fake.req.Target != "qwen" {
|
|
t.Fatalf("dispatch mismatch: adapter=%q target=%q", fake.req.Adapter, fake.req.Target)
|
|
}
|
|
if fake.req.ModelGroupKey != "model-a" {
|
|
t.Fatalf("model group key: got %q, want model-a", fake.req.ModelGroupKey)
|
|
}
|
|
}
|
|
|
|
func TestResponsesRouteCatalogDispatchesQueuePolicy(t *testing.T) {
|
|
fake := &fakeRunService{events: make(chan *iop.RunEvent, 2)}
|
|
fake.events <- &iop.RunEvent{Type: "delta", Delta: "ok"}
|
|
fake.events <- &iop.RunEvent{Type: "complete"}
|
|
|
|
srv := NewServer(config.EdgeOpenAIConf{
|
|
ModelRoutes: []config.OpenAIRouteEntry{
|
|
{Model: "model-a", Adapter: "ollama", Target: "route-target", MaxQueue: 3, QueueTimeoutMS: 1500},
|
|
},
|
|
}, fake, nil)
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(`{
|
|
"model":"model-a",
|
|
"input":"test"
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.routes().ServeHTTP(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
if fake.req.Target != "route-target" {
|
|
t.Fatalf("expected route-target, got %q", fake.req.Target)
|
|
}
|
|
if fake.req.ModelGroupKey != "model-a" {
|
|
t.Fatalf("model group key: got %q, want model-a", fake.req.ModelGroupKey)
|
|
}
|
|
if fake.req.MaxQueue != 3 || fake.req.QueueTimeoutMS != 1500 {
|
|
t.Fatalf("queue policy mismatch: MaxQueue=%d, QueueTimeoutMS=%d", fake.req.MaxQueue, fake.req.QueueTimeoutMS)
|
|
}
|
|
}
|
|
|
|
func TestChatCompletionsRejectsNumCtxOptionsWrapper(t *testing.T) {
|
|
srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama", Target: "llama"}, &fakeRunService{}, nil)
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"from-request",
|
|
"messages":[{"role":"user","content":"hi"}],
|
|
"options":{"num_ctx":8192}
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
if w.Code != http.StatusBadRequest {
|
|
t.Fatalf("got %d want 400, body=%s", w.Code, w.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestResolveRouteDispatchPreservesWorkspaceRequired(t *testing.T) {
|
|
srv := NewServer(config.EdgeOpenAIConf{
|
|
ModelRoutes: []config.OpenAIRouteEntry{
|
|
{Model: "codex", Adapter: "cli", Target: "codex", WorkspaceRequired: true},
|
|
{Model: "llama3", Adapter: "ollama", Target: "llama3:8b"},
|
|
},
|
|
}, &fakeRunService{}, nil)
|
|
|
|
dispatch, ok := srv.resolveRouteDispatch("codex")
|
|
if !ok {
|
|
t.Fatal("expected dispatch to succeed for codex route")
|
|
}
|
|
if !dispatch.WorkspaceRequired {
|
|
t.Fatalf("expected workspace_required=true for codex route, got false")
|
|
}
|
|
|
|
dispatch, ok = srv.resolveRouteDispatch("llama3")
|
|
if !ok {
|
|
t.Fatal("expected dispatch to succeed for llama3 route")
|
|
}
|
|
if dispatch.WorkspaceRequired {
|
|
t.Fatalf("expected workspace_required=false for llama3 route, got true")
|
|
}
|
|
}
|
|
|
|
func TestResolveRouteDispatchFallbackWorkspaceRequiredFalse(t *testing.T) {
|
|
srv := NewServer(config.EdgeOpenAIConf{
|
|
Adapter: "ollama",
|
|
Target: "llama3",
|
|
ModelRoutes: []config.OpenAIRouteEntry{
|
|
{Model: "codex", Adapter: "cli", Target: "codex", WorkspaceRequired: true},
|
|
},
|
|
}, &fakeRunService{}, nil)
|
|
|
|
dispatch, ok := srv.resolveRouteDispatch("unknown-model")
|
|
if !ok {
|
|
t.Fatal("expected fallback dispatch to succeed")
|
|
}
|
|
if dispatch.WorkspaceRequired {
|
|
t.Fatalf("expected workspace_required=false for fallback route, got true")
|
|
}
|
|
}
|
|
|
|
func TestCollectRunResultFailsWhenEventStreamCloses(t *testing.T) {
|
|
events := make(chan *iop.RunEvent)
|
|
close(events)
|
|
handle := &edgeservice.RunHandle{
|
|
RunDispatch: edgeservice.RunDispatch{TimeoutSec: 60},
|
|
RunStream: edgeservice.RunStream{
|
|
Events: events,
|
|
NodeEvents: make(chan *iop.EdgeNodeEvent),
|
|
},
|
|
}
|
|
|
|
_, _, _, _, _, _, err := collectRunResult(context.Background(), handle.Stream(), handle.WaitTimeout())
|
|
if err == nil {
|
|
t.Fatal("expected closed stream error")
|
|
}
|
|
if !strings.Contains(err.Error(), "run stream closed") {
|
|
t.Fatalf("expected run stream closed error, got %v", err)
|
|
}
|
|
}
|
|
|
|
// workspaceBoundCfg returns a server config with a workspace-required codex route and a
|
|
// non-required llama3 route for workspace validation tests.
|
|
func workspaceBoundCfg() config.EdgeOpenAIConf {
|
|
return config.EdgeOpenAIConf{
|
|
Adapter: "ollama",
|
|
Target: "llama3",
|
|
ModelRoutes: []config.OpenAIRouteEntry{
|
|
{Model: "codex", Adapter: "cli", Target: "codex", WorkspaceRequired: true},
|
|
{Model: "llama3", Adapter: "ollama", Target: "llama3"},
|
|
},
|
|
}
|
|
}
|
|
|
|
func TestResponsesWorkspaceRequiredRouteMissingWorkspace400(t *testing.T) {
|
|
fake := &fakeRunService{events: make(chan *iop.RunEvent)}
|
|
srv := NewServer(workspaceBoundCfg(), fake, nil)
|
|
body := `{"model":"codex","input":"hello"}`
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(body))
|
|
w := httptest.NewRecorder()
|
|
srv.handleResponses(w, req)
|
|
if w.Code != http.StatusBadRequest {
|
|
t.Fatalf("missing workspace: want 400, got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
if !strings.Contains(w.Body.String(), "workspace is required") {
|
|
t.Fatalf("expected workspace error, got %s", w.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestResponsesWorkspaceRequiredRouteRelativeWorkspace400(t *testing.T) {
|
|
fake := &fakeRunService{events: make(chan *iop.RunEvent)}
|
|
srv := NewServer(workspaceBoundCfg(), fake, nil)
|
|
body := `{"model":"codex","input":"hello","metadata":{"workspace":"relative/path"}}`
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(body))
|
|
w := httptest.NewRecorder()
|
|
srv.handleResponses(w, req)
|
|
if w.Code != http.StatusBadRequest {
|
|
t.Fatalf("relative workspace: want 400, got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
if !strings.Contains(w.Body.String(), "absolute path") {
|
|
t.Fatalf("expected absolute path error, got %s", w.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestResponsesWorkspaceRequiredRouteAbsoluteWorkspaceOK(t *testing.T) {
|
|
fake := &fakeRunService{events: make(chan *iop.RunEvent, 2)}
|
|
fake.events <- &iop.RunEvent{Type: "delta", Delta: "ok"}
|
|
fake.events <- &iop.RunEvent{Type: "complete"}
|
|
srv := NewServer(workspaceBoundCfg(), fake, nil)
|
|
body := `{"model":"codex","input":"hello","metadata":{"workspace":"/abs/path"}}`
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(body))
|
|
w := httptest.NewRecorder()
|
|
srv.handleResponses(w, req)
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("absolute workspace: want 200, got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
if fake.req.Workspace != "/abs/path" {
|
|
t.Fatalf("workspace not preserved: got %q", fake.req.Workspace)
|
|
}
|
|
}
|
|
|
|
func TestResponsesNonRequiredRouteNoWorkspaceOK(t *testing.T) {
|
|
fake := &fakeRunService{events: make(chan *iop.RunEvent, 2)}
|
|
fake.events <- &iop.RunEvent{Type: "delta", Delta: "ok"}
|
|
fake.events <- &iop.RunEvent{Type: "complete"}
|
|
srv := NewServer(workspaceBoundCfg(), fake, nil)
|
|
body := `{"model":"llama3","input":"hello"}`
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(body))
|
|
w := httptest.NewRecorder()
|
|
srv.handleResponses(w, req)
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("non-required route no workspace: want 200, got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestResponsesSurfacesDistinctRunFailures(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
runError string
|
|
want []string
|
|
forbidAny []string
|
|
}{
|
|
{
|
|
name: "missing workspace path from node",
|
|
runError: "cli adapter: workspace not found: /abs/missing",
|
|
want: []string{`"type":"run_error"`, "cli adapter: workspace not found: /abs/missing"},
|
|
},
|
|
{
|
|
name: "inaccessible workspace path from node",
|
|
runError: "cli adapter: workspace inaccessible: /abs/private: permission denied",
|
|
want: []string{`"type":"run_error"`, "cli adapter: workspace inaccessible: /abs/private"},
|
|
},
|
|
{
|
|
name: "agent process exit failure",
|
|
runError: "command failed: exit status 7",
|
|
want: []string{`"type":"run_error"`, "command failed: exit status 7"},
|
|
forbidAny: []string{"workspace not found", "workspace inaccessible", "workspace is not a directory"},
|
|
},
|
|
}
|
|
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
fake := &fakeRunService{events: make(chan *iop.RunEvent, 1)}
|
|
fake.events <- &iop.RunEvent{Type: "error", Error: tc.runError}
|
|
srv := NewServer(workspaceBoundCfg(), fake, nil)
|
|
body := `{"model":"codex","input":"hello","metadata":{"workspace":"/abs/workspace"}}`
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(body))
|
|
w := httptest.NewRecorder()
|
|
srv.handleResponses(w, req)
|
|
|
|
if w.Code != http.StatusBadGateway {
|
|
t.Fatalf("run failure: want 502, got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
for _, want := range tc.want {
|
|
if !strings.Contains(w.Body.String(), want) {
|
|
t.Fatalf("expected body to contain %q, got %s", want, w.Body.String())
|
|
}
|
|
}
|
|
for _, forbidden := range tc.forbidAny {
|
|
if strings.Contains(w.Body.String(), forbidden) {
|
|
t.Fatalf("body should not contain %q, got %s", forbidden, w.Body.String())
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestChatCompletionsWorkspaceRequiredRouteMissingWorkspace400(t *testing.T) {
|
|
fake := &fakeRunService{events: make(chan *iop.RunEvent)}
|
|
srv := NewServer(workspaceBoundCfg(), fake, nil)
|
|
body := `{"model":"codex","messages":[{"role":"user","content":"hi"}]}`
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(body))
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
if w.Code != http.StatusBadRequest {
|
|
t.Fatalf("missing workspace: want 400, got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
if !strings.Contains(w.Body.String(), "workspace is required") {
|
|
t.Fatalf("expected workspace error, got %s", w.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestChatCompletionsWorkspaceRequiredRouteRelativeWorkspace400(t *testing.T) {
|
|
fake := &fakeRunService{events: make(chan *iop.RunEvent)}
|
|
srv := NewServer(workspaceBoundCfg(), fake, nil)
|
|
body := `{"model":"codex","messages":[{"role":"user","content":"hi"}],"metadata":{"workspace":"some/relative"}}`
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(body))
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
if w.Code != http.StatusBadRequest {
|
|
t.Fatalf("relative workspace: want 400, got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
if !strings.Contains(w.Body.String(), "absolute path") {
|
|
t.Fatalf("expected absolute path error, got %s", w.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestChatCompletionsWorkspaceRequiredRouteAbsoluteWorkspaceOK(t *testing.T) {
|
|
fake := &fakeRunService{events: make(chan *iop.RunEvent, 2)}
|
|
fake.events <- &iop.RunEvent{Type: "delta", Delta: "ok"}
|
|
fake.events <- &iop.RunEvent{Type: "complete"}
|
|
srv := NewServer(workspaceBoundCfg(), fake, nil)
|
|
body := `{"model":"codex","messages":[{"role":"user","content":"hi"}],"metadata":{"workspace":"/abs/workspace"}}`
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(body))
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("absolute workspace: want 200, got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
if fake.req.Workspace != "/abs/workspace" {
|
|
t.Fatalf("workspace not preserved: got %q", fake.req.Workspace)
|
|
}
|
|
}
|
|
|
|
func TestChatCompletionsNonRequiredRouteNoWorkspaceOK(t *testing.T) {
|
|
fake := &fakeRunService{events: make(chan *iop.RunEvent, 2)}
|
|
fake.events <- &iop.RunEvent{Type: "delta", Delta: "ok"}
|
|
fake.events <- &iop.RunEvent{Type: "complete"}
|
|
srv := NewServer(workspaceBoundCfg(), fake, nil)
|
|
body := `{"model":"llama3","messages":[{"role":"user","content":"hi"}]}`
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(body))
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("non-required route no workspace: want 200, got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
}
|
|
|
|
// TestHandleModelsProviderPoolCatalog verifies that /v1/models returns the
|
|
// provider-pool catalog IDs when a catalog is set, ignoring legacy model_routes.
|
|
func TestHandleModelsProviderPoolCatalog(t *testing.T) {
|
|
catalog := []config.ModelCatalogEntry{
|
|
{ID: "qwen3.6:35b", Providers: map[string]string{"prov-1": "Qwen3-35B-A22B"}},
|
|
{ID: "llama3.3:70b", Providers: map[string]string{"prov-2": "llama-3.3-70b"}},
|
|
}
|
|
srv := NewServer(config.EdgeOpenAIConf{
|
|
ModelRoutes: []config.OpenAIRouteEntry{
|
|
{Model: "legacy-model", Target: "legacy-target"},
|
|
},
|
|
}, &fakeRunService{}, nil)
|
|
srv.SetModelCatalog(catalog)
|
|
|
|
req := httptest.NewRequest(http.MethodGet, "/v1/models", nil)
|
|
w := httptest.NewRecorder()
|
|
srv.handleModels(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
body := w.Body.String()
|
|
if !strings.Contains(body, "qwen3.6:35b") || !strings.Contains(body, "llama3.3:70b") {
|
|
t.Fatalf("catalog models not listed: %s", body)
|
|
}
|
|
if strings.Contains(body, "legacy-model") {
|
|
t.Fatalf("legacy model_routes should be suppressed when catalog is set: %s", body)
|
|
}
|
|
}
|
|
|
|
// TestChatCompletionsProviderPoolDispatch verifies that when a request model
|
|
// matches the provider-pool catalog with omitted response mode, the request is
|
|
// dispatched over the provider tunnel with ProviderPool=true and Adapter/Target
|
|
// left empty for service-layer resolution (pure passthrough default, SDD D02).
|
|
func TestChatCompletionsProviderPoolDispatch(t *testing.T) {
|
|
frames := make(chan *iop.ProviderTunnelFrame, 3)
|
|
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START, StatusCode: 200}
|
|
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(`{"ok":true}`)}
|
|
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true}
|
|
close(frames)
|
|
fake := &fakeRunService{tunnelFrames: frames}
|
|
|
|
catalog := []config.ModelCatalogEntry{
|
|
{ID: "qwen3.6:35b", Providers: map[string]string{"prov-vllm": "Qwen3-35B-A22B"}},
|
|
}
|
|
srv := NewServer(config.EdgeOpenAIConf{}, fake, nil)
|
|
srv.SetModelCatalog(catalog)
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"qwen3.6:35b",
|
|
"messages":[{"role":"user","content":"hello"}]
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
tunnelReqs := fake.tunnelReqsSnapshot()
|
|
if len(tunnelReqs) != 1 {
|
|
t.Fatalf("expected 1 tunnel dispatch, got %d (SubmitRun calls: %d)", len(tunnelReqs), len(fake.reqsSnapshot()))
|
|
}
|
|
treq := tunnelReqs[0]
|
|
if !treq.ProviderPool {
|
|
t.Error("ProviderPool should be true for catalog-matched model")
|
|
}
|
|
if treq.ModelGroupKey != "qwen3.6:35b" {
|
|
t.Errorf("ModelGroupKey: got %q, want qwen3.6:35b", treq.ModelGroupKey)
|
|
}
|
|
if treq.Adapter != "" || treq.Target != "" {
|
|
t.Errorf("Adapter/Target should be empty for provider-pool dispatch, got %q/%q", treq.Adapter, treq.Target)
|
|
}
|
|
if treq.Method != http.MethodPost || treq.Path != "/v1/chat/completions" {
|
|
t.Errorf("tunnel method/path: got %q %q", treq.Method, treq.Path)
|
|
}
|
|
if len(fake.reqsSnapshot()) != 0 {
|
|
t.Errorf("normalized SubmitRun must not be called for omitted-mode provider route")
|
|
}
|
|
}
|
|
|
|
func TestChatCompletionsStrictOutputProviderPoolDefaultKeepsPassthroughPath(t *testing.T) {
|
|
frames := make(chan *iop.ProviderTunnelFrame, 3)
|
|
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START, StatusCode: 200}
|
|
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(`{"ok":true}`)}
|
|
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true}
|
|
close(frames)
|
|
fake := &fakeRunService{tunnelFrames: frames}
|
|
|
|
catalog := []config.ModelCatalogEntry{
|
|
{ID: "qwen3.6:35b", Providers: map[string]string{"prov-vllm": "Qwen3-35B-A22B"}},
|
|
}
|
|
srv := NewServer(config.EdgeOpenAIConf{StrictOutput: true}, fake, nil)
|
|
srv.SetModelCatalog(catalog)
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"qwen3.6:35b",
|
|
"messages":[{"role":"user","content":"hello"}],
|
|
"include_reasoning": true
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
if len(fake.tunnelReqsSnapshot()) != 1 {
|
|
t.Fatalf("strict output default must keep provider routes on passthrough, got %d tunnel dispatches", len(fake.tunnelReqsSnapshot()))
|
|
}
|
|
if len(fake.reqsSnapshot()) != 0 {
|
|
t.Fatalf("strict output provider route must not fall back to normalized SubmitRun, got %d calls", len(fake.reqsSnapshot()))
|
|
}
|
|
}
|
|
|
|
// TestChatCompletionsProviderPoolIgnoresSelectorLikeMetadata verifies SDD S01:
|
|
// arbitrary caller metadata — including keys or values that resemble a legacy
|
|
// response-mode selector — is opaque context. It never rejects the request or
|
|
// switches a provider-pool route off pure passthrough. (The former selector key
|
|
// literal is deliberately not used here; the deterministic surface check forbids
|
|
// it anywhere under apps.)
|
|
func TestChatCompletionsProviderPoolIgnoresSelectorLikeMetadata(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
metadata string
|
|
}{
|
|
{name: "selector-like value is ignored", metadata: `"metadata":{"response_hint":"raw"}`},
|
|
{name: "arbitrary metadata is ignored", metadata: `"metadata":{"experiment":"mode-x","team":"search"}`},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
providerBody := `{"ok":true}`
|
|
fake := &fakeRunService{
|
|
poolDispatchPath: string(edgeservice.ProviderPoolPathTunnel),
|
|
tunnelFrames: staticProviderTunnelFrames(providerBody),
|
|
}
|
|
srv := NewServer(config.EdgeOpenAIConf{}, fake, nil)
|
|
srv.SetModelCatalog([]config.ModelCatalogEntry{
|
|
{ID: "pool-model", Providers: map[string]string{"prov-1": "served-model"}},
|
|
})
|
|
|
|
body := `{
|
|
"model":"pool-model",
|
|
"messages":[{"role":"user","content":"hello"}],
|
|
` + tc.metadata + `
|
|
}`
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(body))
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("selector-like metadata must not be rejected: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
if got := w.Body.String(); got != providerBody {
|
|
t.Fatalf("provider body must be relayed byte-identically: got %q want %q", got, providerBody)
|
|
}
|
|
if len(fake.tunnelReqsSnapshot()) != 1 {
|
|
t.Fatalf("expected 1 pure passthrough tunnel dispatch, got %d", len(fake.tunnelReqsSnapshot()))
|
|
}
|
|
if len(fake.reqsSnapshot()) != 0 {
|
|
t.Fatalf("arbitrary metadata must not switch to normalized SubmitRun, got %d", len(fake.reqsSnapshot()))
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// chatSidebandServer returns a direct OpenAI-compatible provider route backed by
|
|
// the given tunnel frames. It routes via the legacy openai_compat model route
|
|
// (not provider-pool) so the raw provider tunnel passthrough path is exercised.
|
|
func chatSidebandServer(frames chan *iop.ProviderTunnelFrame) (*Server, *fakeRunService) {
|
|
fake := &fakeRunService{
|
|
tunnelFrames: frames,
|
|
}
|
|
srv := chatProviderRouteServer(fake, nil)
|
|
return srv, fake
|
|
}
|
|
|
|
func chatProviderRouteServer(fake *fakeRunService, logger *zap.Logger) *Server {
|
|
return NewServer(config.EdgeOpenAIConf{
|
|
ModelRoutes: []config.OpenAIRouteEntry{
|
|
{Model: "pool-model", Adapter: "openai_compat", Target: "served-model"},
|
|
},
|
|
}, fake, logger)
|
|
}
|
|
|
|
func TestChatCompletionsPassthroughDoesNotExposeSideband(t *testing.T) {
|
|
providerBody := "data: {\"choices\":[{\"delta\":{\"content\":\"pure\"}}]}\n\ndata: [DONE]\n\n"
|
|
frames := make(chan *iop.ProviderTunnelFrame, 4)
|
|
frames <- &iop.ProviderTunnelFrame{
|
|
Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START,
|
|
StatusCode: 200,
|
|
Headers: map[string]string{"Content-Type": "text/event-stream"},
|
|
}
|
|
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(providerBody)}
|
|
frames <- &iop.ProviderTunnelFrame{
|
|
Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_USAGE,
|
|
Usage: &iop.Usage{InputTokens: 3, OutputTokens: 5},
|
|
}
|
|
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true}
|
|
close(frames)
|
|
srv, _ := chatSidebandServer(frames)
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"pool-model",
|
|
"stream":true,
|
|
"messages":[{"role":"user","content":"hello"}]
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
// Byte-identity with the provider body proves no IOP extension markers or
|
|
// response-mode labels are injected into the pure passthrough response.
|
|
if got := w.Body.String(); got != providerBody {
|
|
t.Fatalf("pure passthrough body not byte-identical:\n got: %q\nwant: %q", got, providerBody)
|
|
}
|
|
}
|
|
|
|
// chatProviderAuthServer builds a provider-pool tunnel server with the given
|
|
// provider auth forwarding config, mirroring chatSidebandServer's catalog so
|
|
// pool-model routes over the provider tunnel path.
|
|
func chatProviderAuthServer(frames chan *iop.ProviderTunnelFrame, auth config.EdgeOpenAIProviderAuthConf) (*Server, *fakeRunService) {
|
|
fake := &fakeRunService{
|
|
poolDispatchPath: string(edgeservice.ProviderPoolPathTunnel),
|
|
tunnelFrames: frames,
|
|
}
|
|
srv := NewServer(config.EdgeOpenAIConf{ProviderAuth: auth}, fake, nil)
|
|
srv.SetModelCatalog([]config.ModelCatalogEntry{
|
|
{ID: "pool-model", Providers: map[string]string{"prov-1": "served-model"}},
|
|
})
|
|
return srv, fake
|
|
}
|
|
|
|
func staticOKTunnelFrames(body string) chan *iop.ProviderTunnelFrame {
|
|
frames := make(chan *iop.ProviderTunnelFrame, 3)
|
|
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START, StatusCode: 200}
|
|
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(body)}
|
|
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true}
|
|
close(frames)
|
|
return frames
|
|
}
|
|
|
|
// TestChatProviderTunnelForwardsProviderAuthHeader verifies SDD S03: a raw user
|
|
// token supplied in the configured provider auth request header is forwarded to
|
|
// the provider tunnel request as the configured target header, formatted with
|
|
// the configured scheme, while already-prefixed values are preserved.
|
|
func TestChatProviderTunnelForwardsProviderAuthHeader(t *testing.T) {
|
|
auth := config.EdgeOpenAIProviderAuthConf{
|
|
Enabled: true,
|
|
FromHeader: "X-IOP-Provider-Authorization",
|
|
TargetHeader: "Authorization",
|
|
Scheme: "Bearer",
|
|
Required: true,
|
|
}
|
|
for _, tc := range []struct {
|
|
name string
|
|
callerAuth string
|
|
wantHeader string
|
|
}{
|
|
{name: "raw token gets scheme", callerAuth: "user-token", wantHeader: "Bearer user-token"},
|
|
{name: "already prefixed preserved", callerAuth: "Bearer user-token", wantHeader: "Bearer user-token"},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
srv, fake := chatProviderAuthServer(staticOKTunnelFrames(`{"ok":true}`), auth)
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"pool-model",
|
|
"messages":[{"role":"user","content":"hello"}]
|
|
}`))
|
|
req.Header.Set("X-IOP-Provider-Authorization", tc.callerAuth)
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
reqs := fake.tunnelReqsSnapshot()
|
|
if len(reqs) != 1 {
|
|
t.Fatalf("expected 1 tunnel dispatch, got %d", len(reqs))
|
|
}
|
|
// The handler sets auth headers on the tunnel request in the
|
|
// tunnel path of handleChatCompletionsProviderPool.
|
|
if got := reqs[0].Headers["Authorization"]; got != tc.wantHeader {
|
|
t.Fatalf("provider Authorization header: got %q, want %q", got, tc.wantHeader)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestChatProviderTunnelProviderAuthRequired verifies SDD S03: a missing
|
|
// required provider auth header is rejected. For provider-pool tunnel path,
|
|
// auth is checked after SubmitProviderPool returns. The tunnel handle is closed
|
|
// and a 400 is returned (SDD D01).
|
|
func TestChatProviderTunnelProviderAuthRequired(t *testing.T) {
|
|
auth := config.EdgeOpenAIProviderAuthConf{
|
|
Enabled: true,
|
|
FromHeader: "X-IOP-Provider-Authorization",
|
|
TargetHeader: "Authorization",
|
|
Scheme: "Bearer",
|
|
Required: true,
|
|
}
|
|
srv, fake := chatProviderAuthServer(staticOKTunnelFrames(`{"ok":true}`), auth)
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"pool-model",
|
|
"messages":[{"role":"user","content":"hello"}]
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
// Auth required check happens in the tunnel path of handleChatCompletionsProviderPool.
|
|
// Since auth is missing, we get 400 and no tunnel request is sent (SDD S03 pre-dispatch).
|
|
if w.Code != http.StatusBadRequest {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
if strings.Contains(w.Body.String(), "user-token") {
|
|
t.Fatalf("error body must not echo a raw token: %s", w.Body.String())
|
|
}
|
|
// No tunnel request should be recorded: PrepareTunnel failed before dispatch.
|
|
if got := len(fake.tunnelReqsSnapshot()); got != 0 {
|
|
t.Fatalf("missing required provider auth must not dispatch tunnel: got %d tunnel requests", got)
|
|
}
|
|
}
|
|
|
|
// TestChatCompletionsProviderPoolTunnelProviderAuthForwardedBeforeDispatch verifies
|
|
// SDD S03: when provider auth is configured and present, the auth header lands on
|
|
// the actual tunnel request that the service sends, not via late handle mutation.
|
|
// The auth check runs inside SubmitProviderPool via PrepareTunnel before the
|
|
// tunnel request is built and dispatched.
|
|
func TestChatCompletionsProviderPoolTunnelProviderAuthForwardedBeforeDispatch(t *testing.T) {
|
|
auth := config.EdgeOpenAIProviderAuthConf{
|
|
Enabled: true,
|
|
FromHeader: "X-IOP-Provider-Authorization",
|
|
TargetHeader: "Authorization",
|
|
Scheme: "Bearer",
|
|
Required: true,
|
|
}
|
|
srv, fake := chatProviderAuthServer(staticOKTunnelFrames(`{"ok":true}`), auth)
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"pool-model",
|
|
"messages":[{"role":"user","content":"hello"}]
|
|
}`))
|
|
req.Header.Set("X-IOP-Provider-Authorization", "user-token")
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
reqs := fake.tunnelReqsSnapshot()
|
|
if len(reqs) != 1 {
|
|
t.Fatalf("expected 1 tunnel dispatch, got %d", len(reqs))
|
|
}
|
|
// Auth header must be on the recorded tunnel request (i.e. pre-dispatch),
|
|
// not on a late handle mutation.
|
|
if got := reqs[0].Headers["Authorization"]; got != "Bearer user-token" {
|
|
t.Fatalf("tunnel request Authorization header: got %q, want %q", got, "Bearer user-token")
|
|
}
|
|
}
|
|
|
|
// TestChatCompletionsProviderPoolOllamaSelectionAllowsMissingProviderAuth verifies
|
|
// SDD D01: a missing required provider auth does NOT block the normalized
|
|
// (Ollama/CLI) execution path. Only tunnel dispatch requires the auth header.
|
|
func TestChatCompletionsProviderPoolOllamaSelectionAllowsMissingProviderAuth(t *testing.T) {
|
|
auth := config.EdgeOpenAIProviderAuthConf{
|
|
Enabled: true,
|
|
FromHeader: "X-IOP-Provider-Authorization",
|
|
TargetHeader: "Authorization",
|
|
Scheme: "Bearer",
|
|
Required: true,
|
|
}
|
|
// Ollama path: poolDispatchPath=normalized, no tunnel frames needed.
|
|
fake := &fakeRunService{
|
|
poolDispatchPath: string(edgeservice.ProviderPoolPathNormalized),
|
|
}
|
|
catalog := []config.ModelCatalogEntry{
|
|
{ID: "ollama-model", Providers: map[string]string{"prov-ollama": "qwen3.6:35b"}},
|
|
}
|
|
srv := NewServer(config.EdgeOpenAIConf{ProviderAuth: auth}, fake, nil)
|
|
srv.SetModelCatalog(catalog)
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"ollama-model",
|
|
"messages":[{"role":"user","content":"hello"}]
|
|
}`))
|
|
// No X-IOP-Provider-Authorization header — should still succeed.
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
// No tunnel dispatch for Ollama-normalized path.
|
|
reqs := fake.tunnelReqsSnapshot()
|
|
if len(reqs) != 0 {
|
|
t.Fatalf("expected 0 tunnel dispatches for Ollama path, got %d", len(reqs))
|
|
}
|
|
}
|
|
|
|
// TestChatProviderTunnelProviderAuthDoesNotReplaceInboundAuth verifies the
|
|
// provider token is sourced only from the configured provider auth header, not
|
|
// from the inbound IOP Authorization header, keeping IOP inbound auth and
|
|
// outbound provider auth separate.
|
|
func TestChatProviderTunnelProviderAuthDoesNotReplaceInboundAuth(t *testing.T) {
|
|
auth := config.EdgeOpenAIProviderAuthConf{
|
|
Enabled: true,
|
|
FromHeader: "X-IOP-Provider-Authorization",
|
|
TargetHeader: "Authorization",
|
|
Scheme: "Bearer",
|
|
Required: true,
|
|
}
|
|
srv, fake := chatProviderAuthServer(staticOKTunnelFrames(`{"ok":true}`), auth)
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"pool-model",
|
|
"messages":[{"role":"user","content":"hello"}]
|
|
}`))
|
|
// Inbound IOP auth and the configured provider auth header carry different
|
|
// tokens; the provider header must come from the provider auth header only.
|
|
req.Header.Set("Authorization", "Bearer iop-inbound-token")
|
|
req.Header.Set("X-IOP-Provider-Authorization", "provider-token")
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
reqs := fake.tunnelReqsSnapshot()
|
|
if len(reqs) != 1 {
|
|
t.Fatalf("expected 1 tunnel dispatch, got %d", len(reqs))
|
|
}
|
|
// The tunnel auth headers should come from the provider auth header only,
|
|
// not from the inbound IOP Authorization header.
|
|
if got := reqs[0].Headers["Authorization"]; got != "Bearer provider-token" {
|
|
t.Fatalf("provider Authorization must derive from provider auth header, got %q", got)
|
|
}
|
|
}
|
|
|
|
func TestChatCompletionsPassthroughDoesNotLabelTransformed(t *testing.T) {
|
|
providerBody := `{"ok":true}`
|
|
frames := make(chan *iop.ProviderTunnelFrame, 3)
|
|
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START, StatusCode: 200}
|
|
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(providerBody)}
|
|
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true}
|
|
close(frames)
|
|
srv, _ := chatSidebandServer(frames)
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"pool-model",
|
|
"messages":[{"role":"user","content":"hello"}]
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
// Byte-identity with the provider body proves the pure passthrough response
|
|
// carries no IOP response-mode label.
|
|
if got := w.Body.String(); got != providerBody {
|
|
t.Fatalf("pure passthrough body not byte-identical: %q", got)
|
|
}
|
|
}
|
|
|
|
// chatPassthroughServer returns an openai.Server whose fake service relays the
|
|
// tunnel to the given provider fixture, emulating the Node relay.
|
|
func chatPassthroughServer(providerURL string) (*Server, *fakeRunService) {
|
|
fake := &fakeRunService{
|
|
poolDispatchPath: string(edgeservice.ProviderPoolPathTunnel),
|
|
tunnelProviderURL: providerURL,
|
|
tunnelServedTarget: "served-model",
|
|
}
|
|
srv := NewServer(config.EdgeOpenAIConf{}, fake, nil)
|
|
srv.SetModelCatalog([]config.ModelCatalogEntry{
|
|
{ID: "pool-model", Providers: map[string]string{"prov-1": "served-model"}},
|
|
})
|
|
return srv, fake
|
|
}
|
|
|
|
// TestChatCompletionsPassthroughNonStreamingByteIdentity verifies SDD S04 for
|
|
// the non-streaming path: when the provider omits a top-level model echo, the
|
|
// caller receives the provider JSON body byte-identically, along with provider
|
|
// status and headers.
|
|
func TestChatCompletionsPassthroughNonStreamingByteIdentity(t *testing.T) {
|
|
// Provider-original body with fields the normalized path would rewrite or
|
|
// drop (reasoning_content, provider-specific key order and spacing).
|
|
providerBody := "{\"id\":\"cmpl-provider-1\",\"object\":\"chat.completion\", \"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"hi\",\"reasoning_content\":\"because\"},\"finish_reason\":\"stop\"}],\"provider_extra\":{\"a\":1}}"
|
|
var gotProviderReq []byte
|
|
provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
gotProviderReq, _ = io.ReadAll(r.Body)
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.Header().Set("X-Provider-Marker", "prov-42")
|
|
w.WriteHeader(http.StatusOK)
|
|
_, _ = w.Write([]byte(providerBody))
|
|
}))
|
|
defer provider.Close()
|
|
|
|
srv, fake := chatPassthroughServer(provider.URL)
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"pool-model",
|
|
"messages":[{"role":"user","content":"hello"}]
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
if got := w.Body.String(); got != providerBody {
|
|
t.Fatalf("body not byte-identical to provider:\n got: %q\nwant: %q", got, providerBody)
|
|
}
|
|
if got := w.Header().Get("X-Provider-Marker"); got != "prov-42" {
|
|
t.Errorf("provider header not relayed: %q", got)
|
|
}
|
|
if got := w.Header().Get("Content-Type"); got != "application/json" {
|
|
t.Errorf("content type not relayed: %q", got)
|
|
}
|
|
// The provider request body carries the served model (model rewrite only).
|
|
if !strings.Contains(string(gotProviderReq), `"model":"served-model"`) {
|
|
t.Errorf("provider request body missing served model: %s", gotProviderReq)
|
|
}
|
|
if !strings.Contains(string(gotProviderReq), `"content":"hello"`) {
|
|
t.Errorf("provider request body lost caller messages: %s", gotProviderReq)
|
|
}
|
|
if len(fake.reqsSnapshot()) != 0 {
|
|
t.Error("normalized SubmitRun must not be called on passthrough")
|
|
}
|
|
}
|
|
|
|
func TestChatCompletionsPassthroughNonStreamingRewritesModelEcho(t *testing.T) {
|
|
providerBody := `{"id":"cmpl-provider-1","object":"chat.completion","created":123,"model":"served-model","choices":[{"index":0,"message":{"role":"assistant","content":"hi","reasoning_content":"because"},"finish_reason":"stop"}],"provider_extra":{"a":1}}`
|
|
var gotProviderReq []byte
|
|
provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
gotProviderReq, _ = io.ReadAll(r.Body)
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(http.StatusOK)
|
|
_, _ = w.Write([]byte(providerBody))
|
|
}))
|
|
defer provider.Close()
|
|
|
|
srv, _ := chatPassthroughServer(provider.URL)
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"pool-model",
|
|
"messages":[{"role":"user","content":"hello"}]
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
var resp struct {
|
|
Model string `json:"model"`
|
|
Choices []struct {
|
|
Message struct {
|
|
Content string `json:"content"`
|
|
ReasoningContent string `json:"reasoning_content"`
|
|
} `json:"message"`
|
|
} `json:"choices"`
|
|
ProviderExtra map[string]int `json:"provider_extra"`
|
|
}
|
|
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
|
t.Fatalf("decode response: %v body=%s", err, w.Body.String())
|
|
}
|
|
if resp.Model != "pool-model" {
|
|
t.Fatalf("response model: got %q, want pool-model; body=%s", resp.Model, w.Body.String())
|
|
}
|
|
if resp.Choices[0].Message.Content != "hi" || resp.Choices[0].Message.ReasoningContent != "because" {
|
|
t.Fatalf("provider content/reasoning not preserved: %+v", resp.Choices[0].Message)
|
|
}
|
|
if resp.ProviderExtra["a"] != 1 {
|
|
t.Fatalf("provider_extra not preserved: %+v", resp.ProviderExtra)
|
|
}
|
|
if !strings.Contains(string(gotProviderReq), `"model":"served-model"`) {
|
|
t.Fatalf("provider request body missing served model: %s", gotProviderReq)
|
|
}
|
|
}
|
|
|
|
// TestChatCompletionsPassthroughProviderPoolGenerationPolicy verifies that
|
|
// provider-pool omitted-mode passthrough requests apply the catalog entry's
|
|
// generation policy (default_max_tokens, min_max_tokens, default_thinking_token_budget)
|
|
// before dispatching to the provider. The fixture omits a top-level model echo,
|
|
// so the response body remains byte-identical.
|
|
func TestChatCompletionsPassthroughProviderPoolGenerationPolicy(t *testing.T) {
|
|
providerBody := `{"id":"cmpl-provider-1","object":"chat.completion","choices":[{"index":0,"message":{"role":"assistant","content":"hi"},"finish_reason":"stop"}]}`
|
|
var gotProviderReq []byte
|
|
provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
gotProviderReq, _ = io.ReadAll(r.Body)
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(http.StatusOK)
|
|
_, _ = w.Write([]byte(providerBody))
|
|
}))
|
|
defer provider.Close()
|
|
|
|
fake := &fakeRunService{
|
|
tunnelProviderURL: provider.URL,
|
|
tunnelServedTarget: "served-model",
|
|
}
|
|
// Enable strict output to trigger think=true from policy
|
|
srv := NewServer(config.EdgeOpenAIConf{StrictOutput: true}, fake, nil)
|
|
srv.SetModelCatalog([]config.ModelCatalogEntry{
|
|
{
|
|
ID: "pool-model",
|
|
Providers: map[string]string{"prov-1": "served-model"},
|
|
DefaultMaxTokens: 100,
|
|
MinMaxTokens: 50,
|
|
DefaultThinkingTokenBudget: 30,
|
|
},
|
|
})
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"pool-model",
|
|
"messages":[{"role":"user","content":"hello"}],
|
|
"max_completion_tokens":20
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
if got := w.Body.String(); got != providerBody {
|
|
t.Fatalf("body not byte-identical to provider:\n got: %q\nwant: %q", got, providerBody)
|
|
}
|
|
|
|
var parsedReq map[string]interface{}
|
|
if err := json.Unmarshal(gotProviderReq, &parsedReq); err != nil {
|
|
t.Fatalf("failed to parse provider request: %v", err)
|
|
}
|
|
|
|
if model, ok := parsedReq["model"].(string); !ok || model != "served-model" {
|
|
t.Errorf("expected model served-model, got %v", parsedReq["model"])
|
|
}
|
|
|
|
// MinMaxTokens (50) should reach max_tokens (overriding max_completion_tokens 20)
|
|
if maxTokens, ok := parsedReq["max_tokens"].(float64); !ok || maxTokens != 50 {
|
|
t.Errorf("expected max_tokens 50, got %v", parsedReq["max_tokens"])
|
|
}
|
|
if _, ok := parsedReq["max_completion_tokens"]; ok {
|
|
t.Errorf("max_completion_tokens should be deleted")
|
|
}
|
|
|
|
// DefaultThinkingTokenBudget (30) and Think (true) should reach provider
|
|
if budget, ok := parsedReq["thinking_token_budget"].(float64); !ok || budget != 30 {
|
|
t.Errorf("expected thinking_token_budget 30, got %v", parsedReq["thinking_token_budget"])
|
|
}
|
|
if think, ok := parsedReq["think"].(bool); !ok || !think {
|
|
t.Errorf("expected think true, got %v", parsedReq["think"])
|
|
}
|
|
}
|
|
|
|
// TestChatCompletionsPassthroughStreamingByteIdentity verifies SDD S04 for the
|
|
// streaming path: provider SSE bytes without top-level model echoes, including
|
|
// provider-specific fields like reasoning_content and native tool_calls chunks,
|
|
// reach the caller byte-identically.
|
|
func TestChatCompletionsPassthroughStreamingByteIdentity(t *testing.T) {
|
|
providerBody := "data: {\"choices\":[{\"delta\":{\"role\":\"assistant\"}}]}\n\n" +
|
|
"data: {\"choices\":[{\"delta\":{\"reasoning_content\":\"thinking...\"}}]}\n\n" +
|
|
"data: {\"choices\":[{\"delta\":{\"content\":\"<think>not rewritten</think>\"}}]}\n\n" +
|
|
"data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"name\":\"f\",\"arguments\":\"{}\"}}]}}]}\n\n" +
|
|
"data: {\"choices\":[{\"delta\":{},\"finish_reason\":\"tool_calls\"}],\"usage\":{\"prompt_tokens\":3}}\n\n" +
|
|
"data: [DONE]\n\n"
|
|
provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
w.WriteHeader(http.StatusOK)
|
|
_, _ = w.Write([]byte(providerBody))
|
|
}))
|
|
defer provider.Close()
|
|
|
|
srv, _ := chatPassthroughServer(provider.URL)
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"pool-model",
|
|
"stream":true,
|
|
"messages":[{"role":"user","content":"hello"}]
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
if got := w.Body.String(); got != providerBody {
|
|
t.Fatalf("SSE stream not byte-identical to provider:\n got: %q\nwant: %q", got, providerBody)
|
|
}
|
|
if got := w.Header().Get("Content-Type"); got != "text/event-stream" {
|
|
t.Errorf("content type not relayed: %q", got)
|
|
}
|
|
}
|
|
|
|
func TestChatCompletionsPassthroughStreamingRewritesModelEcho(t *testing.T) {
|
|
providerBody := "data: {\"id\":\"chunk-1\",\"object\":\"chat.completion.chunk\",\"created\":123,\"model\":\"served-model\",\"choices\":[{\"delta\":{\"role\":\"assistant\"}}]}\n\n" +
|
|
"data: {\"id\":\"chunk-1\",\"object\":\"chat.completion.chunk\",\"created\":123,\"model\":\"served-model\",\"choices\":[{\"delta\":{\"reasoning\":\"thinking...\"}}]}\n\n" +
|
|
"data: {\"id\":\"chunk-1\",\"object\":\"chat.completion.chunk\",\"created\":123,\"model\":\"served-model\",\"choices\":[{\"delta\":{\"content\":\"hi\"}}]}\n\n" +
|
|
"data: [DONE]\n\n"
|
|
provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
w.WriteHeader(http.StatusOK)
|
|
_, _ = w.Write([]byte(providerBody))
|
|
}))
|
|
defer provider.Close()
|
|
|
|
srv, _ := chatPassthroughServer(provider.URL)
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"pool-model",
|
|
"stream":true,
|
|
"messages":[{"role":"user","content":"hello"}]
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
body := w.Body.String()
|
|
if strings.Contains(body, `"model":"served-model"`) {
|
|
t.Fatalf("response leaked provider-served model: %s", body)
|
|
}
|
|
if strings.Count(body, `"model":"pool-model"`) != 3 {
|
|
t.Fatalf("response model aliases not rewritten in every chunk: %s", body)
|
|
}
|
|
if !strings.Contains(body, `"reasoning":"thinking..."`) || !strings.Contains(body, `"content":"hi"`) {
|
|
t.Fatalf("provider reasoning/content not preserved: %s", body)
|
|
}
|
|
if !strings.Contains(body, "data: [DONE]") {
|
|
t.Fatalf("done marker not preserved: %s", body)
|
|
}
|
|
if got := w.Header().Get("Content-Type"); got != "text/event-stream" {
|
|
t.Errorf("content type not relayed: %q", got)
|
|
}
|
|
}
|
|
|
|
// TestChatCompletionsPassthroughProviderErrorStatusRelayed verifies a provider
|
|
// error status/body is passed to the caller unmodified instead of being
|
|
// converted into an IOP error envelope.
|
|
func TestChatCompletionsPassthroughProviderErrorStatusRelayed(t *testing.T) {
|
|
providerBody := `{"error":{"message":"rate limited by provider","type":"rate_limit"}}`
|
|
provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(http.StatusTooManyRequests)
|
|
_, _ = w.Write([]byte(providerBody))
|
|
}))
|
|
defer provider.Close()
|
|
|
|
srv, _ := chatPassthroughServer(provider.URL)
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"pool-model",
|
|
"messages":[{"role":"user","content":"hello"}]
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
if w.Code != http.StatusTooManyRequests {
|
|
t.Fatalf("provider status not relayed: got %d", w.Code)
|
|
}
|
|
if got := w.Body.String(); got != providerBody {
|
|
t.Fatalf("provider error body not byte-identical: %q", got)
|
|
}
|
|
}
|
|
|
|
// TestChatCompletionsPassthroughLegacyProviderRouteUsesTunnel verifies that a
|
|
// legacy model_routes entry with an openai_compat adapter also defaults to the
|
|
// tunnel passthrough path (direct route, no provider pool).
|
|
func TestChatCompletionsPassthroughLegacyProviderRouteUsesTunnel(t *testing.T) {
|
|
frames := make(chan *iop.ProviderTunnelFrame, 3)
|
|
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START, StatusCode: 200}
|
|
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(`{"ok":true}`)}
|
|
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true}
|
|
close(frames)
|
|
fake := &fakeRunService{tunnelFrames: frames}
|
|
srv := NewServer(config.EdgeOpenAIConf{
|
|
ModelRoutes: []config.OpenAIRouteEntry{
|
|
{Model: "compat-model", Adapter: "openai_compat", Target: "backend-model"},
|
|
},
|
|
}, fake, nil)
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"compat-model",
|
|
"messages":[{"role":"user","content":"hello"}]
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
tunnelReqs := fake.tunnelReqsSnapshot()
|
|
if len(tunnelReqs) != 1 {
|
|
t.Fatalf("expected tunnel dispatch for openai_compat route, got %d", len(tunnelReqs))
|
|
}
|
|
if tunnelReqs[0].Adapter != "openai_compat" || tunnelReqs[0].Target != "backend-model" {
|
|
t.Errorf("tunnel adapter/target: got %q/%q", tunnelReqs[0].Adapter, tunnelReqs[0].Target)
|
|
}
|
|
if tunnelReqs[0].ProviderPool {
|
|
t.Error("legacy route must not set ProviderPool")
|
|
}
|
|
bodies := fake.tunnelBodiesSnapshot()
|
|
if len(bodies) != 1 || !strings.Contains(string(bodies[0]), `"model":"backend-model"`) {
|
|
t.Errorf("legacy route body must carry route target: %s", bodies)
|
|
}
|
|
}
|
|
|
|
func TestChatCompletionsPassthroughNonProviderRouteKeepsNormalizedPath(t *testing.T) {
|
|
fake := &fakeRunService{events: bufferedRunEvents(
|
|
&iop.RunEvent{Type: "delta", Delta: "ok"},
|
|
&iop.RunEvent{Type: "complete"},
|
|
)}
|
|
srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama", Target: "llama-fixed"}, fake, nil)
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"any-model",
|
|
"messages":[{"role":"user","content":"hello"}]
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
if len(fake.tunnelReqsSnapshot()) != 0 {
|
|
t.Error("ollama route must not use the tunnel")
|
|
}
|
|
if len(fake.reqsSnapshot()) != 1 {
|
|
t.Fatalf("expected normalized SubmitRun dispatch, got %d", len(fake.reqsSnapshot()))
|
|
}
|
|
}
|
|
|
|
// TestChatCompletionsPassthroughTunnelErrorBeforeStartReturns502 verifies an
|
|
// ERROR frame before response-start becomes a JSON error response.
|
|
func TestChatCompletionsPassthroughTunnelErrorBeforeStartReturns502(t *testing.T) {
|
|
frames := make(chan *iop.ProviderTunnelFrame, 1)
|
|
frames <- &iop.ProviderTunnelFrame{
|
|
Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_ERROR,
|
|
Error: "provider connect refused",
|
|
}
|
|
close(frames)
|
|
fake := &fakeRunService{tunnelFrames: frames}
|
|
srv := NewServer(config.EdgeOpenAIConf{}, fake, nil)
|
|
srv.SetModelCatalog([]config.ModelCatalogEntry{
|
|
{ID: "pool-model", Providers: map[string]string{"prov-1": "served-model"}},
|
|
})
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"pool-model",
|
|
"messages":[{"role":"user","content":"hello"}]
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
if w.Code != http.StatusBadGateway {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
if !strings.Contains(w.Body.String(), "provider connect refused") {
|
|
t.Fatalf("error message not surfaced: %s", w.Body.String())
|
|
}
|
|
}
|
|
|
|
// TestChatCompletionsPassthroughContextCancelSendsCancelRun verifies caller
|
|
// disconnect propagates to the Node cancel path for tunnel dispatches.
|
|
func TestChatCompletionsPassthroughContextCancelSendsCancelRun(t *testing.T) {
|
|
// Frames channel stays open and empty: the tunnel is in-flight when the
|
|
// caller context is cancelled.
|
|
fake := &fakeRunService{tunnelFrames: make(chan *iop.ProviderTunnelFrame)}
|
|
srv := NewServer(config.EdgeOpenAIConf{}, fake, nil)
|
|
srv.SetModelCatalog([]config.ModelCatalogEntry{
|
|
{ID: "pool-model", Providers: map[string]string{"prov-1": "served-model"}},
|
|
})
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"pool-model",
|
|
"messages":[{"role":"user","content":"hello"}]
|
|
}`)).WithContext(ctx)
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
calls := fake.cancelCallsSnapshot()
|
|
if len(calls) != 1 {
|
|
t.Fatalf("expected 1 CancelRun call, got %d: %+v", len(calls), calls)
|
|
}
|
|
if calls[0].RunID != "run-tunnel" || calls[0].NodeRef != "node-1" {
|
|
t.Fatalf("unexpected cancel request: %+v", calls[0])
|
|
}
|
|
}
|
|
|
|
// flakyResponseWriter is a tunnel-path ResponseWriter double: it accepts
|
|
// failAfterWrites successful Write calls, then fails every subsequent Write,
|
|
// optionally sleeping writeDelay per call to emulate a slow client.
|
|
type flakyResponseWriter struct {
|
|
mu sync.Mutex
|
|
header http.Header
|
|
status int
|
|
body bytes.Buffer
|
|
failAfterWrites int // -1: never fail
|
|
writeDelay time.Duration
|
|
writes int
|
|
failedWrites int
|
|
}
|
|
|
|
func newFlakyResponseWriter(failAfterWrites int) *flakyResponseWriter {
|
|
return &flakyResponseWriter{header: make(http.Header), failAfterWrites: failAfterWrites}
|
|
}
|
|
|
|
func (w *flakyResponseWriter) Header() http.Header { return w.header }
|
|
|
|
func (w *flakyResponseWriter) WriteHeader(code int) {
|
|
w.mu.Lock()
|
|
defer w.mu.Unlock()
|
|
if w.status == 0 {
|
|
w.status = code
|
|
}
|
|
}
|
|
|
|
func (w *flakyResponseWriter) Write(p []byte) (int, error) {
|
|
if w.writeDelay > 0 {
|
|
time.Sleep(w.writeDelay)
|
|
}
|
|
w.mu.Lock()
|
|
defer w.mu.Unlock()
|
|
if w.status == 0 {
|
|
w.status = http.StatusOK
|
|
}
|
|
if w.failAfterWrites >= 0 && w.writes >= w.failAfterWrites {
|
|
w.failedWrites++
|
|
return 0, errors.New("client connection write failed")
|
|
}
|
|
w.writes++
|
|
w.body.Write(p)
|
|
return len(p), nil
|
|
}
|
|
|
|
func (w *flakyResponseWriter) Flush() {}
|
|
|
|
func (w *flakyResponseWriter) snapshot() (status int, body string, failedWrites int) {
|
|
w.mu.Lock()
|
|
defer w.mu.Unlock()
|
|
return w.status, w.body.String(), w.failedWrites
|
|
}
|
|
|
|
// TestChatCompletionsPassthroughWriteFailureSendsCancelRunOnce verifies S09
|
|
// for a caller write failure after response start: exactly one CancelRun is
|
|
// propagated to the Node, the ordered prefix written before the failure is
|
|
// preserved, and no tunnel bytes are written after the failed write even
|
|
// though more frames (including END) are available.
|
|
func TestChatCompletionsPassthroughWriteFailureSendsCancelRunOnce(t *testing.T) {
|
|
frames := make(chan *iop.ProviderTunnelFrame, 5)
|
|
frames <- &iop.ProviderTunnelFrame{
|
|
Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START,
|
|
StatusCode: 200,
|
|
Headers: map[string]string{"Content-Type": "text/event-stream"},
|
|
}
|
|
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte("data: chunk-1\n\n")}
|
|
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte("data: chunk-2\n\n")}
|
|
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte("data: chunk-3\n\n")}
|
|
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true}
|
|
close(frames)
|
|
srv, fake := chatSidebandServer(frames)
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"pool-model",
|
|
"stream":true,
|
|
"messages":[{"role":"user","content":"hello"}]
|
|
}`))
|
|
w := newFlakyResponseWriter(1) // first body write succeeds, second fails
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
status, body, failedWrites := w.snapshot()
|
|
if status != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", status, body)
|
|
}
|
|
if body != "data: chunk-1\n\n" {
|
|
t.Fatalf("body after write failure must stay the ordered prefix:\n got: %q\nwant: %q", body, "data: chunk-1\n\n")
|
|
}
|
|
if failedWrites == 0 {
|
|
t.Fatal("fixture did not exercise the write failure path")
|
|
}
|
|
calls := fake.cancelCallsSnapshot()
|
|
if len(calls) != 1 {
|
|
t.Fatalf("expected exactly 1 CancelRun call on write failure, got %d: %+v", len(calls), calls)
|
|
}
|
|
if calls[0].RunID != "run-tunnel" || calls[0].NodeRef != "node-1" {
|
|
t.Fatalf("unexpected cancel request: %+v", calls[0])
|
|
}
|
|
}
|
|
|
|
// TestChatCompletionsPassthroughWaitTimeoutSendsCancelRun verifies the tunnel
|
|
// wait timeout before response start returns a 502 and propagates exactly one
|
|
// CancelRun to the Node.
|
|
func TestChatCompletionsPassthroughWaitTimeoutSendsCancelRun(t *testing.T) {
|
|
fake := &fakeRunService{
|
|
tunnelFrames: make(chan *iop.ProviderTunnelFrame),
|
|
tunnelWaitTimeout: 50 * time.Millisecond,
|
|
}
|
|
srv := NewServer(config.EdgeOpenAIConf{}, fake, nil)
|
|
srv.SetModelCatalog([]config.ModelCatalogEntry{
|
|
{ID: "pool-model", Providers: map[string]string{"prov-1": "served-model"}},
|
|
})
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"pool-model",
|
|
"messages":[{"role":"user","content":"hello"}]
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
if w.Code != http.StatusBadGateway {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
if !strings.Contains(w.Body.String(), "run timed out") {
|
|
t.Fatalf("timeout error not surfaced: %s", w.Body.String())
|
|
}
|
|
calls := fake.cancelCallsSnapshot()
|
|
if len(calls) != 1 {
|
|
t.Fatalf("expected exactly 1 CancelRun call on wait timeout, got %d: %+v", len(calls), calls)
|
|
}
|
|
if calls[0].RunID != "run-tunnel" {
|
|
t.Fatalf("unexpected cancel request: %+v", calls[0])
|
|
}
|
|
}
|
|
|
|
// TestChatCompletionsPassthroughProviderErrorAfterStartStopsBodyWrites
|
|
// verifies an ERROR frame after response start truncates the stream: bytes
|
|
// written before the error are preserved in order, later frames are never
|
|
// written, and no spurious CancelRun is sent for a provider-terminal failure.
|
|
func TestChatCompletionsPassthroughProviderErrorAfterStartStopsBodyWrites(t *testing.T) {
|
|
frames := make(chan *iop.ProviderTunnelFrame, 4)
|
|
frames <- &iop.ProviderTunnelFrame{
|
|
Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START,
|
|
StatusCode: 200,
|
|
Headers: map[string]string{"Content-Type": "text/event-stream"},
|
|
}
|
|
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte("data: before-error\n\n")}
|
|
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_ERROR, Error: "provider stream failed"}
|
|
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte("data: after-error-must-not-write\n\n")}
|
|
close(frames)
|
|
srv, fake := chatSidebandServer(frames)
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"pool-model",
|
|
"stream":true,
|
|
"messages":[{"role":"user","content":"hello"}]
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status already committed before the error must stay 200, got %d", w.Code)
|
|
}
|
|
if got := w.Body.String(); got != "data: before-error\n\n" {
|
|
t.Fatalf("provider error must truncate after the ordered prefix:\n got: %q\nwant: %q", got, "data: before-error\n\n")
|
|
}
|
|
if calls := fake.cancelCallsSnapshot(); len(calls) != 0 {
|
|
t.Fatalf("provider-terminal error must not trigger CancelRun, got %+v", calls)
|
|
}
|
|
}
|
|
|
|
// TestChatCompletionsPassthroughSlowClientByteIdentity verifies a slow caller
|
|
// applies backpressure without corrupting opaque relay data: every body frame
|
|
// reaches the caller exactly once, in order, and a slow-but-alive client never
|
|
// triggers CancelRun.
|
|
func TestChatCompletionsPassthroughSlowClientByteIdentity(t *testing.T) {
|
|
const chunkCount = 40
|
|
var want strings.Builder
|
|
frames := make(chan *iop.ProviderTunnelFrame, chunkCount+2)
|
|
frames <- &iop.ProviderTunnelFrame{
|
|
Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START,
|
|
StatusCode: 200,
|
|
Headers: map[string]string{"Content-Type": "text/event-stream"},
|
|
}
|
|
for i := 0; i < chunkCount; i++ {
|
|
chunk := fmt.Sprintf("data: chunk-%02d\n\n", i)
|
|
want.WriteString(chunk)
|
|
frames <- &iop.ProviderTunnelFrame{
|
|
Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY,
|
|
Sequence: int64(i + 1),
|
|
Body: []byte(chunk),
|
|
}
|
|
}
|
|
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true}
|
|
close(frames)
|
|
srv, fake := chatSidebandServer(frames)
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"pool-model",
|
|
"stream":true,
|
|
"messages":[{"role":"user","content":"hello"}]
|
|
}`))
|
|
w := newFlakyResponseWriter(-1)
|
|
w.writeDelay = 2 * time.Millisecond
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
status, body, _ := w.snapshot()
|
|
if status != http.StatusOK {
|
|
t.Fatalf("status: got %d", status)
|
|
}
|
|
if body != want.String() {
|
|
t.Fatalf("slow client stream not byte-identical/ordered:\n got: %q\nwant: %q", body, want.String())
|
|
}
|
|
if calls := fake.cancelCallsSnapshot(); len(calls) != 0 {
|
|
t.Fatalf("slow but alive client must not trigger CancelRun, got %+v", calls)
|
|
}
|
|
}
|
|
|
|
func TestChatCompletionsProviderPoolAppliesGenerationPolicy(t *testing.T) {
|
|
fake := &fakeRunService{
|
|
tunnelFrames: staticProviderTunnelFrames(`{"ok":true}`),
|
|
tunnelServedTarget: "Ornith-1.0-35B",
|
|
}
|
|
catalog := []config.ModelCatalogEntry{{
|
|
ID: "ornith:35b",
|
|
DefaultMaxTokens: 32768,
|
|
MinMaxTokens: 32768,
|
|
DefaultThinkingTokenBudget: 8192,
|
|
Providers: map[string]string{"prov-vllm": "Ornith-1.0-35B"},
|
|
}}
|
|
srv := NewServer(config.EdgeOpenAIConf{}, fake, nil)
|
|
srv.SetModelCatalog(catalog)
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"ornith:35b",
|
|
"messages":[{"role":"user","content":"hello"}],
|
|
"max_tokens":4096
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
if len(fake.reqsSnapshot()) != 0 {
|
|
t.Fatalf("provider-pool generation policy must stay on tunnel path, got %d SubmitRun calls", len(fake.reqsSnapshot()))
|
|
}
|
|
bodies := fake.tunnelBodiesSnapshot()
|
|
if len(bodies) != 1 {
|
|
t.Fatalf("expected 1 tunnel body, got %d", len(bodies))
|
|
}
|
|
var providerReq map[string]any
|
|
if err := json.Unmarshal(bodies[0], &providerReq); err != nil {
|
|
t.Fatalf("provider body JSON: %v body=%s", err, bodies[0])
|
|
}
|
|
if providerReq["model"] != "Ornith-1.0-35B" {
|
|
t.Fatalf("served model rewrite not applied: %+v", providerReq["model"])
|
|
}
|
|
if providerReq["max_tokens"].(float64) != 32768 {
|
|
t.Fatalf("max_tokens policy not applied: %+v", providerReq)
|
|
}
|
|
if providerReq["thinking_token_budget"].(float64) != 8192 {
|
|
t.Fatalf("thinking_token_budget policy not applied: %+v", providerReq)
|
|
}
|
|
}
|
|
|
|
func TestChatCompletionsProviderPoolThinkingPolicyOverridesStrictOutputDisable(t *testing.T) {
|
|
fake := &fakeRunService{
|
|
tunnelFrames: staticProviderTunnelFrames(`{"ok":true}`),
|
|
tunnelServedTarget: "Ornith-1.0-35B",
|
|
}
|
|
catalog := []config.ModelCatalogEntry{{
|
|
ID: "ornith:35b",
|
|
DefaultThinkingTokenBudget: 8192,
|
|
Providers: map[string]string{"prov-vllm": "Ornith-1.0-35B"},
|
|
}}
|
|
srv := NewServer(config.EdgeOpenAIConf{StrictOutput: true}, fake, nil)
|
|
srv.SetModelCatalog(catalog)
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"ornith:35b",
|
|
"messages":[{"role":"user","content":"hello"}]
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
if len(fake.reqsSnapshot()) != 0 {
|
|
t.Fatalf("provider-pool thinking policy must stay on tunnel path, got %d SubmitRun calls", len(fake.reqsSnapshot()))
|
|
}
|
|
bodies := fake.tunnelBodiesSnapshot()
|
|
if len(bodies) != 1 {
|
|
t.Fatalf("expected 1 tunnel body, got %d", len(bodies))
|
|
}
|
|
var providerReq map[string]any
|
|
if err := json.Unmarshal(bodies[0], &providerReq); err != nil {
|
|
t.Fatalf("provider body JSON: %v body=%s", err, bodies[0])
|
|
}
|
|
if providerReq["think"] != true {
|
|
t.Fatalf("provider-pool thinking policy should keep thinking enabled under strict output: %+v", providerReq)
|
|
}
|
|
if providerReq["thinking_token_budget"].(float64) != 8192 {
|
|
t.Fatalf("thinking_token_budget policy not applied: %+v", providerReq)
|
|
}
|
|
}
|
|
|
|
func TestChatCompletionsProviderPoolProviderNativeThinkingDisablesDefaultBudgetInjection(t *testing.T) {
|
|
fake := &fakeRunService{
|
|
tunnelFrames: staticProviderTunnelFrames(`{"ok":true}`),
|
|
tunnelServedTarget: "Ornith-1.0-35B",
|
|
}
|
|
catalog := []config.ModelCatalogEntry{{
|
|
ID: "ornith:35b",
|
|
DefaultThinkingTokenBudget: 8192,
|
|
Providers: map[string]string{"prov-vllm": "Ornith-1.0-35B"},
|
|
}}
|
|
srv := NewServer(config.EdgeOpenAIConf{StrictOutput: true}, fake, nil)
|
|
srv.SetModelCatalog(catalog)
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"ornith:35b",
|
|
"messages":[{"role":"user","content":"hello"}],
|
|
"chat_template_kwargs":{"enable_thinking":false}
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
bodies := fake.tunnelBodiesSnapshot()
|
|
if len(bodies) != 1 {
|
|
t.Fatalf("expected 1 tunnel body, got %d", len(bodies))
|
|
}
|
|
var providerReq map[string]any
|
|
if err := json.Unmarshal(bodies[0], &providerReq); err != nil {
|
|
t.Fatalf("provider body JSON: %v body=%s", err, bodies[0])
|
|
}
|
|
ctk, ok := providerReq["chat_template_kwargs"].(map[string]any)
|
|
if !ok {
|
|
t.Fatalf("chat_template_kwargs not preserved as object: %+v", providerReq)
|
|
}
|
|
if ctk["enable_thinking"] != false {
|
|
t.Fatalf("provider-native enable_thinking must stay false: %+v", ctk)
|
|
}
|
|
if _, ok := providerReq["think"]; ok {
|
|
t.Fatalf("catalog policy must not inject top-level think over provider-native thinking: %+v", providerReq)
|
|
}
|
|
if _, ok := providerReq["thinking_token_budget"]; ok {
|
|
t.Fatalf("catalog policy must not inject thinking_token_budget over provider-native thinking: %+v", providerReq)
|
|
}
|
|
}
|
|
|
|
func TestChatCompletionsProviderPoolPreservesLargerGenerationPolicyValues(t *testing.T) {
|
|
fake := &fakeRunService{
|
|
tunnelFrames: staticProviderTunnelFrames(`{"ok":true}`),
|
|
tunnelServedTarget: "Ornith-1.0-35B",
|
|
}
|
|
catalog := []config.ModelCatalogEntry{{
|
|
ID: "ornith:35b",
|
|
DefaultMaxTokens: 32768,
|
|
MinMaxTokens: 32768,
|
|
DefaultThinkingTokenBudget: 8192,
|
|
Providers: map[string]string{"prov-vllm": "Ornith-1.0-35B"},
|
|
}}
|
|
srv := NewServer(config.EdgeOpenAIConf{}, fake, nil)
|
|
srv.SetModelCatalog(catalog)
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"ornith:35b",
|
|
"messages":[{"role":"user","content":"hello"}],
|
|
"max_tokens":40000,
|
|
"thinking_token_budget":2048
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
if len(fake.reqsSnapshot()) != 0 {
|
|
t.Fatalf("provider-pool generation policy must stay on tunnel path, got %d SubmitRun calls", len(fake.reqsSnapshot()))
|
|
}
|
|
bodies := fake.tunnelBodiesSnapshot()
|
|
if len(bodies) != 1 {
|
|
t.Fatalf("expected 1 tunnel body, got %d", len(bodies))
|
|
}
|
|
var providerReq map[string]any
|
|
if err := json.Unmarshal(bodies[0], &providerReq); err != nil {
|
|
t.Fatalf("provider body JSON: %v body=%s", err, bodies[0])
|
|
}
|
|
if providerReq["max_tokens"].(float64) != 40000 {
|
|
t.Fatalf("larger max_tokens should be preserved: %+v", providerReq)
|
|
}
|
|
if providerReq["thinking_token_budget"].(float64) != 2048 {
|
|
t.Fatalf("explicit thinking_token_budget should be preserved: %+v", providerReq)
|
|
}
|
|
}
|
|
|
|
// TestChatCompletionsLegacyRouteSetsProviderPoolFalse verifies that when the
|
|
// server is configured with Adapter + Target (no catalog), the dispatched
|
|
// SubmitRunRequest has ProviderPool=false so the service uses direct dispatch
|
|
// (not the queue admission gate that requires ProviderPool=true).
|
|
func TestChatCompletionsLegacyRouteSetsProviderPoolFalse(t *testing.T) {
|
|
fake := &fakeRunService{events: make(chan *iop.RunEvent, 2)}
|
|
fake.events <- &iop.RunEvent{Type: "delta", Delta: "hi"}
|
|
fake.events <- &iop.RunEvent{Type: "complete"}
|
|
|
|
// Server with adapter+target only, no catalog — classic legacy route.
|
|
srv := NewServer(config.EdgeOpenAIConf{
|
|
Adapter: "cli",
|
|
Target: "codex",
|
|
TimeoutSec: 10,
|
|
}, fake, nil)
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"any-model",
|
|
"messages":[{"role":"user","content":"hello"}]
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
if fake.req.ProviderPool {
|
|
t.Error("ProviderPool must be false for a legacy adapter+target route with no catalog")
|
|
}
|
|
if fake.req.Adapter != "cli" || fake.req.Target != "codex" {
|
|
t.Errorf("Adapter/Target: got %q/%q, want cli/codex", fake.req.Adapter, fake.req.Target)
|
|
}
|
|
}
|
|
|
|
// responsesProviderTunnelServer builds a server whose catalog routes
|
|
// "pool-model" through the OpenAI-compatible provider tunnel, wired to the given
|
|
// provider frames and served target.
|
|
func responsesProviderTunnelServer(frames chan *iop.ProviderTunnelFrame, servedTarget string) (*Server, *fakeRunService) {
|
|
fake := &fakeRunService{
|
|
poolDispatchPath: string(edgeservice.ProviderPoolPathTunnel),
|
|
tunnelFrames: frames,
|
|
tunnelServedTarget: servedTarget,
|
|
}
|
|
srv := NewServer(config.EdgeOpenAIConf{}, fake, nil)
|
|
srv.SetModelCatalog([]config.ModelCatalogEntry{
|
|
{ID: "pool-model", Providers: map[string]string{"prov-1": "served-model"}},
|
|
})
|
|
return srv, fake
|
|
}
|
|
|
|
// TestResponsesProviderPoolTunnelSelectionUsesPassthrough verifies that when a
|
|
// provider-pool catalog match is resolved and the selected provider is an
|
|
// OpenAI-compatible type (tunnel executionPath), the Responses handler
|
|
// dispatches via tunnel passthrough with the raw body preserved (model rewrite
|
|
// only). This mirrors TestChatCompletionsProviderPoolTunnelSelectionUsesPassthrough
|
|
// for /v1/responses.
|
|
func TestResponsesProviderPoolTunnelSelectionUsesPassthrough(t *testing.T) {
|
|
frames := make(chan *iop.ProviderTunnelFrame, 3)
|
|
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START, StatusCode: 200}
|
|
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(`{"ok":true}`)}
|
|
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true}
|
|
close(frames)
|
|
|
|
fake := &fakeRunService{
|
|
poolDispatchPath: string(edgeservice.ProviderPoolPathTunnel),
|
|
tunnelFrames: frames,
|
|
}
|
|
catalog := []config.ModelCatalogEntry{
|
|
{ID: "vllm-resp-model", Providers: map[string]string{"prov-vllm": "Qwen3-35B"}},
|
|
}
|
|
srv := NewServer(config.EdgeOpenAIConf{}, fake, nil)
|
|
srv.SetModelCatalog(catalog)
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(`{
|
|
"model":"vllm-resp-model",
|
|
"input":"hello world"
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleResponses(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
// Tunnel passthrough must use SubmitProviderPool tunnel path.
|
|
tunnelReqs := fake.tunnelReqsSnapshot()
|
|
reqs := fake.reqsSnapshot()
|
|
if len(tunnelReqs) != 1 {
|
|
t.Fatalf("expected 1 tunnel dispatch for vLLM path, got %d", len(tunnelReqs))
|
|
}
|
|
if len(reqs) != 0 {
|
|
t.Fatalf("expected 0 normalized SubmitRun for vLLM tunnel path, got %d", len(reqs))
|
|
}
|
|
if !tunnelReqs[0].ProviderPool {
|
|
t.Error("tunnel dispatch must have ProviderPool=true")
|
|
}
|
|
if tunnelReqs[0].Path != "/v1/responses" {
|
|
t.Fatalf("tunnel path: got %q want /v1/responses", tunnelReqs[0].Path)
|
|
}
|
|
}
|
|
|
|
// TestResponsesProviderPoolOllamaSelectionUsesNormalizedRun verifies that when a
|
|
// provider-pool catalog match is resolved and the selected provider is an
|
|
// Ollama type (normalized executionPath), the Responses handler dispatches via
|
|
// normalized RunEvent path with the decoded prompt/input, not via tunnel. This
|
|
// mirrors TestChatCompletionsProviderPoolOllamaSelectionUsesNormalizedRun for
|
|
// /v1/responses.
|
|
func TestResponsesProviderPoolOllamaSelectionUsesNormalizedRun(t *testing.T) {
|
|
fake := &fakeRunService{
|
|
poolDispatchPath: string(edgeservice.ProviderPoolPathNormalized),
|
|
}
|
|
catalog := []config.ModelCatalogEntry{
|
|
{ID: "ollama-resp-model", Providers: map[string]string{"prov-ollama": "qwen3.6:35b"}},
|
|
}
|
|
srv := NewServer(config.EdgeOpenAIConf{}, fake, nil)
|
|
srv.SetModelCatalog(catalog)
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(`{
|
|
"model":"ollama-resp-model",
|
|
"input":"hello world"
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleResponses(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
// Normalized Ollama path must use SubmitProviderPool normalized path.
|
|
reqs := fake.reqsSnapshot()
|
|
tunnelReqs := fake.tunnelReqsSnapshot()
|
|
if len(reqs) != 1 {
|
|
t.Fatalf("expected 1 normalized SubmitRun for Ollama path, got %d", len(reqs))
|
|
}
|
|
if len(tunnelReqs) != 0 {
|
|
t.Fatalf("expected 0 tunnel dispatch for Ollama normalized path, got %d", len(tunnelReqs))
|
|
}
|
|
if !reqs[0].ProviderPool {
|
|
t.Error("normalized dispatch must have ProviderPool=true")
|
|
}
|
|
if reqs[0].ModelGroupKey != "ollama-resp-model" {
|
|
t.Fatalf("expected model group key %q, got %q", "ollama-resp-model", reqs[0].ModelGroupKey)
|
|
}
|
|
}
|
|
|
|
// TestResponsesMixedProviderPoolTunnelSelection verifies S05: a mixed
|
|
// provider-pool model group that selects an OpenAI-compatible (vLLM) provider
|
|
// dispatches via tunnel passthrough for /v1/responses.
|
|
func TestResponsesMixedProviderPoolTunnelSelection(t *testing.T) {
|
|
tunnelFrames := make(chan *iop.ProviderTunnelFrame, 3)
|
|
tunnelFrames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START, StatusCode: 200}
|
|
tunnelFrames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(`{"id":"resp-mixed","object":"response"}`)}
|
|
tunnelFrames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true}
|
|
close(tunnelFrames)
|
|
|
|
fake := &fakeRunService{
|
|
poolDispatchPath: string(edgeservice.ProviderPoolPathTunnel),
|
|
tunnelFrames: tunnelFrames,
|
|
}
|
|
catalog := []config.ModelCatalogEntry{
|
|
{
|
|
ID: "mixed-resp-model",
|
|
Providers: map[string]string{
|
|
"prov-vllm": "served-vllm",
|
|
"prov-ollama": "served-ollama",
|
|
},
|
|
},
|
|
}
|
|
srv := NewServer(config.EdgeOpenAIConf{}, fake, nil)
|
|
srv.SetModelCatalog(catalog)
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(`{
|
|
"model":"mixed-resp-model",
|
|
"input":"hello mixed"
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleResponses(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
|
|
tunnelReqs := fake.tunnelReqsSnapshot()
|
|
if len(tunnelReqs) != 1 {
|
|
t.Fatalf("expected 1 tunnel dispatch, got %d", len(tunnelReqs))
|
|
}
|
|
if !tunnelReqs[0].ProviderPool {
|
|
t.Error("tunnel request must have ProviderPool=true")
|
|
}
|
|
if tunnelReqs[0].Path != "/v1/responses" {
|
|
t.Fatalf("tunnel path: got %q want /v1/responses", tunnelReqs[0].Path)
|
|
}
|
|
|
|
runReqs := fake.reqsSnapshot()
|
|
if len(runReqs) != 0 {
|
|
t.Errorf("expected 0 normalized SubmitRun calls, got %d", len(runReqs))
|
|
}
|
|
}
|
|
|
|
// TestResponsesMixedProviderPoolOllamaSelection verifies S05: when the mixed
|
|
// model group selects an Ollama provider, the request is dispatched over the
|
|
// normalized path (SubmitRun), and no tunnel dispatch is issued for /v1/responses.
|
|
func TestResponsesMixedProviderPoolOllamaSelection(t *testing.T) {
|
|
events := bufferedRunEvents(
|
|
&iop.RunEvent{Type: "delta", Delta: "ollama-resp"},
|
|
&iop.RunEvent{Type: "complete"},
|
|
)
|
|
|
|
fake := &fakeRunService{
|
|
poolDispatchPath: string(edgeservice.ProviderPoolPathNormalized),
|
|
events: events,
|
|
}
|
|
catalog := []config.ModelCatalogEntry{
|
|
{
|
|
ID: "mixed-resp-model",
|
|
Providers: map[string]string{
|
|
"prov-vllm": "served-vllm",
|
|
"prov-ollama": "served-ollama",
|
|
},
|
|
},
|
|
}
|
|
srv := NewServer(config.EdgeOpenAIConf{}, fake, nil)
|
|
srv.SetModelCatalog(catalog)
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(`{
|
|
"model":"mixed-resp-model",
|
|
"input":"hello mixed"
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleResponses(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
|
|
tunnelReqs := fake.tunnelReqsSnapshot()
|
|
if len(tunnelReqs) != 0 {
|
|
t.Fatalf("expected 0 tunnel dispatches for Ollama selection, got %d", len(tunnelReqs))
|
|
}
|
|
|
|
runReqs := fake.reqsSnapshot()
|
|
if len(runReqs) != 1 {
|
|
t.Fatalf("expected 1 normalized SubmitRun, got %d", len(runReqs))
|
|
}
|
|
if runReqs[0].ModelGroupKey != "mixed-resp-model" {
|
|
t.Errorf("normalized ModelGroupKey: got %q, want mixed-resp-model", runReqs[0].ModelGroupKey)
|
|
}
|
|
if !runReqs[0].ProviderPool {
|
|
t.Error("normalized run must have ProviderPool=true")
|
|
}
|
|
}
|
|
|
|
// TestResponsesProviderPoolDispatch verifies that /v1/responses sends
|
|
// provider-pool models through the raw provider tunnel (POST /v1/responses)
|
|
// instead of the normalized RunEvent path.
|
|
func TestResponsesProviderPoolDispatch(t *testing.T) {
|
|
fake := &fakeRunService{
|
|
tunnelFrames: staticProviderTunnelFrames(`{"id":"resp-1","object":"response"}`),
|
|
tunnelServedTarget: "model-a",
|
|
}
|
|
catalog := []config.ModelCatalogEntry{
|
|
{ID: "prov-vllm:model-a", Providers: map[string]string{"prov-vllm": "model-a"}},
|
|
}
|
|
srv := NewServer(config.EdgeOpenAIConf{TimeoutSec: 5}, fake, nil)
|
|
srv.SetModelCatalog(catalog)
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(`{
|
|
"model":"prov-vllm:model-a",
|
|
"input":"hello"
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleResponses(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
if len(fake.reqsSnapshot()) != 0 {
|
|
t.Fatalf("provider-pool /v1/responses must not call SubmitRun, got %d calls", len(fake.reqsSnapshot()))
|
|
}
|
|
reqs := fake.tunnelReqsSnapshot()
|
|
if len(reqs) != 1 {
|
|
t.Fatalf("expected 1 tunnel dispatch, got %d", len(reqs))
|
|
}
|
|
if reqs[0].Path != "/v1/responses" {
|
|
t.Fatalf("tunnel path: got %q want /v1/responses", reqs[0].Path)
|
|
}
|
|
if reqs[0].Method != http.MethodPost {
|
|
t.Fatalf("tunnel method: got %q want POST", reqs[0].Method)
|
|
}
|
|
}
|
|
|
|
// TestResponsesProviderPoolPreservesRawOutputTokens verifies that raw
|
|
// passthrough preserves the caller's Responses-shaped max_output_tokens and does
|
|
// not inject the Chat-shaped max_tokens or generation policy.
|
|
func TestResponsesProviderPoolPreservesRawOutputTokens(t *testing.T) {
|
|
fake := &fakeRunService{
|
|
tunnelFrames: staticProviderTunnelFrames(`{"ok":true}`),
|
|
tunnelServedTarget: "Ornith-1.0-35B",
|
|
}
|
|
catalog := []config.ModelCatalogEntry{{
|
|
ID: "ornith:35b",
|
|
DefaultMaxTokens: 32768,
|
|
MinMaxTokens: 32768,
|
|
DefaultThinkingTokenBudget: 8192,
|
|
Providers: map[string]string{"prov-vllm": "Ornith-1.0-35B"},
|
|
}}
|
|
srv := NewServer(config.EdgeOpenAIConf{TimeoutSec: 5}, fake, nil)
|
|
srv.SetModelCatalog(catalog)
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(`{
|
|
"model":"ornith:35b",
|
|
"input":"hello",
|
|
"max_output_tokens":4096
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleResponses(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
if len(fake.reqsSnapshot()) != 0 {
|
|
t.Fatalf("provider-pool /v1/responses must not call SubmitRun, got %d calls", len(fake.reqsSnapshot()))
|
|
}
|
|
bodies := fake.tunnelBodiesSnapshot()
|
|
if len(bodies) != 1 {
|
|
t.Fatalf("expected 1 tunnel body, got %d", len(bodies))
|
|
}
|
|
var providerReq map[string]any
|
|
if err := json.Unmarshal(bodies[0], &providerReq); err != nil {
|
|
t.Fatalf("provider body JSON: %v body=%s", err, bodies[0])
|
|
}
|
|
if providerReq["model"] != "Ornith-1.0-35B" {
|
|
t.Fatalf("served model rewrite not applied: %+v", providerReq["model"])
|
|
}
|
|
if providerReq["max_output_tokens"].(float64) != 4096 {
|
|
t.Fatalf("caller max_output_tokens must be preserved: %+v", providerReq)
|
|
}
|
|
if _, ok := providerReq["max_tokens"]; ok {
|
|
t.Fatalf("passthrough must not inject Chat-shaped max_tokens: %+v", providerReq)
|
|
}
|
|
if _, ok := providerReq["thinking_token_budget"]; ok {
|
|
t.Fatalf("passthrough must not inject thinking_token_budget: %+v", providerReq)
|
|
}
|
|
}
|
|
|
|
// TestResponsesProviderPoolPassthroughOmitsThinkingPolicy verifies that raw
|
|
// passthrough forwards the caller body as-is under strict output: the
|
|
// normalized-path thinking policy is not injected into the provider body.
|
|
func TestResponsesProviderPoolPassthroughOmitsThinkingPolicy(t *testing.T) {
|
|
fake := &fakeRunService{
|
|
tunnelFrames: staticProviderTunnelFrames(`{"ok":true}`),
|
|
tunnelServedTarget: "Ornith-1.0-35B",
|
|
}
|
|
catalog := []config.ModelCatalogEntry{{
|
|
ID: "ornith:35b",
|
|
DefaultThinkingTokenBudget: 8192,
|
|
Providers: map[string]string{"prov-vllm": "Ornith-1.0-35B"},
|
|
}}
|
|
srv := NewServer(config.EdgeOpenAIConf{TimeoutSec: 5, StrictOutput: true}, fake, nil)
|
|
srv.SetModelCatalog(catalog)
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(`{
|
|
"model":"ornith:35b",
|
|
"input":"hello"
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleResponses(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
if len(fake.reqsSnapshot()) != 0 {
|
|
t.Fatalf("provider-pool /v1/responses must not call SubmitRun, got %d calls", len(fake.reqsSnapshot()))
|
|
}
|
|
bodies := fake.tunnelBodiesSnapshot()
|
|
if len(bodies) != 1 {
|
|
t.Fatalf("expected 1 tunnel body, got %d", len(bodies))
|
|
}
|
|
var providerReq map[string]any
|
|
if err := json.Unmarshal(bodies[0], &providerReq); err != nil {
|
|
t.Fatalf("provider body JSON: %v body=%s", err, bodies[0])
|
|
}
|
|
if _, ok := providerReq["think"]; ok {
|
|
t.Fatalf("passthrough must not inject think: %+v", providerReq)
|
|
}
|
|
if _, ok := providerReq["thinking_token_budget"]; ok {
|
|
t.Fatalf("passthrough must not inject thinking_token_budget: %+v", providerReq)
|
|
}
|
|
}
|
|
|
|
// TestResponsesProviderTunnelAllowsUnknownFields verifies that Codex/Responses
|
|
// unknown fields (tools, parallel_tool_calls, store) are accepted on the
|
|
// provider passthrough route and preserved in the forwarded tunnel body.
|
|
func TestResponsesProviderTunnelAllowsUnknownFields(t *testing.T) {
|
|
srv, fake := responsesProviderTunnelServer(staticProviderTunnelFrames(`{"ok":true}`), "served-model")
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(`{
|
|
"model":"pool-model",
|
|
"input":"hi",
|
|
"tools":[{"type":"web_search"}],
|
|
"parallel_tool_calls":true,
|
|
"store":false
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleResponses(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("provider passthrough must accept unknown fields: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
bodies := fake.tunnelBodiesSnapshot()
|
|
if len(bodies) != 1 {
|
|
t.Fatalf("expected 1 tunnel body, got %d", len(bodies))
|
|
}
|
|
var providerReq map[string]any
|
|
if err := json.Unmarshal(bodies[0], &providerReq); err != nil {
|
|
t.Fatalf("provider body JSON: %v body=%s", err, bodies[0])
|
|
}
|
|
if _, ok := providerReq["tools"]; !ok {
|
|
t.Fatalf("tools must be preserved on passthrough: %+v", providerReq)
|
|
}
|
|
if providerReq["parallel_tool_calls"] != true {
|
|
t.Fatalf("parallel_tool_calls must be preserved: %+v", providerReq)
|
|
}
|
|
if providerReq["store"] != false {
|
|
t.Fatalf("store must be preserved: %+v", providerReq)
|
|
}
|
|
}
|
|
|
|
// TestResponsesProviderTunnelSubmitsResponsesPath verifies the passthrough
|
|
// dispatches a provider tunnel POST to /v1/responses and never calls SubmitRun.
|
|
func TestResponsesProviderTunnelSubmitsResponsesPath(t *testing.T) {
|
|
srv, fake := responsesProviderTunnelServer(staticProviderTunnelFrames(`{"ok":true}`), "served-model")
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(`{"model":"pool-model","input":"hi"}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleResponses(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
if len(fake.reqsSnapshot()) != 0 {
|
|
t.Fatalf("responses passthrough must not call SubmitRun, got %d", len(fake.reqsSnapshot()))
|
|
}
|
|
reqs := fake.tunnelReqsSnapshot()
|
|
if len(reqs) != 1 {
|
|
t.Fatalf("expected 1 tunnel dispatch, got %d", len(reqs))
|
|
}
|
|
if reqs[0].Path != "/v1/responses" {
|
|
t.Fatalf("tunnel path: got %q want /v1/responses", reqs[0].Path)
|
|
}
|
|
if reqs[0].Method != http.MethodPost {
|
|
t.Fatalf("tunnel method: got %q want POST", reqs[0].Method)
|
|
}
|
|
}
|
|
|
|
// TestResponsesProviderTunnelRewritesOnlyModel verifies the caller model alias
|
|
// is rewritten to the provider-served model while max_output_tokens, tools, and
|
|
// arbitrary fields are preserved verbatim.
|
|
func TestResponsesProviderTunnelRewritesOnlyModel(t *testing.T) {
|
|
srv, fake := responsesProviderTunnelServer(staticProviderTunnelFrames(`{"ok":true}`), "served-model")
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(`{
|
|
"model":"pool-model",
|
|
"input":"hi",
|
|
"max_output_tokens":123,
|
|
"tools":[{"type":"web_search"}],
|
|
"custom_field":"keep-me"
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleResponses(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
bodies := fake.tunnelBodiesSnapshot()
|
|
if len(bodies) != 1 {
|
|
t.Fatalf("expected 1 tunnel body, got %d", len(bodies))
|
|
}
|
|
var providerReq map[string]any
|
|
if err := json.Unmarshal(bodies[0], &providerReq); err != nil {
|
|
t.Fatalf("provider body JSON: %v body=%s", err, bodies[0])
|
|
}
|
|
if providerReq["model"] != "served-model" {
|
|
t.Fatalf("model must be rewritten to served target: %+v", providerReq["model"])
|
|
}
|
|
if providerReq["max_output_tokens"].(float64) != 123 {
|
|
t.Fatalf("max_output_tokens must be preserved: %+v", providerReq)
|
|
}
|
|
if _, ok := providerReq["tools"]; !ok {
|
|
t.Fatalf("tools must be preserved: %+v", providerReq)
|
|
}
|
|
if providerReq["custom_field"] != "keep-me" {
|
|
t.Fatalf("arbitrary fields must be preserved: %+v", providerReq)
|
|
}
|
|
}
|
|
|
|
// TestResponsesProviderTunnelForwardsProviderAuthHeader verifies the provider
|
|
// auth helper is applied to the Responses tunnel: a raw request-header token is
|
|
// forwarded as the configured target header, and a missing required token is
|
|
// rejected before dispatch.
|
|
func TestResponsesProviderTunnelForwardsProviderAuthHeader(t *testing.T) {
|
|
auth := config.EdgeOpenAIProviderAuthConf{
|
|
Enabled: true,
|
|
FromHeader: "X-IOP-Provider-Authorization",
|
|
TargetHeader: "Authorization",
|
|
Scheme: "Bearer",
|
|
Required: true,
|
|
}
|
|
t.Run("forwards raw token with scheme", func(t *testing.T) {
|
|
srv, fake := chatProviderAuthServer(staticOKTunnelFrames(`{"ok":true}`), auth)
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(`{"model":"pool-model","input":"hi"}`))
|
|
req.Header.Set("X-IOP-Provider-Authorization", "user-token")
|
|
w := httptest.NewRecorder()
|
|
srv.handleResponses(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
reqs := fake.tunnelReqsSnapshot()
|
|
if len(reqs) != 1 {
|
|
t.Fatalf("expected 1 tunnel dispatch, got %d", len(reqs))
|
|
}
|
|
if got := reqs[0].Headers["Authorization"]; got != "Bearer user-token" {
|
|
t.Fatalf("provider Authorization header: got %q want %q", got, "Bearer user-token")
|
|
}
|
|
})
|
|
t.Run("missing required token rejected", func(t *testing.T) {
|
|
srv, fake := chatProviderAuthServer(staticOKTunnelFrames(`{"ok":true}`), auth)
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(`{"model":"pool-model","input":"hi"}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleResponses(w, req)
|
|
|
|
if w.Code != http.StatusBadRequest {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
if got := len(fake.tunnelReqsSnapshot()); got != 0 {
|
|
t.Fatalf("missing required provider auth must not dispatch: got %d tunnel requests", got)
|
|
}
|
|
})
|
|
}
|
|
|
|
// TestResponsesProviderTunnelStreaming verifies a stream:true Responses request
|
|
// sets Stream=true on the tunnel dispatch and relays raw SSE bytes verbatim.
|
|
func TestResponsesProviderTunnelStreaming(t *testing.T) {
|
|
frames := make(chan *iop.ProviderTunnelFrame, 4)
|
|
frames <- &iop.ProviderTunnelFrame{
|
|
Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START,
|
|
StatusCode: 200,
|
|
Headers: map[string]string{"Content-Type": "text/event-stream"},
|
|
}
|
|
frames <- &iop.ProviderTunnelFrame{
|
|
Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY,
|
|
Body: []byte("data: {\"type\":\"response.output_text.delta\",\"delta\":\"hi\"}\n\n"),
|
|
}
|
|
frames <- &iop.ProviderTunnelFrame{
|
|
Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY,
|
|
Body: []byte("data: [DONE]\n\n"),
|
|
}
|
|
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true}
|
|
close(frames)
|
|
|
|
srv, fake := responsesProviderTunnelServer(frames, "served-model")
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(`{"model":"pool-model","input":"hi","stream":true}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleResponses(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
reqs := fake.tunnelReqsSnapshot()
|
|
if len(reqs) != 1 {
|
|
t.Fatalf("expected 1 tunnel dispatch, got %d", len(reqs))
|
|
}
|
|
if !reqs[0].Stream {
|
|
t.Fatalf("tunnel request Stream must be true for a stream:true responses request")
|
|
}
|
|
body := w.Body.String()
|
|
if !strings.Contains(body, "response.output_text.delta") || !strings.Contains(body, "[DONE]") {
|
|
t.Fatalf("SSE body must be relayed verbatim, got %q", body)
|
|
}
|
|
}
|
|
|
|
// TestResponsesProviderPoolIgnoresSelectorLikeMetadata verifies SDD S01/S04 for
|
|
// /v1/responses: arbitrary caller metadata never rejects the request or moves a
|
|
// provider route off pure passthrough. (The former selector key literal is
|
|
// deliberately not used; the deterministic surface check forbids it under apps.)
|
|
func TestResponsesProviderPoolIgnoresSelectorLikeMetadata(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
metadata string
|
|
}{
|
|
{name: "selector-like value is ignored", metadata: `"metadata":{"response_hint":"raw"}`},
|
|
{name: "arbitrary metadata is ignored", metadata: `"metadata":{"experiment":"mode-x"}`},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
providerBody := `{"ok":true}`
|
|
srv, fake := responsesProviderTunnelServer(staticProviderTunnelFrames(providerBody), "served-model")
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(fmt.Sprintf(`{
|
|
"model":"pool-model",
|
|
"input":"hi",
|
|
%s
|
|
}`, tc.metadata)))
|
|
w := httptest.NewRecorder()
|
|
srv.handleResponses(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("selector-like metadata must not be rejected: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
if got := w.Body.String(); got != providerBody {
|
|
t.Fatalf("provider body must be relayed byte-identically: got %q want %q", got, providerBody)
|
|
}
|
|
if len(fake.tunnelReqsSnapshot()) != 1 {
|
|
t.Fatalf("expected 1 pure passthrough tunnel dispatch, got %d", len(fake.tunnelReqsSnapshot()))
|
|
}
|
|
if len(fake.reqsSnapshot()) != 0 {
|
|
t.Fatalf("responses provider route must not call SubmitRun, got %d", len(fake.reqsSnapshot()))
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestChatCompletionsProviderPoolFallsBackToLegacyRoute(t *testing.T) {
|
|
fake := &fakeRunService{events: make(chan *iop.RunEvent, 2)}
|
|
fake.events <- &iop.RunEvent{Type: "delta", Delta: "ok"}
|
|
fake.events <- &iop.RunEvent{Type: "complete"}
|
|
|
|
catalog := []config.ModelCatalogEntry{
|
|
{ID: "qwen3.6:35b", Providers: map[string]string{"prov-vllm": "Qwen3-35B-A22B"}},
|
|
}
|
|
srv := NewServer(config.EdgeOpenAIConf{
|
|
ModelRoutes: []config.OpenAIRouteEntry{
|
|
{Model: "ollama-model", Adapter: "ollama", Target: "llama3"},
|
|
},
|
|
}, fake, nil)
|
|
srv.SetModelCatalog(catalog)
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"ollama-model",
|
|
"messages":[{"role":"user","content":"hi"}]
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
if fake.req.ProviderPool {
|
|
t.Error("ProviderPool should be false for non-catalog model")
|
|
}
|
|
if fake.req.Target != "llama3" {
|
|
t.Errorf("Target: got %q, want llama3", fake.req.Target)
|
|
}
|
|
}
|
|
|
|
func TestChatCompletionsRetriesUnknownTextToolCallBeforeResponse(t *testing.T) {
|
|
// Attempt 1: unknown tool call (triggers retry)
|
|
run1Events := make(chan *iop.RunEvent, 2)
|
|
run1Events <- &iop.RunEvent{Type: "delta", Delta: "<tool_call><function=delete_everything><parameter=confirm>true</parameter></function></tool_call>"}
|
|
run1Events <- &iop.RunEvent{Type: "complete"}
|
|
|
|
// Attempt 2: valid tool call (succeeds)
|
|
run2Events := make(chan *iop.RunEvent, 2)
|
|
run2Events <- &iop.RunEvent{Type: "delta", Delta: "<tool_call><function=run_commands><parameter=commands>[\"ls\"]\n</parameter></function></tool_call>"}
|
|
run2Events <- &iop.RunEvent{Type: "complete"}
|
|
|
|
fake := &fakeRunService{eventRuns: []chan *iop.RunEvent{run1Events, run2Events}}
|
|
srv := NewServer(config.EdgeOpenAIConf{Adapter: "cli", Target: "codex"}, fake, nil)
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"qwen3.6:35b",
|
|
"messages":[{"role":"user","content":"status"}],
|
|
"tools":[{"type":"function","function":{"name":"run_commands","parameters":{"type":"object","properties":{"commands":{"type":"array","items":{"type":"string"}}}}}}]
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
var resp chatCompletionResponse
|
|
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
|
t.Fatalf("decode response: %v", err)
|
|
}
|
|
if len(resp.Choices) == 0 {
|
|
t.Fatal("empty choices")
|
|
}
|
|
choice := resp.Choices[0]
|
|
if choice.FinishReason != "tool_calls" || len(choice.Message.ToolCalls) == 0 {
|
|
t.Fatalf("expected synthesized tool calls: %+v", choice)
|
|
}
|
|
}
|
|
|
|
func TestChatCompletionsFailsUnknownTextToolCallAfterRetryLimit(t *testing.T) {
|
|
run1Events := make(chan *iop.RunEvent, 2)
|
|
run1Events <- &iop.RunEvent{Type: "delta", Delta: "<tool_call><function=unknown><parameter=arg>val</parameter></function></tool_call>"}
|
|
run1Events <- &iop.RunEvent{Type: "complete"}
|
|
|
|
run2Events := make(chan *iop.RunEvent, 2)
|
|
run2Events <- &iop.RunEvent{Type: "delta", Delta: "<tool_call><function=unknown><parameter=arg>val</parameter></function></tool_call>"}
|
|
run2Events <- &iop.RunEvent{Type: "complete"}
|
|
|
|
fake := &fakeRunService{eventRuns: []chan *iop.RunEvent{run1Events, run2Events}}
|
|
srv := NewServer(config.EdgeOpenAIConf{Adapter: "cli", Target: "codex"}, fake, nil)
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"qwen3.6:35b",
|
|
"messages":[{"role":"user","content":"status"}],
|
|
"tools":[{"type":"function","function":{"name":"run_commands"}}]
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
if w.Code != http.StatusBadGateway {
|
|
t.Fatalf("status: got %d, want 502; body=%s", w.Code, w.Body.String())
|
|
}
|
|
var resp errorResponse
|
|
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
|
t.Fatalf("decode response: %v", err)
|
|
}
|
|
if resp.Error.Type != "tool_validation_error" {
|
|
t.Fatalf("unexpected error type: got %q, want tool_validation_error", resp.Error.Type)
|
|
}
|
|
}
|
|
|
|
func TestChatCompletionsProviderStreamSynthesizesTextToolCalls(t *testing.T) {
|
|
fake := &fakeRunService{events: make(chan *iop.RunEvent, 3)}
|
|
fake.events <- &iop.RunEvent{Type: "delta", Delta: "thought text "}
|
|
fake.events <- &iop.RunEvent{Type: "delta", Delta: "<tool_call><function=run_commands><parameter=commands>[\"echo 1\"]</parameter></function></tool_call>"}
|
|
fake.events <- &iop.RunEvent{Type: "complete"}
|
|
|
|
srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama"}, fake, nil)
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"metadata":{},
|
|
"model":"qwen3.6:35b",
|
|
"messages":[{"role":"user","content":"status"}],
|
|
"tools":[{"type":"function","function":{"name":"run_commands","parameters":{"type":"object"}}}],
|
|
"stream": true
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
body := w.Body.String()
|
|
if !strings.Contains(body, `"tool_calls"`) {
|
|
t.Fatalf("expected stream to contain tool_calls: %s", body)
|
|
}
|
|
if strings.Contains(body, `"<tool_call>"`) {
|
|
t.Fatalf("stream should not contain raw tool_call block: %s", body)
|
|
}
|
|
}
|
|
|
|
func TestChatCompletionsProviderStreamBlocksUnknownTextToolCall(t *testing.T) {
|
|
invalidRun := func() chan *iop.RunEvent {
|
|
return bufferedRunEvents(
|
|
&iop.RunEvent{Type: "delta", Delta: "thought text "},
|
|
&iop.RunEvent{Type: "delta", Delta: "<tool_call><function=unknown_tool><parameter=arg>val</parameter></function></tool_call>"},
|
|
&iop.RunEvent{Type: "complete"},
|
|
)
|
|
}
|
|
fake := &fakeRunService{
|
|
eventRuns: []chan *iop.RunEvent{
|
|
invalidRun(),
|
|
invalidRun(),
|
|
},
|
|
runIDs: []string{"run-bad-1", "run-bad-2"},
|
|
}
|
|
|
|
srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama"}, fake, nil)
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"metadata":{},
|
|
"model":"qwen3.6:35b",
|
|
"messages":[{"role":"user","content":"status"}],
|
|
"tools":[{"type":"function","function":{"name":"run_commands"}}],
|
|
"stream": true
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
body := w.Body.String()
|
|
if !strings.Contains(body, "tool_validation_error") {
|
|
t.Fatalf("expected stream to end with tool_validation_error: %s", body)
|
|
}
|
|
if strings.Contains(body, `\u003ctool_call\u003e`) {
|
|
t.Fatalf("unknown tool call leaked as content: %s", body)
|
|
}
|
|
if reqs := fake.reqsSnapshot(); len(reqs) != 2 {
|
|
t.Fatalf("submit attempts: got %d want 2", len(reqs))
|
|
}
|
|
}
|
|
|
|
func TestChatCompletionsStrictBufferedStreamFailsUnknownTextToolCallAfterRetryLimit(t *testing.T) {
|
|
run1Events := make(chan *iop.RunEvent, 2)
|
|
run1Events <- &iop.RunEvent{Type: "delta", Delta: "<tool_call><function=unknown><parameter=arg>val</parameter></function></tool_call>"}
|
|
run1Events <- &iop.RunEvent{Type: "complete"}
|
|
|
|
run2Events := make(chan *iop.RunEvent, 2)
|
|
run2Events <- &iop.RunEvent{Type: "delta", Delta: "<tool_call><function=unknown><parameter=arg>val</parameter></function></tool_call>"}
|
|
run2Events <- &iop.RunEvent{Type: "complete"}
|
|
|
|
fake := &fakeRunService{eventRuns: []chan *iop.RunEvent{run1Events, run2Events}}
|
|
srv := NewServer(config.EdgeOpenAIConf{Adapter: "cli", Target: "codex", StrictOutput: true, StrictStreamBuffer: true}, fake, nil)
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"qwen3.6:35b",
|
|
"messages":[{"role":"user","content":"status"}],
|
|
"tools":[{"type":"function","function":{"name":"run_commands"}}],
|
|
"stream": true
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
body := w.Body.String()
|
|
if !strings.Contains(body, "tool_validation_error") {
|
|
t.Fatalf("expected stream to end with tool_validation_error: %s", body)
|
|
}
|
|
}
|
|
|
|
func TestChatCompletionsFailsMalformedTextToolCallAfterRetryLimit(t *testing.T) {
|
|
invalidComplete := func() chan *iop.RunEvent {
|
|
ch := make(chan *iop.RunEvent, 2)
|
|
ch <- &iop.RunEvent{Type: "delta", Delta: "여기 툴 콜입니다. <tool_call><function=run_commands><parameter=commands>"}
|
|
ch <- &iop.RunEvent{Type: "complete"}
|
|
return ch
|
|
}
|
|
fake := &fakeRunService{
|
|
eventRuns: []chan *iop.RunEvent{
|
|
invalidComplete(),
|
|
invalidComplete(),
|
|
},
|
|
runIDs: []string{"run-bad-1", "run-bad-2"},
|
|
}
|
|
|
|
srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama"}, fake, nil)
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"metadata":{},
|
|
"model":"qwen3.6:35b",
|
|
"messages":[{"role":"user","content":"status"}],
|
|
"tools":[{"type":"function","function":{"name":"run_commands","parameters":{"type":"object"}}}],
|
|
"tool_choice":"auto"
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
if w.Code != http.StatusBadGateway {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
if !strings.Contains(w.Body.String(), `"type":"tool_validation_error"`) || !strings.Contains(w.Body.String(), "malformed tool call: unclosed tool_call block") {
|
|
t.Fatalf("expected tool validation error body, got %s", w.Body.String())
|
|
}
|
|
if reqs := fake.reqsSnapshot(); len(reqs) != 2 {
|
|
t.Fatalf("submit attempts: got %d want 2", len(reqs))
|
|
}
|
|
}
|
|
|
|
func TestChatCompletionsProviderStreamBlocksMalformedTextToolCall(t *testing.T) {
|
|
invalidRun := func() chan *iop.RunEvent {
|
|
return bufferedRunEvents(
|
|
&iop.RunEvent{Type: "delta", Delta: "이것은 정상 텍스트 "},
|
|
&iop.RunEvent{Type: "delta", Delta: "<tool_call><function=run_commands>"},
|
|
&iop.RunEvent{Type: "complete"},
|
|
)
|
|
}
|
|
fake := &fakeRunService{
|
|
eventRuns: []chan *iop.RunEvent{
|
|
invalidRun(),
|
|
invalidRun(),
|
|
},
|
|
runIDs: []string{"run-bad-1", "run-bad-2"},
|
|
}
|
|
|
|
srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama"}, fake, nil)
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"metadata":{},
|
|
"model":"qwen3.6:35b",
|
|
"messages":[{"role":"user","content":"status"}],
|
|
"tools":[{"type":"function","function":{"name":"run_commands","parameters":{"type":"object"}}}],
|
|
"tool_choice":"auto",
|
|
"stream": true
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
body := w.Body.String()
|
|
if strings.Contains(body, "<tool_call") {
|
|
t.Fatalf("body should not contain tool_call leak, got: %s", body)
|
|
}
|
|
if !strings.Contains(body, "tool_validation_error") {
|
|
t.Fatalf("expected tool_validation_error, got: %s", body)
|
|
}
|
|
if reqs := fake.reqsSnapshot(); len(reqs) != 2 {
|
|
t.Fatalf("submit attempts: got %d want 2", len(reqs))
|
|
}
|
|
}
|
|
|
|
func TestValidateToolCallResponseValidatesEveryToolCall(t *testing.T) {
|
|
contract := toolValidationContract{
|
|
enabled: true,
|
|
specs: map[string]textToolSpec{
|
|
"run_commands": {
|
|
parameters: map[string]any{
|
|
"type": "object",
|
|
"properties": map[string]any{
|
|
"commands": map[string]any{
|
|
"type": "array",
|
|
"items": map[string]any{
|
|
"type": "string",
|
|
},
|
|
},
|
|
},
|
|
"required": []any{"commands"},
|
|
},
|
|
},
|
|
"read_file": {
|
|
parameters: map[string]any{
|
|
"type": "object",
|
|
"properties": map[string]any{
|
|
"path": map[string]any{
|
|
"type": "string",
|
|
},
|
|
},
|
|
"required": []any{"path"},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
invalidFirst := []any{
|
|
map[string]any{
|
|
"id": "call_1",
|
|
"type": "object",
|
|
"function": map[string]any{
|
|
"name": "run_commands",
|
|
"arguments": `{"not_commands":[]}`,
|
|
},
|
|
},
|
|
map[string]any{
|
|
"id": "call_2",
|
|
"type": "object",
|
|
"function": map[string]any{
|
|
"name": "read_file",
|
|
"arguments": `{"path":"/a/b"}`,
|
|
},
|
|
},
|
|
}
|
|
|
|
err := validateToolCallResponse(contract, invalidFirst, "native")
|
|
if err == nil {
|
|
t.Fatal("expected validation error because first tool call is invalid")
|
|
}
|
|
if !strings.Contains(err.Error(), "tool call 0") || !strings.Contains(err.Error(), "commands is required") {
|
|
t.Fatalf("unexpected error message: %v", err)
|
|
}
|
|
|
|
allValid := []any{
|
|
map[string]any{
|
|
"id": "call_1",
|
|
"type": "object",
|
|
"function": map[string]any{
|
|
"name": "run_commands",
|
|
"arguments": `{"commands":["ls"]}`,
|
|
},
|
|
},
|
|
map[string]any{
|
|
"id": "call_2",
|
|
"type": "object",
|
|
"function": map[string]any{
|
|
"name": "read_file",
|
|
"arguments": `{"path":"/a/b"}`,
|
|
},
|
|
},
|
|
}
|
|
if err := validateToolCallResponse(contract, allValid, "native"); err != nil {
|
|
t.Fatalf("expected no validation error, got: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestChatCompletionsFailsWhenAnySynthesizedTextToolCallViolatesSchema(t *testing.T) {
|
|
invalidComplete := func() chan *iop.RunEvent {
|
|
ch := make(chan *iop.RunEvent, 2)
|
|
ch <- &iop.RunEvent{Type: "delta", Delta: `<tool_call><function=run_commands><parameter=not_commands>["ls"]</parameter></function></tool_call><tool_call><function=read_file><parameter=path>"/a/b"</parameter></function></tool_call>`}
|
|
ch <- &iop.RunEvent{Type: "complete"}
|
|
return ch
|
|
}
|
|
fake := &fakeRunService{
|
|
eventRuns: []chan *iop.RunEvent{
|
|
invalidComplete(),
|
|
invalidComplete(),
|
|
},
|
|
runIDs: []string{"run-bad-1", "run-bad-2"},
|
|
}
|
|
|
|
srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama"}, fake, nil)
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"metadata":{},
|
|
"model":"qwen3.6:35b",
|
|
"messages":[{"role":"user","content":"status"}],
|
|
"tools":[
|
|
{"type":"function","function":{"name":"run_commands","parameters":{"type":"object","properties":{"commands":{"type":"array","items":{"type":"string"}}},"required":["commands"]}}},
|
|
{"type":"function","function":{"name":"read_file","parameters":{"type":"object","properties":{"path":{"type":"string"}},"required":["path"]}}}
|
|
],
|
|
"tool_choice":"auto"
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
if w.Code != http.StatusBadGateway {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
if !strings.Contains(w.Body.String(), `"type":"tool_validation_error"`) {
|
|
t.Fatalf("expected tool validation error body, got %s", w.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestResponsesMetadataIncludesTypedEstimateAndClassification(t *testing.T) {
|
|
fake := &fakeRunService{events: make(chan *iop.RunEvent, 2)}
|
|
fake.events <- &iop.RunEvent{Type: "delta", Delta: "ok"}
|
|
fake.events <- &iop.RunEvent{Type: "complete"}
|
|
|
|
srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama"}, fake, nil)
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(`{
|
|
"model":"client-model",
|
|
"input":"test",
|
|
"metadata":{"request_id":"req-001"}
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.routes().ServeHTTP(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
estStr := fake.req.Metadata["estimated_input_tokens"]
|
|
if estStr == "" {
|
|
t.Fatalf("estimated_input_tokens should be present")
|
|
}
|
|
ctxClass := fake.req.Metadata["context_class"]
|
|
if ctxClass != "normal" {
|
|
t.Fatalf("context_class: got %q, want normal", ctxClass)
|
|
}
|
|
est := fake.req.EstimatedInputTokens
|
|
if est <= 0 {
|
|
t.Fatalf("EstimatedInputTokens: got %d", est)
|
|
}
|
|
if fake.req.ContextClass != "normal" {
|
|
t.Fatalf("ContextClass: got %q, want normal", fake.req.ContextClass)
|
|
}
|
|
}
|
|
|
|
func TestResponsesMetadataIncludesTypedEstimateAndClassification_LargePayload(t *testing.T) {
|
|
largeInput := make([]byte, 400000)
|
|
for i := range largeInput {
|
|
largeInput[i] = 'x'
|
|
}
|
|
|
|
fake := &fakeRunService{events: make(chan *iop.RunEvent, 2)}
|
|
fake.events <- &iop.RunEvent{Type: "delta", Delta: "ok"}
|
|
fake.events <- &iop.RunEvent{Type: "complete"}
|
|
|
|
srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama"}, fake, nil)
|
|
body := `{"model":"client-model","input":"` + string(largeInput) + `"}`
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(body))
|
|
w := httptest.NewRecorder()
|
|
srv.routes().ServeHTTP(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
if fake.req.ContextClass != "long" {
|
|
t.Fatalf("context_class: got %q, want long", fake.req.ContextClass)
|
|
}
|
|
if fake.req.EstimatedInputTokens < 100000 {
|
|
t.Fatalf("EstimatedInputTokens: got %d, want >= 100000", fake.req.EstimatedInputTokens)
|
|
}
|
|
}
|
|
|
|
func TestChatCompletionsMetadataIncludesTypedEstimateAndClassification(t *testing.T) {
|
|
fake := &fakeRunService{events: make(chan *iop.RunEvent, 2)}
|
|
fake.events <- &iop.RunEvent{Type: "delta", Delta: "ok"}
|
|
fake.events <- &iop.RunEvent{Type: "complete"}
|
|
|
|
srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama"}, fake, nil)
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"client-model",
|
|
"messages":[{"role":"user","content":"hi"}]
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
estStr := fake.req.Metadata["estimated_input_tokens"]
|
|
if estStr == "" {
|
|
t.Fatalf("estimated_input_tokens should be present")
|
|
}
|
|
ctxClass := fake.req.Metadata["context_class"]
|
|
if ctxClass != "normal" {
|
|
t.Fatalf("context_class: got %q, want normal", ctxClass)
|
|
}
|
|
est := fake.req.EstimatedInputTokens
|
|
if est <= 0 {
|
|
t.Fatalf("EstimatedInputTokens: got %d", est)
|
|
}
|
|
if fake.req.ContextClass != "normal" {
|
|
t.Fatalf("ContextClass: got %q, want normal", fake.req.ContextClass)
|
|
}
|
|
}
|
|
|
|
func TestChatCompletionsAssembledLogs(t *testing.T) {
|
|
core, observed := observer.New(zap.DebugLevel)
|
|
logger := zap.New(core)
|
|
|
|
fake := &fakeRunService{events: make(chan *iop.RunEvent, 4)}
|
|
fake.events <- &iop.RunEvent{Type: "delta", Delta: "hello"}
|
|
fake.events <- &iop.RunEvent{Type: "reasoning_delta", Delta: "reasoning..."}
|
|
fake.events <- &iop.RunEvent{Type: "complete", Metadata: map[string]string{
|
|
"openai_tool_calls": `[{"id":"call_001","type":"function","function":{"name":"get_weather","arguments":"{}"}}]`,
|
|
}}
|
|
|
|
srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama"}, fake, logger)
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"client-model",
|
|
"messages":[{"role":"user","content":"hi"}],
|
|
"tools":[{"type":"function","function":{"name":"get_weather"}}]
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
found := false
|
|
for _, entry := range observed.All() {
|
|
if entry.Message == "openai chat completion output" {
|
|
found = true
|
|
m := entry.ContextMap()
|
|
if m["response_mode"] != "normalized" {
|
|
t.Errorf("expected response_mode=normalized, got %v", m["response_mode"])
|
|
}
|
|
if m["assembled_content"] != "hello" {
|
|
t.Errorf("expected assembled_content=hello, got %v", m["assembled_content"])
|
|
}
|
|
if m["assembled_reasoning"] != "reasoning..." {
|
|
t.Errorf("expected assembled_reasoning=reasoning..., got %v", m["assembled_reasoning"])
|
|
}
|
|
tc, ok := m["assembled_tool_calls"].([]interface{})
|
|
if !ok || len(tc) != 1 || tc[0] != "get_weather" {
|
|
t.Errorf("expected assembled_tool_calls=[get_weather], got %v", m["assembled_tool_calls"])
|
|
}
|
|
if count, ok := m["assembled_tool_call_count"].(int64); !ok || count != 1 {
|
|
t.Errorf("expected assembled_tool_call_count=1, got %v", m["assembled_tool_call_count"])
|
|
}
|
|
}
|
|
}
|
|
if !found {
|
|
t.Error("expected log message 'openai chat completion output' not found")
|
|
}
|
|
}
|
|
|
|
// TestChatCompletionsAssembledLogsNormalized verifies the normalized-path
|
|
// completion log reports the internal response_mode execution label
|
|
// (normalized) and the assembled observation fields.
|
|
func TestChatCompletionsAssembledLogsNormalized(t *testing.T) {
|
|
core, observed := observer.New(zap.DebugLevel)
|
|
logger := zap.New(core)
|
|
|
|
fake := &fakeRunService{events: make(chan *iop.RunEvent, 4)}
|
|
fake.events <- &iop.RunEvent{Type: "delta", Delta: "hello normalized"}
|
|
fake.events <- &iop.RunEvent{Type: "reasoning_delta", Delta: "reasoning normalized..."}
|
|
fake.events <- &iop.RunEvent{Type: "complete", Metadata: map[string]string{
|
|
"openai_tool_calls": `[{"id":"call_002","type":"function","function":{"name":"run_code","arguments":"{}"}}]`,
|
|
}}
|
|
|
|
srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama"}, fake, logger)
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"client-model",
|
|
"messages":[{"role":"user","content":"hi"}],
|
|
"tools":[{"type":"function","function":{"name":"run_code"}}]
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
found := false
|
|
for _, entry := range observed.All() {
|
|
if entry.Message == "openai chat completion output" {
|
|
found = true
|
|
m := entry.ContextMap()
|
|
if m["response_mode"] != "normalized" {
|
|
t.Errorf("expected response_mode=normalized, got %v", m["response_mode"])
|
|
}
|
|
if m["assembled_content"] != "hello normalized" {
|
|
t.Errorf("expected assembled_content=hello normalized, got %v", m["assembled_content"])
|
|
}
|
|
if m["assembled_reasoning"] != "reasoning normalized..." {
|
|
t.Errorf("expected assembled_reasoning=reasoning normalized..., got %v", m["assembled_reasoning"])
|
|
}
|
|
tc, ok := m["assembled_tool_calls"].([]interface{})
|
|
if !ok || len(tc) != 1 || tc[0] != "run_code" {
|
|
t.Errorf("expected assembled_tool_calls=[run_code], got %v", m["assembled_tool_calls"])
|
|
}
|
|
if count, ok := m["assembled_tool_call_count"].(int64); !ok || count != 1 {
|
|
t.Errorf("expected assembled_tool_call_count=1, got %v", m["assembled_tool_call_count"])
|
|
}
|
|
}
|
|
}
|
|
if !found {
|
|
t.Error("expected log message 'openai chat completion output' not found")
|
|
}
|
|
}
|
|
|
|
func TestChatCompletionsAssembledLogsPassthrough(t *testing.T) {
|
|
core, observed := observer.New(zap.DebugLevel)
|
|
logger := zap.New(core)
|
|
|
|
providerBody := `{"choices":[{"message":{"content":"passthrough content","reasoning_content":"passthrough reasoning","tool_calls":[{"function":{"name":"call_passthrough"}}]}}]}`
|
|
frames := make(chan *iop.ProviderTunnelFrame, 3)
|
|
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START, StatusCode: 200}
|
|
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(providerBody)}
|
|
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true}
|
|
close(frames)
|
|
|
|
fake := &fakeRunService{tunnelFrames: frames}
|
|
catalog := []config.ModelCatalogEntry{
|
|
{ID: "pool-model", Providers: map[string]string{"prov-1": "served-model"}},
|
|
}
|
|
srv := NewServer(config.EdgeOpenAIConf{}, fake, logger)
|
|
srv.SetModelCatalog(catalog)
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"pool-model",
|
|
"messages":[{"role":"user","content":"hello"}]
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
found := false
|
|
for _, entry := range observed.All() {
|
|
if entry.Message == "openai chat completion passthrough closed" {
|
|
found = true
|
|
m := entry.ContextMap()
|
|
if m["assembled_content"] != "passthrough content" {
|
|
t.Errorf("expected assembled_content='passthrough content', got %v", m["assembled_content"])
|
|
}
|
|
if m["assembled_reasoning"] != "passthrough reasoning" {
|
|
t.Errorf("expected assembled_reasoning='passthrough reasoning', got %v", m["assembled_reasoning"])
|
|
}
|
|
tc, ok := m["assembled_tool_calls"].([]interface{})
|
|
if !ok || len(tc) != 1 || tc[0] != "call_passthrough" {
|
|
t.Errorf("expected assembled_tool_calls=[call_passthrough], got %v", m["assembled_tool_calls"])
|
|
}
|
|
if count, ok := m["assembled_tool_call_count"].(int64); !ok || count != 1 {
|
|
t.Errorf("expected assembled_tool_call_count=1, got %v", m["assembled_tool_call_count"])
|
|
}
|
|
}
|
|
}
|
|
if !found {
|
|
t.Error("expected log message 'openai chat completion passthrough closed' not found")
|
|
}
|
|
}
|
|
|
|
func TestChatCompletionsProviderPoolOllamaSelectionUsesNormalizedRun(t *testing.T) {
|
|
runEvents := make(chan *iop.RunEvent, 2)
|
|
runEvents <- &iop.RunEvent{Type: "complete"}
|
|
close(runEvents)
|
|
|
|
fake := &fakeRunService{
|
|
poolDispatchPath: string(edgeservice.ProviderPoolPathNormalized),
|
|
poolRunFrames: runEvents,
|
|
}
|
|
catalog := []config.ModelCatalogEntry{
|
|
{ID: "ollama-model", Providers: map[string]string{"prov-ollama": "qwen3.6:35b"}},
|
|
}
|
|
srv := NewServer(config.EdgeOpenAIConf{}, fake, nil)
|
|
srv.SetModelCatalog(catalog)
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"ollama-model",
|
|
"messages":[{"role":"user","content":"hello"}]
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
|
|
// Ollama-selected provider-pool should use normalized path (SubmitRun),
|
|
// NOT tunnel passthrough.
|
|
tunnelReqs := fake.tunnelReqsSnapshot()
|
|
reqs := fake.reqsSnapshot()
|
|
if len(tunnelReqs) != 0 {
|
|
t.Fatalf("expected 0 tunnel dispatches for Ollama-normalized path, got %d", len(tunnelReqs))
|
|
}
|
|
if len(reqs) != 1 {
|
|
t.Fatalf("expected exactly 1 normalized SubmitRun for Ollama path, got %d", len(reqs))
|
|
}
|
|
}
|
|
|
|
// TestChatCompletionsProviderPoolTunnelSelectionUsesPassthrough verifies that
|
|
// when a provider-pool catalog match is resolved and the selected provider is
|
|
// an OpenAI-compatible type (tunnel executionPath), the Chat handler dispatches
|
|
// via tunnel passthrough.
|
|
func TestChatCompletionsProviderPoolTunnelSelectionUsesPassthrough(t *testing.T) {
|
|
frames := make(chan *iop.ProviderTunnelFrame, 3)
|
|
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START, StatusCode: 200}
|
|
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(`{"ok":true}`)}
|
|
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true}
|
|
close(frames)
|
|
|
|
fake := &fakeRunService{
|
|
poolDispatchPath: string(edgeservice.ProviderPoolPathTunnel),
|
|
tunnelFrames: frames,
|
|
}
|
|
catalog := []config.ModelCatalogEntry{
|
|
{ID: "vllm-model", Providers: map[string]string{"prov-vllm": "Qwen3-35B"}},
|
|
}
|
|
srv := NewServer(config.EdgeOpenAIConf{}, fake, nil)
|
|
srv.SetModelCatalog(catalog)
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"vllm-model",
|
|
"messages":[{"role":"user","content":"hello"}]
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
// OpenAI-compatible provider-pool should use tunnel passthrough.
|
|
tunnelReqs := fake.tunnelReqsSnapshot()
|
|
reqs := fake.reqsSnapshot()
|
|
if len(tunnelReqs) != 1 {
|
|
t.Fatalf("expected 1 tunnel dispatch for vLLM path, got %d", len(tunnelReqs))
|
|
}
|
|
if !tunnelReqs[0].ProviderPool {
|
|
t.Error("tunnel dispatch must have ProviderPool=true")
|
|
}
|
|
if len(reqs) != 0 {
|
|
t.Fatalf("expected 0 normalized SubmitRun for vLLM tunnel path, got %d", len(reqs))
|
|
}
|
|
}
|
|
|
|
// TestChatCompletionsProviderPoolNormalizedToolValidationRetryPreservesPoolRequest
|
|
// verifies SDD S01: when a provider-pool normalized run fails tool validation,
|
|
// the retry calls SubmitProviderPool (not SubmitRun) to preserve ModelGroupKey,
|
|
// provider-pool metadata, and input. The retry metadata includes
|
|
// iop_tool_validation_attempt=2.
|
|
func TestChatCompletionsProviderPoolNormalizedToolValidationRetryPreservesPoolRequest(t *testing.T) {
|
|
badToolEvent := &iop.RunEvent{
|
|
Type: "complete",
|
|
Metadata: map[string]string{
|
|
"finish_reason": "tool_calls",
|
|
runtimeMetadataOpenAIToolCalls: `[{"id":"call_bad","type":"function","function":{"name":"run_commands","arguments":"{\"commands\":\"git status\"}"}}]`,
|
|
},
|
|
}
|
|
goodToolEvent := &iop.RunEvent{
|
|
Type: "complete",
|
|
Metadata: map[string]string{
|
|
"finish_reason": "tool_calls",
|
|
runtimeMetadataOpenAIToolCalls: `[{"id":"call_good","type":"function","function":{"name":"run_commands","arguments":"{\"commands\":[\"git status\"]}"}}]`,
|
|
},
|
|
}
|
|
|
|
badRun := &fakeRunResult{
|
|
dispatch: edgeservice.RunDispatch{
|
|
RunID: "run-pool-bad",
|
|
NodeID: "node-pool",
|
|
ModelGroupKey: "pool-group",
|
|
Adapter: "ollama",
|
|
Target: "served",
|
|
},
|
|
events: bufferedRunEvents(badToolEvent),
|
|
}
|
|
goodRun := &fakeRunResult{
|
|
dispatch: edgeservice.RunDispatch{
|
|
RunID: "run-pool-good",
|
|
NodeID: "node-pool",
|
|
ModelGroupKey: "pool-group",
|
|
Adapter: "ollama",
|
|
Target: "served",
|
|
},
|
|
events: bufferedRunEvents(goodToolEvent),
|
|
}
|
|
|
|
fake := &fakeRunService{
|
|
poolSubmitResults: []edgeservice.ProviderPoolDispatchResult{
|
|
{Path: edgeservice.ProviderPoolPathNormalized, Run: badRun, DispatchInfo: edgeservice.RunDispatch{RunID: "run-pool-bad", NodeID: "node-pool", ModelGroupKey: "pool-group", Adapter: "ollama", Target: "served"}},
|
|
{Path: edgeservice.ProviderPoolPathNormalized, Run: goodRun, DispatchInfo: edgeservice.RunDispatch{RunID: "run-pool-good", NodeID: "node-pool", ModelGroupKey: "pool-group", Adapter: "ollama", Target: "served"}},
|
|
},
|
|
}
|
|
catalog := []config.ModelCatalogEntry{
|
|
{ID: "pool-ollama-model", Providers: map[string]string{"prov-ollama": "qwen3.6:35b"}},
|
|
}
|
|
srv := NewServer(config.EdgeOpenAIConf{}, fake, nil)
|
|
srv.SetModelCatalog(catalog)
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"pool-ollama-model",
|
|
"messages":[{"role":"user","content":"status"}],
|
|
"tools":[{"type":"function","function":{"name":"run_commands","parameters":{"type":"object","properties":{"commands":{"type":"array","items":{"type":"string"}}},"required":["commands"],"additionalProperties":false}}}],
|
|
"tool_choice":"auto"
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
|
|
// Two SubmitProviderPool calls: initial + 1 retry.
|
|
reqs := fake.reqsSnapshot()
|
|
if len(reqs) != 2 {
|
|
t.Fatalf("expected 2 SubmitRun (initial + retry), got %d", len(reqs))
|
|
}
|
|
// Both must be provider-pool dispatches.
|
|
if !reqs[0].ProviderPool {
|
|
t.Error("first req must have ProviderPool=true")
|
|
}
|
|
if !reqs[1].ProviderPool {
|
|
t.Error("retry req must have ProviderPool=true")
|
|
}
|
|
// The model field must be preserved (submitted via SubmitProviderPool, not SubmitRun).
|
|
if reqs[0].ModelGroupKey != "pool-ollama-model" {
|
|
t.Errorf("first req ModelGroupKey: got %q", reqs[0].ModelGroupKey)
|
|
}
|
|
if reqs[1].ModelGroupKey != "pool-ollama-model" {
|
|
t.Errorf("retry req ModelGroupKey: got %q", reqs[1].ModelGroupKey)
|
|
}
|
|
// Retry metadata must indicate attempt 2.
|
|
if got := reqs[1].Metadata[runtimeMetadataToolValidationAttempt]; got != "2" {
|
|
t.Errorf("retry attempt metadata: got %q", got)
|
|
}
|
|
if got := reqs[1].Metadata[runtimeMetadataToolValidationRetryOf]; got != "run-pool-bad" {
|
|
t.Errorf("retry retry-of metadata: got %q", got)
|
|
}
|
|
}
|
|
|
|
// TestChatCompletionsProviderPoolToolValidationRetryRejectsTunnelResult verifies
|
|
// that when a provider-pool retry returns a tunnel-path result (which would
|
|
// cause a nil Run panic downstream), the handler closes the tunnel handle and
|
|
// returns a tool_validation_retry_error instead of panicking.
|
|
func TestChatCompletionsProviderPoolToolValidationRetryRejectsTunnelResult(t *testing.T) {
|
|
// First run: produces a tool validation failure.
|
|
badToolEvent := &iop.RunEvent{
|
|
Type: "complete",
|
|
Metadata: map[string]string{
|
|
"finish_reason": "tool_calls",
|
|
runtimeMetadataOpenAIToolCalls: `[{"id":"call_bad","type":"function","function":{"name":"run_commands","arguments":"{\"commands\":\"git status\"}"}}]`,
|
|
},
|
|
}
|
|
badRun := &fakeRunResult{
|
|
dispatch: edgeservice.RunDispatch{
|
|
RunID: "run-pool-bad",
|
|
NodeID: "node-pool",
|
|
ModelGroupKey: "pool-group",
|
|
Adapter: "ollama",
|
|
Target: "served",
|
|
},
|
|
events: bufferedRunEvents(badToolEvent),
|
|
}
|
|
|
|
// The retry result is tunnel-path: this would set Run=nil and cause a panic
|
|
// without the guard. We create a fake tunnel handle so the guard can close it.
|
|
tunnelFrames := make(chan *iop.ProviderTunnelFrame, 4)
|
|
tunnelFrames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START, StatusCode: 200}
|
|
tunnelFrames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(`{"ok":true}`)}
|
|
tunnelFrames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true}
|
|
close(tunnelFrames)
|
|
|
|
fakeTunnel := &fakeTunnelHandle{
|
|
dispatch: edgeservice.RunDispatch{
|
|
RunID: "run-pool-tunnel-retry",
|
|
NodeID: "node-pool",
|
|
ModelGroupKey: "pool-group",
|
|
Adapter: "vllm",
|
|
Target: "served",
|
|
},
|
|
headers: map[string]string{},
|
|
frames: tunnelFrames,
|
|
waitTimeout: 5 * time.Second,
|
|
}
|
|
|
|
fake := &fakeRunService{
|
|
poolSubmitResults: []edgeservice.ProviderPoolDispatchResult{
|
|
{Path: edgeservice.ProviderPoolPathNormalized, Run: badRun, DispatchInfo: edgeservice.RunDispatch{RunID: "run-pool-bad", NodeID: "node-pool", ModelGroupKey: "pool-group", Adapter: "ollama", Target: "served"}},
|
|
{Path: edgeservice.ProviderPoolPathTunnel, Tunnel: fakeTunnel, DispatchInfo: edgeservice.RunDispatch{RunID: "run-pool-tunnel-retry", NodeID: "node-pool", ModelGroupKey: "pool-group", Adapter: "vllm", Target: "served"}},
|
|
},
|
|
}
|
|
catalog := []config.ModelCatalogEntry{
|
|
{ID: "pool-ollama-model", Providers: map[string]string{"prov-ollama": "qwen3.6:35b"}},
|
|
}
|
|
srv := NewServer(config.EdgeOpenAIConf{}, fake, nil)
|
|
srv.SetModelCatalog(catalog)
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"pool-ollama-model",
|
|
"messages":[{"role":"user","content":"status"}],
|
|
"tools":[{"type":"function","function":{"name":"run_commands","parameters":{"type":"object","properties":{"commands":{"type":"array","items":{"type":"string"}}},"required":["commands"],"additionalProperties":false}}}],
|
|
"tool_choice":"auto"
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
// The handler must reject the tunnel-path retry result with 502.
|
|
if w.Code != http.StatusBadGateway {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
var resp map[string]any
|
|
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
|
t.Fatalf("decode response: %v", err)
|
|
}
|
|
errMap, ok := resp["error"].(map[string]any)
|
|
if !ok {
|
|
t.Fatalf("expected error map in response: %s", w.Body.String())
|
|
}
|
|
if errMap["type"] != "tool_validation_retry_error" {
|
|
t.Errorf("error type: got %q", errMap["type"])
|
|
}
|
|
if errMap["message"] != "provider-pool retry selected tunnel path" {
|
|
t.Errorf("error message: got %q", errMap["message"])
|
|
}
|
|
}
|
|
|
|
// --- MIXED_GROUP-2: Mixed/Ollama-Only Provider Pool Fixtures ---
|
|
|
|
// TestChatCompletionsMixedProviderPoolTunnelSelection verifies S05: a mixed
|
|
// model group containing both OpenAI-compatible and Ollama providers dispatches
|
|
// the selected tunnel-capable provider over the provider tunnel (passthrough)
|
|
// path. Only one tunnel request is sent, no normalized SubmitRun is called.
|
|
func TestChatCompletionsMixedProviderPoolTunnelSelection(t *testing.T) {
|
|
// Tunnel frames for the tunnel path.
|
|
tunnelFrames := make(chan *iop.ProviderTunnelFrame, 3)
|
|
tunnelFrames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START, StatusCode: 200}
|
|
tunnelFrames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(`{"id":"cmpl-mixed","choices":[{"index":0,"message":{"role":"assistant","content":"tunnel"}}],"finish_reason":"stop"}`)}
|
|
tunnelFrames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true}
|
|
close(tunnelFrames)
|
|
|
|
// Fake service selects tunnel path (simulating vLLM provider selection).
|
|
fake := &fakeRunService{
|
|
poolDispatchPath: string(edgeservice.ProviderPoolPathTunnel),
|
|
tunnelFrames: tunnelFrames,
|
|
}
|
|
|
|
// Mixed catalog: OpenAI-compatible (vLLM) + Ollama in same group.
|
|
catalog := []config.ModelCatalogEntry{
|
|
{
|
|
ID: "mixed-model",
|
|
Providers: map[string]string{
|
|
"prov-vllm": "served-vllm",
|
|
"prov-ollama": "served-ollama",
|
|
},
|
|
},
|
|
}
|
|
srv := NewServer(config.EdgeOpenAIConf{}, fake, nil)
|
|
srv.SetModelCatalog(catalog)
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"mixed-model",
|
|
"messages":[{"role":"user","content":"hello"}]
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
|
|
// Tunnel path: exactly 1 tunnel dispatch, 0 normalized SubmitRun.
|
|
tunnelReqs := fake.tunnelReqsSnapshot()
|
|
if len(tunnelReqs) != 1 {
|
|
t.Fatalf("expected 1 tunnel dispatch, got %d", len(tunnelReqs))
|
|
}
|
|
if !tunnelReqs[0].ProviderPool {
|
|
t.Error("tunnel request must have ProviderPool=true")
|
|
}
|
|
if tunnelReqs[0].ModelGroupKey != "mixed-model" {
|
|
t.Errorf("tunnel ModelGroupKey: got %q, want mixed-model", tunnelReqs[0].ModelGroupKey)
|
|
}
|
|
|
|
runReqs := fake.reqsSnapshot()
|
|
if len(runReqs) != 0 {
|
|
t.Errorf("expected 0 normalized SubmitRun calls, got %d", len(runReqs))
|
|
}
|
|
}
|
|
|
|
// TestChatCompletionsMixedProviderPoolOllamaSelection verifies S05: when the
|
|
// mixed model group selects an Ollama provider, the request is dispatched over
|
|
// the normalized path (SubmitRun), and no tunnel dispatch is issued.
|
|
func TestChatCompletionsMixedProviderPoolOllamaSelection(t *testing.T) {
|
|
// Normalized path events.
|
|
events := bufferedRunEvents(
|
|
&iop.RunEvent{Type: "delta", Delta: "ollama"},
|
|
&iop.RunEvent{Type: "complete"},
|
|
)
|
|
|
|
// Fake service selects normalized path (simulating Ollama provider selection).
|
|
fake := &fakeRunService{
|
|
poolDispatchPath: string(edgeservice.ProviderPoolPathNormalized),
|
|
events: events,
|
|
}
|
|
|
|
// Same mixed catalog: vLLM + Ollama.
|
|
catalog := []config.ModelCatalogEntry{
|
|
{
|
|
ID: "mixed-model",
|
|
Providers: map[string]string{
|
|
"prov-vllm": "served-vllm",
|
|
"prov-ollama": "served-ollama",
|
|
},
|
|
},
|
|
}
|
|
srv := NewServer(config.EdgeOpenAIConf{}, fake, nil)
|
|
srv.SetModelCatalog(catalog)
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"mixed-model",
|
|
"messages":[{"role":"user","content":"hello"}]
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
|
|
// Normalized path: 0 tunnel dispatches, 1 normalized SubmitRun.
|
|
tunnelReqs := fake.tunnelReqsSnapshot()
|
|
if len(tunnelReqs) != 0 {
|
|
t.Fatalf("expected 0 tunnel dispatches for Ollama selection, got %d", len(tunnelReqs))
|
|
}
|
|
|
|
runReqs := fake.reqsSnapshot()
|
|
if len(runReqs) != 1 {
|
|
t.Fatalf("expected 1 normalized SubmitRun, got %d", len(runReqs))
|
|
}
|
|
// Adapter/Target are rewritten by the service's admitted candidate; the
|
|
// fake here records the pre-admission request so the handler-level path
|
|
// selection (tunnel vs normalized) is the assertion surface.
|
|
if runReqs[0].ModelGroupKey != "mixed-model" {
|
|
t.Errorf("normalized ModelGroupKey: got %q, want mixed-model", runReqs[0].ModelGroupKey)
|
|
}
|
|
if !runReqs[0].ProviderPool {
|
|
t.Error("normalized run must have ProviderPool=true")
|
|
}
|
|
}
|
|
|
|
// TestChatCompletionsOllamaOnlyProviderPoolUsesNormalizedPath verifies S05:
|
|
// a model group containing only Ollama providers always dispatches over the
|
|
// normalized path. No tunnel request is ever issued.
|
|
func TestChatCompletionsOllamaOnlyProviderPoolUsesNormalizedPath(t *testing.T) {
|
|
events := bufferedRunEvents(
|
|
&iop.RunEvent{Type: "delta", Delta: "ollama-only"},
|
|
&iop.RunEvent{Type: "complete"},
|
|
)
|
|
|
|
// Fake service selects normalized path.
|
|
fake := &fakeRunService{
|
|
poolDispatchPath: string(edgeservice.ProviderPoolPathNormalized),
|
|
events: events,
|
|
}
|
|
|
|
// Ollama-only catalog.
|
|
catalog := []config.ModelCatalogEntry{
|
|
{
|
|
ID: "ollama-only-model",
|
|
Providers: map[string]string{
|
|
"prov-ollama": "qwen3.6:35b",
|
|
},
|
|
},
|
|
}
|
|
srv := NewServer(config.EdgeOpenAIConf{}, fake, nil)
|
|
srv.SetModelCatalog(catalog)
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"ollama-only-model",
|
|
"messages":[{"role":"user","content":"hello"}]
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
|
|
// Must not dispatch tunnel.
|
|
tunnelReqs := fake.tunnelReqsSnapshot()
|
|
if len(tunnelReqs) != 0 {
|
|
t.Fatalf("ollama-only must not dispatch tunnel, got %d", len(tunnelReqs))
|
|
}
|
|
|
|
// Must dispatch exactly one normalized run.
|
|
runReqs := fake.reqsSnapshot()
|
|
if len(runReqs) != 1 {
|
|
t.Fatalf("expected 1 normalized SubmitRun, got %d", len(runReqs))
|
|
}
|
|
if runReqs[0].ModelGroupKey != "ollama-only-model" {
|
|
t.Errorf("ollama-only ModelGroupKey: got %q, want ollama-only-model", runReqs[0].ModelGroupKey)
|
|
}
|
|
if !runReqs[0].ProviderPool {
|
|
t.Error("ollama-only normalized run must have ProviderPool=true")
|
|
}
|
|
}
|
|
|
|
// TestChatCompletionsProviderPoolTunnelPreservesUnknownFields verifies that
|
|
// when the provider-pool dispatch selects the tunnel path, the caller's raw
|
|
// body is forwarded with model rewrite only: unknown top-level fields
|
|
// (store, codex_trace, custom_provider_options, nested unknown object) and
|
|
// Codex/provider-specific fields survive into the tunnel request body. No
|
|
// IOP sideband key is injected into the pure response payload (SURFACE_FIELD).
|
|
func TestChatCompletionsProviderPoolTunnelPreservesUnknownFields(t *testing.T) {
|
|
frames := make(chan *iop.ProviderTunnelFrame, 3)
|
|
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START, StatusCode: 200}
|
|
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(`{"ok":true}`)}
|
|
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true}
|
|
close(frames)
|
|
fake := &fakeRunService{
|
|
tunnelFrames: frames,
|
|
tunnelServedTarget: "served-model",
|
|
}
|
|
|
|
catalog := []config.ModelCatalogEntry{
|
|
{ID: "pool-model", Providers: map[string]string{"prov-vllm": "served-model"}},
|
|
}
|
|
srv := NewServer(config.EdgeOpenAIConf{}, fake, nil)
|
|
srv.SetModelCatalog(catalog)
|
|
|
|
reqBody := `{
|
|
"model":"pool-model",
|
|
"messages":[{"role":"user","content":"hello"}],
|
|
"store":true,
|
|
"parallel_tool_calls":true,
|
|
"codex_trace":{"enabled":true},
|
|
"custom_provider_options":{"timeout_ms":5000},
|
|
"nested":{"unknown":{"deep":{"value":42}}},
|
|
"thinking_token_budget":1024
|
|
}`
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(reqBody))
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
|
|
tunnelReqs := fake.tunnelReqsSnapshot()
|
|
if len(tunnelReqs) != 1 {
|
|
t.Fatalf("expected 1 tunnel dispatch, got %d", len(tunnelReqs))
|
|
}
|
|
|
|
// Parse the tunnel body as a JSON map and verify model rewrite + unknown preservation.
|
|
tunnelBodies := fake.tunnelBodiesSnapshot()
|
|
if len(tunnelBodies) != 1 {
|
|
t.Fatalf("expected 1 tunnel body, got %d", len(tunnelBodies))
|
|
}
|
|
var bodyMap map[string]json.RawMessage
|
|
if err := json.Unmarshal(tunnelBodies[0], &bodyMap); err != nil {
|
|
t.Fatalf("decode tunnel body: %v body=%s", err, string(tunnelBodies[0]))
|
|
}
|
|
|
|
// Model must be rewritten to the served target.
|
|
var bodyModel string
|
|
if err := json.Unmarshal(bodyMap["model"], &bodyModel); err != nil {
|
|
t.Fatalf("decode model field: %v", err)
|
|
}
|
|
if bodyModel != "served-model" {
|
|
t.Errorf("model rewrite: got %q, want served-model", bodyModel)
|
|
}
|
|
|
|
// Known fields must survive.
|
|
if _, ok := bodyMap["messages"]; !ok {
|
|
t.Error("messages must be preserved in tunnel body")
|
|
}
|
|
if _, ok := bodyMap["store"]; !ok {
|
|
t.Error("store field must be preserved in tunnel body")
|
|
}
|
|
if _, ok := bodyMap["parallel_tool_calls"]; !ok {
|
|
t.Error("parallel_tool_calls field must be preserved in tunnel body")
|
|
}
|
|
if _, ok := bodyMap["codex_trace"]; !ok {
|
|
t.Error("codex_trace field must be preserved in tunnel body")
|
|
}
|
|
if _, ok := bodyMap["custom_provider_options"]; !ok {
|
|
t.Error("custom_provider_options field must be preserved in tunnel body")
|
|
}
|
|
if _, ok := bodyMap["nested"]; !ok {
|
|
t.Error("nested unknown object must be preserved in tunnel body")
|
|
}
|
|
if _, ok := bodyMap["thinking_token_budget"]; !ok {
|
|
t.Error("thinking_token_budget must be preserved in tunnel body")
|
|
}
|
|
|
|
// No IOP-internal metadata keys should leak into the pure passthrough body.
|
|
for _, key := range []string{"estimated_input_tokens", "context_class"} {
|
|
if _, ok := bodyMap[key]; ok {
|
|
t.Errorf("IOP internal key %q must not be in tunnel body", key)
|
|
}
|
|
}
|
|
|
|
if len(fake.reqsSnapshot()) != 0 {
|
|
t.Errorf("normalized SubmitRun must not be called for tunnel path, got %d", len(fake.reqsSnapshot()))
|
|
}
|
|
}
|
|
|
|
func TestChatCompletionsProviderPoolTunnelRelaysProviderErrorBody(t *testing.T) {
|
|
providerError := `{"error":{"message":"unknown field custom_provider_options","type":"invalid_request_error","param":"custom_provider_options"}}`
|
|
frames := make(chan *iop.ProviderTunnelFrame, 3)
|
|
frames <- &iop.ProviderTunnelFrame{
|
|
Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START,
|
|
StatusCode: http.StatusUnprocessableEntity,
|
|
Headers: map[string]string{"Content-Type": "application/json"},
|
|
}
|
|
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(providerError)}
|
|
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true}
|
|
close(frames)
|
|
|
|
fake := &fakeRunService{
|
|
tunnelFrames: frames,
|
|
tunnelServedTarget: "served-model",
|
|
}
|
|
srv := NewServer(config.EdgeOpenAIConf{}, fake, nil)
|
|
srv.SetModelCatalog([]config.ModelCatalogEntry{
|
|
{ID: "pool-model", Providers: map[string]string{"prov-vllm": "served-model"}},
|
|
})
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"pool-model",
|
|
"messages":[{"role":"user","content":"hello"}],
|
|
"custom_provider_options":{"reject":true}
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
if w.Code != http.StatusUnprocessableEntity {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
if w.Body.String() != providerError {
|
|
t.Fatalf("provider error body not relayed byte-identically:\n got: %q\nwant: %q", w.Body.String(), providerError)
|
|
}
|
|
if got := w.Header().Get("Content-Type"); !strings.Contains(got, "application/json") {
|
|
t.Fatalf("Content-Type header not relayed: %q", got)
|
|
}
|
|
if len(fake.reqsSnapshot()) != 0 {
|
|
t.Fatalf("provider error must not fall back to normalized SubmitRun, got %d calls", len(fake.reqsSnapshot()))
|
|
}
|
|
}
|
|
|
|
// TestChatCompletionsProviderPoolNormalizedIgnoresUnknownFields verifies that
|
|
// when the provider-pool dispatch selects the normalized path, the lenient
|
|
// ingress accepts unknown/provider-specific fields without rejecting the
|
|
// request, but the normalized RunRequest.Input only reflects standard supported
|
|
// fields. Unknown fields must never be invented or propagated into the adapter
|
|
// native request (SURFACE_FIELD).
|
|
func TestChatCompletionsProviderPoolNormalizedIgnoresUnknownFields(t *testing.T) {
|
|
completeEvent := &iop.RunEvent{
|
|
Type: "delta", Delta: "ok",
|
|
}
|
|
completeEvent2 := &iop.RunEvent{Type: "complete"}
|
|
|
|
fake := &fakeRunService{
|
|
poolDispatchPath: string(edgeservice.ProviderPoolPathNormalized),
|
|
poolRunFrames: bufferedRunEvents(completeEvent, completeEvent2),
|
|
}
|
|
|
|
catalog := []config.ModelCatalogEntry{
|
|
{ID: "ollama-pool-model", Providers: map[string]string{"prov-ollama": "qwen3.6:35b"}},
|
|
}
|
|
srv := NewServer(config.EdgeOpenAIConf{}, fake, nil)
|
|
srv.SetModelCatalog(catalog)
|
|
|
|
reqBody := `{
|
|
"model":"ollama-pool-model",
|
|
"messages":[{"role":"user","content":"hello"}],
|
|
"temperature":0.7,
|
|
"store":true,
|
|
"codex_trace":{"enabled":true},
|
|
"custom_provider_options":{"timeout_ms":5000},
|
|
"nested_unknown":{"deep":true},
|
|
"reasoning_effort":"high"
|
|
}`
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(reqBody))
|
|
w := httptest.NewRecorder()
|
|
srv.handleChatCompletions(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
|
|
// The lenient ingress must NOT reject the request due to unknown fields.
|
|
runReqs := fake.reqsSnapshot()
|
|
if len(runReqs) != 1 {
|
|
t.Fatalf("expected 1 normalized SubmitRun, got %d", len(runReqs))
|
|
}
|
|
|
|
runReq := runReqs[0]
|
|
if !runReq.ProviderPool {
|
|
t.Error("normalized pool run must have ProviderPool=true")
|
|
}
|
|
|
|
// Standard supported fields must be present in Input.
|
|
input := runReq.Input
|
|
if _, ok := input["prompt"]; !ok {
|
|
t.Error("prompt must be in normalized Input")
|
|
}
|
|
if _, ok := input["messages"]; !ok {
|
|
t.Error("messages must be in normalized Input")
|
|
}
|
|
options, ok := input["options"].(map[string]any)
|
|
if !ok {
|
|
t.Fatalf("options must be in normalized Input, got %T", input["options"])
|
|
}
|
|
if options["temperature"] != 0.7 {
|
|
t.Errorf("temperature must be forwarded to normalized options, got %+v", options["temperature"])
|
|
}
|
|
|
|
// Unknown/provider-specific fields must NOT be in the normalized Input.
|
|
for _, key := range []string{"store", "codex_trace", "custom_provider_options", "nested_unknown"} {
|
|
if _, ok := input[key]; ok {
|
|
t.Errorf("unknown field %q must not be in normalized Input", key)
|
|
}
|
|
if _, ok := options[key]; ok {
|
|
t.Errorf("unknown field %q must not be in normalized Input options", key)
|
|
}
|
|
}
|
|
|
|
// The tunnel path must not be taken.
|
|
tunnelReqs := fake.tunnelReqsSnapshot()
|
|
if len(tunnelReqs) != 0 {
|
|
t.Errorf("tunnel must not be dispatched for normalized path, got %d", len(tunnelReqs))
|
|
}
|
|
}
|
|
|
|
// TestProviderPoolDispatchLogObservationFields verifies that provider-pool
|
|
// dispatch logs carry provider_id, provider_type, execution_path, and
|
|
// queue_reason fields populated from the selected candidate (SURFACE_OBS-2).
|
|
func TestProviderPoolDispatchLogObservationFields(t *testing.T) {
|
|
core, observed := observer.New(zap.InfoLevel)
|
|
logger := zap.New(core)
|
|
const rawToken = "sk-pool-obs-token"
|
|
const edgeID = "edge-pool-obs-logs"
|
|
const model = "pool-obs-model"
|
|
|
|
// Build tunnel frames that complete successfully.
|
|
frames := staticProviderTunnelFrames(`{"choices":[{"message":{"role":"assistant","content":"ok"}}],"usage":{"prompt_tokens":4,"completion_tokens":3}}`)
|
|
|
|
fake := &fakeRunService{
|
|
poolDispatchPath: "provider_tunnel",
|
|
tunnelFrames: frames,
|
|
}
|
|
srv := NewServer(config.EdgeOpenAIConf{
|
|
PrincipalTokens: []config.OpenAIPrincipalTokenConf{
|
|
{TokenRef: "iop-tok-alice", TokenHashSHA256: sha256Hex(rawToken), PrincipalRef: "user:alice", PrincipalAlias: "alice"},
|
|
},
|
|
}, fake, logger)
|
|
srv.SetEdgeID(edgeID)
|
|
srv.SetModelCatalog([]config.ModelCatalogEntry{
|
|
{ID: model, Providers: map[string]string{"prov-obs": "served-obs"}},
|
|
})
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"pool-obs-model",
|
|
"messages":[{"role":"user","content":"hello"}]
|
|
}`))
|
|
req.Header.Set("Authorization", "Bearer "+rawToken)
|
|
w := httptest.NewRecorder()
|
|
srv.routes().ServeHTTP(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
|
|
// Verify the observer logger captured the dispatch log line with the
|
|
// expected provider observation fields. The structured log line from
|
|
// handleChatCompletionsProviderPool must carry provider_id, provider_type,
|
|
// execution_path, adapter, target, and queue_reason.
|
|
var found bool
|
|
for _, entry := range observed.All() {
|
|
if entry.Message != "openai chat completion provider-pool dispatch" {
|
|
continue
|
|
}
|
|
found = true
|
|
// Verify provider observation fields exist on the structured log.
|
|
if _, ok := entry.ContextMap()["provider_id"]; !ok {
|
|
t.Error("dispatch log missing provider_id field")
|
|
}
|
|
if _, ok := entry.ContextMap()["provider_type"]; !ok {
|
|
t.Error("dispatch log missing provider_type field")
|
|
}
|
|
if _, ok := entry.ContextMap()["execution_path"]; !ok {
|
|
t.Error("dispatch log missing execution_path field")
|
|
}
|
|
if _, ok := entry.ContextMap()["queue_reason"]; !ok {
|
|
t.Error("dispatch log missing queue_reason field")
|
|
}
|
|
if _, ok := entry.ContextMap()["adapter"]; !ok {
|
|
t.Error("dispatch log missing adapter field")
|
|
}
|
|
if _, ok := entry.ContextMap()["target"]; !ok {
|
|
t.Error("dispatch log missing target field")
|
|
}
|
|
}
|
|
if !found {
|
|
t.Fatal("expected provider-pool dispatch log line in observer")
|
|
}
|
|
}
|
|
|
|
// TestProviderPoolStandardResponseNoExtensionFields verifies that a
|
|
// provider-pool tunnel passthrough standard response carries only
|
|
// provider-original bytes and no IOP extension fields or response-mode label
|
|
// (SURFACE_OBS-2/3 regression).
|
|
func TestProviderPoolStandardResponseNoExtensionFields(t *testing.T) {
|
|
providerBody := `{"choices":[{"message":{"role":"assistant","content":"hello provider"}}],"usage":{"prompt_tokens":5,"completion_tokens":3}}`
|
|
frames := staticProviderTunnelFrames(providerBody)
|
|
|
|
fake := &fakeRunService{
|
|
poolDispatchPath: "provider_tunnel",
|
|
tunnelFrames: frames,
|
|
}
|
|
srv := NewServer(config.EdgeOpenAIConf{
|
|
PrincipalTokens: []config.OpenAIPrincipalTokenConf{
|
|
{TokenRef: "iop-tok-alice", TokenHashSHA256: sha256Hex("sk-no-sb-token"), PrincipalRef: "user:alice", PrincipalAlias: "alice"},
|
|
},
|
|
}, fake, nil)
|
|
srv.SetEdgeID("edge-no-sb")
|
|
srv.SetModelCatalog([]config.ModelCatalogEntry{
|
|
{ID: "sb-model", Providers: map[string]string{"prov-sb": "served-sb"}},
|
|
})
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"sb-model",
|
|
"messages":[{"role":"user","content":"hi"}]
|
|
}`))
|
|
req.Header.Set("Authorization", "Bearer sk-no-sb-token")
|
|
w := httptest.NewRecorder()
|
|
srv.routes().ServeHTTP(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
|
|
// Response body must be the provider-original JSON with only the model
|
|
// field rewritten, carrying no IOP extension markers.
|
|
body := w.Body.String()
|
|
if strings.Contains(body, "iop_") || strings.Contains(body, "iop.") {
|
|
t.Errorf("standard response must not contain IOP extension markers: %s", body)
|
|
}
|
|
|
|
// Verify the tunnel was dispatched.
|
|
if len(fake.tunnelReqsSnapshot()) == 0 {
|
|
t.Error("expected provider tunnel dispatch")
|
|
}
|
|
}
|
|
|
|
// TestResponsesProviderPoolTunnelUnknownFieldsUsesPoolSelection asserts that
|
|
// when the selected provider is tunnel-passthrough (OpenAI-compatible),
|
|
// unknown fields (tools, parallel_tool_calls, store, custom field) are
|
|
// preserved verbatim in the tunnel body, and that the Run sent to
|
|
// SubmitProviderPool carries a non-empty ModelGroupKey.
|
|
func TestResponsesProviderPoolTunnelUnknownFieldsUsesPoolSelection(t *testing.T) {
|
|
fake := &fakeRunService{
|
|
poolDispatchPath: string(edgeservice.ProviderPoolPathTunnel),
|
|
}
|
|
|
|
srv := NewServer(config.EdgeOpenAIConf{}, fake, nil)
|
|
srv.SetModelCatalog([]config.ModelCatalogEntry{
|
|
{ID: "pool-model", Providers: map[string]string{"prov-tunnel": "served-tunnel"}},
|
|
})
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(`{
|
|
"model":"pool-model",
|
|
"input":"hello world",
|
|
"tools":[{"type":"function","function":{"name":"lookup"}}],
|
|
"parallel_tool_calls":true,
|
|
"store":false,
|
|
"custom_field":"keep-me"
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
|
|
srv.handleResponses(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
|
|
// SubmitProviderPool must have been called with a populated Run.
|
|
lastRun := fake.poolLastRunSnapshot()
|
|
if lastRun.ModelGroupKey == "" {
|
|
t.Error("SubmitProviderPool was called with empty Run.ModelGroupKey")
|
|
}
|
|
|
|
// The tunnel body must preserve unknown fields verbatim.
|
|
if len(fake.tunnelBodiesSnapshot()) != 1 {
|
|
t.Fatalf("expected exactly one tunnel body, got %d", len(fake.tunnelBodiesSnapshot()))
|
|
}
|
|
body := fake.tunnelBodiesSnapshot()[0]
|
|
if !strings.Contains(string(body), "\"tools\"") {
|
|
t.Error("tunnel body must preserve tools field")
|
|
}
|
|
if !strings.Contains(string(body), "\"parallel_tool_calls\"") {
|
|
t.Error("tunnel body must preserve parallel_tool_calls field")
|
|
}
|
|
if !strings.Contains(string(body), "\"store\"") {
|
|
t.Error("tunnel body must preserve store field")
|
|
}
|
|
if !strings.Contains(string(body), "\"custom_field\"") {
|
|
t.Error("tunnel body must preserve custom_field")
|
|
}
|
|
}
|
|
|
|
// TestResponsesProviderPoolOllamaSelectionRejectsUnknownFieldsWithoutTunnel
|
|
// asserts that when the selected provider is normalized (Ollama/CLI), an
|
|
// unknown-field request returns HTTP 400 and no tunnel or run dispatch occurs.
|
|
func TestResponsesProviderPoolOllamaSelectionRejectsUnknownFieldsWithoutTunnel(t *testing.T) {
|
|
fake := &fakeRunService{
|
|
poolDispatchPath: string(edgeservice.ProviderPoolPathNormalized),
|
|
}
|
|
|
|
srv := NewServer(config.EdgeOpenAIConf{}, fake, nil)
|
|
srv.SetModelCatalog([]config.ModelCatalogEntry{
|
|
{ID: "ollama-model", Providers: map[string]string{"prov-ollama": "served-ollama"}},
|
|
})
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(`{
|
|
"model":"ollama-model",
|
|
"input":"hello",
|
|
"unknown_field":"reject-me"
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
|
|
srv.handleResponses(w, req)
|
|
|
|
if w.Code != http.StatusBadRequest {
|
|
t.Fatalf("status: got %d body=%s, want 400", w.Code, w.Body.String())
|
|
}
|
|
|
|
// No tunnel or run dispatch should have occurred.
|
|
if len(fake.tunnelReqsSnapshot()) != 0 {
|
|
t.Error("normalized path must not dispatch tunnel for unknown fields")
|
|
}
|
|
if len(fake.reqsSnapshot()) != 0 {
|
|
t.Error("normalized path must not dispatch run for unknown fields")
|
|
}
|
|
|
|
// Verify pool dispatch was made.
|
|
if fake.poolSubmitCountSnapshot() != 1 {
|
|
t.Fatalf("pool submit count: got %d, want 1", fake.poolSubmitCountSnapshot())
|
|
}
|
|
|
|
// Verify PrepareRun was called on the normalized path.
|
|
prepareRunCalled := fake.poolPrepareRunCalledSnapshot()
|
|
if !prepareRunCalled {
|
|
t.Errorf("PrepareRun must be called on normalized path (poolSubmitCount=%d, poolLastRun.ModelGroupKey=%q)", fake.poolSubmitCountSnapshot(), fake.poolLastRunSnapshot().ModelGroupKey)
|
|
}
|
|
}
|
|
|
|
// TestResponsesProviderPoolOllamaSelectionRejectsStreamingWithoutTunnel asserts
|
|
// that when the selected provider is normalized, stream:true returns HTTP 400
|
|
// and no tunnel dispatch occurs.
|
|
func TestResponsesProviderPoolOllamaSelectionRejectsStreamingWithoutTunnel(t *testing.T) {
|
|
fake := &fakeRunService{
|
|
poolDispatchPath: string(edgeservice.ProviderPoolPathNormalized),
|
|
}
|
|
|
|
srv := NewServer(config.EdgeOpenAIConf{}, fake, nil)
|
|
srv.SetModelCatalog([]config.ModelCatalogEntry{
|
|
{ID: "ollama-model", Providers: map[string]string{"prov-ollama": "served-ollama"}},
|
|
})
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(`{
|
|
"model":"ollama-model",
|
|
"input":"hello",
|
|
"stream":true
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
|
|
srv.handleResponses(w, req)
|
|
|
|
if w.Code != http.StatusBadRequest {
|
|
t.Fatalf("status: got %d body=%s, want 400", w.Code, w.Body.String())
|
|
}
|
|
|
|
// No tunnel dispatch for normalized path with stream:true.
|
|
if len(fake.tunnelReqsSnapshot()) != 0 {
|
|
t.Error("normalized path must not dispatch tunnel for stream:true")
|
|
}
|
|
if len(fake.reqsSnapshot()) != 0 {
|
|
t.Error("normalized path must not dispatch run for stream:true")
|
|
}
|
|
|
|
if !fake.poolPrepareRunCalledSnapshot() {
|
|
t.Error("PrepareRun must be called on normalized path")
|
|
}
|
|
}
|
|
|
|
// TestResponsesProviderPoolTunnelStreamingUsesPoolSelection asserts that when
|
|
// the selected provider is tunnel-passthrough, stream:true is relayed through
|
|
// SubmitProviderPool with the raw SSE body preserved.
|
|
func TestResponsesProviderPoolTunnelStreamingUsesPoolSelection(t *testing.T) {
|
|
fake := &fakeRunService{
|
|
poolDispatchPath: string(edgeservice.ProviderPoolPathTunnel),
|
|
tunnelFrames: staticProviderTunnelFrames(`{"id":"resp-1","output":[{"type":"message","role":"assistant","content":[{"type":"output_text","text":"hello"}]}],"stream":true}\n`),
|
|
}
|
|
|
|
srv := NewServer(config.EdgeOpenAIConf{}, fake, nil)
|
|
srv.SetModelCatalog([]config.ModelCatalogEntry{
|
|
{ID: "pool-model", Providers: map[string]string{"prov-tunnel": "served-tunnel"}},
|
|
})
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(`{
|
|
"model":"pool-model",
|
|
"input":"hello",
|
|
"stream":true
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
|
|
srv.handleResponses(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
|
|
// SubmitProviderPool must have been called.
|
|
if fake.poolSubmitCountSnapshot() != 1 {
|
|
t.Fatalf("SubmitProviderPool call count: got %d, want 1", fake.poolSubmitCountSnapshot())
|
|
}
|
|
|
|
// The tunnel body must preserve the raw SSE payload.
|
|
if len(fake.tunnelBodiesSnapshot()) != 1 {
|
|
t.Fatalf("expected one tunnel body, got %d", len(fake.tunnelBodiesSnapshot()))
|
|
}
|
|
body := fake.tunnelBodiesSnapshot()[0]
|
|
if !strings.Contains(string(body), "\"stream\":true") {
|
|
t.Error("tunnel body must preserve stream:true")
|
|
}
|
|
|
|
// PrepareRun must NOT be called on the tunnel path.
|
|
if fake.poolPrepareRunCalledSnapshot() {
|
|
t.Error("PrepareRun must NOT be called on tunnel path")
|
|
}
|
|
}
|
|
|
|
// TestResponsesProviderPoolStrictOutputNormalizesAgentResponse verifies that the
|
|
// provider-pool normalized path preserves the strict-output contract from
|
|
// PrepareRun into completeResponse. A prompt containing an XML completion
|
|
// contract must have its response wrapped in the expected block (SDD D02).
|
|
func TestResponsesProviderPoolStrictOutputNormalizesAgentResponse(t *testing.T) {
|
|
fake := &fakeRunService{
|
|
poolDispatchPath: string(edgeservice.ProviderPoolPathNormalized),
|
|
}
|
|
fake.poolRunFrames = make(chan *iop.RunEvent, 3)
|
|
fake.poolRunFrames <- &iop.RunEvent{Type: "reasoning_delta", Delta: "hidden"}
|
|
fake.poolRunFrames <- &iop.RunEvent{Type: "delta", Delta: "plain answer"}
|
|
fake.poolRunFrames <- &iop.RunEvent{Type: "complete", RunId: "run-pool-normalized"}
|
|
close(fake.poolRunFrames)
|
|
|
|
catalog := []config.ModelCatalogEntry{
|
|
{ID: "ollama-strict-model", Providers: map[string]string{"prov-ollama": "served-ollama"}},
|
|
}
|
|
srv := NewServer(config.EdgeOpenAIConf{StrictOutput: true}, fake, nil)
|
|
srv.SetModelCatalog(catalog)
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(`{
|
|
"model":"ollama-strict-model",
|
|
"input":"Once you've completed the user's task, you must use the attempt_completion tool to present the result.\n\n<attempt_completion>\n<result>done</result>\n</attempt_completion>\n\nfinish"
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleResponses(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
|
|
// PrepareRun must have been invoked (normalized path).
|
|
if !fake.poolPrepareRunCalledSnapshot() {
|
|
t.Fatal("PrepareRun was not called on normalized path")
|
|
}
|
|
|
|
// The prepared request must carry strict_output and the injected contract
|
|
// instruction.
|
|
if len(fake.reqsSnapshot()) != 1 {
|
|
t.Fatalf("expected one prepared req, got %d", len(fake.reqsSnapshot()))
|
|
}
|
|
reqPrep := fake.reqsSnapshot()[0]
|
|
if reqPrep.Metadata["strict_output"] != "true" {
|
|
t.Fatalf("strict_output metadata lost in prepared request: %+v", reqPrep.Metadata)
|
|
}
|
|
if !strings.Contains(reqPrep.Prompt, "output exactly one <attempt_completion> block") {
|
|
t.Fatalf("strict contract instruction not injected into prepared prompt:\n%s", reqPrep.Prompt)
|
|
}
|
|
|
|
var resp responsesResponse
|
|
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
|
t.Fatalf("decode: %v", err)
|
|
}
|
|
want := "<attempt_completion>\n<result>plain answer</result>\n</attempt_completion>"
|
|
if resp.OutputText != want {
|
|
t.Fatalf("output_text:\ngot %q\nwant %q", resp.OutputText, want)
|
|
}
|
|
}
|
|
|
|
// TestResponsesProviderPoolTunnelMetadataAndContextPropagation verifies that the
|
|
// provider-pool tunnel path preserves metadata, estimated_input_tokens, and
|
|
// context_class from the base request into the tunnel dispatch (SDD S02/S03).
|
|
func TestResponsesProviderPoolTunnelMetadataAndContextPropagation(t *testing.T) {
|
|
frames := make(chan *iop.ProviderTunnelFrame, 3)
|
|
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START, StatusCode: 200}
|
|
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(`{"ok":true}`)}
|
|
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true}
|
|
close(frames)
|
|
|
|
fake := &fakeRunService{
|
|
poolDispatchPath: string(edgeservice.ProviderPoolPathTunnel),
|
|
tunnelFrames: frames,
|
|
tunnelServedTarget: "served-model",
|
|
}
|
|
|
|
catalog := []config.ModelCatalogEntry{
|
|
{ID: "pool-model", Providers: map[string]string{"prov-1": "served-model"}},
|
|
}
|
|
srv := NewServer(config.EdgeOpenAIConf{}, fake, nil)
|
|
srv.SetModelCatalog(catalog)
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(`{
|
|
"model":"pool-model",
|
|
"input":"hello",
|
|
"metadata":{"experiment":"pool-test"}
|
|
}`))
|
|
w := httptest.NewRecorder()
|
|
srv.handleResponses(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
|
|
tunnelReqs := fake.tunnelReqsSnapshot()
|
|
if len(tunnelReqs) != 1 {
|
|
t.Fatalf("expected one tunnel req, got %d", len(tunnelReqs))
|
|
}
|
|
tunnelReq := tunnelReqs[0]
|
|
|
|
// Metadata must contain the principal and pool-echoed fields.
|
|
if tunnelReq.Metadata == nil {
|
|
t.Fatalf("tunnel metadata must not be nil")
|
|
}
|
|
if tunnelReq.Metadata["openai_model"] == "" {
|
|
t.Error("openai_model missing from tunnel metadata")
|
|
}
|
|
if tunnelReq.Metadata["openai_stream"] == "" {
|
|
t.Error("openai_stream missing from tunnel metadata")
|
|
}
|
|
if tunnelReq.Metadata["estimated_input_tokens"] == "" {
|
|
t.Error("estimated_input_tokens missing from tunnel metadata")
|
|
}
|
|
if tunnelReq.Metadata["context_class"] == "" {
|
|
t.Error("context_class missing from tunnel metadata")
|
|
}
|
|
if tunnelReq.Metadata["experiment"] != "pool-test" {
|
|
t.Errorf("caller metadata lost: got %v", tunnelReq.Metadata)
|
|
}
|
|
|
|
// EstimatedInputTokens and ContextClass must be non-zero/non-empty on the
|
|
// tunnel request itself, so Node observability preserves the same values
|
|
// as the direct tunnel path.
|
|
if tunnelReq.EstimatedInputTokens <= 0 {
|
|
t.Errorf("EstimatedInputTokens must be > 0: got %d", tunnelReq.EstimatedInputTokens)
|
|
}
|
|
if tunnelReq.ContextClass == "" {
|
|
t.Error("ContextClass must be non-empty on tunnel request")
|
|
}
|
|
}
|