package openai import ( "encoding/json" "fmt" "net/http" "net/http/httptest" "strings" "testing" edgeservice "iop/apps/edge/internal/service" "iop/packages/go/config" iop "iop/proto/gen/iop" ) func TestHandleModelsProviderPoolCatalog(t *testing.T) { catalog := []config.ModelCatalogEntry{ {ID: "qwen3.6:35b", Providers: map[string]string{"prov-1": "Qwen3-35B-A22B"}}, {ID: "llama3.3:70b", Providers: map[string]string{"prov-2": "llama-3.3-70b"}}, } srv := NewServer(config.EdgeOpenAIConf{ ModelRoutes: []config.OpenAIRouteEntry{ {Model: "legacy-model", Target: "legacy-target"}, }, }, &fakeRunService{}, nil) srv.SetModelCatalog(catalog) req := httptest.NewRequest(http.MethodGet, "/v1/models", nil) w := httptest.NewRecorder() srv.handleModels(w, req) if w.Code != http.StatusOK { t.Fatalf("status: got %d body=%s", w.Code, w.Body.String()) } body := w.Body.String() if !strings.Contains(body, "qwen3.6:35b") || !strings.Contains(body, "llama3.3:70b") { t.Fatalf("catalog models not listed: %s", body) } if strings.Contains(body, "legacy-model") { t.Fatalf("legacy model_routes should be suppressed when catalog is set: %s", body) } } // TestChatCompletionsProviderPoolDispatch verifies that when a request model // matches the provider-pool catalog with omitted response mode, the request is // dispatched over the provider tunnel with ProviderPool=true and Adapter/Target // left empty for service-layer resolution (pure passthrough default, SDD D02). func TestChatCompletionsProviderPoolDispatch(t *testing.T) { frames := make(chan *iop.ProviderTunnelFrame, 3) frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START, StatusCode: 200} frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(`{"ok":true}`)} frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true} close(frames) fake := &providerFakeRunService{ tunnelFrames: frames, } catalog := []config.ModelCatalogEntry{ {ID: "qwen3.6:35b", Providers: map[string]string{"prov-vllm": "Qwen3-35B-A22B"}}, } srv := NewServer(config.EdgeOpenAIConf{}, fake, nil) srv.SetModelCatalog(catalog) req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ "model":"qwen3.6:35b", "messages":[{"role":"user","content":"hello"}] }`)) w := httptest.NewRecorder() srv.handleChatCompletions(w, req) if w.Code != http.StatusOK { t.Fatalf("status: got %d body=%s", w.Code, w.Body.String()) } tunnelReqs := fake.tunnelReqsSnapshot() if len(tunnelReqs) != 1 { t.Fatalf("expected 1 tunnel dispatch, got %d (SubmitRun calls: %d)", len(tunnelReqs), len(fake.reqsSnapshot())) } treq := tunnelReqs[0] if !treq.ProviderPool { t.Error("ProviderPool should be true for catalog-matched model") } if treq.ModelGroupKey != "qwen3.6:35b" { t.Errorf("ModelGroupKey: got %q, want qwen3.6:35b", treq.ModelGroupKey) } if treq.Adapter != "" || treq.Target != "" { t.Errorf("Adapter/Target should be empty for provider-pool dispatch, got %q/%q", treq.Adapter, treq.Target) } if treq.Method != http.MethodPost || treq.Path != "/v1/chat/completions" { t.Errorf("tunnel method/path: got %q %q", treq.Method, treq.Path) } if len(fake.reqsSnapshot()) != 0 { t.Errorf("normalized SubmitRun must not be called for omitted-mode provider route") } } func TestChatCompletionsStrictOutputProviderPoolDefaultKeepsPassthroughPath(t *testing.T) { frames := make(chan *iop.ProviderTunnelFrame, 3) frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START, StatusCode: 200} frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(`{"ok":true}`)} frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true} close(frames) fake := &providerFakeRunService{ tunnelFrames: frames, } catalog := []config.ModelCatalogEntry{ {ID: "qwen3.6:35b", Providers: map[string]string{"prov-vllm": "Qwen3-35B-A22B"}}, } srv := NewServer(config.EdgeOpenAIConf{StrictOutput: true}, fake, nil) srv.SetModelCatalog(catalog) req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ "model":"qwen3.6:35b", "messages":[{"role":"user","content":"hello"}], "include_reasoning": true }`)) w := httptest.NewRecorder() srv.handleChatCompletions(w, req) if w.Code != http.StatusOK { t.Fatalf("status: got %d body=%s", w.Code, w.Body.String()) } if len(fake.tunnelReqsSnapshot()) != 1 { t.Fatalf("strict output default must keep provider routes on passthrough, got %d tunnel dispatches", len(fake.tunnelReqsSnapshot())) } if len(fake.reqsSnapshot()) != 0 { t.Fatalf("strict output provider route must not fall back to normalized SubmitRun, got %d calls", len(fake.reqsSnapshot())) } } // TestChatCompletionsProviderPoolIgnoresSelectorLikeMetadata verifies SDD S01: // arbitrary caller metadata — including keys or values that resemble a legacy // response-mode selector — is opaque context. It never rejects the request or // switches a provider-pool route off pure passthrough. (The former selector key // literal is deliberately not used here; the deterministic surface check forbids // it anywhere under apps.) func TestChatCompletionsProviderPoolIgnoresSelectorLikeMetadata(t *testing.T) { cases := []struct { name string metadata string }{ {name: "selector-like value is ignored", metadata: `"metadata":{"response_hint":"raw"}`}, {name: "arbitrary metadata is ignored", metadata: `"metadata":{"experiment":"mode-x","team":"search"}`}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { providerBody := `{"ok":true}` fake := &providerFakeRunService{ poolDispatchPath: string(edgeservice.ProviderPoolPathTunnel), tunnelFrames: staticProviderTunnelFrames(providerBody), } srv := NewServer(config.EdgeOpenAIConf{}, fake, nil) srv.SetModelCatalog([]config.ModelCatalogEntry{ {ID: "pool-model", Providers: map[string]string{"prov-1": "served-model"}}, }) body := `{ "model":"pool-model", "messages":[{"role":"user","content":"hello"}], ` + tc.metadata + ` }` req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(body)) w := httptest.NewRecorder() srv.handleChatCompletions(w, req) if w.Code != http.StatusOK { t.Fatalf("selector-like metadata must not be rejected: got %d body=%s", w.Code, w.Body.String()) } if got := w.Body.String(); got != providerBody { t.Fatalf("provider body must be relayed byte-identically: got %q want %q", got, providerBody) } if len(fake.tunnelReqsSnapshot()) != 1 { t.Fatalf("expected 1 pure passthrough tunnel dispatch, got %d", len(fake.tunnelReqsSnapshot())) } if len(fake.reqsSnapshot()) != 0 { t.Fatalf("arbitrary metadata must not switch to normalized SubmitRun, got %d", len(fake.reqsSnapshot())) } }) } } // TestChatCompletionsLegacyRouteSetsProviderPoolFalse verifies that when the // server is configured with Adapter + Target (no catalog), the dispatched // SubmitRunRequest has ProviderPool=false so the service uses direct dispatch // (not the queue admission gate that requires ProviderPool=true). func TestChatCompletionsLegacyRouteSetsProviderPoolFalse(t *testing.T) { fake := &fakeRunService{events: make(chan *iop.RunEvent, 2)} fake.events <- &iop.RunEvent{Type: "delta", Delta: "hi"} fake.events <- &iop.RunEvent{Type: "complete"} // Server with adapter+target only, no catalog — classic legacy route. srv := NewServer(config.EdgeOpenAIConf{ Adapter: "cli", Target: "codex", TimeoutSec: 10, }, fake, nil) req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ "model":"any-model", "messages":[{"role":"user","content":"hello"}] }`)) w := httptest.NewRecorder() srv.handleChatCompletions(w, req) if w.Code != http.StatusOK { t.Fatalf("status: got %d body=%s", w.Code, w.Body.String()) } if fake.req.ProviderPool { t.Error("ProviderPool must be false for a legacy adapter+target route with no catalog") } if fake.req.Adapter != "cli" || fake.req.Target != "codex" { t.Errorf("Adapter/Target: got %q/%q, want cli/codex", fake.req.Adapter, fake.req.Target) } } // TestResponsesProviderPoolDispatch verifies that /v1/responses sends // provider-pool models through the raw provider tunnel (POST /v1/responses) // instead of the normalized RunEvent path. func TestResponsesProviderPoolDispatch(t *testing.T) { fake := &providerFakeRunService{ tunnelFrames: staticProviderTunnelFrames(`{"id":"resp-1","object":"response"}`), tunnelServedTarget: "model-a", } catalog := []config.ModelCatalogEntry{ {ID: "prov-vllm:model-a", Providers: map[string]string{"prov-vllm": "model-a"}}, } srv := NewServer(config.EdgeOpenAIConf{TimeoutSec: 5}, fake, nil) srv.SetModelCatalog(catalog) req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(`{ "model":"prov-vllm:model-a", "input":"hello" }`)) w := httptest.NewRecorder() srv.handleResponses(w, req) if w.Code != http.StatusOK { t.Fatalf("status: got %d body=%s", w.Code, w.Body.String()) } if len(fake.reqsSnapshot()) != 0 { t.Fatalf("provider-pool /v1/responses must not call SubmitRun, got %d calls", len(fake.reqsSnapshot())) } reqs := fake.tunnelReqsSnapshot() if len(reqs) != 1 { t.Fatalf("expected 1 tunnel dispatch, got %d", len(reqs)) } if reqs[0].Path != "/v1/responses" { t.Fatalf("tunnel path: got %q want /v1/responses", reqs[0].Path) } if reqs[0].Method != http.MethodPost { t.Fatalf("tunnel method: got %q want POST", reqs[0].Method) } } // TestResponsesProviderPoolPreservesRawOutputTokens verifies that raw // passthrough preserves the caller's Responses-shaped max_output_tokens and does // not inject the Chat-shaped max_tokens or generation policy. func TestResponsesProviderPoolPreservesRawOutputTokens(t *testing.T) { fake := &providerFakeRunService{ tunnelFrames: staticProviderTunnelFrames(`{"ok":true}`), tunnelServedTarget: "Ornith-1.0-35B", } catalog := []config.ModelCatalogEntry{{ ID: "ornith:35b", DefaultMaxTokens: 32768, MinMaxTokens: 32768, DefaultThinkingTokenBudget: 8192, Providers: map[string]string{"prov-vllm": "Ornith-1.0-35B"}, }} srv := NewServer(config.EdgeOpenAIConf{TimeoutSec: 5}, fake, nil) srv.SetModelCatalog(catalog) req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(`{ "model":"ornith:35b", "input":"hello", "max_output_tokens":4096 }`)) w := httptest.NewRecorder() srv.handleResponses(w, req) if w.Code != http.StatusOK { t.Fatalf("status: got %d body=%s", w.Code, w.Body.String()) } if len(fake.reqsSnapshot()) != 0 { t.Fatalf("provider-pool /v1/responses must not call SubmitRun, got %d calls", len(fake.reqsSnapshot())) } bodies := fake.tunnelBodiesSnapshot() if len(bodies) != 1 { t.Fatalf("expected 1 tunnel body, got %d", len(bodies)) } var providerReq map[string]any if err := json.Unmarshal(bodies[0], &providerReq); err != nil { t.Fatalf("provider body JSON: %v body=%s", err, bodies[0]) } if providerReq["model"] != "Ornith-1.0-35B" { t.Fatalf("served model rewrite not applied: %+v", providerReq["model"]) } if providerReq["max_output_tokens"].(float64) != 4096 { t.Fatalf("caller max_output_tokens must be preserved: %+v", providerReq) } if _, ok := providerReq["max_tokens"]; ok { t.Fatalf("passthrough must not inject Chat-shaped max_tokens: %+v", providerReq) } if _, ok := providerReq["thinking_token_budget"]; ok { t.Fatalf("passthrough must not inject thinking_token_budget: %+v", providerReq) } } // TestResponsesProviderPoolIgnoresSelectorLikeMetadata verifies SDD S01/S04 for // /v1/responses: arbitrary caller metadata never rejects the request or moves a // provider route off pure passthrough. (The former selector key literal is // deliberately not used; the deterministic surface check forbids it under apps.) func TestResponsesProviderPoolIgnoresSelectorLikeMetadata(t *testing.T) { cases := []struct { name string metadata string }{ {name: "selector-like value is ignored", metadata: `"metadata":{"response_hint":"raw"}`}, {name: "arbitrary metadata is ignored", metadata: `"metadata":{"experiment":"mode-x"}`}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { providerBody := `{"ok":true}` srv, fake := responsesProviderTunnelServer(staticProviderTunnelFrames(providerBody), "served-model") req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(fmt.Sprintf(`{ "model":"pool-model", "input":"hi", %s }`, tc.metadata))) w := httptest.NewRecorder() srv.handleResponses(w, req) if w.Code != http.StatusOK { t.Fatalf("selector-like metadata must not be rejected: got %d body=%s", w.Code, w.Body.String()) } if got := w.Body.String(); got != providerBody { t.Fatalf("provider body must be relayed byte-identically: got %q want %q", got, providerBody) } if len(fake.tunnelReqsSnapshot()) != 1 { t.Fatalf("expected 1 pure passthrough tunnel dispatch, got %d", len(fake.tunnelReqsSnapshot())) } if len(fake.reqsSnapshot()) != 0 { t.Fatalf("responses provider route must not call SubmitRun, got %d", len(fake.reqsSnapshot())) } }) } } func TestChatCompletionsProviderPoolFallsBackToLegacyRoute(t *testing.T) { fake := &fakeRunService{events: make(chan *iop.RunEvent, 2)} fake.events <- &iop.RunEvent{Type: "delta", Delta: "ok"} fake.events <- &iop.RunEvent{Type: "complete"} catalog := []config.ModelCatalogEntry{ {ID: "qwen3.6:35b", Providers: map[string]string{"prov-vllm": "Qwen3-35B-A22B"}}, } srv := NewServer(config.EdgeOpenAIConf{ ModelRoutes: []config.OpenAIRouteEntry{ {Model: "ollama-model", Adapter: "ollama", Target: "llama3"}, }, }, fake, nil) srv.SetModelCatalog(catalog) req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ "model":"ollama-model", "messages":[{"role":"user","content":"hi"}] }`)) w := httptest.NewRecorder() srv.handleChatCompletions(w, req) if w.Code != http.StatusOK { t.Fatalf("status: got %d body=%s", w.Code, w.Body.String()) } if fake.req.ProviderPool { t.Error("ProviderPool should be false for non-catalog model") } if fake.req.Target != "llama3" { t.Errorf("Target: got %q, want llama3", fake.req.Target) } } // TestProviderPoolStandardResponseNoExtensionFields verifies that a // provider-pool tunnel passthrough standard response carries only // provider-original bytes and no IOP extension fields or response-mode label // (SURFACE_OBS-2/3 regression). func TestProviderPoolStandardResponseNoExtensionFields(t *testing.T) { providerBody := `{"choices":[{"message":{"role":"assistant","content":"hello provider"}}],"usage":{"prompt_tokens":5,"completion_tokens":3}}` frames := staticProviderTunnelFrames(providerBody) fake := &providerFakeRunService{ poolDispatchPath: "provider_tunnel", tunnelFrames: frames, } srv := NewServer(config.EdgeOpenAIConf{ PrincipalTokens: []config.OpenAIPrincipalTokenConf{ {TokenRef: "iop-tok-alice", TokenHashSHA256: sha256Hex("sk-no-sb-token"), PrincipalRef: "user:alice", PrincipalAlias: "alice"}, }, }, fake, nil) srv.SetEdgeID("edge-no-sb") srv.SetModelCatalog([]config.ModelCatalogEntry{ {ID: "sb-model", Providers: map[string]string{"prov-sb": "served-sb"}}, }) req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ "model":"sb-model", "messages":[{"role":"user","content":"hi"}] }`)) req.Header.Set("Authorization", "Bearer sk-no-sb-token") w := httptest.NewRecorder() srv.routes().ServeHTTP(w, req) if w.Code != http.StatusOK { t.Fatalf("status: got %d body=%s", w.Code, w.Body.String()) } // Response body must be the provider-original JSON with only the model // field rewritten, carrying no IOP extension markers. body := w.Body.String() if strings.Contains(body, "iop_") || strings.Contains(body, "iop.") { t.Errorf("standard response must not contain IOP extension markers: %s", body) } // Verify the tunnel was dispatched. if len(fake.tunnelReqsSnapshot()) == 0 { t.Error("expected provider tunnel dispatch") } }