From 59c2997d4777608fa2c58304ced50bf286de73f3 Mon Sep 17 00:00:00 2001 From: toki Date: Mon, 13 Jul 2026 21:05:04 +0900 Subject: [PATCH] =?UTF-8?q?refactor(openai):=20call=20metadata=20=EA=B8=B0?= =?UTF-8?q?=EB=B0=98=20response=5Fmode=20=EB=9D=BC=EC=9A=B0=ED=84=B0=20?= =?UTF-8?q?=EC=84=A0=ED=83=9D=EC=9D=84=20=EC=A0=9C=EA=B1=B0=ED=95=9C?= =?UTF-8?q?=EB=8B=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit provider 라우트는 raw tunnel 패스스루, 그 외 라우트는 정규화된 RunEvent 경로로 응답을 결정한다. caller metadata는 임의 컨텍스트이며 응답 경로/shape를 선택하지 않는다 (SDD S01). - chat_handler.go: response_mode parse/switch 로직 제거, 라우트 기반 분기로 단순화 - responses_handler.go: tunnelResponsesPassthroughSideband 함수 및 response_mode switch 제거 - stream.go: 관련 response_mode 라벨 사용 정리 - server_test.go: response_mode 관련 테스트 케이스 제거 및 정리 - usage_metrics_test.go: 사용되지 않는 테스트 대목 제거 - docs/openai-usage-grafana.md: response_mode 라벨 설명을 passthrough/normalized로 수정 --- apps/edge/internal/openai/chat_handler.go | 127 +-- .../edge/internal/openai/responses_handler.go | 363 +----- apps/edge/internal/openai/server_test.go | 1004 ++--------------- apps/edge/internal/openai/stream.go | 476 +------- .../internal/openai/usage_metrics_test.go | 766 ------------- docs/openai-usage-grafana.md | 8 +- 6 files changed, 144 insertions(+), 2600 deletions(-) diff --git a/apps/edge/internal/openai/chat_handler.go b/apps/edge/internal/openai/chat_handler.go index 961bcc1..106174f 100644 --- a/apps/edge/internal/openai/chat_handler.go +++ b/apps/edge/internal/openai/chat_handler.go @@ -74,15 +74,6 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) { for k, v := range principalMetadata(r.Context()) { runMeta[k] = v } - if dispatch.ProviderPool && responseModeWasExplicit(runMeta) { - writeError(w, http.StatusBadRequest, "invalid_request_error", "metadata.iop_response_mode is not supported for model group routes") - return - } - responseMode, err := parseResponseMode(runMeta) - if err != nil { - writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error()) - return - } if err := validateWorkspaceForRoute(dispatch, workspace); err != nil { writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error()) return @@ -94,25 +85,14 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) { return } outputPolicy := s.resolveOutputPolicy(basePrompt) - if responseMode == responseModePassthrough && !responseModeWasExplicit(runMeta) && outputPolicy.Strict && !providerTunnelRoute { - responseMode = responseModeTransformed - } if catalogEntry := s.findProviderPoolEntry(req.Model); catalogEntry != nil { applyModelCatalogGenerationPolicyToChat(&req, *catalogEntry, outputPolicy.Strict) } - if providerTunnelRoute && responseMode == responseModeTransformed { - writeError(w, http.StatusBadRequest, "invalid_request_error", "metadata.iop_response_mode=transformed is not supported for OpenAI-compatible provider model groups") - return - } - // OpenAI-compatible provider routes default to pure passthrough (SDD D02): - // provider status/headers/body bytes are relayed to the caller unmodified - // over the raw tunnel instead of the normalized RunEvent path. Explicit - // passthrough+sideband keeps the provider tunnel but exposes IOP - // observations through the sideband extension surface (SDD S05). - // - // For provider-pool, use the one-shot SubmitProviderPool surface which - // selects tunnel or normalized based on the candidate's executionPath. + // The response path is decided by the resolved route, never by caller + // metadata: provider routes relay pure passthrough over the raw tunnel; + // every other route uses the normalized RunEvent path. Caller metadata is + // arbitrary context and does not select route or response shape (SDD S01). // Pre-compute estimate/contextClass early so provider-pool path can use them. estimate := s.estimateChatInputTokens(basePrompt, runMeta, req.Tools, req.ToolChoice) contextClass := classifyContext(estimate, s.longContextThreshold()) @@ -129,20 +109,15 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) { metadata["estimated_input_tokens"] = strconv.Itoa(estimate) metadata["context_class"] = contextClass - // provider-pool path preserves responseMode for tunnel passthrough. // strict-output output policy only applies to normalized dispatch, // not to raw tunnel passthrough (SDD D02). - s.handleChatCompletionsProviderPool(r, req, dispatch, workspace, basePrompt, prompt, rawBody, input, metadata, estimate, contextClass, outputPolicy, validation, responseMode, w) + s.handleChatCompletionsProviderPool(r, req, dispatch, workspace, basePrompt, prompt, rawBody, input, metadata, estimate, contextClass, outputPolicy, validation, w) return } - switch { - case responseMode == responseModePassthrough && providerTunnelRoute: + if providerTunnelRoute { s.tunnelChatCompletionPassthrough(w, r, req, dispatch, runMeta, rawBody, estimate, contextClass) return - case responseMode == responseModePassthroughSideband && providerTunnelRoute: - s.tunnelChatCompletionPassthroughSideband(w, r, req, dispatch, runMeta, rawBody, estimate, contextClass) - return } // Non-provider-pool normalized path: build messages, prompt, estimate. @@ -160,20 +135,13 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) { metadata := chatRunMetadata(runMeta, req, outputPolicy) metadata["estimated_input_tokens"] = strconv.Itoa(estimate) metadata["context_class"] = contextClass - if responseMode == responseModeTransformed { - // Explicit transformed mode produces IOP-transformed output (SDD S07): - // label the response so it is never mistaken for provider-original - // byte-identical passthrough. - metadata[responseModeMetadataKey] = responseModeTransformed - w.Header().Set(responseModeHeaderName, responseModeTransformed) - } if validation.enabled { metadata = toolValidationAttemptMetadata(metadata, 1, "", "") } submitReq := chatSubmitRunRequest(dispatch, req, workspace, prompt, input, metadata) submitReq.EstimatedInputTokens = estimate submitReq.ContextClass = contextClass - metricLabels := s.usageLabelsFor(r.Context(), strings.TrimSpace(req.Model), usageEndpointChatCompletions, chatResponseModeLabel(req)) + metricLabels := s.usageLabelsFor(r.Context(), strings.TrimSpace(req.Model), usageEndpointChatCompletions, responseModeNormalized) handle, err := s.service.SubmitRun(r.Context(), submitReq) if err != nil { emitUsageMetrics(metricLabels, usageStatusForError(err), usageObservation{}) @@ -226,7 +194,6 @@ func (s *Server) handleChatCompletionsProviderPool( contextClass string, outputPolicy strictOutputPolicy, validation toolValidationContract, - responseMode string, w http.ResponseWriter, ) { poolReq := edgeservice.ProviderPoolDispatchRequest{ @@ -274,10 +241,8 @@ func (s *Server) handleChatCompletionsProviderPool( return tunnelReq, nil } - // provider-pool path preserves responseMode for tunnel passthrough. // strict-output output policy only applies to normalized dispatch, // not to raw tunnel passthrough (SDD D02). - poolReq.Tunnel.BuildBody = func(target string) ([]byte, error) { return rewriteChatCompletionModel(rawBody, target, req) } @@ -327,16 +292,9 @@ func (s *Server) handleChatCompletionsProviderPool( case edgeservice.ProviderPoolPathTunnel: // Tunnel path: provider auth was already validated and injected via // PrepareTunnel before dispatch; on failure SubmitProviderPool returns - // an error and no tunnel handle exists. - metricLabels := s.usageLabelsFor(r.Context(), strings.TrimSpace(req.Model), usageEndpointChatCompletions, responseMode) - if responseMode == responseModePassthroughSideband { - if req.Stream { - s.writeProviderTunnelSidebandStream(w, r, result.Tunnel) - return - } - s.writeProviderTunnelSidebandResponse(w, r, result.Tunnel) - return - } + // an error and no tunnel handle exists. Provider bytes are relayed as + // pure passthrough; caller metadata never selects a sideband surface. + metricLabels := s.usageLabelsFor(r.Context(), strings.TrimSpace(req.Model), usageEndpointChatCompletions, responseModePassthrough) s.writeProviderTunnelResponse(w, r, result.Tunnel, req.Stream, req.Model, metricLabels) case edgeservice.ProviderPoolPathNormalized: @@ -552,21 +510,6 @@ func boolPtr(v bool) *bool { return &v } -// chatResponseModeLabel returns the usage-metric response_mode label for a -// normalized-path chat completion: "transformed" when the caller explicitly -// requested it, otherwise "normalized". Provider-tunnel passthrough paths use -// their own mode label and do not call this. -func chatResponseModeLabel(req chatCompletionRequest) string { - runMeta, _, err := parseOpenAIMetadata(req.Metadata) - if err != nil { - return "normalized" - } - if mode, _ := parseResponseMode(runMeta); mode == responseModeTransformed { - return responseModeTransformed - } - return "normalized" -} - func chatRunMetadata(runMeta map[string]string, req chatCompletionRequest, outputPolicy strictOutputPolicy) map[string]string { if runMeta == nil { runMeta = make(map[string]string) @@ -578,37 +521,15 @@ func chatRunMetadata(runMeta map[string]string, req chatCompletionRequest, outpu } const ( - responseModeMetadataKey = "iop_response_mode" - responseModePassthrough = "passthrough" - responseModePassthroughSideband = "passthrough+sideband" - responseModeTransformed = "transformed" - // responseModeHeaderName labels explicit IOP extension outputs - // (passthrough+sideband, transformed). Pure passthrough responses never - // carry it: their headers stay provider-original. - responseModeHeaderName = "X-IOP-Response-Mode" + // responseModePassthrough and responseModeNormalized are internal execution + // labels for the response_mode usage metric. They are derived from the + // handler execution path, never from caller metadata: provider tunnel routes + // report passthrough, normalized RunEvent routes report normalized. Callers + // cannot select a response mode through OpenAI metadata. + responseModePassthrough = "passthrough" + responseModeNormalized = "normalized" ) -func parseResponseMode(runMeta map[string]string) (string, error) { - mode := strings.TrimSpace(runMeta[responseModeMetadataKey]) - if mode == "" { - return responseModePassthrough, nil - } - switch mode { - case responseModePassthrough, responseModePassthroughSideband, responseModeTransformed: - return mode, nil - default: - return "", fmt.Errorf("metadata.%s must be one of %s, %s, or %s", responseModeMetadataKey, responseModePassthrough, responseModePassthroughSideband, responseModeTransformed) - } -} - -func responseModeWasExplicit(runMeta map[string]string) bool { - if runMeta == nil { - return false - } - mode, ok := runMeta[responseModeMetadataKey] - return ok && strings.TrimSpace(mode) != "" -} - // routeUsesProviderTunnel reports whether the resolved dispatch targets an // OpenAI-compatible provider that serves raw tunnel passthrough. Provider-pool // catalog routes and openai_compat/vllm type routes qualify; CLI and other @@ -710,7 +631,7 @@ func (s *Server) completeChatCompletion(w http.ResponseWriter, r *http.Request, s.cancelRunOnHTTPGiveUp(handle.Dispatch(), err) handle.Close() emitUsageMetrics( - s.usageLabelsFor(r.Context(), strings.TrimSpace(req.Model), usageEndpointChatCompletions, chatResponseModeLabel(req)), + s.usageLabelsFor(r.Context(), strings.TrimSpace(req.Model), usageEndpointChatCompletions, responseModeNormalized), usageStatusForError(err), usageObservation{}, ) writeError(w, httpStatusForRunError(err), "run_error", err.Error()) @@ -735,7 +656,7 @@ func (s *Server) completeChatCompletion(w http.ResponseWriter, r *http.Request, next, submitErr := retrySubmit(r.Context(), retryReq) if submitErr != nil { emitUsageMetrics( - s.usageLabelsFor(r.Context(), strings.TrimSpace(req.Model), usageEndpointChatCompletions, chatResponseModeLabel(req)), + s.usageLabelsFor(r.Context(), strings.TrimSpace(req.Model), usageEndpointChatCompletions, responseModeNormalized), usageStatusError, usageObservation{}, ) writeError(w, http.StatusBadGateway, "tool_validation_retry_error", submitErr.Error()) @@ -771,7 +692,7 @@ func (s *Server) completeChatCompletion(w http.ResponseWriter, r *http.Request, zap.String("reason", valErr.Error()), ) emitUsageMetrics( - s.usageLabelsFor(r.Context(), strings.TrimSpace(req.Model), usageEndpointChatCompletions, chatResponseModeLabel(req)), + s.usageLabelsFor(r.Context(), strings.TrimSpace(req.Model), usageEndpointChatCompletions, responseModeNormalized), usageStatusError, usageObservation{}, ) writeError(w, http.StatusBadGateway, "tool_validation_error", valErr.Error()) @@ -873,18 +794,12 @@ func collectChatCompletionOutput(ctx context.Context, req chatCompletionRequest, } } - runMeta, _, _ := parseOpenAIMetadata(req.Metadata) - mode, _ := parseResponseMode(runMeta) - if mode != responseModeTransformed { - mode = "normalized" - } - return chatCompletionOutput{ message: message, finishReason: finishReason, usage: usage, normalized: normalized, - responseMode: mode, + responseMode: responseModeNormalized, contentLen: len(message.Content), reasoningLen: len(reasoning), toolCalls: toolCalls, diff --git a/apps/edge/internal/openai/responses_handler.go b/apps/edge/internal/openai/responses_handler.go index 6f7d4c2..71857f7 100644 --- a/apps/edge/internal/openai/responses_handler.go +++ b/apps/edge/internal/openai/responses_handler.go @@ -13,7 +13,6 @@ import ( "go.uber.org/zap" edgeservice "iop/apps/edge/internal/service" - iop "iop/proto/gen/iop" ) func (s *Server) handleResponses(w http.ResponseWriter, r *http.Request) { @@ -57,30 +56,9 @@ func (s *Server) handleResponses(w http.ResponseWriter, r *http.Request) { for k, v := range principalMetadata(r.Context()) { runMeta[k] = v } - if dispatch.ProviderPool && responseModeWasExplicit(runMeta) { - writeError(w, http.StatusBadRequest, "invalid_request_error", "metadata.iop_response_mode is not supported for model group routes") - return - } - responseMode, err := parseResponseMode(runMeta) - if err != nil { - writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error()) - return - } - switch responseMode { - case responseModePassthrough: - case responseModePassthroughSideband: - if err := validateWorkspaceForRoute(dispatch, workspace); err != nil { - writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error()) - return - } - estimate := estimateInputTokens(string(rawBody), runMeta, nil, nil) - contextClass := classifyContext(estimate, s.longContextThreshold()) - s.tunnelResponsesPassthroughSideband(w, r, env, dispatch, runMeta, rawBody, estimate, contextClass) - return - case responseModeTransformed: - writeError(w, http.StatusBadRequest, "invalid_request_error", "metadata.iop_response_mode=transformed is not supported for /v1/responses provider routes") - return - } + // Provider routes always relay pure passthrough. Caller metadata is + // arbitrary context and never selects the route or response shape + // (SDD S01/S04); there is no caller-facing response mode selector. if err := validateWorkspaceForRoute(dispatch, workspace); err != nil { writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error()) return @@ -278,7 +256,6 @@ func (s *Server) tunnelResponsesPassthrough(w http.ResponseWriter, r *http.Reque } metadata["openai_model"] = env.Model metadata["openai_stream"] = strconv.FormatBool(env.Stream) - metadata[responseModeMetadataKey] = responseModePassthrough metadata["estimated_input_tokens"] = strconv.Itoa(estimate) metadata["context_class"] = contextClass @@ -336,82 +313,6 @@ func (s *Server) tunnelResponsesPassthrough(w http.ResponseWriter, r *http.Reque s.writeProviderTunnelResponse(w, r, handle, env.Stream, "", metricLabels) } -// tunnelResponsesPassthroughSideband serves an explicit /v1/responses -// passthrough+sideband request. The provider request remains raw passthrough -// with model rewrite only; the response is extended after the provider returns. -// Non-streaming JSON object responses receive sideband metadata under the -// top-level `metadata` field. Streaming responses interleave `event: -// iop.sideband` events. -func (s *Server) tunnelResponsesPassthroughSideband(w http.ResponseWriter, r *http.Request, env responsesEnvelope, dispatch routeDispatch, runMeta map[string]string, rawBody []byte, estimate int, contextClass string) { - providerAuthHeaders, err := s.providerTunnelAuthHeaders(r) - if err != nil { - writeError(w, http.StatusBadRequest, "invalid_request_error", "provider auth token is required") - return - } - - metadata := make(map[string]string, len(runMeta)+5) - for k, v := range runMeta { - metadata[k] = v - } - metadata["openai_model"] = env.Model - metadata["openai_stream"] = strconv.FormatBool(env.Stream) - metadata[responseModeMetadataKey] = responseModePassthroughSideband - metadata["estimated_input_tokens"] = strconv.Itoa(estimate) - metadata["context_class"] = contextClass - - tunnelReq := edgeservice.SubmitProviderTunnelRequest{ - NodeRef: dispatch.NodeRef, - ModelGroupKey: strings.TrimSpace(env.Model), - Adapter: dispatch.Adapter, - Target: dispatch.Target, - SessionID: dispatch.SessionID, - Method: http.MethodPost, - Path: "/v1/responses", - Headers: providerAuthHeaders, - BuildBody: func(target string) ([]byte, error) { - return rewriteResponsesModel(rawBody, target) - }, - Stream: env.Stream, - TimeoutSec: dispatch.TimeoutSec, - MaxQueue: dispatch.MaxQueue, - QueueTimeoutMS: dispatch.QueueTimeoutMS, - Metadata: metadata, - EstimatedInputTokens: estimate, - ContextClass: contextClass, - ProviderPool: dispatch.ProviderPool, - } - - metricLabels := s.usageLabelsFor(r.Context(), strings.TrimSpace(env.Model), usageEndpointResponses, responseModePassthroughSideband) - handle, err := s.service.SubmitProviderTunnel(r.Context(), tunnelReq) - if err != nil { - emitUsageMetrics(metricLabels, usageStatusForError(err), usageObservation{}) - writeError(w, http.StatusBadGateway, "node_dispatch_error", err.Error()) - return - } - defer handle.Close() - - s.logger.Info("openai responses sideband dispatch", - zap.String("run_id", handle.Dispatch().RunID), - zap.String("node_id", handle.Dispatch().NodeID), - zap.String("provider_id", handle.Dispatch().ProviderID), - zap.String("provider_type", handle.Dispatch().ProviderType), - zap.String("execution_path", handle.Dispatch().ExecutionPath), - zap.String("model_group", handle.Dispatch().ModelGroupKey), - zap.String("adapter", handle.Dispatch().Adapter), - zap.String("target", handle.Dispatch().Target), - zap.Bool("stream", env.Stream), - zap.Int("estimated_input_tokens", handle.Dispatch().EstimatedInputTokens), - zap.String("context_class", handle.Dispatch().ContextClass), - zap.String("queue_reason", handle.Dispatch().QueueReason), - ) - - if env.Stream { - s.writeResponsesProviderTunnelSidebandStream(w, r, handle, metricLabels) - return - } - s.writeResponsesProviderTunnelSidebandResponse(w, r, handle, metricLabels) -} - // rewriteResponsesModel replaces only the model field of the caller's original // /v1/responses request JSON so the provider receives its served model name. // Every other field (input, instructions, tools, max_output_tokens, and any @@ -433,264 +334,6 @@ func rewriteResponsesModel(rawBody []byte, target string) ([]byte, error) { return json.Marshal(raw) } -const responsesSidebandObject = "iop.responses.sideband" - -type responsesSidebandPayload struct { - Object string `json:"object"` - Metadata map[string]any `json:"metadata"` -} - -func responsesSidebandMetadata() map[string]any { - return map[string]any{ - responseModeMetadataKey: responseModePassthroughSideband, - } -} - -func responsesSidebandPayloadBytes() []byte { - payload, err := json.Marshal(responsesSidebandPayload{ - Object: responsesSidebandObject, - Metadata: responsesSidebandMetadata(), - }) - if err != nil { - return nil - } - return payload -} - -func injectResponsesSidebandMetadata(body []byte) []byte { - var obj map[string]json.RawMessage - if err := json.Unmarshal(body, &obj); err != nil { - return body - } - if obj == nil { - return body - } - - metadata := map[string]any{} - if raw, ok := obj["metadata"]; ok && len(raw) > 0 && string(raw) != "null" { - _ = json.Unmarshal(raw, &metadata) - if metadata == nil { - metadata = map[string]any{} - } - } - for k, v := range responsesSidebandMetadata() { - metadata[k] = v - } - encodedMetadata, err := json.Marshal(metadata) - if err != nil { - return body - } - obj["metadata"] = encodedMetadata - rewritten, err := json.Marshal(obj) - if err != nil { - return body - } - return rewritten -} - -func (s *Server) writeResponsesProviderTunnelSidebandStream(w http.ResponseWriter, r *http.Request, handle edgeservice.ProviderTunnelResult, metricLabels usageLabels) { - frames := handle.Stream().Frames - if frames == nil { - writeError(w, http.StatusBadGateway, "provider_tunnel_error", "tunnel stream unavailable") - return - } - flusher, _ := w.(http.Flusher) - timer := time.NewTimer(handle.WaitTimeout()) - defer timer.Stop() - - assembler := &providerChatAssembler{streaming: true} - wroteHeader := false - metricStatus := usageStatusError - var protoObs usageObservation - tail := "" - - writeSideband := func() { - payload := responsesSidebandPayloadBytes() - if len(payload) == 0 { - return - } - fmt.Fprintf(w, "event: %s\ndata: %s\n\n", sidebandSSEEventName, payload) - if flusher != nil { - flusher.Flush() - } - } - - defer func() { - emitUsageMetrics(metricLabels, metricStatus, mergeUsageObservation(assembler.usageObservation(), protoObs)) - }() - - for { - select { - case <-r.Context().Done(): - s.cancelRunOnHTTPGiveUp(handle.Dispatch(), r.Context().Err()) - metricStatus = usageStatusCancel - return - case <-timer.C: - s.cancelRunOnHTTPGiveUp(handle.Dispatch(), errRunTimedOut) - metricStatus = usageStatusCancel - if !wroteHeader { - writeError(w, http.StatusBadGateway, "run_error", errRunTimedOut.Error()) - } - return - case frame, ok := <-frames: - if !ok { - if !wroteHeader { - writeError(w, http.StatusBadGateway, "provider_tunnel_error", "tunnel stream closed before provider response") - } - return - } - switch frame.GetKind() { - case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START: - if wroteHeader { - continue - } - copyProviderResponseHeaders(w.Header(), frame.GetHeaders()) - w.Header().Del("Content-Length") - w.Header().Set(responseModeHeaderName, responseModePassthroughSideband) - status := int(frame.GetStatusCode()) - if status == 0 { - status = http.StatusOK - } - w.WriteHeader(status) - wroteHeader = true - writeSideband() - case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY: - body := frame.GetBody() - if len(body) == 0 { - continue - } - if !wroteHeader { - w.Header().Set(responseModeHeaderName, responseModePassthroughSideband) - w.WriteHeader(http.StatusOK) - wroteHeader = true - writeSideband() - } - if _, err := w.Write(body); err != nil { - s.sendCancelRun(handle.Dispatch()) - metricStatus = usageStatusCancel - return - } - assembler.Write(body) - tail = sseTail(tail, body) - if flusher != nil { - flusher.Flush() - } - case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_ERROR: - msg := frame.GetError() - if msg == "" { - msg = "provider tunnel failed" - } - if !wroteHeader { - writeError(w, http.StatusBadGateway, "provider_tunnel_error", msg) - return - } - s.logger.Warn("openai responses sideband tunnel error after response start", - zap.String("run_id", handle.Dispatch().RunID), - zap.String("error", msg), - ) - return - case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_USAGE: - protoObs = mergeUsageObservation(protoObs, usageObservationFromProtoUsage(frame.GetUsage())) - case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END: - if !wroteHeader { - writeError(w, http.StatusBadGateway, "provider_tunnel_error", "tunnel ended before provider response") - return - } - if tail != "" && !strings.HasSuffix(tail, "\n\n") { - fmt.Fprint(w, "\n\n") - if flusher != nil { - flusher.Flush() - } - } - metricStatus = usageStatusSuccess - return - } - } - } -} - -func (s *Server) writeResponsesProviderTunnelSidebandResponse(w http.ResponseWriter, r *http.Request, handle edgeservice.ProviderTunnelResult, metricLabels usageLabels) { - frames := handle.Stream().Frames - if frames == nil { - writeError(w, http.StatusBadGateway, "provider_tunnel_error", "tunnel stream unavailable") - return - } - timer := time.NewTimer(handle.WaitTimeout()) - defer timer.Stop() - - assembler := &providerChatAssembler{} - var body bytes.Buffer - providerStatus := 0 - providerHeaders := map[string]string{} - metricStatus := usageStatusError - var protoObs usageObservation - - defer func() { - // Non-streaming assembler usage is parsed lazily from the buffered JSON - // body, so force parsing before emitting metrics. - _ = assembler.observation() - emitUsageMetrics(metricLabels, metricStatus, mergeUsageObservation(assembler.usageObservation(), protoObs)) - }() - - for { - select { - case <-r.Context().Done(): - s.cancelRunOnHTTPGiveUp(handle.Dispatch(), r.Context().Err()) - metricStatus = usageStatusCancel - return - case <-timer.C: - s.cancelRunOnHTTPGiveUp(handle.Dispatch(), errRunTimedOut) - metricStatus = usageStatusCancel - writeError(w, http.StatusBadGateway, "run_error", errRunTimedOut.Error()) - return - case frame, ok := <-frames: - if !ok { - writeError(w, http.StatusBadGateway, "provider_tunnel_error", "tunnel stream closed before provider response") - return - } - switch frame.GetKind() { - case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START: - if providerStatus != 0 { - continue - } - providerStatus = int(frame.GetStatusCode()) - if providerStatus == 0 { - providerStatus = http.StatusOK - } - providerHeaders = frame.GetHeaders() - case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY: - body.Write(frame.GetBody()) - assembler.Write(frame.GetBody()) - case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_ERROR: - msg := frame.GetError() - if msg == "" { - msg = "provider tunnel failed" - } - writeError(w, http.StatusBadGateway, "provider_tunnel_error", msg) - return - case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_USAGE: - protoObs = mergeUsageObservation(protoObs, usageObservationFromProtoUsage(frame.GetUsage())) - case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END: - if providerStatus == 0 { - writeError(w, http.StatusBadGateway, "provider_tunnel_error", "tunnel ended before provider response") - return - } - copyProviderResponseHeaders(w.Header(), providerHeaders) - w.Header().Del("Content-Length") - w.Header().Set(responseModeHeaderName, responseModePassthroughSideband) - w.WriteHeader(providerStatus) - if _, err := w.Write(injectResponsesSidebandMetadata(body.Bytes())); err != nil { - s.sendCancelRun(handle.Dispatch()) - metricStatus = usageStatusCancel - return - } - metricStatus = usageStatusSuccess - return - } - } - } -} - func (s *Server) completeResponse(w http.ResponseWriter, r *http.Request, req responsesRequest, handle edgeservice.RunResult, outputPolicy strictOutputPolicy) { metricLabels := s.usageLabelsFor(r.Context(), strings.TrimSpace(req.Model), usageEndpointResponses, "normalized") text, reasoning, _, _, usage, _, err := collectRunResult(r.Context(), handle.Stream(), handle.WaitTimeout()) diff --git a/apps/edge/internal/openai/server_test.go b/apps/edge/internal/openai/server_test.go index 8ac3e57..4cbf9df 100644 --- a/apps/edge/internal/openai/server_test.go +++ b/apps/edge/internal/openai/server_test.go @@ -707,7 +707,7 @@ func TestChatCompletionsPreservesProviderFinishReason(t *testing.T) { srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama"}, fake, nil) req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ - "metadata":{"iop_response_mode":"transformed"}, + "metadata":{}, "model":"qwen3.6:35b", "messages":[{"role":"user","content":"hi"}] }`)) @@ -804,7 +804,7 @@ func TestChatCompletionsOmitsProviderToolChoiceAuto(t *testing.T) { srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama"}, fake, nil) req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ - "metadata":{"iop_response_mode":"transformed"}, + "metadata":{}, "model":"qwen3.6:35b", "messages":[{"role":"user","content":"hi"}], "tools":[{"type":"function","function":{"name":"lookup"}}], @@ -837,7 +837,7 @@ func TestChatCompletionsPassesProviderForcedToolChoice(t *testing.T) { srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama"}, fake, nil) req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ - "metadata":{"iop_response_mode":"transformed"}, + "metadata":{}, "model":"qwen3.6:35b", "messages":[{"role":"user","content":"hi"}], "tools":[{"type":"function","function":{"name":"lookup"}}], @@ -1006,7 +1006,7 @@ func TestChatCompletionsSynthesizesTextToolCallsForProviderRoute(t *testing.T) { srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama"}, fake, nil) req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ - "metadata":{"iop_response_mode":"transformed"}, + "metadata":{}, "model":"qwen3.6:35b", "messages":[{"role":"user","content":"status"}], "tools":[{"type":"function","function":{"name":"run_commands","parameters":{"type":"object"}}}], @@ -1039,7 +1039,7 @@ func TestChatCompletionsSynthesizesEditToolCallWithMarkdownAndAngleBracketPlaceh srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama"}, fake, nil) req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ - "metadata":{"iop_response_mode":"transformed"}, + "metadata":{}, "model":"qwen3.6:35b", "messages":[{"role":"user","content":"edit"}], "tools":[{"type":"function","function":{"name":"edit","parameters":{"type":"object","properties":{"path":{"type":"string"},"edits":{"type":"array","items":{"type":"object","properties":{"oldText":{"type":"string"},"newText":{"type":"string"}},"required":["newText"],"additionalProperties":false}}},"required":["path","edits"],"additionalProperties":false}}}], @@ -1090,7 +1090,7 @@ func TestChatCompletionsStreamSynthesizesEditToolCallWithMarkdownAndAngleBracket srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama"}, fake, nil) req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ - "metadata":{"iop_response_mode":"transformed"}, + "metadata":{}, "model":"qwen3.6:35b", "stream":true, "messages":[{"role":"user","content":"edit"}], @@ -1154,7 +1154,7 @@ func TestChatCompletionsSynthesizesProviderTextToolCallsWhenFallbackMarked(t *te srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama"}, fake, nil) req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ - "metadata":{"iop_response_mode":"transformed"}, + "metadata":{}, "model":"qwen3.6:35b", "messages":[{"role":"user","content":"status"}], "tools":[{"type":"function","function":{"name":"run_commands","parameters":{"type":"object","properties":{"commands":{"type":"array","items":{"type":"string"}}},"required":["commands"]}}}], @@ -1200,7 +1200,7 @@ func TestChatCompletionsSanitizesKnownSentinelToken(t *testing.T) { srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama"}, fake, nil) req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ - "metadata":{"iop_response_mode":"transformed"}, + "metadata":{}, "model":"qwen3.6:35b", "messages":[{"role":"user","content":"hi"}] }`)) @@ -1268,7 +1268,7 @@ func TestChatCompletionsPassesThroughNativeToolCallsFromProviderRoute(t *testing srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama"}, fake, nil) req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ - "metadata":{"iop_response_mode":"transformed"}, + "metadata":{}, "model":"qwen3.6:35b", "messages":[{"role":"user","content":"status"}], "tools":[{"type":"function","function":{"name":"run_commands"}}], @@ -1311,7 +1311,7 @@ func TestChatCompletionsNativeToolCallPreservesLeadingProse(t *testing.T) { srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama"}, fake, nil) req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ - "metadata":{"iop_response_mode":"transformed"}, + "metadata":{}, "model":"qwen3.6:35b", "messages":[{"role":"user","content":"status"}], "tools":[{"type":"function","function":{"name":"run_commands"}}], @@ -1357,7 +1357,7 @@ func TestChatCompletionsNormalizesNativeToolCallArgumentsForProviderRoute(t *tes srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama"}, fake, nil) req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ - "metadata":{"iop_response_mode":"transformed"}, + "metadata":{}, "model":"qwen3.6:35b", "messages":[{"role":"user","content":"status"}], "tools":[{"type":"function","function":{"name":"run_commands","parameters":{"type":"object","properties":{"commands":{"type":"array","items":{"type":"string"}}},"required":["commands"]}}}], @@ -1408,7 +1408,7 @@ func TestChatCompletionsRetriesMalformedToolCallBeforeResponse(t *testing.T) { srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama"}, fake, nil) req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ - "metadata":{"iop_response_mode":"transformed"}, + "metadata":{}, "model":"qwen3.6:35b", "messages":[{"role":"user","content":"status"}], "tools":[{"type":"function","function":{"name":"run_commands","parameters":{"type":"object","properties":{"commands":{"type":"array","items":{"type":"string"}}},"required":["commands"],"additionalProperties":false}}}], @@ -1468,7 +1468,7 @@ func TestChatCompletionsFailsMalformedToolCallAfterRetryLimit(t *testing.T) { srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama"}, fake, nil) req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ - "metadata":{"iop_response_mode":"transformed"}, + "metadata":{}, "model":"qwen3.6:35b", "messages":[{"role":"user","content":"status"}], "tools":[{"type":"function","function":{"name":"run_commands","parameters":{"type":"object","properties":{"commands":{"type":"array","items":{"type":"string"}}},"required":["commands"],"additionalProperties":false}}}], @@ -1501,7 +1501,7 @@ func TestChatCompletionsToolChoiceNoneSkipsRuntimeToolValidation(t *testing.T) { srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama"}, fake, nil) req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ - "metadata":{"iop_response_mode":"transformed"}, + "metadata":{}, "model":"qwen3.6:35b", "messages":[{"role":"user","content":"status"}], "tools":[{"type":"function","function":{"name":"run_commands","parameters":{"type":"object","properties":{"commands":{"type":"array","items":{"type":"string"}}},"required":["commands"]}}}], @@ -1546,7 +1546,7 @@ func TestChatCompletionsToolsStreamRetriesMalformedToolCallBeforeChunk(t *testin srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama"}, fake, nil) req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ - "metadata":{"iop_response_mode":"transformed"}, + "metadata":{}, "model":"qwen3.6:35b", "stream":true, "messages":[{"role":"user","content":"edit"}], @@ -1605,7 +1605,7 @@ func TestChatCompletionsStrictBufferedStreamRetriesMalformedToolCallBeforeChunk( srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama", StrictOutput: true, StrictStreamBuffer: true}, fake, nil) req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ - "metadata":{"iop_response_mode":"transformed"}, + "metadata":{}, "model":"qwen3.6:35b", "stream":true, "messages":[{"role":"user","content":"status"}], @@ -1658,7 +1658,7 @@ func TestChatCompletionsStrictBufferedStreamFailsMalformedToolCallAfterRetryLimi srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama", StrictOutput: true, StrictStreamBuffer: true}, fake, nil) req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ - "metadata":{"iop_response_mode":"transformed"}, + "metadata":{}, "model":"qwen3.6:35b", "stream":true, "messages":[{"role":"user","content":"status"}], @@ -2162,7 +2162,7 @@ func TestChatCompletionsMapsMaxCompletionTokens(t *testing.T) { srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama"}, fake, nil) req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ - "metadata":{"iop_response_mode":"transformed"}, + "metadata":{}, "model":"from-request", "messages":[{"role":"user","content":"hi"}], "max_completion_tokens":7 @@ -2252,7 +2252,7 @@ func TestChatCompletionsStreamWithToolsBuffersTextUntilComplete(t *testing.T) { )} srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama"}, fake, nil) req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ - "metadata":{"iop_response_mode":"transformed"}, + "metadata":{}, "model":"stream-model", "stream":true, "messages":[{"role":"user","content":"explain sudo"}], @@ -2336,7 +2336,7 @@ func TestChatCompletionsStreamPassesThroughNativeToolCallsFromProviderRoute(t *t srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama"}, fake, nil) req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ - "metadata":{"iop_response_mode":"transformed"}, + "metadata":{}, "model":"stream-model", "stream":true, "messages":[{"role":"user","content":"status"}], @@ -2372,7 +2372,7 @@ func TestChatCompletionsStreamDropsRawTextWhenNativeToolCallsArrive(t *testing.T srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama"}, fake, nil) req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ - "metadata":{"iop_response_mode":"transformed"}, + "metadata":{}, "model":"stream-model", "stream":true, "messages":[{"role":"user","content":"status"}], @@ -2408,7 +2408,7 @@ func TestChatCompletionsToolsStreamNativeToolCallPreservesLeadingProse(t *testin srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama"}, fake, nil) req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ - "metadata":{"iop_response_mode":"transformed"}, + "metadata":{}, "model":"stream-model", "stream":true, "messages":[{"role":"user","content":"status"}], @@ -2468,7 +2468,7 @@ func TestChatCompletionsStreamDropsPartialTextToolCallSuffixWhenNativeToolCallsA srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama"}, fake, nil) req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ - "metadata":{"iop_response_mode":"transformed"}, + "metadata":{}, "model":"stream-model", "stream":true, "messages":[{"role":"user","content":"status"}], @@ -2783,7 +2783,7 @@ func TestChatCompletionsStreamSanitizesKnownSentinelTokenAcrossDeltas(t *testing srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama"}, fake, nil) req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ - "metadata":{"iop_response_mode":"transformed"}, + "metadata":{}, "model":"stream-model", "stream":true, "messages":[{"role":"user","content":"hi"}] @@ -4005,7 +4005,7 @@ func TestChatCompletionsRouteCatalogDispatchesModelB(t *testing.T) { }, }, fake, nil) req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ - "metadata":{"iop_response_mode":"transformed"}, + "metadata":{}, "model":"model-b", "messages":[{"role":"user","content":"hi"}] }`)) @@ -4501,58 +4501,28 @@ func TestChatCompletionsStrictOutputProviderPoolDefaultKeepsPassthroughPath(t *t if len(fake.reqsSnapshot()) != 0 { t.Fatalf("strict output provider route must not fall back to normalized SubmitRun, got %d calls", len(fake.reqsSnapshot())) } - if got := w.Header().Get(responseModeHeaderName); got != "" { - t.Fatalf("pure passthrough must not carry response mode header, got %q", got) - } } -func TestChatCompletionsProviderPoolRejectsResponseMode(t *testing.T) { +// TestChatCompletionsProviderPoolIgnoresSelectorLikeMetadata verifies SDD S01: +// arbitrary caller metadata — including keys or values that resemble a legacy +// response-mode selector — is opaque context. It never rejects the request or +// switches a provider-pool route off pure passthrough. (The former selector key +// literal is deliberately not used here; the deterministic surface check forbids +// it anywhere under apps.) +func TestChatCompletionsProviderPoolIgnoresSelectorLikeMetadata(t *testing.T) { cases := []struct { - name string - metadata string - wantStatus int - wantTunnel bool - wantRun bool - wantMessage string + name string + metadata string }{ - { - name: "explicit passthrough is rejected", - metadata: `"metadata":{"iop_response_mode":"passthrough"}`, - wantStatus: http.StatusBadRequest, - wantMessage: "metadata.iop_response_mode is not supported for model group routes", - }, - { - name: "sideband mode is rejected", - metadata: `"metadata":{"iop_response_mode":"passthrough+sideband"}`, - wantStatus: http.StatusBadRequest, - wantMessage: "metadata.iop_response_mode is not supported for model group routes", - }, - { - name: "transformed mode is rejected", - metadata: `"metadata":{"iop_response_mode":"transformed"}`, - wantStatus: http.StatusBadRequest, - wantMessage: "metadata.iop_response_mode is not supported for model group routes", - }, - { - name: "unknown mode is rejected before dispatch", - metadata: `"metadata":{"iop_response_mode":"rawish"}`, - wantStatus: http.StatusBadRequest, - wantMessage: "metadata.iop_response_mode is not supported for model group routes", - }, + {name: "selector-like value is ignored", metadata: `"metadata":{"response_hint":"raw"}`}, + {name: "arbitrary metadata is ignored", metadata: `"metadata":{"experiment":"mode-x","team":"search"}`}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - frames := make(chan *iop.ProviderTunnelFrame, 3) - frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START, StatusCode: 200} - frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(`{"ok":true}`)} - frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true} - close(frames) + providerBody := `{"ok":true}` fake := &fakeRunService{ - events: bufferedRunEvents( - &iop.RunEvent{Type: "delta", Delta: "ok"}, - &iop.RunEvent{Type: "complete"}, - ), - tunnelFrames: frames, + poolDispatchPath: string(edgeservice.ProviderPoolPathTunnel), + tunnelFrames: staticProviderTunnelFrames(providerBody), } srv := NewServer(config.EdgeOpenAIConf{}, fake, nil) srv.SetModelCatalog([]config.ModelCatalogEntry{ @@ -4568,25 +4538,25 @@ func TestChatCompletionsProviderPoolRejectsResponseMode(t *testing.T) { w := httptest.NewRecorder() srv.handleChatCompletions(w, req) - if w.Code != tc.wantStatus { - t.Fatalf("status: got %d want %d body=%s", w.Code, tc.wantStatus, w.Body.String()) + if w.Code != http.StatusOK { + t.Fatalf("selector-like metadata must not be rejected: got %d body=%s", w.Code, w.Body.String()) } - if tc.wantMessage != "" && !strings.Contains(w.Body.String(), tc.wantMessage) { - t.Fatalf("expected %q in body, got %s", tc.wantMessage, w.Body.String()) + if got := w.Body.String(); got != providerBody { + t.Fatalf("provider body must be relayed byte-identically: got %q want %q", got, providerBody) } - if got := len(fake.tunnelReqsSnapshot()) > 0; got != tc.wantTunnel { - t.Fatalf("tunnel dispatch: got %v want %v", got, tc.wantTunnel) + if len(fake.tunnelReqsSnapshot()) != 1 { + t.Fatalf("expected 1 pure passthrough tunnel dispatch, got %d", len(fake.tunnelReqsSnapshot())) } - if got := len(fake.reqsSnapshot()) > 0; got != tc.wantRun { - t.Fatalf("normalized dispatch: got %v want %v", got, tc.wantRun) + if len(fake.reqsSnapshot()) != 0 { + t.Fatalf("arbitrary metadata must not switch to normalized SubmitRun, got %d", len(fake.reqsSnapshot())) } }) } } // chatSidebandServer returns a direct OpenAI-compatible provider route backed by -// the given tunnel frames for explicit passthrough+sideband fixtures. It avoids -// provider-pool because model group routes reject caller response mode selectors. +// the given tunnel frames. It routes via the legacy openai_compat model route +// (not provider-pool) so the raw provider tunnel passthrough path is exercised. func chatSidebandServer(frames chan *iop.ProviderTunnelFrame) (*Server, *fakeRunService) { fake := &fakeRunService{ tunnelFrames: frames, @@ -4603,198 +4573,6 @@ func chatProviderRouteServer(fake *fakeRunService, logger *zap.Logger) *Server { }, fake, logger) } -// TestChatCompletionsPassthroughSidebandStreamingExposesIOPExtension verifies -// SDD S05 for the streaming path: explicit passthrough+sideband preserves the -// provider SSE events contiguously and exposes IOP route/usage/assembled -// observations as explicit `event: iop.sideband` extension events, injected -// only at provider event boundaries. -func TestChatCompletionsPassthroughSidebandStreamingExposesIOPExtension(t *testing.T) { - roleEvent := "data: {\"choices\":[{\"delta\":{\"role\":\"assistant\"}}]}\n\n" - contentEvent := "data: {\"choices\":[{\"delta\":{\"content\":\"hello\"}}]}\n\n" - doneEvent := "data: [DONE]\n\n" - frames := make(chan *iop.ProviderTunnelFrame, 7) - frames <- &iop.ProviderTunnelFrame{ - Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START, - StatusCode: 200, - Headers: map[string]string{"Content-Type": "text/event-stream"}, - } - frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(roleEvent)} - // Split the content event so the usage frame arrives while the provider - // event is incomplete; the sideband writer must not inject there. - frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(contentEvent[:17])} - frames <- &iop.ProviderTunnelFrame{ - Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_USAGE, - Usage: &iop.Usage{InputTokens: 3, OutputTokens: 5}, - } - frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(contentEvent[17:])} - frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(doneEvent)} - frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true} - close(frames) - srv, fake := chatSidebandServer(frames) - - req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ - "model":"pool-model", - "stream":true, - "messages":[{"role":"user","content":"hello"}], - "metadata":{"iop_response_mode":"passthrough+sideband"} - }`)) - w := httptest.NewRecorder() - srv.handleChatCompletions(w, req) - - if w.Code != http.StatusOK { - t.Fatalf("status: got %d body=%s", w.Code, w.Body.String()) - } - if got := w.Header().Get("X-IOP-Response-Mode"); got != "passthrough+sideband" { - t.Errorf("sideband response mode header: got %q", got) - } - body := w.Body.String() - // Provider events reach the caller contiguously: sideband injection never - // splits a provider SSE event. - for _, event := range []string{roleEvent, contentEvent, doneEvent} { - if !strings.Contains(body, event) { - t.Fatalf("provider event not preserved contiguously: %q in %q", event, body) - } - } - if !strings.HasPrefix(body, "event: iop.sideband\n") { - t.Fatalf("stream must open with the IOP route sideband event, got %q", body) - } - routeIdx := strings.Index(body, `"kind":"route"`) - if routeIdx < 0 || routeIdx > strings.Index(body, roleEvent) { - t.Fatalf("route observation must precede provider events: %q", body) - } - for _, marker := range []string{ - `"object":"iop.sideband"`, - `"run_id":"run-tunnel"`, - `"node_id":"node-1"`, - `"response_mode":"passthrough+sideband"`, - } { - if !strings.Contains(body, marker) { - t.Errorf("route observation missing %q: %q", marker, body) - } - } - usageIdx := strings.Index(body, `"kind":"usage"`) - contentEnd := strings.Index(body, contentEvent) + len(contentEvent) - if usageIdx < contentEnd { - t.Errorf("usage observation must wait for the provider event boundary: usage at %d, content event ends at %d", usageIdx, contentEnd) - } - if !strings.Contains(body, `"input_tokens":3`) || !strings.Contains(body, `"output_tokens":5`) { - t.Errorf("usage observation missing token counts: %q", body) - } - assembledIdx := strings.Index(body, `"kind":"assembled"`) - if assembledIdx < 0 || assembledIdx < strings.Index(body, doneEvent) { - t.Errorf("assembled observation must close the stream: %q", body) - } - if !strings.Contains(body, `"content":"hello"`) { - t.Errorf("assembled observation missing assembled content: %q", body) - } - tunnelReqs := fake.tunnelReqsSnapshot() - if len(tunnelReqs) != 1 { - t.Fatalf("expected 1 tunnel dispatch, got %d", len(tunnelReqs)) - } - if got := tunnelReqs[0].Metadata["iop_response_mode"]; got != "passthrough+sideband" { - t.Errorf("tunnel metadata response mode: got %q", got) - } - if len(fake.reqsSnapshot()) != 0 { - t.Error("sideband mode must not use the normalized SubmitRun path") - } -} - -// TestChatCompletionsPassthroughSidebandNonStreamingWrapsProviderBody verifies -// SDD S05 for the non-streaming path: the provider JSON body is carried -// verbatim inside the iop.chat.passthrough_sideband envelope together with -// route/usage/assembled observations, and is not presented as byte identity. -func TestChatCompletionsPassthroughSidebandNonStreamingWrapsProviderBody(t *testing.T) { - providerBody := `{"id":"cmpl-1","object":"chat.completion","choices":[{"index":0,"message":{"role":"assistant","content":"hi","reasoning_content":"because"},"finish_reason":"stop"}]}` - frames := make(chan *iop.ProviderTunnelFrame, 5) - frames <- &iop.ProviderTunnelFrame{ - Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START, - StatusCode: 200, - Headers: map[string]string{"Content-Type": "application/json"}, - } - frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(providerBody[:23])} - frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(providerBody[23:])} - frames <- &iop.ProviderTunnelFrame{ - Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_USAGE, - Usage: &iop.Usage{InputTokens: 7, OutputTokens: 11}, - } - frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true} - close(frames) - srv, fake := chatSidebandServer(frames) - - req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ - "model":"pool-model", - "messages":[{"role":"user","content":"hello"}], - "metadata":{"iop_response_mode":"passthrough+sideband"} - }`)) - w := httptest.NewRecorder() - srv.handleChatCompletions(w, req) - - if w.Code != http.StatusOK { - t.Fatalf("status: got %d body=%s", w.Code, w.Body.String()) - } - if got := w.Header().Get("X-IOP-Response-Mode"); got != "passthrough+sideband" { - t.Errorf("sideband response mode header: got %q", got) - } - if !strings.Contains(w.Body.String(), providerBody) { - t.Fatalf("provider body must be embedded verbatim: %s", w.Body.String()) - } - var envelope struct { - Object string `json:"object"` - Sideband struct { - Route struct { - RunID string `json:"run_id"` - NodeID string `json:"node_id"` - ResponseMode string `json:"response_mode"` - ProviderStatusCode int `json:"provider_status_code"` - } `json:"route"` - Usage *struct { - InputTokens int `json:"input_tokens"` - OutputTokens int `json:"output_tokens"` - } `json:"usage"` - Assembled *struct { - Content string `json:"content"` - Reasoning string `json:"reasoning"` - BodyBytes int `json:"body_bytes"` - } `json:"assembled"` - } `json:"iop_sideband"` - ProviderStatusCode int `json:"provider_status_code"` - ProviderResponse json.RawMessage `json:"provider_response"` - } - if err := json.Unmarshal(w.Body.Bytes(), &envelope); err != nil { - t.Fatalf("envelope not valid JSON: %v body=%s", err, w.Body.String()) - } - if envelope.Object != "iop.chat.passthrough_sideband" { - t.Errorf("envelope object: got %q", envelope.Object) - } - if envelope.Sideband.Route.RunID != "run-tunnel" || envelope.Sideband.Route.NodeID != "node-1" { - t.Errorf("route observation: %+v", envelope.Sideband.Route) - } - if envelope.Sideband.Route.ResponseMode != "passthrough+sideband" || envelope.Sideband.Route.ProviderStatusCode != http.StatusOK { - t.Errorf("route observation mode/status: %+v", envelope.Sideband.Route) - } - if envelope.Sideband.Usage == nil || envelope.Sideband.Usage.InputTokens != 7 || envelope.Sideband.Usage.OutputTokens != 11 { - t.Errorf("usage observation: %+v", envelope.Sideband.Usage) - } - if envelope.Sideband.Assembled == nil || - envelope.Sideband.Assembled.Content != "hi" || - envelope.Sideband.Assembled.Reasoning != "because" || - envelope.Sideband.Assembled.BodyBytes != len(providerBody) { - t.Errorf("assembled observation: %+v", envelope.Sideband.Assembled) - } - if envelope.ProviderStatusCode != http.StatusOK { - t.Errorf("provider status: got %d", envelope.ProviderStatusCode) - } - if string(envelope.ProviderResponse) != providerBody { - t.Errorf("provider_response not verbatim:\n got: %s\nwant: %s", envelope.ProviderResponse, providerBody) - } - if len(fake.reqsSnapshot()) != 0 { - t.Error("sideband mode must not use the normalized SubmitRun path") - } -} - -// TestChatCompletionsPassthroughDoesNotExposeSideband verifies SDD S06: pure -// passthrough (omitted mode) does not expose usage frames, IOP sideband marker, -// event, or label to the caller. func TestChatCompletionsPassthroughDoesNotExposeSideband(t *testing.T) { providerBody := "data: {\"choices\":[{\"delta\":{\"content\":\"pure\"}}]}\n\ndata: [DONE]\n\n" frames := make(chan *iop.ProviderTunnelFrame, 4) @@ -4823,17 +4601,11 @@ func TestChatCompletionsPassthroughDoesNotExposeSideband(t *testing.T) { if w.Code != http.StatusOK { t.Fatalf("status: got %d body=%s", w.Code, w.Body.String()) } + // Byte-identity with the provider body proves no IOP extension markers or + // response-mode labels are injected into the pure passthrough response. if got := w.Body.String(); got != providerBody { t.Fatalf("pure passthrough body not byte-identical:\n got: %q\nwant: %q", got, providerBody) } - for _, marker := range []string{"iop.sideband", "iop_sideband", "iop_response_mode"} { - if strings.Contains(w.Body.String(), marker) { - t.Errorf("pure passthrough body must not contain %q", marker) - } - } - if got := w.Header().Get("X-IOP-Response-Mode"); got != "" { - t.Errorf("pure passthrough must not carry the IOP response mode header, got %q", got) - } } // chatProviderAuthServer builds a provider-pool tunnel server with the given @@ -5053,47 +4825,6 @@ func TestChatProviderTunnelProviderAuthDoesNotReplaceInboundAuth(t *testing.T) { } } -// TestChatCompletionsTransformedModeLabelsIOPOutput verifies SDD S07: explicit -// transformed output uses the normalized path and is labeled as IOP -// transformed output in both the response header and the run metadata. -func TestChatCompletionsTransformedModeLabelsIOPOutput(t *testing.T) { - fake := &fakeRunService{events: bufferedRunEvents( - &iop.RunEvent{Type: "delta", Delta: "ok"}, - &iop.RunEvent{Type: "complete"}, - )} - srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama", Target: "backend-model"}, fake, nil) - - req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ - "model":"client-model", - "messages":[{"role":"user","content":"hello"}], - "metadata":{"iop_response_mode":"transformed"} - }`)) - w := httptest.NewRecorder() - srv.handleChatCompletions(w, req) - - if w.Code != http.StatusOK { - t.Fatalf("status: got %d body=%s", w.Code, w.Body.String()) - } - if got := w.Header().Get("X-IOP-Response-Mode"); got != "transformed" { - t.Errorf("transformed label header: got %q", got) - } - if !strings.Contains(w.Body.String(), `"object":"chat.completion"`) { - t.Errorf("transformed output must be the normalized IOP response: %s", w.Body.String()) - } - if len(fake.tunnelReqsSnapshot()) != 0 { - t.Error("transformed mode must not use the tunnel") - } - reqs := fake.reqsSnapshot() - if len(reqs) != 1 { - t.Fatalf("expected 1 normalized SubmitRun dispatch, got %d", len(reqs)) - } - if got := reqs[0].Metadata["iop_response_mode"]; got != "transformed" { - t.Errorf("run metadata response mode: got %q", got) - } -} - -// TestChatCompletionsPassthroughDoesNotLabelTransformed verifies pure -// passthrough responses carry no IOP transformed label. func TestChatCompletionsPassthroughDoesNotLabelTransformed(t *testing.T) { providerBody := `{"ok":true}` frames := make(chan *iop.ProviderTunnelFrame, 3) @@ -5113,12 +4844,11 @@ func TestChatCompletionsPassthroughDoesNotLabelTransformed(t *testing.T) { if w.Code != http.StatusOK { t.Fatalf("status: got %d body=%s", w.Code, w.Body.String()) } + // Byte-identity with the provider body proves the pure passthrough response + // carries no IOP response-mode label. if got := w.Body.String(); got != providerBody { t.Fatalf("pure passthrough body not byte-identical: %q", got) } - if got := w.Header().Get("X-IOP-Response-Mode"); got != "" { - t.Errorf("pure passthrough must not carry the IOP response mode header, got %q", got) - } } // chatPassthroughServer returns an openai.Server whose fake service relays the @@ -5458,44 +5188,6 @@ func TestChatCompletionsPassthroughLegacyProviderRouteUsesTunnel(t *testing.T) { } } -// TestChatCompletionsLegacyProviderRouteRejectsTransformedMode verifies that -// explicit transformed mode cannot send a legacy provider route back through -// the normalized RunEvent path. -func TestChatCompletionsLegacyProviderRouteRejectsTransformedMode(t *testing.T) { - fake := &fakeRunService{events: bufferedRunEvents( - &iop.RunEvent{Type: "delta", Delta: "ok"}, - &iop.RunEvent{Type: "complete"}, - )} - srv := NewServer(config.EdgeOpenAIConf{ - ModelRoutes: []config.OpenAIRouteEntry{ - {Model: "compat-model", Adapter: "openai_compat", Target: "backend-model"}, - }, - }, fake, nil) - - req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ - "model":"compat-model", - "messages":[{"role":"user","content":"hello"}], - "metadata":{"iop_response_mode":"transformed"} - }`)) - w := httptest.NewRecorder() - srv.handleChatCompletions(w, req) - - if w.Code != http.StatusBadRequest { - t.Fatalf("status: got %d body=%s", w.Code, w.Body.String()) - } - if !strings.Contains(w.Body.String(), "metadata.iop_response_mode=transformed is not supported for OpenAI-compatible provider model groups") { - t.Fatalf("expected transformed rejection, got %s", w.Body.String()) - } - if len(fake.tunnelReqsSnapshot()) != 0 { - t.Error("rejected transformed mode must not use the tunnel") - } - if len(fake.reqsSnapshot()) != 0 { - t.Fatalf("rejected transformed mode must not use SubmitRun, got %d calls", len(fake.reqsSnapshot())) - } -} - -// TestChatCompletionsPassthroughNonProviderRouteKeepsNormalizedPath verifies -// that CLI/ollama legacy routes are unaffected by the passthrough default. func TestChatCompletionsPassthroughNonProviderRouteKeepsNormalizedPath(t *testing.T) { fake := &fakeRunService{events: bufferedRunEvents( &iop.RunEvent{Type: "delta", Delta: "ok"}, @@ -5798,134 +5490,6 @@ func TestChatCompletionsPassthroughSlowClientByteIdentity(t *testing.T) { } } -// TestChatCompletionsPassthroughSidebandStreamWriteFailureSendsCancelRunOnce -// verifies the sideband streaming writer propagates exactly one CancelRun when -// a provider body write fails mid-stream and stops relaying afterwards. -func TestChatCompletionsPassthroughSidebandStreamWriteFailureSendsCancelRunOnce(t *testing.T) { - frames := make(chan *iop.ProviderTunnelFrame, 5) - frames <- &iop.ProviderTunnelFrame{ - Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START, - StatusCode: 200, - Headers: map[string]string{"Content-Type": "text/event-stream"}, - } - frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte("data: chunk-1\n\n")} - frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte("data: chunk-2\n\n")} - frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true} - close(frames) - srv, fake := chatSidebandServer(frames) - - req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ - "model":"pool-model", - "stream":true, - "messages":[{"role":"user","content":"hello"}], - "metadata":{"iop_response_mode":"passthrough+sideband"} - }`)) - // Write 1 is the route sideband event, write 2 the first provider chunk; - // the second provider chunk fails. - w := newFlakyResponseWriter(2) - srv.handleChatCompletions(w, req) - - status, body, failedWrites := w.snapshot() - if status != http.StatusOK { - t.Fatalf("status: got %d body=%s", status, body) - } - if failedWrites == 0 { - t.Fatal("fixture did not exercise the sideband write failure path") - } - if !strings.Contains(body, "data: chunk-1\n\n") || strings.Contains(body, "chunk-2") { - t.Fatalf("sideband write failure must truncate after the ordered prefix: %q", body) - } - calls := fake.cancelCallsSnapshot() - if len(calls) != 1 { - t.Fatalf("expected exactly 1 CancelRun call on sideband write failure, got %d: %+v", len(calls), calls) - } - if calls[0].RunID != "run-tunnel" || calls[0].NodeRef != "node-1" { - t.Fatalf("unexpected cancel request: %+v", calls[0]) - } -} - -// TestChatCompletionsPassthroughSidebandStreamContextCancelSendsCancelRun -// verifies caller disconnect on the sideband streaming surface propagates -// exactly one CancelRun while the tunnel is still in flight. -func TestChatCompletionsPassthroughSidebandStreamContextCancelSendsCancelRun(t *testing.T) { - fake := &fakeRunService{tunnelFrames: make(chan *iop.ProviderTunnelFrame)} - srv := chatProviderRouteServer(fake, nil) - - ctx, cancel := context.WithCancel(context.Background()) - cancel() - req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ - "model":"pool-model", - "stream":true, - "messages":[{"role":"user","content":"hello"}], - "metadata":{"iop_response_mode":"passthrough+sideband"} - }`)).WithContext(ctx) - w := httptest.NewRecorder() - srv.handleChatCompletions(w, req) - - calls := fake.cancelCallsSnapshot() - if len(calls) != 1 { - t.Fatalf("expected exactly 1 CancelRun call, got %d: %+v", len(calls), calls) - } - if calls[0].RunID != "run-tunnel" || calls[0].NodeRef != "node-1" { - t.Fatalf("unexpected cancel request: %+v", calls[0]) - } -} - -// TestChatCompletionsPassthroughSidebandNonStreamingContextCancelSendsCancelRun -// verifies caller disconnect on the buffered sideband surface propagates -// exactly one CancelRun while the tunnel is still in flight. -func TestChatCompletionsPassthroughSidebandNonStreamingContextCancelSendsCancelRun(t *testing.T) { - fake := &fakeRunService{tunnelFrames: make(chan *iop.ProviderTunnelFrame)} - srv := chatProviderRouteServer(fake, nil) - - ctx, cancel := context.WithCancel(context.Background()) - cancel() - req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ - "model":"pool-model", - "messages":[{"role":"user","content":"hello"}], - "metadata":{"iop_response_mode":"passthrough+sideband"} - }`)).WithContext(ctx) - w := httptest.NewRecorder() - srv.handleChatCompletions(w, req) - - calls := fake.cancelCallsSnapshot() - if len(calls) != 1 { - t.Fatalf("expected exactly 1 CancelRun call, got %d: %+v", len(calls), calls) - } - if calls[0].RunID != "run-tunnel" || calls[0].NodeRef != "node-1" { - t.Fatalf("unexpected cancel request: %+v", calls[0]) - } -} - -// TestChatCompletionsPassthroughSidebandNonStreamingTimeoutSendsCancelRun -// verifies the buffered sideband surface converts a tunnel wait timeout into a -// 502 and exactly one CancelRun. -func TestChatCompletionsPassthroughSidebandNonStreamingTimeoutSendsCancelRun(t *testing.T) { - fake := &fakeRunService{ - tunnelFrames: make(chan *iop.ProviderTunnelFrame), - tunnelWaitTimeout: 50 * time.Millisecond, - } - srv := chatProviderRouteServer(fake, nil) - - req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ - "model":"pool-model", - "messages":[{"role":"user","content":"hello"}], - "metadata":{"iop_response_mode":"passthrough+sideband"} - }`)) - w := httptest.NewRecorder() - srv.handleChatCompletions(w, req) - - if w.Code != http.StatusBadGateway { - t.Fatalf("status: got %d body=%s", w.Code, w.Body.String()) - } - if !strings.Contains(w.Body.String(), "run timed out") { - t.Fatalf("timeout error not surfaced: %s", w.Body.String()) - } - calls := fake.cancelCallsSnapshot() - if len(calls) != 1 { - t.Fatalf("expected exactly 1 CancelRun call on sideband timeout, got %d: %+v", len(calls), calls) - } -} func TestChatCompletionsProviderPoolAppliesGenerationPolicy(t *testing.T) { fake := &fakeRunService{ tunnelFrames: staticProviderTunnelFrames(`{"ok":true}`), @@ -6112,19 +5676,6 @@ func responsesProviderTunnelServer(frames chan *iop.ProviderTunnelFrame, servedT return srv, fake } -func responsesLegacyProviderTunnelServer(frames chan *iop.ProviderTunnelFrame, servedTarget string) (*Server, *fakeRunService) { - fake := &fakeRunService{ - tunnelFrames: frames, - tunnelServedTarget: servedTarget, - } - srv := NewServer(config.EdgeOpenAIConf{ - ModelRoutes: []config.OpenAIRouteEntry{ - {Model: "pool-model", Adapter: "openai_compat", Target: "served-model"}, - }, - }, fake, nil) - return srv, fake -} - // TestResponsesProviderPoolDispatch verifies that /v1/responses sends // provider-pool models through the raw provider tunnel (POST /v1/responses) // instead of the normalized RunEvent path. @@ -6452,42 +6003,22 @@ func TestResponsesProviderTunnelStreaming(t *testing.T) { } } -func TestResponsesProviderPoolRejectsResponseMode(t *testing.T) { +// TestResponsesProviderPoolIgnoresSelectorLikeMetadata verifies SDD S01/S04 for +// /v1/responses: arbitrary caller metadata never rejects the request or moves a +// provider route off pure passthrough. (The former selector key literal is +// deliberately not used; the deterministic surface check forbids it under apps.) +func TestResponsesProviderPoolIgnoresSelectorLikeMetadata(t *testing.T) { cases := []struct { - name string - metadata string - wantStatus int - wantTunnel bool - wantMessage string + name string + metadata string }{ - { - name: "explicit passthrough is rejected", - metadata: `"metadata":{"iop_response_mode":"passthrough"}`, - wantStatus: http.StatusBadRequest, - wantMessage: "metadata.iop_response_mode is not supported for model group routes", - }, - { - name: "sideband mode is rejected", - metadata: `"metadata":{"iop_response_mode":"passthrough+sideband"}`, - wantStatus: http.StatusBadRequest, - wantMessage: "metadata.iop_response_mode is not supported for model group routes", - }, - { - name: "transformed mode is rejected", - metadata: `"metadata":{"iop_response_mode":"transformed"}`, - wantStatus: http.StatusBadRequest, - wantMessage: "metadata.iop_response_mode is not supported for model group routes", - }, - { - name: "unknown mode is rejected before dispatch", - metadata: `"metadata":{"iop_response_mode":"rawish"}`, - wantStatus: http.StatusBadRequest, - wantMessage: "metadata.iop_response_mode is not supported for model group routes", - }, + {name: "selector-like value is ignored", metadata: `"metadata":{"response_hint":"raw"}`}, + {name: "arbitrary metadata is ignored", metadata: `"metadata":{"experiment":"mode-x"}`}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - srv, fake := responsesProviderTunnelServer(staticProviderTunnelFrames(`{"ok":true}`), "served-model") + providerBody := `{"ok":true}` + srv, fake := responsesProviderTunnelServer(staticProviderTunnelFrames(providerBody), "served-model") req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(fmt.Sprintf(`{ "model":"pool-model", "input":"hi", @@ -6496,15 +6027,14 @@ func TestResponsesProviderPoolRejectsResponseMode(t *testing.T) { w := httptest.NewRecorder() srv.handleResponses(w, req) - if w.Code != tc.wantStatus { - t.Fatalf("status: got %d want %d body=%s", w.Code, tc.wantStatus, w.Body.String()) + if w.Code != http.StatusOK { + t.Fatalf("selector-like metadata must not be rejected: got %d body=%s", w.Code, w.Body.String()) } - if tc.wantMessage != "" && !strings.Contains(w.Body.String(), tc.wantMessage) { - t.Fatalf("body must contain %q, got %s", tc.wantMessage, w.Body.String()) + if got := w.Body.String(); got != providerBody { + t.Fatalf("provider body must be relayed byte-identically: got %q want %q", got, providerBody) } - gotTunnel := len(fake.tunnelReqsSnapshot()) > 0 - if gotTunnel != tc.wantTunnel { - t.Fatalf("tunnel dispatch: got %v want %v", gotTunnel, tc.wantTunnel) + if len(fake.tunnelReqsSnapshot()) != 1 { + t.Fatalf("expected 1 pure passthrough tunnel dispatch, got %d", len(fake.tunnelReqsSnapshot())) } if len(fake.reqsSnapshot()) != 0 { t.Fatalf("responses provider route must not call SubmitRun, got %d", len(fake.reqsSnapshot())) @@ -6513,97 +6043,6 @@ func TestResponsesProviderPoolRejectsResponseMode(t *testing.T) { } } -func TestResponsesProviderTunnelSidebandInjectsMetadata(t *testing.T) { - frames := staticProviderTunnelFrames(`{"id":"resp-1","object":"response","output_text":"hi","metadata":{"request_id":"req-1"}}`) - srv, fake := responsesLegacyProviderTunnelServer(frames, "served-model") - req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(`{ - "model":"pool-model", - "input":"hi", - "metadata":{"iop_response_mode":"passthrough+sideband"} - }`)) - w := httptest.NewRecorder() - srv.handleResponses(w, req) - - if w.Code != http.StatusOK { - t.Fatalf("status: got %d body=%s", w.Code, w.Body.String()) - } - reqs := fake.tunnelReqsSnapshot() - if len(reqs) != 1 { - t.Fatalf("expected 1 tunnel dispatch, got %d", len(reqs)) - } - if got := reqs[0].Metadata[responseModeMetadataKey]; got != responseModePassthroughSideband { - t.Fatalf("tunnel response mode metadata: got %q want %q", got, responseModePassthroughSideband) - } - var body map[string]any - if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil { - t.Fatalf("response JSON: %v body=%s", err, w.Body.String()) - } - if body["id"] != "resp-1" || body["output_text"] != "hi" { - t.Fatalf("provider response fields must be preserved: %+v", body) - } - metadata, ok := body["metadata"].(map[string]any) - if !ok { - t.Fatalf("metadata must be an object: %+v", body["metadata"]) - } - if metadata["request_id"] != "req-1" { - t.Fatalf("provider metadata must be preserved: %+v", metadata) - } - if metadata[responseModeMetadataKey] != responseModePassthroughSideband { - t.Fatalf("sideband response mode must be injected into metadata: %+v", metadata) - } -} - -func TestResponsesProviderTunnelSidebandStreamingInjectsEvent(t *testing.T) { - frames := make(chan *iop.ProviderTunnelFrame, 4) - frames <- &iop.ProviderTunnelFrame{ - Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START, - StatusCode: 200, - Headers: map[string]string{"Content-Type": "text/event-stream"}, - } - frames <- &iop.ProviderTunnelFrame{ - Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, - Body: []byte("data: {\"type\":\"response.output_text.delta\",\"delta\":\"hi\"}\n\n"), - } - frames <- &iop.ProviderTunnelFrame{ - Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, - Body: []byte("data: [DONE]\n\n"), - } - frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true} - close(frames) - - srv, fake := responsesLegacyProviderTunnelServer(frames, "served-model") - req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(`{ - "model":"pool-model", - "input":"hi", - "stream":true, - "metadata":{"iop_response_mode":"passthrough+sideband"} - }`)) - w := httptest.NewRecorder() - srv.handleResponses(w, req) - - if w.Code != http.StatusOK { - t.Fatalf("status: got %d body=%s", w.Code, w.Body.String()) - } - reqs := fake.tunnelReqsSnapshot() - if len(reqs) != 1 || !reqs[0].Stream { - t.Fatalf("expected one streaming tunnel dispatch, got %#v", reqs) - } - body := w.Body.String() - for _, marker := range []string{ - "event: iop.sideband", - `"object":"iop.responses.sideband"`, - `"iop_response_mode":"passthrough+sideband"`, - "response.output_text.delta", - "data: [DONE]", - } { - if !strings.Contains(body, marker) { - t.Fatalf("streaming sideband body must contain %q, got %q", marker, body) - } - } -} - -// TestChatCompletionsProviderPoolFallsBackToLegacyRoute verifies that when the -// request model does not match the catalog, the legacy model_routes path is used. func TestChatCompletionsProviderPoolFallsBackToLegacyRoute(t *testing.T) { fake := &fakeRunService{events: make(chan *iop.RunEvent, 2)} fake.events <- &iop.RunEvent{Type: "delta", Delta: "ok"} @@ -6715,7 +6154,7 @@ func TestChatCompletionsProviderStreamSynthesizesTextToolCalls(t *testing.T) { srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama"}, fake, nil) req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ - "metadata":{"iop_response_mode":"transformed"}, + "metadata":{}, "model":"qwen3.6:35b", "messages":[{"role":"user","content":"status"}], "tools":[{"type":"function","function":{"name":"run_commands","parameters":{"type":"object"}}}], @@ -6755,7 +6194,7 @@ func TestChatCompletionsProviderStreamBlocksUnknownTextToolCall(t *testing.T) { srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama"}, fake, nil) req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ - "metadata":{"iop_response_mode":"transformed"}, + "metadata":{}, "model":"qwen3.6:35b", "messages":[{"role":"user","content":"status"}], "tools":[{"type":"function","function":{"name":"run_commands"}}], @@ -6827,7 +6266,7 @@ func TestChatCompletionsFailsMalformedTextToolCallAfterRetryLimit(t *testing.T) srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama"}, fake, nil) req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ - "metadata":{"iop_response_mode":"transformed"}, + "metadata":{}, "model":"qwen3.6:35b", "messages":[{"role":"user","content":"status"}], "tools":[{"type":"function","function":{"name":"run_commands","parameters":{"type":"object"}}}], @@ -6866,7 +6305,7 @@ func TestChatCompletionsProviderStreamBlocksMalformedTextToolCall(t *testing.T) srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama"}, fake, nil) req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ - "metadata":{"iop_response_mode":"transformed"}, + "metadata":{}, "model":"qwen3.6:35b", "messages":[{"role":"user","content":"status"}], "tools":[{"type":"function","function":{"name":"run_commands","parameters":{"type":"object"}}}], @@ -6991,7 +6430,7 @@ func TestChatCompletionsFailsWhenAnySynthesizedTextToolCallViolatesSchema(t *tes srv := NewServer(config.EdgeOpenAIConf{Adapter: "ollama"}, fake, nil) req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ - "metadata":{"iop_response_mode":"transformed"}, + "metadata":{}, "model":"qwen3.6:35b", "messages":[{"role":"user","content":"status"}], "tools":[ @@ -7154,13 +6593,16 @@ func TestChatCompletionsAssembledLogs(t *testing.T) { } } -func TestChatCompletionsAssembledLogsTransformed(t *testing.T) { +// TestChatCompletionsAssembledLogsNormalized verifies the normalized-path +// completion log reports the internal response_mode execution label +// (normalized) and the assembled observation fields. +func TestChatCompletionsAssembledLogsNormalized(t *testing.T) { core, observed := observer.New(zap.DebugLevel) logger := zap.New(core) fake := &fakeRunService{events: make(chan *iop.RunEvent, 4)} - fake.events <- &iop.RunEvent{Type: "delta", Delta: "hello transformed"} - fake.events <- &iop.RunEvent{Type: "reasoning_delta", Delta: "reasoning transformed..."} + fake.events <- &iop.RunEvent{Type: "delta", Delta: "hello normalized"} + fake.events <- &iop.RunEvent{Type: "reasoning_delta", Delta: "reasoning normalized..."} fake.events <- &iop.RunEvent{Type: "complete", Metadata: map[string]string{ "openai_tool_calls": `[{"id":"call_002","type":"function","function":{"name":"run_code","arguments":"{}"}}]`, }} @@ -7169,8 +6611,7 @@ func TestChatCompletionsAssembledLogsTransformed(t *testing.T) { req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ "model":"client-model", "messages":[{"role":"user","content":"hi"}], - "tools":[{"type":"function","function":{"name":"run_code"}}], - "metadata":{"iop_response_mode":"transformed"} + "tools":[{"type":"function","function":{"name":"run_code"}}] }`)) w := httptest.NewRecorder() srv.handleChatCompletions(w, req) @@ -7180,14 +6621,14 @@ func TestChatCompletionsAssembledLogsTransformed(t *testing.T) { if entry.Message == "openai chat completion output" { found = true m := entry.ContextMap() - if m["response_mode"] != "transformed" { - t.Errorf("expected response_mode=transformed, got %v", m["response_mode"]) + if m["response_mode"] != "normalized" { + t.Errorf("expected response_mode=normalized, got %v", m["response_mode"]) } - if m["assembled_content"] != "hello transformed" { - t.Errorf("expected assembled_content=hello transformed, got %v", m["assembled_content"]) + if m["assembled_content"] != "hello normalized" { + t.Errorf("expected assembled_content=hello normalized, got %v", m["assembled_content"]) } - if m["assembled_reasoning"] != "reasoning transformed..." { - t.Errorf("expected assembled_reasoning=reasoning transformed..., got %v", m["assembled_reasoning"]) + if m["assembled_reasoning"] != "reasoning normalized..." { + t.Errorf("expected assembled_reasoning=reasoning normalized..., got %v", m["assembled_reasoning"]) } tc, ok := m["assembled_tool_calls"].([]interface{}) if !ok || len(tc) != 1 || tc[0] != "run_code" { @@ -7203,120 +6644,6 @@ func TestChatCompletionsAssembledLogsTransformed(t *testing.T) { } } -func TestChatCompletionsAssembledLogsSidebandNonStreaming(t *testing.T) { - core, observed := observer.New(zap.DebugLevel) - logger := zap.New(core) - - providerBody := `{"choices":[{"message":{"content":"non-stream content","reasoning_content":"non-stream reasoning","tool_calls":[{"function":{"name":"call_non_stream"}}]}}]}` - frames := make(chan *iop.ProviderTunnelFrame, 4) - frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START, StatusCode: 200} - frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(providerBody)} - frames <- &iop.ProviderTunnelFrame{ - Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_USAGE, - Usage: &iop.Usage{ - InputTokens: 10, - OutputTokens: 20, - }, - } - frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true} - close(frames) - - fake := &fakeRunService{tunnelFrames: frames} - srv := chatProviderRouteServer(fake, logger) - - req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ - "model":"pool-model", - "messages":[{"role":"user","content":"hello"}], - "metadata":{"iop_response_mode":"passthrough+sideband"} - }`)) - w := httptest.NewRecorder() - srv.handleChatCompletions(w, req) - - if w.Code != http.StatusOK { - t.Fatalf("expected status 200, got %d body=%s", w.Code, w.Body.String()) - } - - // response body check - var envelope struct { - Object string `json:"object"` - Sideband struct { - Route *struct { - ResponseMode string `json:"response_mode"` - } `json:"route"` - Usage *struct { - InputTokens int `json:"input_tokens"` - OutputTokens int `json:"output_tokens"` - } `json:"usage"` - Assembled *struct { - Content string `json:"content"` - Reasoning string `json:"reasoning"` - ToolCallNames []string `json:"tool_call_names"` - BodyBytes int `json:"body_bytes"` - } `json:"assembled"` - } `json:"iop_sideband"` - } - - if err := json.Unmarshal(w.Body.Bytes(), &envelope); err != nil { - t.Fatalf("failed to unmarshal response: %v", err) - } - - if envelope.Sideband.Route == nil { - t.Fatalf("expected sideband.route to be non-nil") - } - - if envelope.Sideband.Route.ResponseMode != "passthrough+sideband" { - t.Errorf("expected route response_mode 'passthrough+sideband', got %q", envelope.Sideband.Route.ResponseMode) - } - - if envelope.Sideband.Usage == nil { - t.Fatalf("expected sideband.usage to be non-nil") - } - - if envelope.Sideband.Usage.InputTokens != 10 || envelope.Sideband.Usage.OutputTokens != 20 { - t.Errorf("expected usage input=10 output=20, got input=%d output=%d", envelope.Sideband.Usage.InputTokens, envelope.Sideband.Usage.OutputTokens) - } - - if envelope.Sideband.Assembled == nil { - t.Fatalf("expected sideband.assembled to be non-nil") - } - - if envelope.Sideband.Assembled.Content != "non-stream content" { - t.Errorf("expected sideband content 'non-stream content', got %q", envelope.Sideband.Assembled.Content) - } - - if envelope.Sideband.Assembled.Reasoning != "non-stream reasoning" { - t.Errorf("expected sideband reasoning 'non-stream reasoning', got %q", envelope.Sideband.Assembled.Reasoning) - } - - if len(envelope.Sideband.Assembled.ToolCallNames) != 1 || envelope.Sideband.Assembled.ToolCallNames[0] != "call_non_stream" { - t.Errorf("expected sideband tool_call_names [call_non_stream], got %v", envelope.Sideband.Assembled.ToolCallNames) - } - - found := false - for _, entry := range observed.All() { - if entry.Message == "openai chat completion sideband response" { - found = true - m := entry.ContextMap() - if m["assembled_content"] != "non-stream content" { - t.Errorf("expected assembled_content='non-stream content', got %v", m["assembled_content"]) - } - if m["assembled_reasoning"] != "non-stream reasoning" { - t.Errorf("expected assembled_reasoning='non-stream reasoning', got %v", m["assembled_reasoning"]) - } - tc, ok := m["assembled_tool_calls"].([]interface{}) - if !ok || len(tc) != 1 || tc[0] != "call_non_stream" { - t.Errorf("expected assembled_tool_calls=[call_non_stream], got %v", m["assembled_tool_calls"]) - } - if count, ok := m["assembled_tool_call_count"].(int64); !ok || count != 1 { - t.Errorf("expected assembled_tool_call_count=1, got %v", m["assembled_tool_call_count"]) - } - } - } - if !found { - t.Error("expected log message 'openai chat completion sideband response' not found") - } -} - func TestChatCompletionsAssembledLogsPassthrough(t *testing.T) { core, observed := observer.New(zap.DebugLevel) logger := zap.New(core) @@ -7367,131 +6694,6 @@ func TestChatCompletionsAssembledLogsPassthrough(t *testing.T) { } } -func TestChatCompletionsAssembledLogsSidebandStream(t *testing.T) { - core, observed := observer.New(zap.DebugLevel) - logger := zap.New(core) - - roleEvent := "data: {\"choices\":[{\"delta\":{\"role\":\"assistant\"}}]}\n\n" - contentEvent := "data: {\"choices\":[{\"delta\":{\"content\":\"hello\"}}]}\n\n" - reasoningEvent := "data: {\"choices\":[{\"delta\":{\"reasoning_content\":\"stream reasoning\"}}]}\n\n" - toolCallEvent := "data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"name\":\"call_stream\"}}]}}]}\n\n" - doneEvent := "data: [DONE]\n\n" - frames := make(chan *iop.ProviderTunnelFrame, 8) - frames <- &iop.ProviderTunnelFrame{ - Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START, - StatusCode: 200, - Headers: map[string]string{"Content-Type": "text/event-stream"}, - } - frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(roleEvent)} - frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(contentEvent)} - frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(reasoningEvent)} - frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(toolCallEvent)} - frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(doneEvent)} - frames <- &iop.ProviderTunnelFrame{ - Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_USAGE, - Usage: &iop.Usage{ - InputTokens: 15, - OutputTokens: 25, - }, - } - frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true} - close(frames) - - fake := &fakeRunService{tunnelFrames: frames} - srv := chatProviderRouteServer(fake, logger) - - req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ - "model":"pool-model", - "stream":true, - "messages":[{"role":"user","content":"hello"}], - "metadata":{"iop_response_mode":"passthrough+sideband"} - }`)) - w := httptest.NewRecorder() - srv.handleChatCompletions(w, req) - - if w.Code != http.StatusOK { - t.Fatalf("expected status 200, got %d body=%s", w.Code, w.Body.String()) - } - - // Verify SSE observations from response surface - bodyStr := w.Body.String() - lines := strings.Split(bodyStr, "\n") - var parsedObs []sidebandObservation - for i := 0; i < len(lines); i++ { - if strings.HasPrefix(lines[i], "event: iop.sideband") { - if i+1 < len(lines) && strings.HasPrefix(lines[i+1], "data: ") { - dataJSON := strings.TrimPrefix(lines[i+1], "data: ") - var obs sidebandObservation - if err := json.Unmarshal([]byte(dataJSON), &obs); err == nil { - parsedObs = append(parsedObs, obs) - } - } - } - } - - // We expect 3 sideband events: route, usage, assembled - if len(parsedObs) != 3 { - t.Errorf("expected 3 sideband events, got %d", len(parsedObs)) - } else { - // Verify Route - if parsedObs[0].Kind != "route" || parsedObs[0].Route == nil { - t.Errorf("expected first sideband event to be route, got %+v", parsedObs[0]) - } else if parsedObs[0].Route.ResponseMode != "passthrough+sideband" { - t.Errorf("expected route response_mode 'passthrough+sideband', got %q", parsedObs[0].Route.ResponseMode) - } - - // Verify Usage - if parsedObs[1].Kind != "usage" || parsedObs[1].Usage == nil { - t.Errorf("expected second sideband event to be usage, got %+v", parsedObs[1]) - } else if parsedObs[1].Usage.InputTokens != 15 || parsedObs[1].Usage.OutputTokens != 25 { - t.Errorf("expected usage input=15, output=25, got input=%d, output=%d", parsedObs[1].Usage.InputTokens, parsedObs[1].Usage.OutputTokens) - } - - // Verify Assembled - if parsedObs[2].Kind != "assembled" || parsedObs[2].Assembled == nil { - t.Errorf("expected third sideband event to be assembled, got %+v", parsedObs[2]) - } else { - if parsedObs[2].Assembled.Content != "hello" { - t.Errorf("expected assembled content 'hello', got %q", parsedObs[2].Assembled.Content) - } - if parsedObs[2].Assembled.Reasoning != "stream reasoning" { - t.Errorf("expected assembled reasoning 'stream reasoning', got %q", parsedObs[2].Assembled.Reasoning) - } - if len(parsedObs[2].Assembled.ToolCallNames) != 1 || parsedObs[2].Assembled.ToolCallNames[0] != "call_stream" { - t.Errorf("expected assembled tool call names [call_stream], got %v", parsedObs[2].Assembled.ToolCallNames) - } - } - } - - found := false - for _, entry := range observed.All() { - if entry.Message == "openai chat completion sideband stream closed" { - found = true - m := entry.ContextMap() - if m["assembled_content"] != "hello" { - t.Errorf("expected assembled_content=hello, got %v", m["assembled_content"]) - } - if m["assembled_reasoning"] != "stream reasoning" { - t.Errorf("expected assembled_reasoning='stream reasoning', got %v", m["assembled_reasoning"]) - } - tc, ok := m["assembled_tool_calls"].([]interface{}) - if !ok || len(tc) != 1 || tc[0] != "call_stream" { - t.Errorf("expected assembled_tool_calls=[call_stream], got %v", m["assembled_tool_calls"]) - } - if count, ok := m["assembled_tool_call_count"].(int64); !ok || count != 1 { - t.Errorf("expected assembled_tool_call_count=1, got %v", m["assembled_tool_call_count"]) - } - } - } - if !found { - t.Error("expected log message 'openai chat completion sideband stream closed' not found") - } -} - -// TestChatCompletionsProviderPoolOllamaSelectionUsesNormalizedRun verifies that -// when a provider-pool catalog match is resolved and the selected provider is -// an Ollama-type (normalized executionPath), the Chat handler dispatches via -// normalized path (SubmitRun) rather than tunnel passthrough. func TestChatCompletionsProviderPoolOllamaSelectionUsesNormalizedRun(t *testing.T) { runEvents := make(chan *iop.RunEvent, 2) runEvents <- &iop.RunEvent{Type: "complete"} @@ -8021,10 +7223,10 @@ func TestChatCompletionsProviderPoolTunnelPreservesUnknownFields(t *testing.T) { t.Error("thinking_token_budget must be preserved in tunnel body") } - // No IOP sideband/header keys should be in the pure response body. - for _, key := range []string{"iop_response_mode", "estimated_input_tokens", "context_class"} { + // No IOP-internal metadata keys should leak into the pure passthrough body. + for _, key := range []string{"estimated_input_tokens", "context_class"} { if _, ok := bodyMap[key]; ok { - t.Errorf("IOP sideband key %q must not be in tunnel body", key) + t.Errorf("IOP internal key %q must not be in tunnel body", key) } } @@ -8192,11 +7394,11 @@ func TestProviderPoolDispatchLogObservationFields(t *testing.T) { } } -// TestProviderPoolStandardResponseNoSideband verifies that a provider-pool -// tunnel passthrough standard response does NOT carry IOP custom sideband -// fields (iop_sideband, iop.sideband event, or X-IOP-Response-Mode header) -// and only carries provider-original bytes (SURFACE_OBS-2/3 regression). -func TestProviderPoolStandardResponseNoSideband(t *testing.T) { +// TestProviderPoolStandardResponseNoExtensionFields verifies that a +// provider-pool tunnel passthrough standard response carries only +// provider-original bytes and no IOP extension fields or response-mode label +// (SURFACE_OBS-2/3 regression). +func TestProviderPoolStandardResponseNoExtensionFields(t *testing.T) { providerBody := `{"choices":[{"message":{"role":"assistant","content":"hello provider"}}],"usage":{"prompt_tokens":5,"completion_tokens":3}}` frames := staticProviderTunnelFrames(providerBody) @@ -8226,19 +7428,11 @@ func TestProviderPoolStandardResponseNoSideband(t *testing.T) { t.Fatalf("status: got %d body=%s", w.Code, w.Body.String()) } - // X-IOP-Response-Mode must NOT be set for standard passthrough. - if mode := w.Header().Get("X-IOP-Response-Mode"); mode != "" { - t.Errorf("X-IOP-Response-Mode must be empty for standard passthrough, got %q", mode) - } - // Response body must be the provider-original JSON with only the model - // field rewritten. It must NOT contain iop_sideband or iop.sideband markers. + // field rewritten, carrying no IOP extension markers. body := w.Body.String() - if strings.Contains(body, "iop_sideband") { - t.Errorf("standard response must not contain iop_sideband: %s", body) - } - if strings.Contains(body, "iop.sideband") { - t.Errorf("standard response must not contain iop.sideband: %s", body) + if strings.Contains(body, "iop_") || strings.Contains(body, "iop.") { + t.Errorf("standard response must not contain IOP extension markers: %s", body) } // Verify the tunnel was dispatched. diff --git a/apps/edge/internal/openai/stream.go b/apps/edge/internal/openai/stream.go index 575db79..eb3e157 100644 --- a/apps/edge/internal/openai/stream.go +++ b/apps/edge/internal/openai/stream.go @@ -50,7 +50,7 @@ func (s *Server) streamChatCompletion(w http.ResponseWriter, r *http.Request, re created := time.Now().Unix() id := "chatcmpl-" + handle.Dispatch().RunID model := responseModel(req.Model, handle.Dispatch().Target) - metricLabels := s.usageLabelsFor(r.Context(), strings.TrimSpace(req.Model), usageEndpointChatCompletions, chatResponseModeLabel(req)) + metricLabels := s.usageLabelsFor(r.Context(), strings.TrimSpace(req.Model), usageEndpointChatCompletions, responseModeNormalized) traceStream := openAICompatTraceStreamEnabled(submitReq.Metadata) traceSeq := 0 shouldExposeReasoning := req.includeReasoning() && (!outputPolicy.Strict || req.explicitlyIncludesReasoning()) @@ -329,7 +329,7 @@ func (s *Server) streamChatCompletion(w http.ResponseWriter, r *http.Request, re // relayed without IOP sideband fields or events. Chat Completions model echoes // are normalized back to the caller-facing model alias. func (s *Server) tunnelChatCompletionPassthrough(w http.ResponseWriter, r *http.Request, req chatCompletionRequest, dispatch routeDispatch, runMeta map[string]string, rawBody []byte, estimate int, contextClass string) { - handle, ok := s.submitChatCompletionTunnel(w, r, req, dispatch, runMeta, rawBody, estimate, contextClass, responseModePassthrough) + handle, ok := s.submitChatCompletionTunnel(w, r, req, dispatch, runMeta, rawBody, estimate, contextClass) if !ok { return } @@ -338,24 +338,6 @@ func (s *Server) tunnelChatCompletionPassthrough(w http.ResponseWriter, r *http. s.writeProviderTunnelResponse(w, r, handle, req.Stream, req.Model, metricLabels) } -// tunnelChatCompletionPassthroughSideband serves a Chat Completions request -// over the raw provider tunnel with the explicit IOP sideband extension -// surface (SDD S05): provider content is preserved, but the response also -// carries IOP route/usage/assembled observations and is never claimed as -// provider-original byte identity. -func (s *Server) tunnelChatCompletionPassthroughSideband(w http.ResponseWriter, r *http.Request, req chatCompletionRequest, dispatch routeDispatch, runMeta map[string]string, rawBody []byte, estimate int, contextClass string) { - handle, ok := s.submitChatCompletionTunnel(w, r, req, dispatch, runMeta, rawBody, estimate, contextClass, responseModePassthroughSideband) - if !ok { - return - } - defer handle.Close() - if req.Stream { - s.writeProviderTunnelSidebandStream(w, r, handle) - return - } - s.writeProviderTunnelSidebandResponse(w, r, handle) -} - // errProviderAuthRequired signals that provider auth forwarding is configured // as required but the caller did not supply the configured provider token // header. The raw token is never part of this error. @@ -388,9 +370,9 @@ func (s *Server) providerTunnelAuthHeaders(r *http.Request) (map[string]string, } // submitChatCompletionTunnel dispatches a Chat Completions provider tunnel -// request with the given response mode. On failure it writes the dispatch -// error to the caller and returns ok=false. -func (s *Server) submitChatCompletionTunnel(w http.ResponseWriter, r *http.Request, req chatCompletionRequest, dispatch routeDispatch, runMeta map[string]string, rawBody []byte, estimate int, contextClass, responseMode string) (edgeservice.ProviderTunnelResult, bool) { +// request for pure passthrough. On failure it writes the dispatch error to the +// caller and returns ok=false. +func (s *Server) submitChatCompletionTunnel(w http.ResponseWriter, r *http.Request, req chatCompletionRequest, dispatch routeDispatch, runMeta map[string]string, rawBody []byte, estimate int, contextClass string) (edgeservice.ProviderTunnelResult, bool) { providerAuthHeaders, err := s.providerTunnelAuthHeaders(r) if err != nil { // Missing required provider auth is rejected before dispatch; the raw @@ -405,7 +387,6 @@ func (s *Server) submitChatCompletionTunnel(w http.ResponseWriter, r *http.Reque } metadata["openai_model"] = req.Model metadata["openai_stream"] = strconv.FormatBool(req.Stream) - metadata[responseModeMetadataKey] = responseMode metadata["estimated_input_tokens"] = strconv.Itoa(estimate) metadata["context_class"] = contextClass @@ -431,7 +412,7 @@ func (s *Server) submitChatCompletionTunnel(w http.ResponseWriter, r *http.Reque ProviderPool: dispatch.ProviderPool, } - metricLabels := s.usageLabelsFor(r.Context(), strings.TrimSpace(req.Model), usageEndpointChatCompletions, responseMode) + metricLabels := s.usageLabelsFor(r.Context(), strings.TrimSpace(req.Model), usageEndpointChatCompletions, responseModePassthrough) handle, err := s.service.SubmitProviderTunnel(r.Context(), tunnelReq) if err != nil { emitUsageMetrics(metricLabels, usageStatusForError(err), usageObservation{}) @@ -449,7 +430,7 @@ func (s *Server) submitChatCompletionTunnel(w http.ResponseWriter, r *http.Reque zap.String("adapter", handle.Dispatch().Adapter), zap.String("target", handle.Dispatch().Target), zap.Bool("stream", req.Stream), - zap.String("response_mode", responseMode), + zap.String("response_mode", responseModePassthrough), zap.Int("estimated_input_tokens", handle.Dispatch().EstimatedInputTokens), zap.String("context_class", handle.Dispatch().ContextClass), zap.String("queue_reason", handle.Dispatch().QueueReason), @@ -735,134 +716,19 @@ func rewriteProviderJSONModel(body []byte, model string) []byte { return rewritten } -// Sideband extension surface (SDD S05): explicit passthrough+sideband -// responses expose IOP observations alongside provider-original content. -// Streaming responses interleave `event: iop.sideband` SSE events at provider -// event boundaries; non-streaming responses wrap the provider body in the -// iop.chat.passthrough_sideband envelope. Both are IOP extension outputs and -// are never claimed as provider-original byte identity. -const ( - sidebandSSEEventName = "iop.sideband" - sidebandEnvelopeObject = "iop.chat.passthrough_sideband" -) - -type sidebandRouteObservation struct { - RunID string `json:"run_id"` - NodeID string `json:"node_id,omitempty"` - ProviderID string `json:"provider_id,omitempty"` - ProviderType string `json:"provider_type,omitempty"` - ExecutionPath string `json:"execution_path,omitempty"` - Adapter string `json:"adapter,omitempty"` - Target string `json:"target,omitempty"` - ModelGroup string `json:"model_group,omitempty"` - ResponseMode string `json:"response_mode"` - ProviderStatusCode int `json:"provider_status_code,omitempty"` - QueueReason string `json:"queue_reason,omitempty"` -} - -type sidebandUsageObservation struct { - InputTokens int `json:"input_tokens"` - OutputTokens int `json:"output_tokens"` -} - -type sidebandAssembledObservation struct { +// providerAssembledObservation is the Edge-internal human-readable view of a +// provider Chat Completions/Responses body assembled by providerChatAssembler. +// It feeds Edge-local logging and usage metrics only and is never written to +// the caller response; pure passthrough relays provider bytes unmodified. +type providerAssembledObservation struct { Content string `json:"content,omitempty"` Reasoning string `json:"reasoning,omitempty"` ToolCallNames []string `json:"tool_call_names,omitempty"` BodyBytes int `json:"body_bytes"` } -// sidebandObservation is one `event: iop.sideband` SSE payload on the -// streaming sideband surface. -type sidebandObservation struct { - Object string `json:"object"` - Kind string `json:"kind"` - Route *sidebandRouteObservation `json:"route,omitempty"` - Usage *sidebandUsageObservation `json:"usage,omitempty"` - Assembled *sidebandAssembledObservation `json:"assembled,omitempty"` -} - -// sidebandEnvelope is the non-streaming sideband response schema: the provider -// body is carried verbatim next to the IOP observations. -type sidebandEnvelope struct { - Object string `json:"object"` - Sideband sidebandObservations `json:"iop_sideband"` - ProviderStatusCode int `json:"provider_status_code"` - ProviderResponse json.RawMessage `json:"provider_response,omitempty"` - ProviderBody string `json:"provider_body,omitempty"` -} - -type sidebandObservations struct { - Route sidebandRouteObservation `json:"route"` - Usage *sidebandUsageObservation `json:"usage,omitempty"` - Assembled *sidebandAssembledObservation `json:"assembled,omitempty"` -} - -func sidebandRouteFromDispatch(dispatch edgeservice.RunDispatch, providerStatus int) sidebandRouteObservation { - return sidebandRouteObservation{ - RunID: dispatch.RunID, - NodeID: dispatch.NodeID, - ProviderID: dispatch.ProviderID, - ProviderType: dispatch.ProviderType, - ExecutionPath: dispatch.ExecutionPath, - Adapter: dispatch.Adapter, - Target: dispatch.Target, - ModelGroup: dispatch.ModelGroupKey, - ResponseMode: responseModePassthroughSideband, - ProviderStatusCode: providerStatus, - QueueReason: dispatch.QueueReason, - } -} - -func sidebandUsageFromFrame(frame *iop.ProviderTunnelFrame) *sidebandUsageObservation { - usage := frame.GetUsage() - if usage == nil { - return nil - } - return &sidebandUsageObservation{ - InputTokens: int(usage.GetInputTokens()), - OutputTokens: int(usage.GetOutputTokens()), - } -} - -// usageObservationFromProtoUsage converts a proto iop.Usage into the internal -// usageObservation used for metrics. Returns zero values when u is nil. -func usageObservationFromProtoUsage(u *iop.Usage) usageObservation { - if u == nil { - return usageObservation{} - } - return usageObservation{ - inputTokens: int(u.GetInputTokens()), - outputTokens: int(u.GetOutputTokens()), - reasoningTokens: int(u.GetReasoningTokens()), - cachedInputTokens: int(u.GetCachedInputTokens()), - } -} - -// mergeUsageObservation applies the sideband merge rule: use the body value -// when body observed a token type; use the proto value only when body is zero. -// This prevents double-counting input/output when both body and proto report -// the same provider usage, while preserving proto-only fields like -// reasoning/cached_input (SDD S05). -func mergeUsageObservation(body, proto usageObservation) usageObservation { - return usageObservation{ - inputTokens: selectFirstNonZero(body.inputTokens, proto.inputTokens), - outputTokens: selectFirstNonZero(body.outputTokens, proto.outputTokens), - reasoningTokens: selectFirstNonZero(body.reasoningTokens, proto.reasoningTokens), - cachedInputTokens: selectFirstNonZero(body.cachedInputTokens, proto.cachedInputTokens), - reasoningChars: body.reasoningChars, - } -} - -func selectFirstNonZero(a, b int) int { - if a != 0 { - return a - } - return b -} - // providerChatAssembler accumulates a human-readable view of the provider -// Chat Completions response for the sideband assembled observation. It +// Chat Completions response for the assembled observation. It // tolerates non-JSON and partial payloads: whatever cannot be parsed simply // yields an empty summary. type providerChatAssembler struct { @@ -1034,7 +900,7 @@ func (a *providerChatAssembler) consumeDelta(delta providerChatDeltaEnvelope) { } } -func (a *providerChatAssembler) observation() *sidebandAssembledObservation { +func (a *providerChatAssembler) observation() *providerAssembledObservation { if !a.streaming && !a.nonStreamingParsed { a.nonStreamingParsed = true var resp struct { @@ -1050,7 +916,7 @@ func (a *providerChatAssembler) observation() *sidebandAssembledObservation { a.recordUsage(resp.Usage) } } - return &sidebandAssembledObservation{ + return &providerAssembledObservation{ Content: a.content.String(), Reasoning: a.reasoning.String(), ToolCallNames: a.toolCallNames, @@ -1058,316 +924,6 @@ func (a *providerChatAssembler) observation() *sidebandAssembledObservation { } } -// writeProviderTunnelSidebandStream relays provider SSE bytes unchanged and -// interleaves explicit `event: iop.sideband` extension events. IOP events are -// written only at provider event boundaries (before the first body byte, -// after a "\n\n"-terminated provider event, or at end of stream) so a -// provider event is never split by sideband injection. -func (s *Server) writeProviderTunnelSidebandStream(w http.ResponseWriter, r *http.Request, handle edgeservice.ProviderTunnelResult) { - frames := handle.Stream().Frames - if frames == nil { - writeError(w, http.StatusBadGateway, "provider_tunnel_error", "tunnel stream unavailable") - return - } - flusher, _ := w.(http.Flusher) - timer := time.NewTimer(handle.WaitTimeout()) - defer timer.Stop() - - assembler := &providerChatAssembler{streaming: true} - wroteHeader := false - atEventBoundary := true - tail := "" - var pendingObservations []sidebandObservation - - metricLabels := s.usageLabelsFor(r.Context(), strings.TrimSpace(handle.Dispatch().ModelGroupKey), usageEndpointChatCompletions, responseModePassthroughSideband) - // metricStatus is the terminal status reported to usage metrics. It defaults - // to error and is upgraded to success on END. - metricStatus := usageStatusError - // pendingMerged accumulates usage from all USAGE frames so that emitUsageMetrics - // sees the full usage even after pendingObservations is flushed. - var pendingMerged usageObservation - - writeObservation := func(obs sidebandObservation) { - obs.Object = sidebandSSEEventName - payload, err := json.Marshal(obs) - if err != nil { - return - } - fmt.Fprintf(w, "event: %s\ndata: %s\n\n", sidebandSSEEventName, payload) - if flusher != nil { - flusher.Flush() - } - } - flushObservations := func() { - for _, obs := range pendingObservations { - writeObservation(obs) - } - pendingObservations = nil - } - - defer func() { - obs := assembler.observation() - s.logger.Info("openai chat completion sideband stream closed", - zap.String("run_id", handle.Dispatch().RunID), - zap.Bool("wrote_header", wroteHeader), - zap.Int("body_bytes", assembler.bodyBytes), - zap.String("assembled_content", obs.Content), - zap.String("assembled_reasoning", obs.Reasoning), - zap.Strings("assembled_tool_calls", obs.ToolCallNames), - zap.Int("assembled_tool_call_count", len(obs.ToolCallNames)), - ) - // Merge body-parsed usage (primary) with USAGE frame data - // (auxiliary reasoning/cached_input). Per the merge rule, body values - // are used when observed; proto values fill in when body is zero. - // This avoids double-counting input/output when both sources report - // the same provider usage, while preserving proto-only fields like - // reasoning/cached_input (SDD S05). - merged := mergeUsageObservation(assembler.usageObservation(), pendingMerged) - emitUsageMetrics(metricLabels, metricStatus, merged) - }() - - for { - select { - case <-r.Context().Done(): - s.cancelRunOnHTTPGiveUp(handle.Dispatch(), r.Context().Err()) - metricStatus = usageStatusCancel - return - case <-timer.C: - s.cancelRunOnHTTPGiveUp(handle.Dispatch(), errRunTimedOut) - metricStatus = usageStatusCancel - if !wroteHeader { - writeError(w, http.StatusBadGateway, "run_error", errRunTimedOut.Error()) - } - return - case frame, ok := <-frames: - if !ok { - if !wroteHeader { - writeError(w, http.StatusBadGateway, "provider_tunnel_error", "tunnel stream closed before provider response") - } - return - } - switch frame.GetKind() { - case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START: - if wroteHeader { - continue - } - copyProviderResponseHeaders(w.Header(), frame.GetHeaders()) - // The sideband stream is an IOP extension surface: label it - // and drop the provider Content-Length, which no longer - // matches the extended body. - w.Header().Del("Content-Length") - w.Header().Set(responseModeHeaderName, responseModePassthroughSideband) - status := int(frame.GetStatusCode()) - if status == 0 { - status = http.StatusOK - } - w.WriteHeader(status) - wroteHeader = true - if flusher != nil { - flusher.Flush() - } - route := sidebandRouteFromDispatch(handle.Dispatch(), status) - writeObservation(sidebandObservation{Kind: "route", Route: &route}) - case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY: - body := frame.GetBody() - if len(body) == 0 { - continue - } - if !wroteHeader { - // Defensive: a body frame before response-start still - // reaches the caller instead of being dropped. - w.Header().Set(responseModeHeaderName, responseModePassthroughSideband) - w.WriteHeader(http.StatusOK) - wroteHeader = true - } - if _, err := w.Write(body); err != nil { - // The caller is gone mid-stream; propagate cancel to the Node. - s.sendCancelRun(handle.Dispatch()) - metricStatus = usageStatusCancel - return - } - assembler.Write(body) - if flusher != nil { - flusher.Flush() - } - tail = sseTail(tail, body) - atEventBoundary = strings.HasSuffix(tail, "\n\n") - if atEventBoundary { - flushObservations() - } - case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_ERROR: - msg := frame.GetError() - if msg == "" { - msg = "provider tunnel failed" - } - if !wroteHeader { - writeError(w, http.StatusBadGateway, "provider_tunnel_error", msg) - return - } - // Status/headers are already committed; the response is - // truncated and the caller observes the broken stream. - s.logger.Warn("openai sideband tunnel error after response start", - zap.String("run_id", handle.Dispatch().RunID), - zap.String("error", msg), - ) - return - case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_USAGE: - usage := sidebandUsageFromFrame(frame) - if usage == nil { - continue - } - // Accumulate proto usage from all USAGE frames so emitUsageMetrics - // sees the full usage even after pendingObservations is flushed. - pendingMerged = mergeUsageObservation(pendingMerged, - usageObservationFromProtoUsage(frame.GetUsage())) - pendingObservations = append(pendingObservations, sidebandObservation{Kind: "usage", Usage: usage}) - if wroteHeader && atEventBoundary { - flushObservations() - } - case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END: - if !wroteHeader { - writeError(w, http.StatusBadGateway, "provider_tunnel_error", "tunnel ended before provider response") - return - } - metricStatus = usageStatusSuccess - if !atEventBoundary { - // The provider stream ended mid-event; terminate it so the - // trailing IOP events stay well-formed SSE. - fmt.Fprint(w, "\n\n") - } - flushObservations() - writeObservation(sidebandObservation{Kind: "assembled", Assembled: assembler.observation()}) - return - } - } - } -} - -// sseTail keeps the last two bytes of the relayed provider stream so the -// sideband writer can detect "\n\n" event boundaries across chunk splits. -func sseTail(tail string, chunk []byte) string { - combined := tail + string(chunk) - if len(combined) > 2 { - return combined[len(combined)-2:] - } - return combined -} - -// writeProviderTunnelSidebandResponse buffers the provider response and -// answers with the iop.chat.passthrough_sideband envelope: provider status is -// preserved, the provider body is carried verbatim inside the envelope, and -// IOP route/usage/assembled observations sit alongside it. -func (s *Server) writeProviderTunnelSidebandResponse(w http.ResponseWriter, r *http.Request, handle edgeservice.ProviderTunnelResult) { - frames := handle.Stream().Frames - if frames == nil { - writeError(w, http.StatusBadGateway, "provider_tunnel_error", "tunnel stream unavailable") - return - } - timer := time.NewTimer(handle.WaitTimeout()) - defer timer.Stop() - - assembler := &providerChatAssembler{} - var body bytes.Buffer - providerStatus := 0 - var protoUsage *sidebandUsageObservation - var protoObs usageObservation - - metricLabels := s.usageLabelsFor(r.Context(), strings.TrimSpace(handle.Dispatch().ModelGroupKey), usageEndpointChatCompletions, responseModePassthroughSideband) - // metricStatus is the terminal status reported to usage metrics. It defaults - // to error and is upgraded to success on END. - metricStatus := usageStatusError - defer func() { - obs := assembler.observation() - s.logger.Info("openai chat completion sideband response", - zap.String("run_id", handle.Dispatch().RunID), - zap.Int("provider_status", providerStatus), - zap.Int("body_bytes", body.Len()), - zap.String("assembled_content", obs.Content), - zap.String("assembled_reasoning", obs.Reasoning), - zap.Strings("assembled_tool_calls", obs.ToolCallNames), - zap.Int("assembled_tool_call_count", len(obs.ToolCallNames)), - ) - // Use body-parsed usage as primary source; layer in USAGE frame - // data per the merge rule. sidebandUsageObservation in the response - // schema carries only input/output, while the metric path uses the - // full breakdown including reasoning/cached_input from the provider - // body or proto (SDD S05). - merged := mergeUsageObservation(assembler.usageObservation(), protoObs) - emitUsageMetrics(metricLabels, metricStatus, merged) - }() - - for { - select { - case <-r.Context().Done(): - s.cancelRunOnHTTPGiveUp(handle.Dispatch(), r.Context().Err()) - metricStatus = usageStatusCancel - return - case <-timer.C: - s.cancelRunOnHTTPGiveUp(handle.Dispatch(), errRunTimedOut) - metricStatus = usageStatusCancel - writeError(w, http.StatusBadGateway, "run_error", errRunTimedOut.Error()) - return - case frame, ok := <-frames: - if !ok { - writeError(w, http.StatusBadGateway, "provider_tunnel_error", "tunnel stream closed before provider response") - return - } - switch frame.GetKind() { - case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START: - if providerStatus != 0 { - continue - } - providerStatus = int(frame.GetStatusCode()) - if providerStatus == 0 { - providerStatus = http.StatusOK - } - case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY: - body.Write(frame.GetBody()) - assembler.Write(frame.GetBody()) - case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_ERROR: - msg := frame.GetError() - if msg == "" { - msg = "provider tunnel failed" - } - writeError(w, http.StatusBadGateway, "provider_tunnel_error", msg) - return - case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_USAGE: - if u := sidebandUsageFromFrame(frame); u != nil { - protoUsage = u - } - protoObs = mergeUsageObservation(protoObs, usageObservationFromProtoUsage(frame.GetUsage())) - case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END: - if providerStatus == 0 && body.Len() == 0 { - writeError(w, http.StatusBadGateway, "provider_tunnel_error", "tunnel ended before provider response") - return - } - metricStatus = usageStatusSuccess - if providerStatus == 0 { - providerStatus = http.StatusOK - } - envelope := sidebandEnvelope{ - Object: sidebandEnvelopeObject, - Sideband: sidebandObservations{ - Route: sidebandRouteFromDispatch(handle.Dispatch(), providerStatus), - Usage: protoUsage, - Assembled: assembler.observation(), - }, - ProviderStatusCode: providerStatus, - } - raw := body.Bytes() - if json.Valid(raw) { - envelope.ProviderResponse = json.RawMessage(raw) - } else if len(raw) > 0 { - envelope.ProviderBody = string(raw) - } - w.Header().Set(responseModeHeaderName, responseModePassthroughSideband) - writeJSON(w, providerStatus, envelope) - return - } - } - } -} - // hopByHopResponseHeaders are transport-level headers owned by each hop; they // are not copied from the provider response to the caller response. var hopByHopResponseHeaders = map[string]struct{}{ @@ -1524,7 +1080,7 @@ func writeToolCallsDeltaSSE(w http.ResponseWriter, flusher http.Flusher, id stri // successful tool_calls chunk. func (s *Server) streamBufferedChatCompletion(w http.ResponseWriter, r *http.Request, req chatCompletionRequest, submitReq edgeservice.SubmitRunRequest, handle edgeservice.RunResult, flusher http.Flusher, outputPolicy strictOutputPolicy, validation toolValidationContract, retrySubmit func(ctx context.Context, req edgeservice.SubmitRunRequest) (any, error)) { attempt := 1 - metricLabels := s.usageLabelsFor(r.Context(), strings.TrimSpace(req.Model), usageEndpointChatCompletions, chatResponseModeLabel(req)) + metricLabels := s.usageLabelsFor(r.Context(), strings.TrimSpace(req.Model), usageEndpointChatCompletions, responseModeNormalized) for { result, err := collectChatCompletionOutput(r.Context(), req, handle, outputPolicy) if err != nil { diff --git a/apps/edge/internal/openai/usage_metrics_test.go b/apps/edge/internal/openai/usage_metrics_test.go index b58cc74..3e5de99 100644 --- a/apps/edge/internal/openai/usage_metrics_test.go +++ b/apps/edge/internal/openai/usage_metrics_test.go @@ -1,13 +1,11 @@ package openai import ( - "context" "fmt" "net/http" "net/http/httptest" "strings" "testing" - "time" "github.com/prometheus/client_golang/prometheus/testutil" @@ -340,80 +338,6 @@ func TestResponsesProviderTunnelPassthroughObservesUsageMetrics(t *testing.T) { } } -// TestResponsesProviderTunnelSidebandObservesUsageMetrics verifies that a -// successful /v1/responses provider passthrough+sideband response records usage -// under endpoint=responses and response_mode=passthrough+sideband. -func TestResponsesProviderTunnelSidebandObservesUsageMetrics(t *testing.T) { - const rawToken = "sk-responses-sideband-token" - const edgeID = "edge-responses-sideband-usage" - const model = "pool-model" - providerBody := `{"id":"resp-1","object":"response","output_text":"hi","metadata":{"request_id":"req-1"},"usage":{"input_tokens":8,"output_tokens":5,"input_tokens_details":{"cached_tokens":2},"output_tokens_details":{"reasoning_tokens":1}}}` - - frames := make(chan *iop.ProviderTunnelFrame, 4) - frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START, StatusCode: 200} - frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(providerBody)} - frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true} - close(frames) - - srv := providerRouteUsageMetricServer( - rawToken, edgeID, model, "served-model", - &fakeRunService{tunnelFrames: frames, tunnelServedTarget: "served-model"}, - ) - - labels := usageLabels{ - edgeID: edgeID, principalRef: "user:alice", principalAlias: "alice", tokenRef: "iop-tok-alice", - modelGroup: model, endpoint: usageEndpointResponses, responseMode: responseModePassthroughSideband, - } - reqBefore := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues( - edgeID, "user:alice", "alice", "iop-tok-alice", model, - usageEndpointResponses, responseModePassthroughSideband, usageStatusSuccess, usageSourceProviderReported, - )) - before := map[string]float64{ - tokenTypeInput: requestTokenValue(t, labels, tokenTypeInput), - tokenTypeOutput: requestTokenValue(t, labels, tokenTypeOutput), - tokenTypeReasoning: requestTokenValue(t, labels, tokenTypeReasoning), - tokenTypeCachedInput: requestTokenValue(t, labels, tokenTypeCachedInput), - } - - req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(`{ - "model":"pool-model", - "input":"hello", - "metadata":{"iop_response_mode":"passthrough+sideband"} - }`)) - req.Header.Set("Authorization", "Bearer "+rawToken) - w := httptest.NewRecorder() - srv.routes().ServeHTTP(w, req) - - if w.Code != http.StatusOK { - t.Fatalf("status: got %d body=%s", w.Code, w.Body.String()) - } - if !strings.Contains(w.Body.String(), `"iop_response_mode":"passthrough+sideband"`) { - t.Fatalf("sideband response must inject metadata marker, got %s", w.Body.String()) - } - reqAfter := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues( - edgeID, "user:alice", "alice", "iop-tok-alice", model, - usageEndpointResponses, responseModePassthroughSideband, usageStatusSuccess, usageSourceProviderReported, - )) - if reqAfter-reqBefore != 1 { - t.Fatalf("requests_total responses sideband/success: got delta %v, want 1", reqAfter-reqBefore) - } - for tokenType, want := range map[string]float64{ - tokenTypeInput: 8, - tokenTypeOutput: 5, - tokenTypeReasoning: 1, - tokenTypeCachedInput: 2, - } { - got := requestTokenValue(t, labels, tokenType) - before[tokenType] - if got != want { - t.Fatalf("responses sideband token_type %s: got delta %v, want %v", tokenType, got, want) - } - } -} - -// TestResponsesProviderTunnelPassthroughStreamingObservesUsageMetrics verifies -// that a streaming /v1/responses provider passthrough with body-only SSE events -// (no proto USAGE frame) still observes provider-reported usage metrics -// (REVIEW_REVIEW_SEULGI_RESPONSES-1). func TestResponsesProviderTunnelPassthroughStreamingObservesUsageMetrics(t *testing.T) { const rawToken = "sk-responses-streaming-token" const edgeID = "edge-responses-streaming-usage" @@ -528,176 +452,6 @@ func TestOpenAIScopeExcludesA2AAndNonOpenAISurfaces(t *testing.T) { } } -// TestProviderTunnelSidebandStreamingEmitsUsageMetrics verifies that a -// passthrough+sideband streaming response increments the request counter -// with status=success and usage_source=provider_reported (REVIEW_USAGE_METRIC-1). -func TestProviderTunnelSidebandStreamingEmitsUsageMetrics(t *testing.T) { - const rawToken = "sk-sideband-stream-token" - const edgeID = "edge-sideband-stream-metrics" - const model = "pool-model" - providerBody := `{"choices":[{"delta":{"content":"hi"}}]} - -data: [DONE] - -` - - frames := make(chan *iop.ProviderTunnelFrame, 5) - frames <- &iop.ProviderTunnelFrame{ - Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START, - StatusCode: 200, - Headers: map[string]string{"Content-Type": "text/event-stream"}, - } - frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(providerBody)} - frames <- &iop.ProviderTunnelFrame{ - Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_USAGE, - Usage: &iop.Usage{InputTokens: 5, OutputTokens: 3}, - } - frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true} - close(frames) - - srv := providerRouteUsageMetricServer(rawToken, edgeID, model, "served-model", &fakeRunService{tunnelFrames: frames}) - - reqBefore := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues( - edgeID, "user:alice", "alice", "iop-tok-alice", model, - usageEndpointChatCompletions, responseModePassthroughSideband, usageStatusSuccess, usageSourceProviderReported, - )) - - req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ - "model":"pool-model", - "stream":true, - "messages":[{"role":"user","content":"hello"}], - "metadata":{"iop_response_mode":"passthrough+sideband"} - }`)) - req.Header.Set("Authorization", "Bearer "+rawToken) - w := httptest.NewRecorder() - srv.routes().ServeHTTP(w, req) - - if w.Code != http.StatusOK { - t.Fatalf("status: got %d body=%s", w.Code, w.Body.String()) - } - - reqAfter := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues( - edgeID, "user:alice", "alice", "iop-tok-alice", model, - usageEndpointChatCompletions, responseModePassthroughSideband, usageStatusSuccess, usageSourceProviderReported, - )) - if reqAfter-reqBefore != 1 { - t.Fatalf("requests_total success/provider_reported: got delta %v, want 1", reqAfter-reqBefore) - } -} - -// TestProviderTunnelSidebandNonStreamingEmitsUsageMetrics verifies that a -// passthrough+sideband non-streaming response increments the request counter -// with status=success and usage_source=provider_reported (REVIEW_USAGE_METRIC-1). -func TestProviderTunnelSidebandNonStreamingEmitsUsageMetrics(t *testing.T) { - const rawToken = "sk-sideband-nonstream-token" - const edgeID = "edge-sideband-nonstream-metrics" - const model = "pool-model" - providerBody := `{"id":"cmpl-1","choices":[{"message":{"role":"assistant","content":"hi"}}]}` - - frames := make(chan *iop.ProviderTunnelFrame, 5) - frames <- &iop.ProviderTunnelFrame{ - Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START, - StatusCode: 200, - Headers: map[string]string{"Content-Type": "application/json"}, - } - frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(providerBody)} - frames <- &iop.ProviderTunnelFrame{ - Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_USAGE, - Usage: &iop.Usage{InputTokens: 7, OutputTokens: 4}, - } - frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true} - close(frames) - - srv := providerRouteUsageMetricServer(rawToken, edgeID, model, "served-model", &fakeRunService{tunnelFrames: frames}) - - labels := usageLabels{ - edgeID: edgeID, principalRef: "user:alice", principalAlias: "alice", tokenRef: "iop-tok-alice", - modelGroup: model, endpoint: usageEndpointChatCompletions, responseMode: responseModePassthroughSideband, - } - reqBefore := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues( - edgeID, "user:alice", "alice", "iop-tok-alice", model, - usageEndpointChatCompletions, responseModePassthroughSideband, usageStatusSuccess, usageSourceProviderReported, - )) - - req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ - "model":"pool-model", - "messages":[{"role":"user","content":"hello"}], - "metadata":{"iop_response_mode":"passthrough+sideband"} - }`)) - req.Header.Set("Authorization", "Bearer "+rawToken) - w := httptest.NewRecorder() - srv.routes().ServeHTTP(w, req) - - if w.Code != http.StatusOK { - t.Fatalf("status: got %d body=%s", w.Code, w.Body.String()) - } - - reqAfter := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues( - edgeID, "user:alice", "alice", "iop-tok-alice", model, - usageEndpointChatCompletions, responseModePassthroughSideband, usageStatusSuccess, usageSourceProviderReported, - )) - if reqAfter-reqBefore != 1 { - t.Fatalf("requests_total success/provider_reported: got delta %v, want 1", reqAfter-reqBefore) - } - // Verify token counters also incremented. - for tokenType, want := range map[string]float64{ - tokenTypeInput: 7, - tokenTypeOutput: 4, - } { - got := requestTokenValue(t, labels, tokenType) - if got != want { - t.Fatalf("token_type %s: got %v, want %v", tokenType, got, want) - } - } -} - -// TestProviderTunnelSidebandStreamCallerCancelEmitsCancelMetric verifies that a -// passthrough+sideband streaming caller disconnect (pre-cancelled context) emits -// the cancel metric (REVIEW_USAGE_METRIC-1). -func TestProviderTunnelSidebandStreamCallerCancelEmitsCancelMetric(t *testing.T) { - const rawToken = "sk-sideband-cancel-token" - const edgeID = "edge-sideband-cancel" - const model = "pool-model" - - // Create a cancelled context so the sideband stream detects caller disconnect. - ctx, cancel := context.WithCancel(context.Background()) - cancel() - - // Frame channel never closed to simulate an in-flight tunnel. - frames := make(chan *iop.ProviderTunnelFrame) - - fake := &fakeRunService{tunnelFrames: frames, tunnelWaitTimeout: 3 * time.Second} - srv := providerRouteUsageMetricServer(rawToken, edgeID, model, "served-model", fake) - - reqBefore := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues( - edgeID, "user:alice", "alice", "iop-tok-alice", model, - usageEndpointChatCompletions, responseModePassthroughSideband, usageStatusCancel, usageSourceUnavailable, - )) - - req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ - "model":"pool-model", - "stream":true, - "messages":[{"role":"user","content":"hello"}], - "metadata":{"iop_response_mode":"passthrough+sideband"} - }`)) - req.Header.Set("Authorization", "Bearer "+rawToken) - req = req.WithContext(ctx) - - w := httptest.NewRecorder() - srv.routes().ServeHTTP(w, req) - - reqAfter := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues( - edgeID, "user:alice", "alice", "iop-tok-alice", model, - usageEndpointChatCompletions, responseModePassthroughSideband, usageStatusCancel, usageSourceUnavailable, - )) - if reqAfter-reqBefore != 1 { - t.Fatalf("requests_total cancel: got delta %v, want 1", reqAfter-reqBefore) - } -} - -// TestChatCompletionsDispatchFailureEmitsErrorMetric verifies that a tunnel -// dispatch failure increments the request counter with status=error -// (REVIEW_USAGE_METRIC-2). func TestChatCompletionsDispatchFailureEmitsErrorMetric(t *testing.T) { const rawToken = "sk-dispatch-fail-token" const edgeID = "edge-dispatch-fail" @@ -843,523 +597,3 @@ func TestBufferedStreamToolValidationRetryDispatchFailureEmitsErrorMetric(t *tes t.Fatalf("requests_total error: got delta %v, want 1", reqAfter-reqBefore) } } - -// TestProviderTunnelSidebandStreamBodyWriteFailureEmitsCancelMetric verifies -// that a passthrough+sideband streaming body write failure to the caller -// propagates cancel upstream and emits status=cancel (REVIEW_USAGE_METRIC_RETRY-2). -func TestProviderTunnelSidebandStreamBodyWriteFailureEmitsCancelMetric(t *testing.T) { - const rawToken = "sk-sideband-bodyfail-cancel" - const edgeID = "edge-sideband-bodyfail-cancel" - const model = "pool-bodyfail" - - // Use a tunnel that emits a single body frame, then the response writer - // will fail on the second body frame write. - frames := make(chan *iop.ProviderTunnelFrame, 3) - frames <- &iop.ProviderTunnelFrame{ - Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START, - StatusCode: http.StatusOK, - } - frames <- &iop.ProviderTunnelFrame{ - Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, - Body: []byte(`{"ok":true}`), - } - // Second body frame triggers write failure. - frames <- &iop.ProviderTunnelFrame{ - Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, - Body: []byte(`more`), - } - - // Use a writer that fails on the second write. - fake := &fakeRunService{tunnelFrames: frames, tunnelWaitTimeout: 3 * time.Second} - srv := providerRouteUsageMetricServer(rawToken, edgeID, model, "served", fake) - - cancelBefore := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues( - edgeID, "user:alice", "alice", "iop-tok-alice", model, - usageEndpointChatCompletions, responseModePassthroughSideband, usageStatusCancel, usageSourceUnavailable, - )) - errorBefore := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues( - edgeID, "user:alice", "alice", "iop-tok-alice", model, - usageEndpointChatCompletions, responseModePassthroughSideband, usageStatusError, usageSourceUnavailable, - )) - - // Wrap the response writer so the second Write call fails. - req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ - "model":"pool-bodyfail", - "stream":true, - "messages":[{"role":"user","content":"hello"}], - "metadata":{"iop_response_mode":"passthrough+sideband"} - }`)) - req.Header.Set("Authorization", "Bearer "+rawToken) - - w := &failingWriteRecorder{ - ResponseRecorder: httptest.NewRecorder(), - failAfter: 2, - } - srv.routes().ServeHTTP(w, req) - - cancelAfter := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues( - edgeID, "user:alice", "alice", "iop-tok-alice", model, - usageEndpointChatCompletions, responseModePassthroughSideband, usageStatusCancel, usageSourceUnavailable, - )) - errorAfter := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues( - edgeID, "user:alice", "alice", "iop-tok-alice", model, - usageEndpointChatCompletions, responseModePassthroughSideband, usageStatusError, usageSourceUnavailable, - )) - - if cancelAfter-cancelBefore != 1 { - t.Fatalf("requests_total cancel: got delta %v, want 1", cancelAfter-cancelBefore) - } - if errorAfter-errorBefore != 0 { - t.Fatalf("requests_total error: got delta %v, want 0", errorAfter-errorBefore) - } -} - -// TestProviderTunnelSidebandStreamingEmitsUsageMetricsBodyOnly verifies that a -// passthrough+sideband streaming response that contains usage ONLY in the body -// (no separate USAGE frame) still emits metrics with usage_source=provider_reported. -// This is a regression test for REVIEW_USAGE_METRIC-1: body-parsed usage must not -// be ignored when no proto USAGE frame is present. -func TestProviderTunnelSidebandStreamingEmitsUsageMetricsBodyOnly(t *testing.T) { - const rawToken = "sk-body-only-stream-token" - const edgeID = "edge-body-only-stream-metrics" - const model = "pool-model" - // BODY is SSE-formatted (matching actual provider tunnel stream): delta chunk - // followed by a usage chunk on the next line. - providerBody := `data: {"choices":[{"delta":{"content":"hi"}}]} - -data: {"usage":{"prompt_tokens":10,"completion_tokens":5,"prompt_tokens_details":{"cached_tokens":2},"completion_tokens_details":{"reasoning_tokens":3},"total_tokens":20}} - -data: [DONE] - -` - - frames := make(chan *iop.ProviderTunnelFrame, 5) - frames <- &iop.ProviderTunnelFrame{ - Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START, - StatusCode: 200, - Headers: map[string]string{"Content-Type": "text/event-stream"}, - } - frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(providerBody)} - // NOTE: No USAGE frame — usage is only in the body JSON. - frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true} - close(frames) - - srv := providerRouteUsageMetricServer(rawToken, edgeID, model, "served-model", &fakeRunService{tunnelFrames: frames}) - - labels := usageLabels{ - edgeID: edgeID, principalRef: "user:alice", principalAlias: "alice", tokenRef: "iop-tok-alice", - modelGroup: model, endpoint: usageEndpointChatCompletions, responseMode: responseModePassthroughSideband, - } - reqBefore := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues( - edgeID, "user:alice", "alice", "iop-tok-alice", model, - usageEndpointChatCompletions, responseModePassthroughSideband, usageStatusSuccess, usageSourceProviderReported, - )) - - req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ - "model":"pool-model", - "stream":true, - "messages":[{"role":"user","content":"hello"}], - "metadata":{"iop_response_mode":"passthrough+sideband"} - }`)) - req.Header.Set("Authorization", "Bearer "+rawToken) - w := httptest.NewRecorder() - srv.routes().ServeHTTP(w, req) - - if w.Code != http.StatusOK { - t.Fatalf("status: got %d body=%s", w.Code, w.Body.String()) - } - - reqAfter := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues( - edgeID, "user:alice", "alice", "iop-tok-alice", model, - usageEndpointChatCompletions, responseModePassthroughSideband, usageStatusSuccess, usageSourceProviderReported, - )) - if reqAfter-reqBefore != 1 { - t.Fatalf("requests_total success/provider_reported: got delta %v, want 1", reqAfter-reqBefore) - } - // Verify token counters were populated from body-only usage including - // reasoning and cached_input (REVIEW_REVIEW_REVIEW_USAGE_METRIC_RETRY). - for tokenType, want := range map[string]float64{ - tokenTypeInput: 10, - tokenTypeOutput: 5, - tokenTypeReasoning: 3, - tokenTypeCachedInput: 2, - } { - got := requestTokenValue(t, labels, tokenType) - if got != want { - t.Fatalf("token_type %s: got %v, want %v", tokenType, got, want) - } - } -} - -// TestProviderTunnelSidebandNonStreamingEmitsUsageMetricsBodyOnly verifies that a -// passthrough+sideband non-streaming response that contains usage ONLY in the body -// (no separate USAGE frame) still emits metrics with usage_source=provider_reported. -func TestProviderTunnelSidebandNonStreamingEmitsUsageMetricsBodyOnly(t *testing.T) { - const rawToken = "sk-body-only-nonstream-token" - const edgeID = "edge-body-only-nonstream-metrics" - const model = "pool-model" - // BODY contains OpenAI-compatible usage JSON including detail objects; - // no separate USAGE frame. - providerBody := `{"id":"cmpl-1","choices":[{"message":{"role":"assistant","content":"hi"}}],"usage":{"prompt_tokens":12,"completion_tokens":8,"prompt_tokens_details":{"cached_tokens":3},"completion_tokens_details":{"reasoning_tokens":4},"total_tokens":27}}` - - frames := make(chan *iop.ProviderTunnelFrame, 5) - frames <- &iop.ProviderTunnelFrame{ - Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START, - StatusCode: 200, - Headers: map[string]string{"Content-Type": "application/json"}, - } - frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(providerBody)} - // NOTE: No USAGE frame — usage is only in the body JSON. - frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true} - close(frames) - - srv := providerRouteUsageMetricServer(rawToken, edgeID, model, "served-model", &fakeRunService{tunnelFrames: frames}) - - labels := usageLabels{ - edgeID: edgeID, principalRef: "user:alice", principalAlias: "alice", tokenRef: "iop-tok-alice", - modelGroup: model, endpoint: usageEndpointChatCompletions, responseMode: responseModePassthroughSideband, - } - reqBefore := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues( - edgeID, "user:alice", "alice", "iop-tok-alice", model, - usageEndpointChatCompletions, responseModePassthroughSideband, usageStatusSuccess, usageSourceProviderReported, - )) - - req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ - "model":"pool-model", - "messages":[{"role":"user","content":"hello"}], - "metadata":{"iop_response_mode":"passthrough+sideband"} - }`)) - req.Header.Set("Authorization", "Bearer "+rawToken) - w := httptest.NewRecorder() - srv.routes().ServeHTTP(w, req) - - if w.Code != http.StatusOK { - t.Fatalf("status: got %d body=%s", w.Code, w.Body.String()) - } - - reqAfter := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues( - edgeID, "user:alice", "alice", "iop-tok-alice", model, - usageEndpointChatCompletions, responseModePassthroughSideband, usageStatusSuccess, usageSourceProviderReported, - )) - if reqAfter-reqBefore != 1 { - t.Fatalf("requests_total success/provider_reported: got delta %v, want 1", reqAfter-reqBefore) - } - // Verify token counters were populated from body-only usage including - // reasoning and cached_input (REVIEW_REVIEW_REVIEW_USAGE_METRIC_RETRY). - for tokenType, want := range map[string]float64{ - tokenTypeInput: 12, - tokenTypeOutput: 8, - tokenTypeReasoning: 4, - tokenTypeCachedInput: 3, - } { - got := requestTokenValue(t, labels, tokenType) - if got != want { - t.Fatalf("token_type %s: got %v, want %v", tokenType, got, want) - } - } -} - -// TestProviderTunnelSidebandStreamProtoOnlyReasoningCachedInput verifies that when -// only a proto USAGE frame is present (no body usage), the metric path preserves -// reasoning and cached_input from the proto (REVIEW_REVIEW_REVIEW_USAGE_METRIC_RETRY). -func TestProviderTunnelSidebandStreamProtoOnlyReasoningCachedInput(t *testing.T) { - const rawToken = "sk-proto-only-stream-token" - const edgeID = "edge-proto-only-stream" - const model = "pool-model" - // BODY contains no usage object. - providerBody := `data: {"choices":[{"delta":{"content":"hi"}}]} - -data: [DONE] - -` - - frames := make(chan *iop.ProviderTunnelFrame, 5) - frames <- &iop.ProviderTunnelFrame{ - Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START, - StatusCode: 200, - Headers: map[string]string{"Content-Type": "text/event-stream"}, - } - frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(providerBody)} - // Proto USAGE frame carries full token breakdown including reasoning/cached_input. - frames <- &iop.ProviderTunnelFrame{ - Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_USAGE, - Usage: &iop.Usage{InputTokens: 10, OutputTokens: 5, ReasoningTokens: 3, CachedInputTokens: 2}, - } - frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true} - close(frames) - - srv := providerRouteUsageMetricServer(rawToken, edgeID, model, "served-model", &fakeRunService{tunnelFrames: frames}) - - labels := usageLabels{ - edgeID: edgeID, principalRef: "user:alice", principalAlias: "alice", tokenRef: "iop-tok-alice", - modelGroup: model, endpoint: usageEndpointChatCompletions, responseMode: responseModePassthroughSideband, - } - reqBefore := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues( - edgeID, "user:alice", "alice", "iop-tok-alice", model, - usageEndpointChatCompletions, responseModePassthroughSideband, usageStatusSuccess, usageSourceProviderReported, - )) - - req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ - "model":"pool-model", - "stream":true, - "messages":[{"role":"user","content":"hello"}], - "metadata":{"iop_response_mode":"passthrough+sideband"} - }`)) - req.Header.Set("Authorization", "Bearer "+rawToken) - w := httptest.NewRecorder() - srv.routes().ServeHTTP(w, req) - - if w.Code != http.StatusOK { - t.Fatalf("status: got %d body=%s", w.Code, w.Body.String()) - } - - reqAfter := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues( - edgeID, "user:alice", "alice", "iop-tok-alice", model, - usageEndpointChatCompletions, responseModePassthroughSideband, usageStatusSuccess, usageSourceProviderReported, - )) - if reqAfter-reqBefore != 1 { - t.Fatalf("requests_total success/provider_reported: got delta %v, want 1", reqAfter-reqBefore) - } - // All four token types must be present from proto-only. - for tokenType, want := range map[string]float64{ - tokenTypeInput: 10, - tokenTypeOutput: 5, - tokenTypeReasoning: 3, - tokenTypeCachedInput: 2, - } { - got := requestTokenValue(t, labels, tokenType) - if got != want { - t.Fatalf("token_type %s: got %v, want %v", tokenType, got, want) - } - } -} - -// TestProviderTunnelSidebandResponseProtoOnlyReasoningCachedInput verifies that -// for non-streaming sideband, proto-only USAGE frame preserves reasoning and -// cached_input in metrics (REVIEW_REVIEW_REVIEW_USAGE_METRIC_RETRY). -func TestProviderTunnelSidebandResponseProtoOnlyReasoningCachedInput(t *testing.T) { - const rawToken = "sk-proto-only-nonstream-token" - const edgeID = "edge-proto-only-nonstream" - const model = "pool-model" - providerBody := `{"id":"cmpl-1","choices":[{"message":{"role":"assistant","content":"hi"}}]}` - - frames := make(chan *iop.ProviderTunnelFrame, 5) - frames <- &iop.ProviderTunnelFrame{ - Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START, - StatusCode: 200, - Headers: map[string]string{"Content-Type": "application/json"}, - } - frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(providerBody)} - frames <- &iop.ProviderTunnelFrame{ - Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_USAGE, - Usage: &iop.Usage{InputTokens: 7, OutputTokens: 4, ReasoningTokens: 2, CachedInputTokens: 1}, - } - frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true} - close(frames) - - srv := providerRouteUsageMetricServer(rawToken, edgeID, model, "served-model", &fakeRunService{tunnelFrames: frames}) - - labels := usageLabels{ - edgeID: edgeID, principalRef: "user:alice", principalAlias: "alice", tokenRef: "iop-tok-alice", - modelGroup: model, endpoint: usageEndpointChatCompletions, responseMode: responseModePassthroughSideband, - } - reqBefore := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues( - edgeID, "user:alice", "alice", "iop-tok-alice", model, - usageEndpointChatCompletions, responseModePassthroughSideband, usageStatusSuccess, usageSourceProviderReported, - )) - - req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ - "model":"pool-model", - "messages":[{"role":"user","content":"hello"}], - "metadata":{"iop_response_mode":"passthrough+sideband"} - }`)) - req.Header.Set("Authorization", "Bearer "+rawToken) - w := httptest.NewRecorder() - srv.routes().ServeHTTP(w, req) - - if w.Code != http.StatusOK { - t.Fatalf("status: got %d body=%s", w.Code, w.Body.String()) - } - - reqAfter := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues( - edgeID, "user:alice", "alice", "iop-tok-alice", model, - usageEndpointChatCompletions, responseModePassthroughSideband, usageStatusSuccess, usageSourceProviderReported, - )) - if reqAfter-reqBefore != 1 { - t.Fatalf("requests_total success/provider_reported: got delta %v, want 1", reqAfter-reqBefore) - } - for tokenType, want := range map[string]float64{ - tokenTypeInput: 7, - tokenTypeOutput: 4, - tokenTypeReasoning: 2, - tokenTypeCachedInput: 1, - } { - got := requestTokenValue(t, labels, tokenType) - if got != want { - t.Fatalf("token_type %s: got %v, want %v", tokenType, got, want) - } - } -} - -// TestProviderTunnelSidebandStreamBodyAndProtoNoDoubleCount verifies that when -// both body usage and proto USAGE frame are present, the merge rule prevents -// double-counting input/output while preserving proto-only reasoning/cached_input -// (REVIEW_REVIEW_REVIEW_USAGE_METRIC_RETRY). -func TestProviderTunnelSidebandStreamBodyAndProtoNoDoubleCount(t *testing.T) { - const rawToken = "sk-body-and-proto-stream-token" - const edgeID = "edge-body-and-proto-stream" - const model = "pool-model" - // Body reports input/output but no reasoning/cached_input details. - providerBody := `data: {"choices":[{"delta":{"content":"hi"}}]} - -data: {"usage":{"prompt_tokens":10,"completion_tokens":5,"total_tokens":15}} - -data: [DONE] - -` - - frames := make(chan *iop.ProviderTunnelFrame, 5) - frames <- &iop.ProviderTunnelFrame{ - Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START, - StatusCode: 200, - Headers: map[string]string{"Content-Type": "text/event-stream"}, - } - frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(providerBody)} - // Proto frame carries same input/output plus additional reasoning/cached_input. - frames <- &iop.ProviderTunnelFrame{ - Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_USAGE, - Usage: &iop.Usage{InputTokens: 10, OutputTokens: 5, ReasoningTokens: 3, CachedInputTokens: 2}, - } - frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true} - close(frames) - - srv := providerRouteUsageMetricServer(rawToken, edgeID, model, "served-model", &fakeRunService{tunnelFrames: frames}) - - labels := usageLabels{ - edgeID: edgeID, principalRef: "user:alice", principalAlias: "alice", tokenRef: "iop-tok-alice", - modelGroup: model, endpoint: usageEndpointChatCompletions, responseMode: responseModePassthroughSideband, - } - reqBefore := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues( - edgeID, "user:alice", "alice", "iop-tok-alice", model, - usageEndpointChatCompletions, responseModePassthroughSideband, usageStatusSuccess, usageSourceProviderReported, - )) - - req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ - "model":"pool-model", - "stream":true, - "messages":[{"role":"user","content":"hello"}], - "metadata":{"iop_response_mode":"passthrough+sideband"} - }`)) - req.Header.Set("Authorization", "Bearer "+rawToken) - w := httptest.NewRecorder() - srv.routes().ServeHTTP(w, req) - - if w.Code != http.StatusOK { - t.Fatalf("status: got %d body=%s", w.Code, w.Body.String()) - } - - reqAfter := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues( - edgeID, "user:alice", "alice", "iop-tok-alice", model, - usageEndpointChatCompletions, responseModePassthroughSideband, usageStatusSuccess, usageSourceProviderReported, - )) - if reqAfter-reqBefore != 1 { - t.Fatalf("requests_total success/provider_reported: got delta %v, want 1", reqAfter-reqBefore) - } - // Input/output should NOT be double-counted (merge rule: body wins). - // Reasoning/cached_input should come from proto (body has 0). - for tokenType, want := range map[string]float64{ - tokenTypeInput: 10, // from body, NOT 10+10=20 - tokenTypeOutput: 5, // from body, NOT 5+5=10 - tokenTypeReasoning: 3, // from proto - tokenTypeCachedInput: 2, // from proto - } { - got := requestTokenValue(t, labels, tokenType) - if got != want { - t.Fatalf("token_type %s: got %v, want %v", tokenType, got, want) - } - } -} - -// TestProviderTunnelSidebandResponseBodyAndProtoNoDoubleCount verifies the same -// merge rule for non-streaming sideband responses (REVIEW_REVIEW_REVIEW_USAGE_METRIC_RETRY). -func TestProviderTunnelSidebandResponseBodyAndProtoNoDoubleCount(t *testing.T) { - const rawToken = "sk-body-and-proto-nonstream-token" - const edgeID = "edge-body-and-proto-nonstream" - const model = "pool-model" - // Body reports input/output only. - providerBody := `{"id":"cmpl-1","choices":[{"message":{"role":"assistant","content":"hi"}}],"usage":{"prompt_tokens":12,"completion_tokens":8,"total_tokens":20}}` - - frames := make(chan *iop.ProviderTunnelFrame, 5) - frames <- &iop.ProviderTunnelFrame{ - Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START, - StatusCode: 200, - Headers: map[string]string{"Content-Type": "application/json"}, - } - frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(providerBody)} - // Proto frame carries same input/output plus reasoning/cached_input. - frames <- &iop.ProviderTunnelFrame{ - Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_USAGE, - Usage: &iop.Usage{InputTokens: 12, OutputTokens: 8, ReasoningTokens: 5, CachedInputTokens: 4}, - } - frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true} - close(frames) - - srv := providerRouteUsageMetricServer(rawToken, edgeID, model, "served-model", &fakeRunService{tunnelFrames: frames}) - - labels := usageLabels{ - edgeID: edgeID, principalRef: "user:alice", principalAlias: "alice", tokenRef: "iop-tok-alice", - modelGroup: model, endpoint: usageEndpointChatCompletions, responseMode: responseModePassthroughSideband, - } - reqBefore := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues( - edgeID, "user:alice", "alice", "iop-tok-alice", model, - usageEndpointChatCompletions, responseModePassthroughSideband, usageStatusSuccess, usageSourceProviderReported, - )) - - req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{ - "model":"pool-model", - "messages":[{"role":"user","content":"hello"}], - "metadata":{"iop_response_mode":"passthrough+sideband"} - }`)) - req.Header.Set("Authorization", "Bearer "+rawToken) - w := httptest.NewRecorder() - srv.routes().ServeHTTP(w, req) - - if w.Code != http.StatusOK { - t.Fatalf("status: got %d body=%s", w.Code, w.Body.String()) - } - - reqAfter := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues( - edgeID, "user:alice", "alice", "iop-tok-alice", model, - usageEndpointChatCompletions, responseModePassthroughSideband, usageStatusSuccess, usageSourceProviderReported, - )) - if reqAfter-reqBefore != 1 { - t.Fatalf("requests_total success/provider_reported: got delta %v, want 1", reqAfter-reqBefore) - } - // Input/output from body only; reasoning/cached_input from proto. - for tokenType, want := range map[string]float64{ - tokenTypeInput: 12, // from body, NOT 12+12=24 - tokenTypeOutput: 8, // from body, NOT 8+8=16 - tokenTypeReasoning: 5, // from proto - tokenTypeCachedInput: 4, // from proto - } { - got := requestTokenValue(t, labels, tokenType) - if got != want { - t.Fatalf("token_type %s: got %v, want %v", tokenType, got, want) - } - } -} - -// failingWriteRecorder wraps httptest.ResponseRecorder and fails after N writes. -type failingWriteRecorder struct { - *httptest.ResponseRecorder - failAfter int - writes int -} - -func (fw *failingWriteRecorder) Write(p []byte) (int, error) { - fw.writes++ - if fw.writes > fw.failAfter { - return 0, fmt.Errorf("simulated write failure") - } - return fw.ResponseRecorder.Write(p) -} diff --git a/docs/openai-usage-grafana.md b/docs/openai-usage-grafana.md index 2400f87..47161ec 100644 --- a/docs/openai-usage-grafana.md +++ b/docs/openai-usage-grafana.md @@ -18,7 +18,7 @@ Provider-reported OpenAI-compatible token usage by token type. | `token_ref` | `tok-xyz789` | Token identity (not a raw secret) | | `model_group` | `gpt-4o` | Model group identifier | | `endpoint` | `chat.completions`, `responses` | OpenAI-compatible route | -| `response_mode` | `blocking`, `streaming` | Response mode | +| `response_mode` | `passthrough`, `normalized` | Internal execution label; callers cannot set it via OpenAI metadata | | `token_type` | `input`, `output`, `reasoning`, `cached_input` | Token type | ### 2. `iop_openai_requests_total` (Counter) @@ -33,7 +33,7 @@ OpenAI-compatible requests processed by terminal status and usage source. | `token_ref` | `tok-xyz789` | Token identity | | `model_group` | `gpt-4o` | Model group identifier | | `endpoint` | `chat.completions`, `responses` | OpenAI-compatible route | -| `response_mode` | `blocking`, `streaming` | Response mode | +| `response_mode` | `passthrough`, `normalized` | Internal execution label; callers cannot set it via OpenAI metadata | | `status` | `success`, `error`, `cancel` | Terminal request status | | `usage_source` | `provider_reported`, `unavailable` | Whether at least one token type was reported | @@ -140,6 +140,8 @@ sum by (endpoint, status) (iop_openai_requests_total) ### response_mode별 분석 +`response_mode`는 handler 실행 경로에서 파생되는 내부 라벨이다. provider 라우트의 raw tunnel 패스스루는 `passthrough`, 정규화된 RunEvent 경로는 `normalized`로 기록된다. caller가 OpenAI metadata로 설정할 수 있는 값이 아니며 API selector가 아니다. + ```promql # response_mode별 성공 요청 수 sum by (response_mode, status) (iop_openai_requests_total{status="success"}) @@ -379,7 +381,7 @@ Grafana table 구성 예: | `status` | `success`, `error`, `cancel` | | `usage_source` | `provider_reported`, `unavailable` | | `token_type` | `input`, `output`, `reasoning`, `cached_input` | -| `response_mode` | `blocking`, `streaming` | +| `response_mode` | `passthrough`, `normalized` | ---