refactor(openai): call metadata 기반 response_mode 라우터 선택을 제거한다
provider 라우트는 raw tunnel 패스스루, 그 외 라우트는 정규화된 RunEvent 경로로 응답을 결정한다. caller metadata는 임의 컨텍스트이며 응답 경로/shape를 선택하지 않는다 (SDD S01). - chat_handler.go: response_mode parse/switch 로직 제거, 라우트 기반 분기로 단순화 - responses_handler.go: tunnelResponsesPassthroughSideband 함수 및 response_mode switch 제거 - stream.go: 관련 response_mode 라벨 사용 정리 - server_test.go: response_mode 관련 테스트 케이스 제거 및 정리 - usage_metrics_test.go: 사용되지 않는 테스트 대목 제거 - docs/openai-usage-grafana.md: response_mode 라벨 설명을 passthrough/normalized로 수정
This commit is contained in:
parent
9f5d15d91c
commit
59c2997d47
6 changed files with 144 additions and 2600 deletions
|
|
@ -74,15 +74,6 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) {
|
|||
for k, v := range principalMetadata(r.Context()) {
|
||||
runMeta[k] = v
|
||||
}
|
||||
if dispatch.ProviderPool && responseModeWasExplicit(runMeta) {
|
||||
writeError(w, http.StatusBadRequest, "invalid_request_error", "metadata.iop_response_mode is not supported for model group routes")
|
||||
return
|
||||
}
|
||||
responseMode, err := parseResponseMode(runMeta)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error())
|
||||
return
|
||||
}
|
||||
if err := validateWorkspaceForRoute(dispatch, workspace); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error())
|
||||
return
|
||||
|
|
@ -94,25 +85,14 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) {
|
|||
return
|
||||
}
|
||||
outputPolicy := s.resolveOutputPolicy(basePrompt)
|
||||
if responseMode == responseModePassthrough && !responseModeWasExplicit(runMeta) && outputPolicy.Strict && !providerTunnelRoute {
|
||||
responseMode = responseModeTransformed
|
||||
}
|
||||
if catalogEntry := s.findProviderPoolEntry(req.Model); catalogEntry != nil {
|
||||
applyModelCatalogGenerationPolicyToChat(&req, *catalogEntry, outputPolicy.Strict)
|
||||
}
|
||||
if providerTunnelRoute && responseMode == responseModeTransformed {
|
||||
writeError(w, http.StatusBadRequest, "invalid_request_error", "metadata.iop_response_mode=transformed is not supported for OpenAI-compatible provider model groups")
|
||||
return
|
||||
}
|
||||
|
||||
// OpenAI-compatible provider routes default to pure passthrough (SDD D02):
|
||||
// provider status/headers/body bytes are relayed to the caller unmodified
|
||||
// over the raw tunnel instead of the normalized RunEvent path. Explicit
|
||||
// passthrough+sideband keeps the provider tunnel but exposes IOP
|
||||
// observations through the sideband extension surface (SDD S05).
|
||||
//
|
||||
// For provider-pool, use the one-shot SubmitProviderPool surface which
|
||||
// selects tunnel or normalized based on the candidate's executionPath.
|
||||
// The response path is decided by the resolved route, never by caller
|
||||
// metadata: provider routes relay pure passthrough over the raw tunnel;
|
||||
// every other route uses the normalized RunEvent path. Caller metadata is
|
||||
// arbitrary context and does not select route or response shape (SDD S01).
|
||||
// Pre-compute estimate/contextClass early so provider-pool path can use them.
|
||||
estimate := s.estimateChatInputTokens(basePrompt, runMeta, req.Tools, req.ToolChoice)
|
||||
contextClass := classifyContext(estimate, s.longContextThreshold())
|
||||
|
|
@ -129,20 +109,15 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) {
|
|||
metadata["estimated_input_tokens"] = strconv.Itoa(estimate)
|
||||
metadata["context_class"] = contextClass
|
||||
|
||||
// provider-pool path preserves responseMode for tunnel passthrough.
|
||||
// strict-output output policy only applies to normalized dispatch,
|
||||
// not to raw tunnel passthrough (SDD D02).
|
||||
s.handleChatCompletionsProviderPool(r, req, dispatch, workspace, basePrompt, prompt, rawBody, input, metadata, estimate, contextClass, outputPolicy, validation, responseMode, w)
|
||||
s.handleChatCompletionsProviderPool(r, req, dispatch, workspace, basePrompt, prompt, rawBody, input, metadata, estimate, contextClass, outputPolicy, validation, w)
|
||||
return
|
||||
}
|
||||
|
||||
switch {
|
||||
case responseMode == responseModePassthrough && providerTunnelRoute:
|
||||
if providerTunnelRoute {
|
||||
s.tunnelChatCompletionPassthrough(w, r, req, dispatch, runMeta, rawBody, estimate, contextClass)
|
||||
return
|
||||
case responseMode == responseModePassthroughSideband && providerTunnelRoute:
|
||||
s.tunnelChatCompletionPassthroughSideband(w, r, req, dispatch, runMeta, rawBody, estimate, contextClass)
|
||||
return
|
||||
}
|
||||
|
||||
// Non-provider-pool normalized path: build messages, prompt, estimate.
|
||||
|
|
@ -160,20 +135,13 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) {
|
|||
metadata := chatRunMetadata(runMeta, req, outputPolicy)
|
||||
metadata["estimated_input_tokens"] = strconv.Itoa(estimate)
|
||||
metadata["context_class"] = contextClass
|
||||
if responseMode == responseModeTransformed {
|
||||
// Explicit transformed mode produces IOP-transformed output (SDD S07):
|
||||
// label the response so it is never mistaken for provider-original
|
||||
// byte-identical passthrough.
|
||||
metadata[responseModeMetadataKey] = responseModeTransformed
|
||||
w.Header().Set(responseModeHeaderName, responseModeTransformed)
|
||||
}
|
||||
if validation.enabled {
|
||||
metadata = toolValidationAttemptMetadata(metadata, 1, "", "")
|
||||
}
|
||||
submitReq := chatSubmitRunRequest(dispatch, req, workspace, prompt, input, metadata)
|
||||
submitReq.EstimatedInputTokens = estimate
|
||||
submitReq.ContextClass = contextClass
|
||||
metricLabels := s.usageLabelsFor(r.Context(), strings.TrimSpace(req.Model), usageEndpointChatCompletions, chatResponseModeLabel(req))
|
||||
metricLabels := s.usageLabelsFor(r.Context(), strings.TrimSpace(req.Model), usageEndpointChatCompletions, responseModeNormalized)
|
||||
handle, err := s.service.SubmitRun(r.Context(), submitReq)
|
||||
if err != nil {
|
||||
emitUsageMetrics(metricLabels, usageStatusForError(err), usageObservation{})
|
||||
|
|
@ -226,7 +194,6 @@ func (s *Server) handleChatCompletionsProviderPool(
|
|||
contextClass string,
|
||||
outputPolicy strictOutputPolicy,
|
||||
validation toolValidationContract,
|
||||
responseMode string,
|
||||
w http.ResponseWriter,
|
||||
) {
|
||||
poolReq := edgeservice.ProviderPoolDispatchRequest{
|
||||
|
|
@ -274,10 +241,8 @@ func (s *Server) handleChatCompletionsProviderPool(
|
|||
return tunnelReq, nil
|
||||
}
|
||||
|
||||
// provider-pool path preserves responseMode for tunnel passthrough.
|
||||
// strict-output output policy only applies to normalized dispatch,
|
||||
// not to raw tunnel passthrough (SDD D02).
|
||||
|
||||
poolReq.Tunnel.BuildBody = func(target string) ([]byte, error) {
|
||||
return rewriteChatCompletionModel(rawBody, target, req)
|
||||
}
|
||||
|
|
@ -327,16 +292,9 @@ func (s *Server) handleChatCompletionsProviderPool(
|
|||
case edgeservice.ProviderPoolPathTunnel:
|
||||
// Tunnel path: provider auth was already validated and injected via
|
||||
// PrepareTunnel before dispatch; on failure SubmitProviderPool returns
|
||||
// an error and no tunnel handle exists.
|
||||
metricLabels := s.usageLabelsFor(r.Context(), strings.TrimSpace(req.Model), usageEndpointChatCompletions, responseMode)
|
||||
if responseMode == responseModePassthroughSideband {
|
||||
if req.Stream {
|
||||
s.writeProviderTunnelSidebandStream(w, r, result.Tunnel)
|
||||
return
|
||||
}
|
||||
s.writeProviderTunnelSidebandResponse(w, r, result.Tunnel)
|
||||
return
|
||||
}
|
||||
// an error and no tunnel handle exists. Provider bytes are relayed as
|
||||
// pure passthrough; caller metadata never selects a sideband surface.
|
||||
metricLabels := s.usageLabelsFor(r.Context(), strings.TrimSpace(req.Model), usageEndpointChatCompletions, responseModePassthrough)
|
||||
s.writeProviderTunnelResponse(w, r, result.Tunnel, req.Stream, req.Model, metricLabels)
|
||||
|
||||
case edgeservice.ProviderPoolPathNormalized:
|
||||
|
|
@ -552,21 +510,6 @@ func boolPtr(v bool) *bool {
|
|||
return &v
|
||||
}
|
||||
|
||||
// chatResponseModeLabel returns the usage-metric response_mode label for a
|
||||
// normalized-path chat completion: "transformed" when the caller explicitly
|
||||
// requested it, otherwise "normalized". Provider-tunnel passthrough paths use
|
||||
// their own mode label and do not call this.
|
||||
func chatResponseModeLabel(req chatCompletionRequest) string {
|
||||
runMeta, _, err := parseOpenAIMetadata(req.Metadata)
|
||||
if err != nil {
|
||||
return "normalized"
|
||||
}
|
||||
if mode, _ := parseResponseMode(runMeta); mode == responseModeTransformed {
|
||||
return responseModeTransformed
|
||||
}
|
||||
return "normalized"
|
||||
}
|
||||
|
||||
func chatRunMetadata(runMeta map[string]string, req chatCompletionRequest, outputPolicy strictOutputPolicy) map[string]string {
|
||||
if runMeta == nil {
|
||||
runMeta = make(map[string]string)
|
||||
|
|
@ -578,37 +521,15 @@ func chatRunMetadata(runMeta map[string]string, req chatCompletionRequest, outpu
|
|||
}
|
||||
|
||||
const (
|
||||
responseModeMetadataKey = "iop_response_mode"
|
||||
responseModePassthrough = "passthrough"
|
||||
responseModePassthroughSideband = "passthrough+sideband"
|
||||
responseModeTransformed = "transformed"
|
||||
// responseModeHeaderName labels explicit IOP extension outputs
|
||||
// (passthrough+sideband, transformed). Pure passthrough responses never
|
||||
// carry it: their headers stay provider-original.
|
||||
responseModeHeaderName = "X-IOP-Response-Mode"
|
||||
// responseModePassthrough and responseModeNormalized are internal execution
|
||||
// labels for the response_mode usage metric. They are derived from the
|
||||
// handler execution path, never from caller metadata: provider tunnel routes
|
||||
// report passthrough, normalized RunEvent routes report normalized. Callers
|
||||
// cannot select a response mode through OpenAI metadata.
|
||||
responseModePassthrough = "passthrough"
|
||||
responseModeNormalized = "normalized"
|
||||
)
|
||||
|
||||
func parseResponseMode(runMeta map[string]string) (string, error) {
|
||||
mode := strings.TrimSpace(runMeta[responseModeMetadataKey])
|
||||
if mode == "" {
|
||||
return responseModePassthrough, nil
|
||||
}
|
||||
switch mode {
|
||||
case responseModePassthrough, responseModePassthroughSideband, responseModeTransformed:
|
||||
return mode, nil
|
||||
default:
|
||||
return "", fmt.Errorf("metadata.%s must be one of %s, %s, or %s", responseModeMetadataKey, responseModePassthrough, responseModePassthroughSideband, responseModeTransformed)
|
||||
}
|
||||
}
|
||||
|
||||
func responseModeWasExplicit(runMeta map[string]string) bool {
|
||||
if runMeta == nil {
|
||||
return false
|
||||
}
|
||||
mode, ok := runMeta[responseModeMetadataKey]
|
||||
return ok && strings.TrimSpace(mode) != ""
|
||||
}
|
||||
|
||||
// routeUsesProviderTunnel reports whether the resolved dispatch targets an
|
||||
// OpenAI-compatible provider that serves raw tunnel passthrough. Provider-pool
|
||||
// catalog routes and openai_compat/vllm type routes qualify; CLI and other
|
||||
|
|
@ -710,7 +631,7 @@ func (s *Server) completeChatCompletion(w http.ResponseWriter, r *http.Request,
|
|||
s.cancelRunOnHTTPGiveUp(handle.Dispatch(), err)
|
||||
handle.Close()
|
||||
emitUsageMetrics(
|
||||
s.usageLabelsFor(r.Context(), strings.TrimSpace(req.Model), usageEndpointChatCompletions, chatResponseModeLabel(req)),
|
||||
s.usageLabelsFor(r.Context(), strings.TrimSpace(req.Model), usageEndpointChatCompletions, responseModeNormalized),
|
||||
usageStatusForError(err), usageObservation{},
|
||||
)
|
||||
writeError(w, httpStatusForRunError(err), "run_error", err.Error())
|
||||
|
|
@ -735,7 +656,7 @@ func (s *Server) completeChatCompletion(w http.ResponseWriter, r *http.Request,
|
|||
next, submitErr := retrySubmit(r.Context(), retryReq)
|
||||
if submitErr != nil {
|
||||
emitUsageMetrics(
|
||||
s.usageLabelsFor(r.Context(), strings.TrimSpace(req.Model), usageEndpointChatCompletions, chatResponseModeLabel(req)),
|
||||
s.usageLabelsFor(r.Context(), strings.TrimSpace(req.Model), usageEndpointChatCompletions, responseModeNormalized),
|
||||
usageStatusError, usageObservation{},
|
||||
)
|
||||
writeError(w, http.StatusBadGateway, "tool_validation_retry_error", submitErr.Error())
|
||||
|
|
@ -771,7 +692,7 @@ func (s *Server) completeChatCompletion(w http.ResponseWriter, r *http.Request,
|
|||
zap.String("reason", valErr.Error()),
|
||||
)
|
||||
emitUsageMetrics(
|
||||
s.usageLabelsFor(r.Context(), strings.TrimSpace(req.Model), usageEndpointChatCompletions, chatResponseModeLabel(req)),
|
||||
s.usageLabelsFor(r.Context(), strings.TrimSpace(req.Model), usageEndpointChatCompletions, responseModeNormalized),
|
||||
usageStatusError, usageObservation{},
|
||||
)
|
||||
writeError(w, http.StatusBadGateway, "tool_validation_error", valErr.Error())
|
||||
|
|
@ -873,18 +794,12 @@ func collectChatCompletionOutput(ctx context.Context, req chatCompletionRequest,
|
|||
}
|
||||
}
|
||||
|
||||
runMeta, _, _ := parseOpenAIMetadata(req.Metadata)
|
||||
mode, _ := parseResponseMode(runMeta)
|
||||
if mode != responseModeTransformed {
|
||||
mode = "normalized"
|
||||
}
|
||||
|
||||
return chatCompletionOutput{
|
||||
message: message,
|
||||
finishReason: finishReason,
|
||||
usage: usage,
|
||||
normalized: normalized,
|
||||
responseMode: mode,
|
||||
responseMode: responseModeNormalized,
|
||||
contentLen: len(message.Content),
|
||||
reasoningLen: len(reasoning),
|
||||
toolCalls: toolCalls,
|
||||
|
|
|
|||
|
|
@ -13,7 +13,6 @@ import (
|
|||
"go.uber.org/zap"
|
||||
|
||||
edgeservice "iop/apps/edge/internal/service"
|
||||
iop "iop/proto/gen/iop"
|
||||
)
|
||||
|
||||
func (s *Server) handleResponses(w http.ResponseWriter, r *http.Request) {
|
||||
|
|
@ -57,30 +56,9 @@ func (s *Server) handleResponses(w http.ResponseWriter, r *http.Request) {
|
|||
for k, v := range principalMetadata(r.Context()) {
|
||||
runMeta[k] = v
|
||||
}
|
||||
if dispatch.ProviderPool && responseModeWasExplicit(runMeta) {
|
||||
writeError(w, http.StatusBadRequest, "invalid_request_error", "metadata.iop_response_mode is not supported for model group routes")
|
||||
return
|
||||
}
|
||||
responseMode, err := parseResponseMode(runMeta)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error())
|
||||
return
|
||||
}
|
||||
switch responseMode {
|
||||
case responseModePassthrough:
|
||||
case responseModePassthroughSideband:
|
||||
if err := validateWorkspaceForRoute(dispatch, workspace); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error())
|
||||
return
|
||||
}
|
||||
estimate := estimateInputTokens(string(rawBody), runMeta, nil, nil)
|
||||
contextClass := classifyContext(estimate, s.longContextThreshold())
|
||||
s.tunnelResponsesPassthroughSideband(w, r, env, dispatch, runMeta, rawBody, estimate, contextClass)
|
||||
return
|
||||
case responseModeTransformed:
|
||||
writeError(w, http.StatusBadRequest, "invalid_request_error", "metadata.iop_response_mode=transformed is not supported for /v1/responses provider routes")
|
||||
return
|
||||
}
|
||||
// Provider routes always relay pure passthrough. Caller metadata is
|
||||
// arbitrary context and never selects the route or response shape
|
||||
// (SDD S01/S04); there is no caller-facing response mode selector.
|
||||
if err := validateWorkspaceForRoute(dispatch, workspace); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error())
|
||||
return
|
||||
|
|
@ -278,7 +256,6 @@ func (s *Server) tunnelResponsesPassthrough(w http.ResponseWriter, r *http.Reque
|
|||
}
|
||||
metadata["openai_model"] = env.Model
|
||||
metadata["openai_stream"] = strconv.FormatBool(env.Stream)
|
||||
metadata[responseModeMetadataKey] = responseModePassthrough
|
||||
metadata["estimated_input_tokens"] = strconv.Itoa(estimate)
|
||||
metadata["context_class"] = contextClass
|
||||
|
||||
|
|
@ -336,82 +313,6 @@ func (s *Server) tunnelResponsesPassthrough(w http.ResponseWriter, r *http.Reque
|
|||
s.writeProviderTunnelResponse(w, r, handle, env.Stream, "", metricLabels)
|
||||
}
|
||||
|
||||
// tunnelResponsesPassthroughSideband serves an explicit /v1/responses
|
||||
// passthrough+sideband request. The provider request remains raw passthrough
|
||||
// with model rewrite only; the response is extended after the provider returns.
|
||||
// Non-streaming JSON object responses receive sideband metadata under the
|
||||
// top-level `metadata` field. Streaming responses interleave `event:
|
||||
// iop.sideband` events.
|
||||
func (s *Server) tunnelResponsesPassthroughSideband(w http.ResponseWriter, r *http.Request, env responsesEnvelope, dispatch routeDispatch, runMeta map[string]string, rawBody []byte, estimate int, contextClass string) {
|
||||
providerAuthHeaders, err := s.providerTunnelAuthHeaders(r)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid_request_error", "provider auth token is required")
|
||||
return
|
||||
}
|
||||
|
||||
metadata := make(map[string]string, len(runMeta)+5)
|
||||
for k, v := range runMeta {
|
||||
metadata[k] = v
|
||||
}
|
||||
metadata["openai_model"] = env.Model
|
||||
metadata["openai_stream"] = strconv.FormatBool(env.Stream)
|
||||
metadata[responseModeMetadataKey] = responseModePassthroughSideband
|
||||
metadata["estimated_input_tokens"] = strconv.Itoa(estimate)
|
||||
metadata["context_class"] = contextClass
|
||||
|
||||
tunnelReq := edgeservice.SubmitProviderTunnelRequest{
|
||||
NodeRef: dispatch.NodeRef,
|
||||
ModelGroupKey: strings.TrimSpace(env.Model),
|
||||
Adapter: dispatch.Adapter,
|
||||
Target: dispatch.Target,
|
||||
SessionID: dispatch.SessionID,
|
||||
Method: http.MethodPost,
|
||||
Path: "/v1/responses",
|
||||
Headers: providerAuthHeaders,
|
||||
BuildBody: func(target string) ([]byte, error) {
|
||||
return rewriteResponsesModel(rawBody, target)
|
||||
},
|
||||
Stream: env.Stream,
|
||||
TimeoutSec: dispatch.TimeoutSec,
|
||||
MaxQueue: dispatch.MaxQueue,
|
||||
QueueTimeoutMS: dispatch.QueueTimeoutMS,
|
||||
Metadata: metadata,
|
||||
EstimatedInputTokens: estimate,
|
||||
ContextClass: contextClass,
|
||||
ProviderPool: dispatch.ProviderPool,
|
||||
}
|
||||
|
||||
metricLabels := s.usageLabelsFor(r.Context(), strings.TrimSpace(env.Model), usageEndpointResponses, responseModePassthroughSideband)
|
||||
handle, err := s.service.SubmitProviderTunnel(r.Context(), tunnelReq)
|
||||
if err != nil {
|
||||
emitUsageMetrics(metricLabels, usageStatusForError(err), usageObservation{})
|
||||
writeError(w, http.StatusBadGateway, "node_dispatch_error", err.Error())
|
||||
return
|
||||
}
|
||||
defer handle.Close()
|
||||
|
||||
s.logger.Info("openai responses sideband dispatch",
|
||||
zap.String("run_id", handle.Dispatch().RunID),
|
||||
zap.String("node_id", handle.Dispatch().NodeID),
|
||||
zap.String("provider_id", handle.Dispatch().ProviderID),
|
||||
zap.String("provider_type", handle.Dispatch().ProviderType),
|
||||
zap.String("execution_path", handle.Dispatch().ExecutionPath),
|
||||
zap.String("model_group", handle.Dispatch().ModelGroupKey),
|
||||
zap.String("adapter", handle.Dispatch().Adapter),
|
||||
zap.String("target", handle.Dispatch().Target),
|
||||
zap.Bool("stream", env.Stream),
|
||||
zap.Int("estimated_input_tokens", handle.Dispatch().EstimatedInputTokens),
|
||||
zap.String("context_class", handle.Dispatch().ContextClass),
|
||||
zap.String("queue_reason", handle.Dispatch().QueueReason),
|
||||
)
|
||||
|
||||
if env.Stream {
|
||||
s.writeResponsesProviderTunnelSidebandStream(w, r, handle, metricLabels)
|
||||
return
|
||||
}
|
||||
s.writeResponsesProviderTunnelSidebandResponse(w, r, handle, metricLabels)
|
||||
}
|
||||
|
||||
// rewriteResponsesModel replaces only the model field of the caller's original
|
||||
// /v1/responses request JSON so the provider receives its served model name.
|
||||
// Every other field (input, instructions, tools, max_output_tokens, and any
|
||||
|
|
@ -433,264 +334,6 @@ func rewriteResponsesModel(rawBody []byte, target string) ([]byte, error) {
|
|||
return json.Marshal(raw)
|
||||
}
|
||||
|
||||
const responsesSidebandObject = "iop.responses.sideband"
|
||||
|
||||
type responsesSidebandPayload struct {
|
||||
Object string `json:"object"`
|
||||
Metadata map[string]any `json:"metadata"`
|
||||
}
|
||||
|
||||
func responsesSidebandMetadata() map[string]any {
|
||||
return map[string]any{
|
||||
responseModeMetadataKey: responseModePassthroughSideband,
|
||||
}
|
||||
}
|
||||
|
||||
func responsesSidebandPayloadBytes() []byte {
|
||||
payload, err := json.Marshal(responsesSidebandPayload{
|
||||
Object: responsesSidebandObject,
|
||||
Metadata: responsesSidebandMetadata(),
|
||||
})
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return payload
|
||||
}
|
||||
|
||||
func injectResponsesSidebandMetadata(body []byte) []byte {
|
||||
var obj map[string]json.RawMessage
|
||||
if err := json.Unmarshal(body, &obj); err != nil {
|
||||
return body
|
||||
}
|
||||
if obj == nil {
|
||||
return body
|
||||
}
|
||||
|
||||
metadata := map[string]any{}
|
||||
if raw, ok := obj["metadata"]; ok && len(raw) > 0 && string(raw) != "null" {
|
||||
_ = json.Unmarshal(raw, &metadata)
|
||||
if metadata == nil {
|
||||
metadata = map[string]any{}
|
||||
}
|
||||
}
|
||||
for k, v := range responsesSidebandMetadata() {
|
||||
metadata[k] = v
|
||||
}
|
||||
encodedMetadata, err := json.Marshal(metadata)
|
||||
if err != nil {
|
||||
return body
|
||||
}
|
||||
obj["metadata"] = encodedMetadata
|
||||
rewritten, err := json.Marshal(obj)
|
||||
if err != nil {
|
||||
return body
|
||||
}
|
||||
return rewritten
|
||||
}
|
||||
|
||||
func (s *Server) writeResponsesProviderTunnelSidebandStream(w http.ResponseWriter, r *http.Request, handle edgeservice.ProviderTunnelResult, metricLabels usageLabels) {
|
||||
frames := handle.Stream().Frames
|
||||
if frames == nil {
|
||||
writeError(w, http.StatusBadGateway, "provider_tunnel_error", "tunnel stream unavailable")
|
||||
return
|
||||
}
|
||||
flusher, _ := w.(http.Flusher)
|
||||
timer := time.NewTimer(handle.WaitTimeout())
|
||||
defer timer.Stop()
|
||||
|
||||
assembler := &providerChatAssembler{streaming: true}
|
||||
wroteHeader := false
|
||||
metricStatus := usageStatusError
|
||||
var protoObs usageObservation
|
||||
tail := ""
|
||||
|
||||
writeSideband := func() {
|
||||
payload := responsesSidebandPayloadBytes()
|
||||
if len(payload) == 0 {
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(w, "event: %s\ndata: %s\n\n", sidebandSSEEventName, payload)
|
||||
if flusher != nil {
|
||||
flusher.Flush()
|
||||
}
|
||||
}
|
||||
|
||||
defer func() {
|
||||
emitUsageMetrics(metricLabels, metricStatus, mergeUsageObservation(assembler.usageObservation(), protoObs))
|
||||
}()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-r.Context().Done():
|
||||
s.cancelRunOnHTTPGiveUp(handle.Dispatch(), r.Context().Err())
|
||||
metricStatus = usageStatusCancel
|
||||
return
|
||||
case <-timer.C:
|
||||
s.cancelRunOnHTTPGiveUp(handle.Dispatch(), errRunTimedOut)
|
||||
metricStatus = usageStatusCancel
|
||||
if !wroteHeader {
|
||||
writeError(w, http.StatusBadGateway, "run_error", errRunTimedOut.Error())
|
||||
}
|
||||
return
|
||||
case frame, ok := <-frames:
|
||||
if !ok {
|
||||
if !wroteHeader {
|
||||
writeError(w, http.StatusBadGateway, "provider_tunnel_error", "tunnel stream closed before provider response")
|
||||
}
|
||||
return
|
||||
}
|
||||
switch frame.GetKind() {
|
||||
case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START:
|
||||
if wroteHeader {
|
||||
continue
|
||||
}
|
||||
copyProviderResponseHeaders(w.Header(), frame.GetHeaders())
|
||||
w.Header().Del("Content-Length")
|
||||
w.Header().Set(responseModeHeaderName, responseModePassthroughSideband)
|
||||
status := int(frame.GetStatusCode())
|
||||
if status == 0 {
|
||||
status = http.StatusOK
|
||||
}
|
||||
w.WriteHeader(status)
|
||||
wroteHeader = true
|
||||
writeSideband()
|
||||
case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY:
|
||||
body := frame.GetBody()
|
||||
if len(body) == 0 {
|
||||
continue
|
||||
}
|
||||
if !wroteHeader {
|
||||
w.Header().Set(responseModeHeaderName, responseModePassthroughSideband)
|
||||
w.WriteHeader(http.StatusOK)
|
||||
wroteHeader = true
|
||||
writeSideband()
|
||||
}
|
||||
if _, err := w.Write(body); err != nil {
|
||||
s.sendCancelRun(handle.Dispatch())
|
||||
metricStatus = usageStatusCancel
|
||||
return
|
||||
}
|
||||
assembler.Write(body)
|
||||
tail = sseTail(tail, body)
|
||||
if flusher != nil {
|
||||
flusher.Flush()
|
||||
}
|
||||
case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_ERROR:
|
||||
msg := frame.GetError()
|
||||
if msg == "" {
|
||||
msg = "provider tunnel failed"
|
||||
}
|
||||
if !wroteHeader {
|
||||
writeError(w, http.StatusBadGateway, "provider_tunnel_error", msg)
|
||||
return
|
||||
}
|
||||
s.logger.Warn("openai responses sideband tunnel error after response start",
|
||||
zap.String("run_id", handle.Dispatch().RunID),
|
||||
zap.String("error", msg),
|
||||
)
|
||||
return
|
||||
case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_USAGE:
|
||||
protoObs = mergeUsageObservation(protoObs, usageObservationFromProtoUsage(frame.GetUsage()))
|
||||
case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END:
|
||||
if !wroteHeader {
|
||||
writeError(w, http.StatusBadGateway, "provider_tunnel_error", "tunnel ended before provider response")
|
||||
return
|
||||
}
|
||||
if tail != "" && !strings.HasSuffix(tail, "\n\n") {
|
||||
fmt.Fprint(w, "\n\n")
|
||||
if flusher != nil {
|
||||
flusher.Flush()
|
||||
}
|
||||
}
|
||||
metricStatus = usageStatusSuccess
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) writeResponsesProviderTunnelSidebandResponse(w http.ResponseWriter, r *http.Request, handle edgeservice.ProviderTunnelResult, metricLabels usageLabels) {
|
||||
frames := handle.Stream().Frames
|
||||
if frames == nil {
|
||||
writeError(w, http.StatusBadGateway, "provider_tunnel_error", "tunnel stream unavailable")
|
||||
return
|
||||
}
|
||||
timer := time.NewTimer(handle.WaitTimeout())
|
||||
defer timer.Stop()
|
||||
|
||||
assembler := &providerChatAssembler{}
|
||||
var body bytes.Buffer
|
||||
providerStatus := 0
|
||||
providerHeaders := map[string]string{}
|
||||
metricStatus := usageStatusError
|
||||
var protoObs usageObservation
|
||||
|
||||
defer func() {
|
||||
// Non-streaming assembler usage is parsed lazily from the buffered JSON
|
||||
// body, so force parsing before emitting metrics.
|
||||
_ = assembler.observation()
|
||||
emitUsageMetrics(metricLabels, metricStatus, mergeUsageObservation(assembler.usageObservation(), protoObs))
|
||||
}()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-r.Context().Done():
|
||||
s.cancelRunOnHTTPGiveUp(handle.Dispatch(), r.Context().Err())
|
||||
metricStatus = usageStatusCancel
|
||||
return
|
||||
case <-timer.C:
|
||||
s.cancelRunOnHTTPGiveUp(handle.Dispatch(), errRunTimedOut)
|
||||
metricStatus = usageStatusCancel
|
||||
writeError(w, http.StatusBadGateway, "run_error", errRunTimedOut.Error())
|
||||
return
|
||||
case frame, ok := <-frames:
|
||||
if !ok {
|
||||
writeError(w, http.StatusBadGateway, "provider_tunnel_error", "tunnel stream closed before provider response")
|
||||
return
|
||||
}
|
||||
switch frame.GetKind() {
|
||||
case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START:
|
||||
if providerStatus != 0 {
|
||||
continue
|
||||
}
|
||||
providerStatus = int(frame.GetStatusCode())
|
||||
if providerStatus == 0 {
|
||||
providerStatus = http.StatusOK
|
||||
}
|
||||
providerHeaders = frame.GetHeaders()
|
||||
case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY:
|
||||
body.Write(frame.GetBody())
|
||||
assembler.Write(frame.GetBody())
|
||||
case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_ERROR:
|
||||
msg := frame.GetError()
|
||||
if msg == "" {
|
||||
msg = "provider tunnel failed"
|
||||
}
|
||||
writeError(w, http.StatusBadGateway, "provider_tunnel_error", msg)
|
||||
return
|
||||
case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_USAGE:
|
||||
protoObs = mergeUsageObservation(protoObs, usageObservationFromProtoUsage(frame.GetUsage()))
|
||||
case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END:
|
||||
if providerStatus == 0 {
|
||||
writeError(w, http.StatusBadGateway, "provider_tunnel_error", "tunnel ended before provider response")
|
||||
return
|
||||
}
|
||||
copyProviderResponseHeaders(w.Header(), providerHeaders)
|
||||
w.Header().Del("Content-Length")
|
||||
w.Header().Set(responseModeHeaderName, responseModePassthroughSideband)
|
||||
w.WriteHeader(providerStatus)
|
||||
if _, err := w.Write(injectResponsesSidebandMetadata(body.Bytes())); err != nil {
|
||||
s.sendCancelRun(handle.Dispatch())
|
||||
metricStatus = usageStatusCancel
|
||||
return
|
||||
}
|
||||
metricStatus = usageStatusSuccess
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) completeResponse(w http.ResponseWriter, r *http.Request, req responsesRequest, handle edgeservice.RunResult, outputPolicy strictOutputPolicy) {
|
||||
metricLabels := s.usageLabelsFor(r.Context(), strings.TrimSpace(req.Model), usageEndpointResponses, "normalized")
|
||||
text, reasoning, _, _, usage, _, err := collectRunResult(r.Context(), handle.Stream(), handle.WaitTimeout())
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -50,7 +50,7 @@ func (s *Server) streamChatCompletion(w http.ResponseWriter, r *http.Request, re
|
|||
created := time.Now().Unix()
|
||||
id := "chatcmpl-" + handle.Dispatch().RunID
|
||||
model := responseModel(req.Model, handle.Dispatch().Target)
|
||||
metricLabels := s.usageLabelsFor(r.Context(), strings.TrimSpace(req.Model), usageEndpointChatCompletions, chatResponseModeLabel(req))
|
||||
metricLabels := s.usageLabelsFor(r.Context(), strings.TrimSpace(req.Model), usageEndpointChatCompletions, responseModeNormalized)
|
||||
traceStream := openAICompatTraceStreamEnabled(submitReq.Metadata)
|
||||
traceSeq := 0
|
||||
shouldExposeReasoning := req.includeReasoning() && (!outputPolicy.Strict || req.explicitlyIncludesReasoning())
|
||||
|
|
@ -329,7 +329,7 @@ func (s *Server) streamChatCompletion(w http.ResponseWriter, r *http.Request, re
|
|||
// relayed without IOP sideband fields or events. Chat Completions model echoes
|
||||
// are normalized back to the caller-facing model alias.
|
||||
func (s *Server) tunnelChatCompletionPassthrough(w http.ResponseWriter, r *http.Request, req chatCompletionRequest, dispatch routeDispatch, runMeta map[string]string, rawBody []byte, estimate int, contextClass string) {
|
||||
handle, ok := s.submitChatCompletionTunnel(w, r, req, dispatch, runMeta, rawBody, estimate, contextClass, responseModePassthrough)
|
||||
handle, ok := s.submitChatCompletionTunnel(w, r, req, dispatch, runMeta, rawBody, estimate, contextClass)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
|
@ -338,24 +338,6 @@ func (s *Server) tunnelChatCompletionPassthrough(w http.ResponseWriter, r *http.
|
|||
s.writeProviderTunnelResponse(w, r, handle, req.Stream, req.Model, metricLabels)
|
||||
}
|
||||
|
||||
// tunnelChatCompletionPassthroughSideband serves a Chat Completions request
|
||||
// over the raw provider tunnel with the explicit IOP sideband extension
|
||||
// surface (SDD S05): provider content is preserved, but the response also
|
||||
// carries IOP route/usage/assembled observations and is never claimed as
|
||||
// provider-original byte identity.
|
||||
func (s *Server) tunnelChatCompletionPassthroughSideband(w http.ResponseWriter, r *http.Request, req chatCompletionRequest, dispatch routeDispatch, runMeta map[string]string, rawBody []byte, estimate int, contextClass string) {
|
||||
handle, ok := s.submitChatCompletionTunnel(w, r, req, dispatch, runMeta, rawBody, estimate, contextClass, responseModePassthroughSideband)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
defer handle.Close()
|
||||
if req.Stream {
|
||||
s.writeProviderTunnelSidebandStream(w, r, handle)
|
||||
return
|
||||
}
|
||||
s.writeProviderTunnelSidebandResponse(w, r, handle)
|
||||
}
|
||||
|
||||
// errProviderAuthRequired signals that provider auth forwarding is configured
|
||||
// as required but the caller did not supply the configured provider token
|
||||
// header. The raw token is never part of this error.
|
||||
|
|
@ -388,9 +370,9 @@ func (s *Server) providerTunnelAuthHeaders(r *http.Request) (map[string]string,
|
|||
}
|
||||
|
||||
// submitChatCompletionTunnel dispatches a Chat Completions provider tunnel
|
||||
// request with the given response mode. On failure it writes the dispatch
|
||||
// error to the caller and returns ok=false.
|
||||
func (s *Server) submitChatCompletionTunnel(w http.ResponseWriter, r *http.Request, req chatCompletionRequest, dispatch routeDispatch, runMeta map[string]string, rawBody []byte, estimate int, contextClass, responseMode string) (edgeservice.ProviderTunnelResult, bool) {
|
||||
// request for pure passthrough. On failure it writes the dispatch error to the
|
||||
// caller and returns ok=false.
|
||||
func (s *Server) submitChatCompletionTunnel(w http.ResponseWriter, r *http.Request, req chatCompletionRequest, dispatch routeDispatch, runMeta map[string]string, rawBody []byte, estimate int, contextClass string) (edgeservice.ProviderTunnelResult, bool) {
|
||||
providerAuthHeaders, err := s.providerTunnelAuthHeaders(r)
|
||||
if err != nil {
|
||||
// Missing required provider auth is rejected before dispatch; the raw
|
||||
|
|
@ -405,7 +387,6 @@ func (s *Server) submitChatCompletionTunnel(w http.ResponseWriter, r *http.Reque
|
|||
}
|
||||
metadata["openai_model"] = req.Model
|
||||
metadata["openai_stream"] = strconv.FormatBool(req.Stream)
|
||||
metadata[responseModeMetadataKey] = responseMode
|
||||
metadata["estimated_input_tokens"] = strconv.Itoa(estimate)
|
||||
metadata["context_class"] = contextClass
|
||||
|
||||
|
|
@ -431,7 +412,7 @@ func (s *Server) submitChatCompletionTunnel(w http.ResponseWriter, r *http.Reque
|
|||
ProviderPool: dispatch.ProviderPool,
|
||||
}
|
||||
|
||||
metricLabels := s.usageLabelsFor(r.Context(), strings.TrimSpace(req.Model), usageEndpointChatCompletions, responseMode)
|
||||
metricLabels := s.usageLabelsFor(r.Context(), strings.TrimSpace(req.Model), usageEndpointChatCompletions, responseModePassthrough)
|
||||
handle, err := s.service.SubmitProviderTunnel(r.Context(), tunnelReq)
|
||||
if err != nil {
|
||||
emitUsageMetrics(metricLabels, usageStatusForError(err), usageObservation{})
|
||||
|
|
@ -449,7 +430,7 @@ func (s *Server) submitChatCompletionTunnel(w http.ResponseWriter, r *http.Reque
|
|||
zap.String("adapter", handle.Dispatch().Adapter),
|
||||
zap.String("target", handle.Dispatch().Target),
|
||||
zap.Bool("stream", req.Stream),
|
||||
zap.String("response_mode", responseMode),
|
||||
zap.String("response_mode", responseModePassthrough),
|
||||
zap.Int("estimated_input_tokens", handle.Dispatch().EstimatedInputTokens),
|
||||
zap.String("context_class", handle.Dispatch().ContextClass),
|
||||
zap.String("queue_reason", handle.Dispatch().QueueReason),
|
||||
|
|
@ -735,134 +716,19 @@ func rewriteProviderJSONModel(body []byte, model string) []byte {
|
|||
return rewritten
|
||||
}
|
||||
|
||||
// Sideband extension surface (SDD S05): explicit passthrough+sideband
|
||||
// responses expose IOP observations alongside provider-original content.
|
||||
// Streaming responses interleave `event: iop.sideband` SSE events at provider
|
||||
// event boundaries; non-streaming responses wrap the provider body in the
|
||||
// iop.chat.passthrough_sideband envelope. Both are IOP extension outputs and
|
||||
// are never claimed as provider-original byte identity.
|
||||
const (
|
||||
sidebandSSEEventName = "iop.sideband"
|
||||
sidebandEnvelopeObject = "iop.chat.passthrough_sideband"
|
||||
)
|
||||
|
||||
type sidebandRouteObservation struct {
|
||||
RunID string `json:"run_id"`
|
||||
NodeID string `json:"node_id,omitempty"`
|
||||
ProviderID string `json:"provider_id,omitempty"`
|
||||
ProviderType string `json:"provider_type,omitempty"`
|
||||
ExecutionPath string `json:"execution_path,omitempty"`
|
||||
Adapter string `json:"adapter,omitempty"`
|
||||
Target string `json:"target,omitempty"`
|
||||
ModelGroup string `json:"model_group,omitempty"`
|
||||
ResponseMode string `json:"response_mode"`
|
||||
ProviderStatusCode int `json:"provider_status_code,omitempty"`
|
||||
QueueReason string `json:"queue_reason,omitempty"`
|
||||
}
|
||||
|
||||
type sidebandUsageObservation struct {
|
||||
InputTokens int `json:"input_tokens"`
|
||||
OutputTokens int `json:"output_tokens"`
|
||||
}
|
||||
|
||||
type sidebandAssembledObservation struct {
|
||||
// providerAssembledObservation is the Edge-internal human-readable view of a
|
||||
// provider Chat Completions/Responses body assembled by providerChatAssembler.
|
||||
// It feeds Edge-local logging and usage metrics only and is never written to
|
||||
// the caller response; pure passthrough relays provider bytes unmodified.
|
||||
type providerAssembledObservation struct {
|
||||
Content string `json:"content,omitempty"`
|
||||
Reasoning string `json:"reasoning,omitempty"`
|
||||
ToolCallNames []string `json:"tool_call_names,omitempty"`
|
||||
BodyBytes int `json:"body_bytes"`
|
||||
}
|
||||
|
||||
// sidebandObservation is one `event: iop.sideband` SSE payload on the
|
||||
// streaming sideband surface.
|
||||
type sidebandObservation struct {
|
||||
Object string `json:"object"`
|
||||
Kind string `json:"kind"`
|
||||
Route *sidebandRouteObservation `json:"route,omitempty"`
|
||||
Usage *sidebandUsageObservation `json:"usage,omitempty"`
|
||||
Assembled *sidebandAssembledObservation `json:"assembled,omitempty"`
|
||||
}
|
||||
|
||||
// sidebandEnvelope is the non-streaming sideband response schema: the provider
|
||||
// body is carried verbatim next to the IOP observations.
|
||||
type sidebandEnvelope struct {
|
||||
Object string `json:"object"`
|
||||
Sideband sidebandObservations `json:"iop_sideband"`
|
||||
ProviderStatusCode int `json:"provider_status_code"`
|
||||
ProviderResponse json.RawMessage `json:"provider_response,omitempty"`
|
||||
ProviderBody string `json:"provider_body,omitempty"`
|
||||
}
|
||||
|
||||
type sidebandObservations struct {
|
||||
Route sidebandRouteObservation `json:"route"`
|
||||
Usage *sidebandUsageObservation `json:"usage,omitempty"`
|
||||
Assembled *sidebandAssembledObservation `json:"assembled,omitempty"`
|
||||
}
|
||||
|
||||
func sidebandRouteFromDispatch(dispatch edgeservice.RunDispatch, providerStatus int) sidebandRouteObservation {
|
||||
return sidebandRouteObservation{
|
||||
RunID: dispatch.RunID,
|
||||
NodeID: dispatch.NodeID,
|
||||
ProviderID: dispatch.ProviderID,
|
||||
ProviderType: dispatch.ProviderType,
|
||||
ExecutionPath: dispatch.ExecutionPath,
|
||||
Adapter: dispatch.Adapter,
|
||||
Target: dispatch.Target,
|
||||
ModelGroup: dispatch.ModelGroupKey,
|
||||
ResponseMode: responseModePassthroughSideband,
|
||||
ProviderStatusCode: providerStatus,
|
||||
QueueReason: dispatch.QueueReason,
|
||||
}
|
||||
}
|
||||
|
||||
func sidebandUsageFromFrame(frame *iop.ProviderTunnelFrame) *sidebandUsageObservation {
|
||||
usage := frame.GetUsage()
|
||||
if usage == nil {
|
||||
return nil
|
||||
}
|
||||
return &sidebandUsageObservation{
|
||||
InputTokens: int(usage.GetInputTokens()),
|
||||
OutputTokens: int(usage.GetOutputTokens()),
|
||||
}
|
||||
}
|
||||
|
||||
// usageObservationFromProtoUsage converts a proto iop.Usage into the internal
|
||||
// usageObservation used for metrics. Returns zero values when u is nil.
|
||||
func usageObservationFromProtoUsage(u *iop.Usage) usageObservation {
|
||||
if u == nil {
|
||||
return usageObservation{}
|
||||
}
|
||||
return usageObservation{
|
||||
inputTokens: int(u.GetInputTokens()),
|
||||
outputTokens: int(u.GetOutputTokens()),
|
||||
reasoningTokens: int(u.GetReasoningTokens()),
|
||||
cachedInputTokens: int(u.GetCachedInputTokens()),
|
||||
}
|
||||
}
|
||||
|
||||
// mergeUsageObservation applies the sideband merge rule: use the body value
|
||||
// when body observed a token type; use the proto value only when body is zero.
|
||||
// This prevents double-counting input/output when both body and proto report
|
||||
// the same provider usage, while preserving proto-only fields like
|
||||
// reasoning/cached_input (SDD S05).
|
||||
func mergeUsageObservation(body, proto usageObservation) usageObservation {
|
||||
return usageObservation{
|
||||
inputTokens: selectFirstNonZero(body.inputTokens, proto.inputTokens),
|
||||
outputTokens: selectFirstNonZero(body.outputTokens, proto.outputTokens),
|
||||
reasoningTokens: selectFirstNonZero(body.reasoningTokens, proto.reasoningTokens),
|
||||
cachedInputTokens: selectFirstNonZero(body.cachedInputTokens, proto.cachedInputTokens),
|
||||
reasoningChars: body.reasoningChars,
|
||||
}
|
||||
}
|
||||
|
||||
func selectFirstNonZero(a, b int) int {
|
||||
if a != 0 {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// providerChatAssembler accumulates a human-readable view of the provider
|
||||
// Chat Completions response for the sideband assembled observation. It
|
||||
// Chat Completions response for the assembled observation. It
|
||||
// tolerates non-JSON and partial payloads: whatever cannot be parsed simply
|
||||
// yields an empty summary.
|
||||
type providerChatAssembler struct {
|
||||
|
|
@ -1034,7 +900,7 @@ func (a *providerChatAssembler) consumeDelta(delta providerChatDeltaEnvelope) {
|
|||
}
|
||||
}
|
||||
|
||||
func (a *providerChatAssembler) observation() *sidebandAssembledObservation {
|
||||
func (a *providerChatAssembler) observation() *providerAssembledObservation {
|
||||
if !a.streaming && !a.nonStreamingParsed {
|
||||
a.nonStreamingParsed = true
|
||||
var resp struct {
|
||||
|
|
@ -1050,7 +916,7 @@ func (a *providerChatAssembler) observation() *sidebandAssembledObservation {
|
|||
a.recordUsage(resp.Usage)
|
||||
}
|
||||
}
|
||||
return &sidebandAssembledObservation{
|
||||
return &providerAssembledObservation{
|
||||
Content: a.content.String(),
|
||||
Reasoning: a.reasoning.String(),
|
||||
ToolCallNames: a.toolCallNames,
|
||||
|
|
@ -1058,316 +924,6 @@ func (a *providerChatAssembler) observation() *sidebandAssembledObservation {
|
|||
}
|
||||
}
|
||||
|
||||
// writeProviderTunnelSidebandStream relays provider SSE bytes unchanged and
|
||||
// interleaves explicit `event: iop.sideband` extension events. IOP events are
|
||||
// written only at provider event boundaries (before the first body byte,
|
||||
// after a "\n\n"-terminated provider event, or at end of stream) so a
|
||||
// provider event is never split by sideband injection.
|
||||
func (s *Server) writeProviderTunnelSidebandStream(w http.ResponseWriter, r *http.Request, handle edgeservice.ProviderTunnelResult) {
|
||||
frames := handle.Stream().Frames
|
||||
if frames == nil {
|
||||
writeError(w, http.StatusBadGateway, "provider_tunnel_error", "tunnel stream unavailable")
|
||||
return
|
||||
}
|
||||
flusher, _ := w.(http.Flusher)
|
||||
timer := time.NewTimer(handle.WaitTimeout())
|
||||
defer timer.Stop()
|
||||
|
||||
assembler := &providerChatAssembler{streaming: true}
|
||||
wroteHeader := false
|
||||
atEventBoundary := true
|
||||
tail := ""
|
||||
var pendingObservations []sidebandObservation
|
||||
|
||||
metricLabels := s.usageLabelsFor(r.Context(), strings.TrimSpace(handle.Dispatch().ModelGroupKey), usageEndpointChatCompletions, responseModePassthroughSideband)
|
||||
// metricStatus is the terminal status reported to usage metrics. It defaults
|
||||
// to error and is upgraded to success on END.
|
||||
metricStatus := usageStatusError
|
||||
// pendingMerged accumulates usage from all USAGE frames so that emitUsageMetrics
|
||||
// sees the full usage even after pendingObservations is flushed.
|
||||
var pendingMerged usageObservation
|
||||
|
||||
writeObservation := func(obs sidebandObservation) {
|
||||
obs.Object = sidebandSSEEventName
|
||||
payload, err := json.Marshal(obs)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(w, "event: %s\ndata: %s\n\n", sidebandSSEEventName, payload)
|
||||
if flusher != nil {
|
||||
flusher.Flush()
|
||||
}
|
||||
}
|
||||
flushObservations := func() {
|
||||
for _, obs := range pendingObservations {
|
||||
writeObservation(obs)
|
||||
}
|
||||
pendingObservations = nil
|
||||
}
|
||||
|
||||
defer func() {
|
||||
obs := assembler.observation()
|
||||
s.logger.Info("openai chat completion sideband stream closed",
|
||||
zap.String("run_id", handle.Dispatch().RunID),
|
||||
zap.Bool("wrote_header", wroteHeader),
|
||||
zap.Int("body_bytes", assembler.bodyBytes),
|
||||
zap.String("assembled_content", obs.Content),
|
||||
zap.String("assembled_reasoning", obs.Reasoning),
|
||||
zap.Strings("assembled_tool_calls", obs.ToolCallNames),
|
||||
zap.Int("assembled_tool_call_count", len(obs.ToolCallNames)),
|
||||
)
|
||||
// Merge body-parsed usage (primary) with USAGE frame data
|
||||
// (auxiliary reasoning/cached_input). Per the merge rule, body values
|
||||
// are used when observed; proto values fill in when body is zero.
|
||||
// This avoids double-counting input/output when both sources report
|
||||
// the same provider usage, while preserving proto-only fields like
|
||||
// reasoning/cached_input (SDD S05).
|
||||
merged := mergeUsageObservation(assembler.usageObservation(), pendingMerged)
|
||||
emitUsageMetrics(metricLabels, metricStatus, merged)
|
||||
}()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-r.Context().Done():
|
||||
s.cancelRunOnHTTPGiveUp(handle.Dispatch(), r.Context().Err())
|
||||
metricStatus = usageStatusCancel
|
||||
return
|
||||
case <-timer.C:
|
||||
s.cancelRunOnHTTPGiveUp(handle.Dispatch(), errRunTimedOut)
|
||||
metricStatus = usageStatusCancel
|
||||
if !wroteHeader {
|
||||
writeError(w, http.StatusBadGateway, "run_error", errRunTimedOut.Error())
|
||||
}
|
||||
return
|
||||
case frame, ok := <-frames:
|
||||
if !ok {
|
||||
if !wroteHeader {
|
||||
writeError(w, http.StatusBadGateway, "provider_tunnel_error", "tunnel stream closed before provider response")
|
||||
}
|
||||
return
|
||||
}
|
||||
switch frame.GetKind() {
|
||||
case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START:
|
||||
if wroteHeader {
|
||||
continue
|
||||
}
|
||||
copyProviderResponseHeaders(w.Header(), frame.GetHeaders())
|
||||
// The sideband stream is an IOP extension surface: label it
|
||||
// and drop the provider Content-Length, which no longer
|
||||
// matches the extended body.
|
||||
w.Header().Del("Content-Length")
|
||||
w.Header().Set(responseModeHeaderName, responseModePassthroughSideband)
|
||||
status := int(frame.GetStatusCode())
|
||||
if status == 0 {
|
||||
status = http.StatusOK
|
||||
}
|
||||
w.WriteHeader(status)
|
||||
wroteHeader = true
|
||||
if flusher != nil {
|
||||
flusher.Flush()
|
||||
}
|
||||
route := sidebandRouteFromDispatch(handle.Dispatch(), status)
|
||||
writeObservation(sidebandObservation{Kind: "route", Route: &route})
|
||||
case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY:
|
||||
body := frame.GetBody()
|
||||
if len(body) == 0 {
|
||||
continue
|
||||
}
|
||||
if !wroteHeader {
|
||||
// Defensive: a body frame before response-start still
|
||||
// reaches the caller instead of being dropped.
|
||||
w.Header().Set(responseModeHeaderName, responseModePassthroughSideband)
|
||||
w.WriteHeader(http.StatusOK)
|
||||
wroteHeader = true
|
||||
}
|
||||
if _, err := w.Write(body); err != nil {
|
||||
// The caller is gone mid-stream; propagate cancel to the Node.
|
||||
s.sendCancelRun(handle.Dispatch())
|
||||
metricStatus = usageStatusCancel
|
||||
return
|
||||
}
|
||||
assembler.Write(body)
|
||||
if flusher != nil {
|
||||
flusher.Flush()
|
||||
}
|
||||
tail = sseTail(tail, body)
|
||||
atEventBoundary = strings.HasSuffix(tail, "\n\n")
|
||||
if atEventBoundary {
|
||||
flushObservations()
|
||||
}
|
||||
case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_ERROR:
|
||||
msg := frame.GetError()
|
||||
if msg == "" {
|
||||
msg = "provider tunnel failed"
|
||||
}
|
||||
if !wroteHeader {
|
||||
writeError(w, http.StatusBadGateway, "provider_tunnel_error", msg)
|
||||
return
|
||||
}
|
||||
// Status/headers are already committed; the response is
|
||||
// truncated and the caller observes the broken stream.
|
||||
s.logger.Warn("openai sideband tunnel error after response start",
|
||||
zap.String("run_id", handle.Dispatch().RunID),
|
||||
zap.String("error", msg),
|
||||
)
|
||||
return
|
||||
case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_USAGE:
|
||||
usage := sidebandUsageFromFrame(frame)
|
||||
if usage == nil {
|
||||
continue
|
||||
}
|
||||
// Accumulate proto usage from all USAGE frames so emitUsageMetrics
|
||||
// sees the full usage even after pendingObservations is flushed.
|
||||
pendingMerged = mergeUsageObservation(pendingMerged,
|
||||
usageObservationFromProtoUsage(frame.GetUsage()))
|
||||
pendingObservations = append(pendingObservations, sidebandObservation{Kind: "usage", Usage: usage})
|
||||
if wroteHeader && atEventBoundary {
|
||||
flushObservations()
|
||||
}
|
||||
case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END:
|
||||
if !wroteHeader {
|
||||
writeError(w, http.StatusBadGateway, "provider_tunnel_error", "tunnel ended before provider response")
|
||||
return
|
||||
}
|
||||
metricStatus = usageStatusSuccess
|
||||
if !atEventBoundary {
|
||||
// The provider stream ended mid-event; terminate it so the
|
||||
// trailing IOP events stay well-formed SSE.
|
||||
fmt.Fprint(w, "\n\n")
|
||||
}
|
||||
flushObservations()
|
||||
writeObservation(sidebandObservation{Kind: "assembled", Assembled: assembler.observation()})
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sseTail keeps the last two bytes of the relayed provider stream so the
|
||||
// sideband writer can detect "\n\n" event boundaries across chunk splits.
|
||||
func sseTail(tail string, chunk []byte) string {
|
||||
combined := tail + string(chunk)
|
||||
if len(combined) > 2 {
|
||||
return combined[len(combined)-2:]
|
||||
}
|
||||
return combined
|
||||
}
|
||||
|
||||
// writeProviderTunnelSidebandResponse buffers the provider response and
|
||||
// answers with the iop.chat.passthrough_sideband envelope: provider status is
|
||||
// preserved, the provider body is carried verbatim inside the envelope, and
|
||||
// IOP route/usage/assembled observations sit alongside it.
|
||||
func (s *Server) writeProviderTunnelSidebandResponse(w http.ResponseWriter, r *http.Request, handle edgeservice.ProviderTunnelResult) {
|
||||
frames := handle.Stream().Frames
|
||||
if frames == nil {
|
||||
writeError(w, http.StatusBadGateway, "provider_tunnel_error", "tunnel stream unavailable")
|
||||
return
|
||||
}
|
||||
timer := time.NewTimer(handle.WaitTimeout())
|
||||
defer timer.Stop()
|
||||
|
||||
assembler := &providerChatAssembler{}
|
||||
var body bytes.Buffer
|
||||
providerStatus := 0
|
||||
var protoUsage *sidebandUsageObservation
|
||||
var protoObs usageObservation
|
||||
|
||||
metricLabels := s.usageLabelsFor(r.Context(), strings.TrimSpace(handle.Dispatch().ModelGroupKey), usageEndpointChatCompletions, responseModePassthroughSideband)
|
||||
// metricStatus is the terminal status reported to usage metrics. It defaults
|
||||
// to error and is upgraded to success on END.
|
||||
metricStatus := usageStatusError
|
||||
defer func() {
|
||||
obs := assembler.observation()
|
||||
s.logger.Info("openai chat completion sideband response",
|
||||
zap.String("run_id", handle.Dispatch().RunID),
|
||||
zap.Int("provider_status", providerStatus),
|
||||
zap.Int("body_bytes", body.Len()),
|
||||
zap.String("assembled_content", obs.Content),
|
||||
zap.String("assembled_reasoning", obs.Reasoning),
|
||||
zap.Strings("assembled_tool_calls", obs.ToolCallNames),
|
||||
zap.Int("assembled_tool_call_count", len(obs.ToolCallNames)),
|
||||
)
|
||||
// Use body-parsed usage as primary source; layer in USAGE frame
|
||||
// data per the merge rule. sidebandUsageObservation in the response
|
||||
// schema carries only input/output, while the metric path uses the
|
||||
// full breakdown including reasoning/cached_input from the provider
|
||||
// body or proto (SDD S05).
|
||||
merged := mergeUsageObservation(assembler.usageObservation(), protoObs)
|
||||
emitUsageMetrics(metricLabels, metricStatus, merged)
|
||||
}()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-r.Context().Done():
|
||||
s.cancelRunOnHTTPGiveUp(handle.Dispatch(), r.Context().Err())
|
||||
metricStatus = usageStatusCancel
|
||||
return
|
||||
case <-timer.C:
|
||||
s.cancelRunOnHTTPGiveUp(handle.Dispatch(), errRunTimedOut)
|
||||
metricStatus = usageStatusCancel
|
||||
writeError(w, http.StatusBadGateway, "run_error", errRunTimedOut.Error())
|
||||
return
|
||||
case frame, ok := <-frames:
|
||||
if !ok {
|
||||
writeError(w, http.StatusBadGateway, "provider_tunnel_error", "tunnel stream closed before provider response")
|
||||
return
|
||||
}
|
||||
switch frame.GetKind() {
|
||||
case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START:
|
||||
if providerStatus != 0 {
|
||||
continue
|
||||
}
|
||||
providerStatus = int(frame.GetStatusCode())
|
||||
if providerStatus == 0 {
|
||||
providerStatus = http.StatusOK
|
||||
}
|
||||
case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY:
|
||||
body.Write(frame.GetBody())
|
||||
assembler.Write(frame.GetBody())
|
||||
case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_ERROR:
|
||||
msg := frame.GetError()
|
||||
if msg == "" {
|
||||
msg = "provider tunnel failed"
|
||||
}
|
||||
writeError(w, http.StatusBadGateway, "provider_tunnel_error", msg)
|
||||
return
|
||||
case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_USAGE:
|
||||
if u := sidebandUsageFromFrame(frame); u != nil {
|
||||
protoUsage = u
|
||||
}
|
||||
protoObs = mergeUsageObservation(protoObs, usageObservationFromProtoUsage(frame.GetUsage()))
|
||||
case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END:
|
||||
if providerStatus == 0 && body.Len() == 0 {
|
||||
writeError(w, http.StatusBadGateway, "provider_tunnel_error", "tunnel ended before provider response")
|
||||
return
|
||||
}
|
||||
metricStatus = usageStatusSuccess
|
||||
if providerStatus == 0 {
|
||||
providerStatus = http.StatusOK
|
||||
}
|
||||
envelope := sidebandEnvelope{
|
||||
Object: sidebandEnvelopeObject,
|
||||
Sideband: sidebandObservations{
|
||||
Route: sidebandRouteFromDispatch(handle.Dispatch(), providerStatus),
|
||||
Usage: protoUsage,
|
||||
Assembled: assembler.observation(),
|
||||
},
|
||||
ProviderStatusCode: providerStatus,
|
||||
}
|
||||
raw := body.Bytes()
|
||||
if json.Valid(raw) {
|
||||
envelope.ProviderResponse = json.RawMessage(raw)
|
||||
} else if len(raw) > 0 {
|
||||
envelope.ProviderBody = string(raw)
|
||||
}
|
||||
w.Header().Set(responseModeHeaderName, responseModePassthroughSideband)
|
||||
writeJSON(w, providerStatus, envelope)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// hopByHopResponseHeaders are transport-level headers owned by each hop; they
|
||||
// are not copied from the provider response to the caller response.
|
||||
var hopByHopResponseHeaders = map[string]struct{}{
|
||||
|
|
@ -1524,7 +1080,7 @@ func writeToolCallsDeltaSSE(w http.ResponseWriter, flusher http.Flusher, id stri
|
|||
// successful tool_calls chunk.
|
||||
func (s *Server) streamBufferedChatCompletion(w http.ResponseWriter, r *http.Request, req chatCompletionRequest, submitReq edgeservice.SubmitRunRequest, handle edgeservice.RunResult, flusher http.Flusher, outputPolicy strictOutputPolicy, validation toolValidationContract, retrySubmit func(ctx context.Context, req edgeservice.SubmitRunRequest) (any, error)) {
|
||||
attempt := 1
|
||||
metricLabels := s.usageLabelsFor(r.Context(), strings.TrimSpace(req.Model), usageEndpointChatCompletions, chatResponseModeLabel(req))
|
||||
metricLabels := s.usageLabelsFor(r.Context(), strings.TrimSpace(req.Model), usageEndpointChatCompletions, responseModeNormalized)
|
||||
for {
|
||||
result, err := collectChatCompletionOutput(r.Context(), req, handle, outputPolicy)
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -1,13 +1,11 @@
|
|||
package openai
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus/testutil"
|
||||
|
||||
|
|
@ -340,80 +338,6 @@ func TestResponsesProviderTunnelPassthroughObservesUsageMetrics(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// TestResponsesProviderTunnelSidebandObservesUsageMetrics verifies that a
|
||||
// successful /v1/responses provider passthrough+sideband response records usage
|
||||
// under endpoint=responses and response_mode=passthrough+sideband.
|
||||
func TestResponsesProviderTunnelSidebandObservesUsageMetrics(t *testing.T) {
|
||||
const rawToken = "sk-responses-sideband-token"
|
||||
const edgeID = "edge-responses-sideband-usage"
|
||||
const model = "pool-model"
|
||||
providerBody := `{"id":"resp-1","object":"response","output_text":"hi","metadata":{"request_id":"req-1"},"usage":{"input_tokens":8,"output_tokens":5,"input_tokens_details":{"cached_tokens":2},"output_tokens_details":{"reasoning_tokens":1}}}`
|
||||
|
||||
frames := make(chan *iop.ProviderTunnelFrame, 4)
|
||||
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START, StatusCode: 200}
|
||||
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(providerBody)}
|
||||
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true}
|
||||
close(frames)
|
||||
|
||||
srv := providerRouteUsageMetricServer(
|
||||
rawToken, edgeID, model, "served-model",
|
||||
&fakeRunService{tunnelFrames: frames, tunnelServedTarget: "served-model"},
|
||||
)
|
||||
|
||||
labels := usageLabels{
|
||||
edgeID: edgeID, principalRef: "user:alice", principalAlias: "alice", tokenRef: "iop-tok-alice",
|
||||
modelGroup: model, endpoint: usageEndpointResponses, responseMode: responseModePassthroughSideband,
|
||||
}
|
||||
reqBefore := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues(
|
||||
edgeID, "user:alice", "alice", "iop-tok-alice", model,
|
||||
usageEndpointResponses, responseModePassthroughSideband, usageStatusSuccess, usageSourceProviderReported,
|
||||
))
|
||||
before := map[string]float64{
|
||||
tokenTypeInput: requestTokenValue(t, labels, tokenTypeInput),
|
||||
tokenTypeOutput: requestTokenValue(t, labels, tokenTypeOutput),
|
||||
tokenTypeReasoning: requestTokenValue(t, labels, tokenTypeReasoning),
|
||||
tokenTypeCachedInput: requestTokenValue(t, labels, tokenTypeCachedInput),
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(`{
|
||||
"model":"pool-model",
|
||||
"input":"hello",
|
||||
"metadata":{"iop_response_mode":"passthrough+sideband"}
|
||||
}`))
|
||||
req.Header.Set("Authorization", "Bearer "+rawToken)
|
||||
w := httptest.NewRecorder()
|
||||
srv.routes().ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
if !strings.Contains(w.Body.String(), `"iop_response_mode":"passthrough+sideband"`) {
|
||||
t.Fatalf("sideband response must inject metadata marker, got %s", w.Body.String())
|
||||
}
|
||||
reqAfter := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues(
|
||||
edgeID, "user:alice", "alice", "iop-tok-alice", model,
|
||||
usageEndpointResponses, responseModePassthroughSideband, usageStatusSuccess, usageSourceProviderReported,
|
||||
))
|
||||
if reqAfter-reqBefore != 1 {
|
||||
t.Fatalf("requests_total responses sideband/success: got delta %v, want 1", reqAfter-reqBefore)
|
||||
}
|
||||
for tokenType, want := range map[string]float64{
|
||||
tokenTypeInput: 8,
|
||||
tokenTypeOutput: 5,
|
||||
tokenTypeReasoning: 1,
|
||||
tokenTypeCachedInput: 2,
|
||||
} {
|
||||
got := requestTokenValue(t, labels, tokenType) - before[tokenType]
|
||||
if got != want {
|
||||
t.Fatalf("responses sideband token_type %s: got delta %v, want %v", tokenType, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestResponsesProviderTunnelPassthroughStreamingObservesUsageMetrics verifies
|
||||
// that a streaming /v1/responses provider passthrough with body-only SSE events
|
||||
// (no proto USAGE frame) still observes provider-reported usage metrics
|
||||
// (REVIEW_REVIEW_SEULGI_RESPONSES-1).
|
||||
func TestResponsesProviderTunnelPassthroughStreamingObservesUsageMetrics(t *testing.T) {
|
||||
const rawToken = "sk-responses-streaming-token"
|
||||
const edgeID = "edge-responses-streaming-usage"
|
||||
|
|
@ -528,176 +452,6 @@ func TestOpenAIScopeExcludesA2AAndNonOpenAISurfaces(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// TestProviderTunnelSidebandStreamingEmitsUsageMetrics verifies that a
|
||||
// passthrough+sideband streaming response increments the request counter
|
||||
// with status=success and usage_source=provider_reported (REVIEW_USAGE_METRIC-1).
|
||||
func TestProviderTunnelSidebandStreamingEmitsUsageMetrics(t *testing.T) {
|
||||
const rawToken = "sk-sideband-stream-token"
|
||||
const edgeID = "edge-sideband-stream-metrics"
|
||||
const model = "pool-model"
|
||||
providerBody := `{"choices":[{"delta":{"content":"hi"}}]}
|
||||
|
||||
data: [DONE]
|
||||
|
||||
`
|
||||
|
||||
frames := make(chan *iop.ProviderTunnelFrame, 5)
|
||||
frames <- &iop.ProviderTunnelFrame{
|
||||
Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START,
|
||||
StatusCode: 200,
|
||||
Headers: map[string]string{"Content-Type": "text/event-stream"},
|
||||
}
|
||||
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(providerBody)}
|
||||
frames <- &iop.ProviderTunnelFrame{
|
||||
Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_USAGE,
|
||||
Usage: &iop.Usage{InputTokens: 5, OutputTokens: 3},
|
||||
}
|
||||
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true}
|
||||
close(frames)
|
||||
|
||||
srv := providerRouteUsageMetricServer(rawToken, edgeID, model, "served-model", &fakeRunService{tunnelFrames: frames})
|
||||
|
||||
reqBefore := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues(
|
||||
edgeID, "user:alice", "alice", "iop-tok-alice", model,
|
||||
usageEndpointChatCompletions, responseModePassthroughSideband, usageStatusSuccess, usageSourceProviderReported,
|
||||
))
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
||||
"model":"pool-model",
|
||||
"stream":true,
|
||||
"messages":[{"role":"user","content":"hello"}],
|
||||
"metadata":{"iop_response_mode":"passthrough+sideband"}
|
||||
}`))
|
||||
req.Header.Set("Authorization", "Bearer "+rawToken)
|
||||
w := httptest.NewRecorder()
|
||||
srv.routes().ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
|
||||
reqAfter := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues(
|
||||
edgeID, "user:alice", "alice", "iop-tok-alice", model,
|
||||
usageEndpointChatCompletions, responseModePassthroughSideband, usageStatusSuccess, usageSourceProviderReported,
|
||||
))
|
||||
if reqAfter-reqBefore != 1 {
|
||||
t.Fatalf("requests_total success/provider_reported: got delta %v, want 1", reqAfter-reqBefore)
|
||||
}
|
||||
}
|
||||
|
||||
// TestProviderTunnelSidebandNonStreamingEmitsUsageMetrics verifies that a
|
||||
// passthrough+sideband non-streaming response increments the request counter
|
||||
// with status=success and usage_source=provider_reported (REVIEW_USAGE_METRIC-1).
|
||||
func TestProviderTunnelSidebandNonStreamingEmitsUsageMetrics(t *testing.T) {
|
||||
const rawToken = "sk-sideband-nonstream-token"
|
||||
const edgeID = "edge-sideband-nonstream-metrics"
|
||||
const model = "pool-model"
|
||||
providerBody := `{"id":"cmpl-1","choices":[{"message":{"role":"assistant","content":"hi"}}]}`
|
||||
|
||||
frames := make(chan *iop.ProviderTunnelFrame, 5)
|
||||
frames <- &iop.ProviderTunnelFrame{
|
||||
Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START,
|
||||
StatusCode: 200,
|
||||
Headers: map[string]string{"Content-Type": "application/json"},
|
||||
}
|
||||
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(providerBody)}
|
||||
frames <- &iop.ProviderTunnelFrame{
|
||||
Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_USAGE,
|
||||
Usage: &iop.Usage{InputTokens: 7, OutputTokens: 4},
|
||||
}
|
||||
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true}
|
||||
close(frames)
|
||||
|
||||
srv := providerRouteUsageMetricServer(rawToken, edgeID, model, "served-model", &fakeRunService{tunnelFrames: frames})
|
||||
|
||||
labels := usageLabels{
|
||||
edgeID: edgeID, principalRef: "user:alice", principalAlias: "alice", tokenRef: "iop-tok-alice",
|
||||
modelGroup: model, endpoint: usageEndpointChatCompletions, responseMode: responseModePassthroughSideband,
|
||||
}
|
||||
reqBefore := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues(
|
||||
edgeID, "user:alice", "alice", "iop-tok-alice", model,
|
||||
usageEndpointChatCompletions, responseModePassthroughSideband, usageStatusSuccess, usageSourceProviderReported,
|
||||
))
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
||||
"model":"pool-model",
|
||||
"messages":[{"role":"user","content":"hello"}],
|
||||
"metadata":{"iop_response_mode":"passthrough+sideband"}
|
||||
}`))
|
||||
req.Header.Set("Authorization", "Bearer "+rawToken)
|
||||
w := httptest.NewRecorder()
|
||||
srv.routes().ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
|
||||
reqAfter := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues(
|
||||
edgeID, "user:alice", "alice", "iop-tok-alice", model,
|
||||
usageEndpointChatCompletions, responseModePassthroughSideband, usageStatusSuccess, usageSourceProviderReported,
|
||||
))
|
||||
if reqAfter-reqBefore != 1 {
|
||||
t.Fatalf("requests_total success/provider_reported: got delta %v, want 1", reqAfter-reqBefore)
|
||||
}
|
||||
// Verify token counters also incremented.
|
||||
for tokenType, want := range map[string]float64{
|
||||
tokenTypeInput: 7,
|
||||
tokenTypeOutput: 4,
|
||||
} {
|
||||
got := requestTokenValue(t, labels, tokenType)
|
||||
if got != want {
|
||||
t.Fatalf("token_type %s: got %v, want %v", tokenType, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestProviderTunnelSidebandStreamCallerCancelEmitsCancelMetric verifies that a
|
||||
// passthrough+sideband streaming caller disconnect (pre-cancelled context) emits
|
||||
// the cancel metric (REVIEW_USAGE_METRIC-1).
|
||||
func TestProviderTunnelSidebandStreamCallerCancelEmitsCancelMetric(t *testing.T) {
|
||||
const rawToken = "sk-sideband-cancel-token"
|
||||
const edgeID = "edge-sideband-cancel"
|
||||
const model = "pool-model"
|
||||
|
||||
// Create a cancelled context so the sideband stream detects caller disconnect.
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
// Frame channel never closed to simulate an in-flight tunnel.
|
||||
frames := make(chan *iop.ProviderTunnelFrame)
|
||||
|
||||
fake := &fakeRunService{tunnelFrames: frames, tunnelWaitTimeout: 3 * time.Second}
|
||||
srv := providerRouteUsageMetricServer(rawToken, edgeID, model, "served-model", fake)
|
||||
|
||||
reqBefore := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues(
|
||||
edgeID, "user:alice", "alice", "iop-tok-alice", model,
|
||||
usageEndpointChatCompletions, responseModePassthroughSideband, usageStatusCancel, usageSourceUnavailable,
|
||||
))
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
||||
"model":"pool-model",
|
||||
"stream":true,
|
||||
"messages":[{"role":"user","content":"hello"}],
|
||||
"metadata":{"iop_response_mode":"passthrough+sideband"}
|
||||
}`))
|
||||
req.Header.Set("Authorization", "Bearer "+rawToken)
|
||||
req = req.WithContext(ctx)
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
srv.routes().ServeHTTP(w, req)
|
||||
|
||||
reqAfter := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues(
|
||||
edgeID, "user:alice", "alice", "iop-tok-alice", model,
|
||||
usageEndpointChatCompletions, responseModePassthroughSideband, usageStatusCancel, usageSourceUnavailable,
|
||||
))
|
||||
if reqAfter-reqBefore != 1 {
|
||||
t.Fatalf("requests_total cancel: got delta %v, want 1", reqAfter-reqBefore)
|
||||
}
|
||||
}
|
||||
|
||||
// TestChatCompletionsDispatchFailureEmitsErrorMetric verifies that a tunnel
|
||||
// dispatch failure increments the request counter with status=error
|
||||
// (REVIEW_USAGE_METRIC-2).
|
||||
func TestChatCompletionsDispatchFailureEmitsErrorMetric(t *testing.T) {
|
||||
const rawToken = "sk-dispatch-fail-token"
|
||||
const edgeID = "edge-dispatch-fail"
|
||||
|
|
@ -843,523 +597,3 @@ func TestBufferedStreamToolValidationRetryDispatchFailureEmitsErrorMetric(t *tes
|
|||
t.Fatalf("requests_total error: got delta %v, want 1", reqAfter-reqBefore)
|
||||
}
|
||||
}
|
||||
|
||||
// TestProviderTunnelSidebandStreamBodyWriteFailureEmitsCancelMetric verifies
|
||||
// that a passthrough+sideband streaming body write failure to the caller
|
||||
// propagates cancel upstream and emits status=cancel (REVIEW_USAGE_METRIC_RETRY-2).
|
||||
func TestProviderTunnelSidebandStreamBodyWriteFailureEmitsCancelMetric(t *testing.T) {
|
||||
const rawToken = "sk-sideband-bodyfail-cancel"
|
||||
const edgeID = "edge-sideband-bodyfail-cancel"
|
||||
const model = "pool-bodyfail"
|
||||
|
||||
// Use a tunnel that emits a single body frame, then the response writer
|
||||
// will fail on the second body frame write.
|
||||
frames := make(chan *iop.ProviderTunnelFrame, 3)
|
||||
frames <- &iop.ProviderTunnelFrame{
|
||||
Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START,
|
||||
StatusCode: http.StatusOK,
|
||||
}
|
||||
frames <- &iop.ProviderTunnelFrame{
|
||||
Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY,
|
||||
Body: []byte(`{"ok":true}`),
|
||||
}
|
||||
// Second body frame triggers write failure.
|
||||
frames <- &iop.ProviderTunnelFrame{
|
||||
Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY,
|
||||
Body: []byte(`more`),
|
||||
}
|
||||
|
||||
// Use a writer that fails on the second write.
|
||||
fake := &fakeRunService{tunnelFrames: frames, tunnelWaitTimeout: 3 * time.Second}
|
||||
srv := providerRouteUsageMetricServer(rawToken, edgeID, model, "served", fake)
|
||||
|
||||
cancelBefore := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues(
|
||||
edgeID, "user:alice", "alice", "iop-tok-alice", model,
|
||||
usageEndpointChatCompletions, responseModePassthroughSideband, usageStatusCancel, usageSourceUnavailable,
|
||||
))
|
||||
errorBefore := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues(
|
||||
edgeID, "user:alice", "alice", "iop-tok-alice", model,
|
||||
usageEndpointChatCompletions, responseModePassthroughSideband, usageStatusError, usageSourceUnavailable,
|
||||
))
|
||||
|
||||
// Wrap the response writer so the second Write call fails.
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
||||
"model":"pool-bodyfail",
|
||||
"stream":true,
|
||||
"messages":[{"role":"user","content":"hello"}],
|
||||
"metadata":{"iop_response_mode":"passthrough+sideband"}
|
||||
}`))
|
||||
req.Header.Set("Authorization", "Bearer "+rawToken)
|
||||
|
||||
w := &failingWriteRecorder{
|
||||
ResponseRecorder: httptest.NewRecorder(),
|
||||
failAfter: 2,
|
||||
}
|
||||
srv.routes().ServeHTTP(w, req)
|
||||
|
||||
cancelAfter := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues(
|
||||
edgeID, "user:alice", "alice", "iop-tok-alice", model,
|
||||
usageEndpointChatCompletions, responseModePassthroughSideband, usageStatusCancel, usageSourceUnavailable,
|
||||
))
|
||||
errorAfter := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues(
|
||||
edgeID, "user:alice", "alice", "iop-tok-alice", model,
|
||||
usageEndpointChatCompletions, responseModePassthroughSideband, usageStatusError, usageSourceUnavailable,
|
||||
))
|
||||
|
||||
if cancelAfter-cancelBefore != 1 {
|
||||
t.Fatalf("requests_total cancel: got delta %v, want 1", cancelAfter-cancelBefore)
|
||||
}
|
||||
if errorAfter-errorBefore != 0 {
|
||||
t.Fatalf("requests_total error: got delta %v, want 0", errorAfter-errorBefore)
|
||||
}
|
||||
}
|
||||
|
||||
// TestProviderTunnelSidebandStreamingEmitsUsageMetricsBodyOnly verifies that a
|
||||
// passthrough+sideband streaming response that contains usage ONLY in the body
|
||||
// (no separate USAGE frame) still emits metrics with usage_source=provider_reported.
|
||||
// This is a regression test for REVIEW_USAGE_METRIC-1: body-parsed usage must not
|
||||
// be ignored when no proto USAGE frame is present.
|
||||
func TestProviderTunnelSidebandStreamingEmitsUsageMetricsBodyOnly(t *testing.T) {
|
||||
const rawToken = "sk-body-only-stream-token"
|
||||
const edgeID = "edge-body-only-stream-metrics"
|
||||
const model = "pool-model"
|
||||
// BODY is SSE-formatted (matching actual provider tunnel stream): delta chunk
|
||||
// followed by a usage chunk on the next line.
|
||||
providerBody := `data: {"choices":[{"delta":{"content":"hi"}}]}
|
||||
|
||||
data: {"usage":{"prompt_tokens":10,"completion_tokens":5,"prompt_tokens_details":{"cached_tokens":2},"completion_tokens_details":{"reasoning_tokens":3},"total_tokens":20}}
|
||||
|
||||
data: [DONE]
|
||||
|
||||
`
|
||||
|
||||
frames := make(chan *iop.ProviderTunnelFrame, 5)
|
||||
frames <- &iop.ProviderTunnelFrame{
|
||||
Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START,
|
||||
StatusCode: 200,
|
||||
Headers: map[string]string{"Content-Type": "text/event-stream"},
|
||||
}
|
||||
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(providerBody)}
|
||||
// NOTE: No USAGE frame — usage is only in the body JSON.
|
||||
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true}
|
||||
close(frames)
|
||||
|
||||
srv := providerRouteUsageMetricServer(rawToken, edgeID, model, "served-model", &fakeRunService{tunnelFrames: frames})
|
||||
|
||||
labels := usageLabels{
|
||||
edgeID: edgeID, principalRef: "user:alice", principalAlias: "alice", tokenRef: "iop-tok-alice",
|
||||
modelGroup: model, endpoint: usageEndpointChatCompletions, responseMode: responseModePassthroughSideband,
|
||||
}
|
||||
reqBefore := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues(
|
||||
edgeID, "user:alice", "alice", "iop-tok-alice", model,
|
||||
usageEndpointChatCompletions, responseModePassthroughSideband, usageStatusSuccess, usageSourceProviderReported,
|
||||
))
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
||||
"model":"pool-model",
|
||||
"stream":true,
|
||||
"messages":[{"role":"user","content":"hello"}],
|
||||
"metadata":{"iop_response_mode":"passthrough+sideband"}
|
||||
}`))
|
||||
req.Header.Set("Authorization", "Bearer "+rawToken)
|
||||
w := httptest.NewRecorder()
|
||||
srv.routes().ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
|
||||
reqAfter := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues(
|
||||
edgeID, "user:alice", "alice", "iop-tok-alice", model,
|
||||
usageEndpointChatCompletions, responseModePassthroughSideband, usageStatusSuccess, usageSourceProviderReported,
|
||||
))
|
||||
if reqAfter-reqBefore != 1 {
|
||||
t.Fatalf("requests_total success/provider_reported: got delta %v, want 1", reqAfter-reqBefore)
|
||||
}
|
||||
// Verify token counters were populated from body-only usage including
|
||||
// reasoning and cached_input (REVIEW_REVIEW_REVIEW_USAGE_METRIC_RETRY).
|
||||
for tokenType, want := range map[string]float64{
|
||||
tokenTypeInput: 10,
|
||||
tokenTypeOutput: 5,
|
||||
tokenTypeReasoning: 3,
|
||||
tokenTypeCachedInput: 2,
|
||||
} {
|
||||
got := requestTokenValue(t, labels, tokenType)
|
||||
if got != want {
|
||||
t.Fatalf("token_type %s: got %v, want %v", tokenType, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestProviderTunnelSidebandNonStreamingEmitsUsageMetricsBodyOnly verifies that a
|
||||
// passthrough+sideband non-streaming response that contains usage ONLY in the body
|
||||
// (no separate USAGE frame) still emits metrics with usage_source=provider_reported.
|
||||
func TestProviderTunnelSidebandNonStreamingEmitsUsageMetricsBodyOnly(t *testing.T) {
|
||||
const rawToken = "sk-body-only-nonstream-token"
|
||||
const edgeID = "edge-body-only-nonstream-metrics"
|
||||
const model = "pool-model"
|
||||
// BODY contains OpenAI-compatible usage JSON including detail objects;
|
||||
// no separate USAGE frame.
|
||||
providerBody := `{"id":"cmpl-1","choices":[{"message":{"role":"assistant","content":"hi"}}],"usage":{"prompt_tokens":12,"completion_tokens":8,"prompt_tokens_details":{"cached_tokens":3},"completion_tokens_details":{"reasoning_tokens":4},"total_tokens":27}}`
|
||||
|
||||
frames := make(chan *iop.ProviderTunnelFrame, 5)
|
||||
frames <- &iop.ProviderTunnelFrame{
|
||||
Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START,
|
||||
StatusCode: 200,
|
||||
Headers: map[string]string{"Content-Type": "application/json"},
|
||||
}
|
||||
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(providerBody)}
|
||||
// NOTE: No USAGE frame — usage is only in the body JSON.
|
||||
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true}
|
||||
close(frames)
|
||||
|
||||
srv := providerRouteUsageMetricServer(rawToken, edgeID, model, "served-model", &fakeRunService{tunnelFrames: frames})
|
||||
|
||||
labels := usageLabels{
|
||||
edgeID: edgeID, principalRef: "user:alice", principalAlias: "alice", tokenRef: "iop-tok-alice",
|
||||
modelGroup: model, endpoint: usageEndpointChatCompletions, responseMode: responseModePassthroughSideband,
|
||||
}
|
||||
reqBefore := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues(
|
||||
edgeID, "user:alice", "alice", "iop-tok-alice", model,
|
||||
usageEndpointChatCompletions, responseModePassthroughSideband, usageStatusSuccess, usageSourceProviderReported,
|
||||
))
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
||||
"model":"pool-model",
|
||||
"messages":[{"role":"user","content":"hello"}],
|
||||
"metadata":{"iop_response_mode":"passthrough+sideband"}
|
||||
}`))
|
||||
req.Header.Set("Authorization", "Bearer "+rawToken)
|
||||
w := httptest.NewRecorder()
|
||||
srv.routes().ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
|
||||
reqAfter := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues(
|
||||
edgeID, "user:alice", "alice", "iop-tok-alice", model,
|
||||
usageEndpointChatCompletions, responseModePassthroughSideband, usageStatusSuccess, usageSourceProviderReported,
|
||||
))
|
||||
if reqAfter-reqBefore != 1 {
|
||||
t.Fatalf("requests_total success/provider_reported: got delta %v, want 1", reqAfter-reqBefore)
|
||||
}
|
||||
// Verify token counters were populated from body-only usage including
|
||||
// reasoning and cached_input (REVIEW_REVIEW_REVIEW_USAGE_METRIC_RETRY).
|
||||
for tokenType, want := range map[string]float64{
|
||||
tokenTypeInput: 12,
|
||||
tokenTypeOutput: 8,
|
||||
tokenTypeReasoning: 4,
|
||||
tokenTypeCachedInput: 3,
|
||||
} {
|
||||
got := requestTokenValue(t, labels, tokenType)
|
||||
if got != want {
|
||||
t.Fatalf("token_type %s: got %v, want %v", tokenType, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestProviderTunnelSidebandStreamProtoOnlyReasoningCachedInput verifies that when
|
||||
// only a proto USAGE frame is present (no body usage), the metric path preserves
|
||||
// reasoning and cached_input from the proto (REVIEW_REVIEW_REVIEW_USAGE_METRIC_RETRY).
|
||||
func TestProviderTunnelSidebandStreamProtoOnlyReasoningCachedInput(t *testing.T) {
|
||||
const rawToken = "sk-proto-only-stream-token"
|
||||
const edgeID = "edge-proto-only-stream"
|
||||
const model = "pool-model"
|
||||
// BODY contains no usage object.
|
||||
providerBody := `data: {"choices":[{"delta":{"content":"hi"}}]}
|
||||
|
||||
data: [DONE]
|
||||
|
||||
`
|
||||
|
||||
frames := make(chan *iop.ProviderTunnelFrame, 5)
|
||||
frames <- &iop.ProviderTunnelFrame{
|
||||
Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START,
|
||||
StatusCode: 200,
|
||||
Headers: map[string]string{"Content-Type": "text/event-stream"},
|
||||
}
|
||||
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(providerBody)}
|
||||
// Proto USAGE frame carries full token breakdown including reasoning/cached_input.
|
||||
frames <- &iop.ProviderTunnelFrame{
|
||||
Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_USAGE,
|
||||
Usage: &iop.Usage{InputTokens: 10, OutputTokens: 5, ReasoningTokens: 3, CachedInputTokens: 2},
|
||||
}
|
||||
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true}
|
||||
close(frames)
|
||||
|
||||
srv := providerRouteUsageMetricServer(rawToken, edgeID, model, "served-model", &fakeRunService{tunnelFrames: frames})
|
||||
|
||||
labels := usageLabels{
|
||||
edgeID: edgeID, principalRef: "user:alice", principalAlias: "alice", tokenRef: "iop-tok-alice",
|
||||
modelGroup: model, endpoint: usageEndpointChatCompletions, responseMode: responseModePassthroughSideband,
|
||||
}
|
||||
reqBefore := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues(
|
||||
edgeID, "user:alice", "alice", "iop-tok-alice", model,
|
||||
usageEndpointChatCompletions, responseModePassthroughSideband, usageStatusSuccess, usageSourceProviderReported,
|
||||
))
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
||||
"model":"pool-model",
|
||||
"stream":true,
|
||||
"messages":[{"role":"user","content":"hello"}],
|
||||
"metadata":{"iop_response_mode":"passthrough+sideband"}
|
||||
}`))
|
||||
req.Header.Set("Authorization", "Bearer "+rawToken)
|
||||
w := httptest.NewRecorder()
|
||||
srv.routes().ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
|
||||
reqAfter := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues(
|
||||
edgeID, "user:alice", "alice", "iop-tok-alice", model,
|
||||
usageEndpointChatCompletions, responseModePassthroughSideband, usageStatusSuccess, usageSourceProviderReported,
|
||||
))
|
||||
if reqAfter-reqBefore != 1 {
|
||||
t.Fatalf("requests_total success/provider_reported: got delta %v, want 1", reqAfter-reqBefore)
|
||||
}
|
||||
// All four token types must be present from proto-only.
|
||||
for tokenType, want := range map[string]float64{
|
||||
tokenTypeInput: 10,
|
||||
tokenTypeOutput: 5,
|
||||
tokenTypeReasoning: 3,
|
||||
tokenTypeCachedInput: 2,
|
||||
} {
|
||||
got := requestTokenValue(t, labels, tokenType)
|
||||
if got != want {
|
||||
t.Fatalf("token_type %s: got %v, want %v", tokenType, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestProviderTunnelSidebandResponseProtoOnlyReasoningCachedInput verifies that
|
||||
// for non-streaming sideband, proto-only USAGE frame preserves reasoning and
|
||||
// cached_input in metrics (REVIEW_REVIEW_REVIEW_USAGE_METRIC_RETRY).
|
||||
func TestProviderTunnelSidebandResponseProtoOnlyReasoningCachedInput(t *testing.T) {
|
||||
const rawToken = "sk-proto-only-nonstream-token"
|
||||
const edgeID = "edge-proto-only-nonstream"
|
||||
const model = "pool-model"
|
||||
providerBody := `{"id":"cmpl-1","choices":[{"message":{"role":"assistant","content":"hi"}}]}`
|
||||
|
||||
frames := make(chan *iop.ProviderTunnelFrame, 5)
|
||||
frames <- &iop.ProviderTunnelFrame{
|
||||
Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START,
|
||||
StatusCode: 200,
|
||||
Headers: map[string]string{"Content-Type": "application/json"},
|
||||
}
|
||||
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(providerBody)}
|
||||
frames <- &iop.ProviderTunnelFrame{
|
||||
Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_USAGE,
|
||||
Usage: &iop.Usage{InputTokens: 7, OutputTokens: 4, ReasoningTokens: 2, CachedInputTokens: 1},
|
||||
}
|
||||
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true}
|
||||
close(frames)
|
||||
|
||||
srv := providerRouteUsageMetricServer(rawToken, edgeID, model, "served-model", &fakeRunService{tunnelFrames: frames})
|
||||
|
||||
labels := usageLabels{
|
||||
edgeID: edgeID, principalRef: "user:alice", principalAlias: "alice", tokenRef: "iop-tok-alice",
|
||||
modelGroup: model, endpoint: usageEndpointChatCompletions, responseMode: responseModePassthroughSideband,
|
||||
}
|
||||
reqBefore := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues(
|
||||
edgeID, "user:alice", "alice", "iop-tok-alice", model,
|
||||
usageEndpointChatCompletions, responseModePassthroughSideband, usageStatusSuccess, usageSourceProviderReported,
|
||||
))
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
||||
"model":"pool-model",
|
||||
"messages":[{"role":"user","content":"hello"}],
|
||||
"metadata":{"iop_response_mode":"passthrough+sideband"}
|
||||
}`))
|
||||
req.Header.Set("Authorization", "Bearer "+rawToken)
|
||||
w := httptest.NewRecorder()
|
||||
srv.routes().ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
|
||||
reqAfter := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues(
|
||||
edgeID, "user:alice", "alice", "iop-tok-alice", model,
|
||||
usageEndpointChatCompletions, responseModePassthroughSideband, usageStatusSuccess, usageSourceProviderReported,
|
||||
))
|
||||
if reqAfter-reqBefore != 1 {
|
||||
t.Fatalf("requests_total success/provider_reported: got delta %v, want 1", reqAfter-reqBefore)
|
||||
}
|
||||
for tokenType, want := range map[string]float64{
|
||||
tokenTypeInput: 7,
|
||||
tokenTypeOutput: 4,
|
||||
tokenTypeReasoning: 2,
|
||||
tokenTypeCachedInput: 1,
|
||||
} {
|
||||
got := requestTokenValue(t, labels, tokenType)
|
||||
if got != want {
|
||||
t.Fatalf("token_type %s: got %v, want %v", tokenType, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestProviderTunnelSidebandStreamBodyAndProtoNoDoubleCount verifies that when
|
||||
// both body usage and proto USAGE frame are present, the merge rule prevents
|
||||
// double-counting input/output while preserving proto-only reasoning/cached_input
|
||||
// (REVIEW_REVIEW_REVIEW_USAGE_METRIC_RETRY).
|
||||
func TestProviderTunnelSidebandStreamBodyAndProtoNoDoubleCount(t *testing.T) {
|
||||
const rawToken = "sk-body-and-proto-stream-token"
|
||||
const edgeID = "edge-body-and-proto-stream"
|
||||
const model = "pool-model"
|
||||
// Body reports input/output but no reasoning/cached_input details.
|
||||
providerBody := `data: {"choices":[{"delta":{"content":"hi"}}]}
|
||||
|
||||
data: {"usage":{"prompt_tokens":10,"completion_tokens":5,"total_tokens":15}}
|
||||
|
||||
data: [DONE]
|
||||
|
||||
`
|
||||
|
||||
frames := make(chan *iop.ProviderTunnelFrame, 5)
|
||||
frames <- &iop.ProviderTunnelFrame{
|
||||
Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START,
|
||||
StatusCode: 200,
|
||||
Headers: map[string]string{"Content-Type": "text/event-stream"},
|
||||
}
|
||||
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(providerBody)}
|
||||
// Proto frame carries same input/output plus additional reasoning/cached_input.
|
||||
frames <- &iop.ProviderTunnelFrame{
|
||||
Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_USAGE,
|
||||
Usage: &iop.Usage{InputTokens: 10, OutputTokens: 5, ReasoningTokens: 3, CachedInputTokens: 2},
|
||||
}
|
||||
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true}
|
||||
close(frames)
|
||||
|
||||
srv := providerRouteUsageMetricServer(rawToken, edgeID, model, "served-model", &fakeRunService{tunnelFrames: frames})
|
||||
|
||||
labels := usageLabels{
|
||||
edgeID: edgeID, principalRef: "user:alice", principalAlias: "alice", tokenRef: "iop-tok-alice",
|
||||
modelGroup: model, endpoint: usageEndpointChatCompletions, responseMode: responseModePassthroughSideband,
|
||||
}
|
||||
reqBefore := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues(
|
||||
edgeID, "user:alice", "alice", "iop-tok-alice", model,
|
||||
usageEndpointChatCompletions, responseModePassthroughSideband, usageStatusSuccess, usageSourceProviderReported,
|
||||
))
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
||||
"model":"pool-model",
|
||||
"stream":true,
|
||||
"messages":[{"role":"user","content":"hello"}],
|
||||
"metadata":{"iop_response_mode":"passthrough+sideband"}
|
||||
}`))
|
||||
req.Header.Set("Authorization", "Bearer "+rawToken)
|
||||
w := httptest.NewRecorder()
|
||||
srv.routes().ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
|
||||
reqAfter := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues(
|
||||
edgeID, "user:alice", "alice", "iop-tok-alice", model,
|
||||
usageEndpointChatCompletions, responseModePassthroughSideband, usageStatusSuccess, usageSourceProviderReported,
|
||||
))
|
||||
if reqAfter-reqBefore != 1 {
|
||||
t.Fatalf("requests_total success/provider_reported: got delta %v, want 1", reqAfter-reqBefore)
|
||||
}
|
||||
// Input/output should NOT be double-counted (merge rule: body wins).
|
||||
// Reasoning/cached_input should come from proto (body has 0).
|
||||
for tokenType, want := range map[string]float64{
|
||||
tokenTypeInput: 10, // from body, NOT 10+10=20
|
||||
tokenTypeOutput: 5, // from body, NOT 5+5=10
|
||||
tokenTypeReasoning: 3, // from proto
|
||||
tokenTypeCachedInput: 2, // from proto
|
||||
} {
|
||||
got := requestTokenValue(t, labels, tokenType)
|
||||
if got != want {
|
||||
t.Fatalf("token_type %s: got %v, want %v", tokenType, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestProviderTunnelSidebandResponseBodyAndProtoNoDoubleCount verifies the same
|
||||
// merge rule for non-streaming sideband responses (REVIEW_REVIEW_REVIEW_USAGE_METRIC_RETRY).
|
||||
func TestProviderTunnelSidebandResponseBodyAndProtoNoDoubleCount(t *testing.T) {
|
||||
const rawToken = "sk-body-and-proto-nonstream-token"
|
||||
const edgeID = "edge-body-and-proto-nonstream"
|
||||
const model = "pool-model"
|
||||
// Body reports input/output only.
|
||||
providerBody := `{"id":"cmpl-1","choices":[{"message":{"role":"assistant","content":"hi"}}],"usage":{"prompt_tokens":12,"completion_tokens":8,"total_tokens":20}}`
|
||||
|
||||
frames := make(chan *iop.ProviderTunnelFrame, 5)
|
||||
frames <- &iop.ProviderTunnelFrame{
|
||||
Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START,
|
||||
StatusCode: 200,
|
||||
Headers: map[string]string{"Content-Type": "application/json"},
|
||||
}
|
||||
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(providerBody)}
|
||||
// Proto frame carries same input/output plus reasoning/cached_input.
|
||||
frames <- &iop.ProviderTunnelFrame{
|
||||
Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_USAGE,
|
||||
Usage: &iop.Usage{InputTokens: 12, OutputTokens: 8, ReasoningTokens: 5, CachedInputTokens: 4},
|
||||
}
|
||||
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true}
|
||||
close(frames)
|
||||
|
||||
srv := providerRouteUsageMetricServer(rawToken, edgeID, model, "served-model", &fakeRunService{tunnelFrames: frames})
|
||||
|
||||
labels := usageLabels{
|
||||
edgeID: edgeID, principalRef: "user:alice", principalAlias: "alice", tokenRef: "iop-tok-alice",
|
||||
modelGroup: model, endpoint: usageEndpointChatCompletions, responseMode: responseModePassthroughSideband,
|
||||
}
|
||||
reqBefore := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues(
|
||||
edgeID, "user:alice", "alice", "iop-tok-alice", model,
|
||||
usageEndpointChatCompletions, responseModePassthroughSideband, usageStatusSuccess, usageSourceProviderReported,
|
||||
))
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
||||
"model":"pool-model",
|
||||
"messages":[{"role":"user","content":"hello"}],
|
||||
"metadata":{"iop_response_mode":"passthrough+sideband"}
|
||||
}`))
|
||||
req.Header.Set("Authorization", "Bearer "+rawToken)
|
||||
w := httptest.NewRecorder()
|
||||
srv.routes().ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
|
||||
reqAfter := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues(
|
||||
edgeID, "user:alice", "alice", "iop-tok-alice", model,
|
||||
usageEndpointChatCompletions, responseModePassthroughSideband, usageStatusSuccess, usageSourceProviderReported,
|
||||
))
|
||||
if reqAfter-reqBefore != 1 {
|
||||
t.Fatalf("requests_total success/provider_reported: got delta %v, want 1", reqAfter-reqBefore)
|
||||
}
|
||||
// Input/output from body only; reasoning/cached_input from proto.
|
||||
for tokenType, want := range map[string]float64{
|
||||
tokenTypeInput: 12, // from body, NOT 12+12=24
|
||||
tokenTypeOutput: 8, // from body, NOT 8+8=16
|
||||
tokenTypeReasoning: 5, // from proto
|
||||
tokenTypeCachedInput: 4, // from proto
|
||||
} {
|
||||
got := requestTokenValue(t, labels, tokenType)
|
||||
if got != want {
|
||||
t.Fatalf("token_type %s: got %v, want %v", tokenType, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// failingWriteRecorder wraps httptest.ResponseRecorder and fails after N writes.
|
||||
type failingWriteRecorder struct {
|
||||
*httptest.ResponseRecorder
|
||||
failAfter int
|
||||
writes int
|
||||
}
|
||||
|
||||
func (fw *failingWriteRecorder) Write(p []byte) (int, error) {
|
||||
fw.writes++
|
||||
if fw.writes > fw.failAfter {
|
||||
return 0, fmt.Errorf("simulated write failure")
|
||||
}
|
||||
return fw.ResponseRecorder.Write(p)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ Provider-reported OpenAI-compatible token usage by token type.
|
|||
| `token_ref` | `tok-xyz789` | Token identity (not a raw secret) |
|
||||
| `model_group` | `gpt-4o` | Model group identifier |
|
||||
| `endpoint` | `chat.completions`, `responses` | OpenAI-compatible route |
|
||||
| `response_mode` | `blocking`, `streaming` | Response mode |
|
||||
| `response_mode` | `passthrough`, `normalized` | Internal execution label; callers cannot set it via OpenAI metadata |
|
||||
| `token_type` | `input`, `output`, `reasoning`, `cached_input` | Token type |
|
||||
|
||||
### 2. `iop_openai_requests_total` (Counter)
|
||||
|
|
@ -33,7 +33,7 @@ OpenAI-compatible requests processed by terminal status and usage source.
|
|||
| `token_ref` | `tok-xyz789` | Token identity |
|
||||
| `model_group` | `gpt-4o` | Model group identifier |
|
||||
| `endpoint` | `chat.completions`, `responses` | OpenAI-compatible route |
|
||||
| `response_mode` | `blocking`, `streaming` | Response mode |
|
||||
| `response_mode` | `passthrough`, `normalized` | Internal execution label; callers cannot set it via OpenAI metadata |
|
||||
| `status` | `success`, `error`, `cancel` | Terminal request status |
|
||||
| `usage_source` | `provider_reported`, `unavailable` | Whether at least one token type was reported |
|
||||
|
||||
|
|
@ -140,6 +140,8 @@ sum by (endpoint, status) (iop_openai_requests_total)
|
|||
|
||||
### response_mode별 분석
|
||||
|
||||
`response_mode`는 handler 실행 경로에서 파생되는 내부 라벨이다. provider 라우트의 raw tunnel 패스스루는 `passthrough`, 정규화된 RunEvent 경로는 `normalized`로 기록된다. caller가 OpenAI metadata로 설정할 수 있는 값이 아니며 API selector가 아니다.
|
||||
|
||||
```promql
|
||||
# response_mode별 성공 요청 수
|
||||
sum by (response_mode, status) (iop_openai_requests_total{status="success"})
|
||||
|
|
@ -379,7 +381,7 @@ Grafana table 구성 예:
|
|||
| `status` | `success`, `error`, `cancel` |
|
||||
| `usage_source` | `provider_reported`, `unavailable` |
|
||||
| `token_type` | `input`, `output`, `reasoning`, `cached_input` |
|
||||
| `response_mode` | `blocking`, `streaming` |
|
||||
| `response_mode` | `passthrough`, `normalized` |
|
||||
|
||||
---
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue