797 lines
27 KiB
Go
797 lines
27 KiB
Go
package openai
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"go.uber.org/zap"
|
|
|
|
edgeservice "iop/apps/edge/internal/service"
|
|
iop "iop/proto/gen/iop"
|
|
)
|
|
|
|
func (s *Server) handleResponses(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodPost {
|
|
writeError(w, http.StatusMethodNotAllowed, "method_not_allowed", "method not allowed")
|
|
return
|
|
}
|
|
defer r.Body.Close()
|
|
|
|
// The raw body is preserved for the provider tunnel passthrough path, which
|
|
// forwards the caller's own Responses payload (model rewritten to the served
|
|
// target) without strict normalization.
|
|
rawBody, err := io.ReadAll(r.Body)
|
|
if err != nil {
|
|
writeError(w, http.StatusBadRequest, "invalid_request_error", "failed to read request body")
|
|
return
|
|
}
|
|
|
|
// The route decision runs before strict normalization: only the
|
|
// routing-relevant envelope fields are decoded leniently so provider
|
|
// passthrough can preserve Codex/Responses unknown fields (max_output_tokens,
|
|
// tools, store, ...). Strict field validation stays on the normalized path.
|
|
env, err := decodeResponsesEnvelope(rawBody)
|
|
if err != nil {
|
|
writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error())
|
|
return
|
|
}
|
|
dispatch, ok := s.resolveRouteDispatch(env.Model)
|
|
if !ok {
|
|
writeError(w, http.StatusBadRequest, "invalid_request_error", "model is required")
|
|
return
|
|
}
|
|
if routeUsesProviderTunnel(dispatch) {
|
|
runMeta, workspace, err := parseOpenAIMetadata(env.Metadata)
|
|
if err != nil {
|
|
writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error())
|
|
return
|
|
}
|
|
// Overwrite (not merge-if-absent): the authenticated caller identity must
|
|
// win over any caller-supplied metadata.iop_principal_* spoof attempt.
|
|
for k, v := range principalMetadata(r.Context()) {
|
|
runMeta[k] = v
|
|
}
|
|
responseMode, err := parseResponseMode(runMeta)
|
|
if err != nil {
|
|
writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error())
|
|
return
|
|
}
|
|
switch responseMode {
|
|
case responseModePassthrough:
|
|
case responseModePassthroughSideband:
|
|
if err := validateWorkspaceForRoute(dispatch, workspace); err != nil {
|
|
writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error())
|
|
return
|
|
}
|
|
estimate := estimateInputTokens(string(rawBody), runMeta, nil, nil)
|
|
contextClass := classifyContext(estimate, s.longContextThreshold())
|
|
s.tunnelResponsesPassthroughSideband(w, r, env, dispatch, runMeta, rawBody, estimate, contextClass)
|
|
return
|
|
case responseModeTransformed:
|
|
writeError(w, http.StatusBadRequest, "invalid_request_error", "metadata.iop_response_mode=transformed is not supported for /v1/responses provider routes")
|
|
return
|
|
}
|
|
if err := validateWorkspaceForRoute(dispatch, workspace); err != nil {
|
|
writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error())
|
|
return
|
|
}
|
|
estimate := estimateInputTokens(string(rawBody), runMeta, nil, nil)
|
|
contextClass := classifyContext(estimate, s.longContextThreshold())
|
|
s.tunnelResponsesPassthrough(w, r, env, dispatch, runMeta, rawBody, estimate, contextClass)
|
|
return
|
|
}
|
|
|
|
// Non-provider routes keep the normalized RunEvent path: strict decode,
|
|
// stream/background rejection, prompt build, and SubmitRun.
|
|
var req responsesRequest
|
|
if err := decodeResponsesRequest(json.NewDecoder(bytes.NewReader(rawBody)), &req); err != nil {
|
|
writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error())
|
|
return
|
|
}
|
|
|
|
if req.Stream {
|
|
writeError(w, http.StatusBadRequest, "invalid_request_error", "streaming is not supported for /v1/responses")
|
|
return
|
|
}
|
|
if req.Background {
|
|
writeError(w, http.StatusBadRequest, "invalid_request_error", "background is not supported for /v1/responses")
|
|
return
|
|
}
|
|
|
|
inputStr, err := parseResponsesInput(req.Input)
|
|
if err != nil {
|
|
writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error())
|
|
return
|
|
}
|
|
|
|
prompt := buildResponsesPrompt(req.Instructions, inputStr)
|
|
outputPolicy := s.resolveOutputPolicy(prompt)
|
|
if instruction := strictOutputContractInstruction(outputPolicy); instruction != "" {
|
|
prompt = instruction + "\n" + prompt
|
|
}
|
|
|
|
runMeta, workspace, err := parseOpenAIMetadata(req.Metadata)
|
|
if err != nil {
|
|
writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error())
|
|
return
|
|
}
|
|
// Overwrite (not merge-if-absent): the authenticated caller identity must
|
|
// win over any caller-supplied metadata.iop_principal_* spoof attempt.
|
|
for k, v := range principalMetadata(r.Context()) {
|
|
runMeta[k] = v
|
|
}
|
|
|
|
if err := validateWorkspaceForRoute(dispatch, workspace); err != nil {
|
|
writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error())
|
|
return
|
|
}
|
|
var defaultThinkingTokenBudget int
|
|
if catalogEntry := s.findProviderPoolEntry(req.Model); catalogEntry != nil {
|
|
applyModelCatalogGenerationPolicyToResponses(&req, *catalogEntry)
|
|
if catalogEntry.DefaultThinkingTokenBudget > 0 {
|
|
defaultThinkingTokenBudget = catalogEntry.DefaultThinkingTokenBudget
|
|
}
|
|
}
|
|
|
|
runMeta["openai_model"] = req.Model
|
|
runMeta["openai_stream"] = fmt.Sprintf("%t", req.Stream)
|
|
runMeta["strict_output"] = fmt.Sprintf("%t", outputPolicy.Strict)
|
|
input := map[string]any{"prompt": prompt}
|
|
if defaultThinkingTokenBudget > 0 {
|
|
input["think"] = true
|
|
input["thinking_token_budget"] = defaultThinkingTokenBudget
|
|
} else if outputPolicy.Strict {
|
|
input["think"] = false
|
|
}
|
|
if options := req.providerOptions(); len(options) > 0 {
|
|
input["options"] = options
|
|
}
|
|
|
|
s.logger.Info("openai responses input",
|
|
zap.String("model", req.Model),
|
|
zap.String("target", dispatch.Target),
|
|
zap.String("adapter", dispatch.Adapter),
|
|
zap.Bool("strict_output", outputPolicy.Strict),
|
|
zap.String("xml_completion_tool", outputPolicy.XMLCompletionTool),
|
|
zap.Bool("contract_instruction", outputPolicy.ContractInstruction),
|
|
zap.Int("prompt_len", len(prompt)),
|
|
)
|
|
|
|
estimate := estimateInputTokens(prompt, runMeta, nil, nil)
|
|
contextClass := classifyContext(estimate, s.longContextThreshold())
|
|
runMeta["estimated_input_tokens"] = strconv.Itoa(estimate)
|
|
runMeta["context_class"] = contextClass
|
|
|
|
handle, err := s.service.SubmitRun(r.Context(), edgeservice.SubmitRunRequest{
|
|
NodeRef: dispatch.NodeRef,
|
|
ModelGroupKey: strings.TrimSpace(req.Model),
|
|
Adapter: dispatch.Adapter,
|
|
Target: dispatch.Target,
|
|
SessionID: dispatch.SessionID,
|
|
Workspace: workspace,
|
|
Prompt: prompt,
|
|
Input: input,
|
|
TimeoutSec: dispatch.TimeoutSec,
|
|
MaxQueue: dispatch.MaxQueue,
|
|
QueueTimeoutMS: dispatch.QueueTimeoutMS,
|
|
Metadata: runMeta,
|
|
EstimatedInputTokens: estimate,
|
|
ContextClass: contextClass,
|
|
ProviderPool: dispatch.ProviderPool,
|
|
})
|
|
if err != nil {
|
|
writeError(w, http.StatusBadGateway, "node_dispatch_error", err.Error())
|
|
return
|
|
}
|
|
defer handle.Close()
|
|
|
|
s.logger.Info("openai responses dispatch",
|
|
zap.String("run_id", handle.Dispatch().RunID),
|
|
zap.String("node_id", handle.Dispatch().NodeID),
|
|
zap.String("model_group", handle.Dispatch().ModelGroupKey),
|
|
zap.String("adapter", handle.Dispatch().Adapter),
|
|
zap.String("target", handle.Dispatch().Target),
|
|
zap.Int("estimated_input_tokens", handle.Dispatch().EstimatedInputTokens),
|
|
zap.String("context_class", handle.Dispatch().ContextClass),
|
|
zap.String("queue_reason", handle.Dispatch().QueueReason),
|
|
)
|
|
|
|
s.completeResponse(w, r, req, handle, outputPolicy)
|
|
}
|
|
|
|
func decodeResponsesRequest(dec *json.Decoder, req *responsesRequest) error {
|
|
var raw map[string]json.RawMessage
|
|
if err := dec.Decode(&raw); err != nil {
|
|
return fmt.Errorf("invalid JSON request")
|
|
}
|
|
for key := range raw {
|
|
switch key {
|
|
case "model", "input", "instructions", "stream", "background", "metadata", "max_output_tokens", "temperature", "top_p":
|
|
default:
|
|
return fmt.Errorf("%s is not supported for /v1/responses", key)
|
|
}
|
|
}
|
|
normalized, err := json.Marshal(raw)
|
|
if err != nil {
|
|
return fmt.Errorf("invalid JSON request")
|
|
}
|
|
if err := json.Unmarshal(normalized, req); err != nil {
|
|
return fmt.Errorf("invalid /v1/responses request format")
|
|
}
|
|
if req.MaxOutputTokens != nil && *req.MaxOutputTokens <= 0 {
|
|
return fmt.Errorf("max_output_tokens must be greater than zero")
|
|
}
|
|
if req.Temperature != nil && (*req.Temperature < 0 || *req.Temperature > 2) {
|
|
return fmt.Errorf("temperature must be between 0 and 2")
|
|
}
|
|
if req.TopP != nil && (*req.TopP < 0 || *req.TopP > 1) {
|
|
return fmt.Errorf("top_p must be between 0 and 1")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// decodeResponsesEnvelope leniently extracts only the routing-relevant fields
|
|
// (model, metadata, stream, background) from a /v1/responses request body. It
|
|
// does not reject unknown fields so the provider tunnel passthrough can forward
|
|
// Codex/Responses payloads verbatim; strict field validation stays on the
|
|
// normalized non-provider path.
|
|
func decodeResponsesEnvelope(rawBody []byte) (responsesEnvelope, error) {
|
|
var env responsesEnvelope
|
|
if err := json.Unmarshal(rawBody, &env); err != nil {
|
|
return responsesEnvelope{}, fmt.Errorf("invalid JSON request")
|
|
}
|
|
return env, nil
|
|
}
|
|
|
|
// tunnelResponsesPassthrough serves a /v1/responses request over the raw
|
|
// provider tunnel (SDD S04): the caller's original body is forwarded with only
|
|
// the model field rewritten to the served target, provider auth is injected
|
|
// from the configured request header, and provider status/headers/body bytes
|
|
// are relayed to the caller unmodified. Streaming is honored when the caller
|
|
// requested it. No IOP sideband fields, model-echo rewrite, or output-token
|
|
// normalization are applied.
|
|
func (s *Server) tunnelResponsesPassthrough(w http.ResponseWriter, r *http.Request, env responsesEnvelope, dispatch routeDispatch, runMeta map[string]string, rawBody []byte, estimate int, contextClass string) {
|
|
providerAuthHeaders, err := s.providerTunnelAuthHeaders(r)
|
|
if err != nil {
|
|
// Missing required provider auth is rejected before dispatch; the raw
|
|
// token is never echoed into the error surface.
|
|
writeError(w, http.StatusBadRequest, "invalid_request_error", "provider auth token is required")
|
|
return
|
|
}
|
|
|
|
metadata := make(map[string]string, len(runMeta)+5)
|
|
for k, v := range runMeta {
|
|
metadata[k] = v
|
|
}
|
|
metadata["openai_model"] = env.Model
|
|
metadata["openai_stream"] = strconv.FormatBool(env.Stream)
|
|
metadata[responseModeMetadataKey] = responseModePassthrough
|
|
metadata["estimated_input_tokens"] = strconv.Itoa(estimate)
|
|
metadata["context_class"] = contextClass
|
|
|
|
tunnelReq := edgeservice.SubmitProviderTunnelRequest{
|
|
NodeRef: dispatch.NodeRef,
|
|
ModelGroupKey: strings.TrimSpace(env.Model),
|
|
Adapter: dispatch.Adapter,
|
|
Target: dispatch.Target,
|
|
SessionID: dispatch.SessionID,
|
|
Method: http.MethodPost,
|
|
Path: "/v1/responses",
|
|
Headers: providerAuthHeaders,
|
|
BuildBody: func(target string) ([]byte, error) {
|
|
return rewriteResponsesModel(rawBody, target)
|
|
},
|
|
Stream: env.Stream,
|
|
TimeoutSec: dispatch.TimeoutSec,
|
|
MaxQueue: dispatch.MaxQueue,
|
|
QueueTimeoutMS: dispatch.QueueTimeoutMS,
|
|
Metadata: metadata,
|
|
EstimatedInputTokens: estimate,
|
|
ContextClass: contextClass,
|
|
ProviderPool: dispatch.ProviderPool,
|
|
}
|
|
|
|
metricLabels := s.usageLabelsFor(r.Context(), strings.TrimSpace(env.Model), usageEndpointResponses, responseModePassthrough)
|
|
handle, err := s.service.SubmitProviderTunnel(r.Context(), tunnelReq)
|
|
if err != nil {
|
|
emitUsageMetrics(metricLabels, usageStatusForError(err), usageObservation{})
|
|
writeError(w, http.StatusBadGateway, "node_dispatch_error", err.Error())
|
|
return
|
|
}
|
|
defer handle.Close()
|
|
|
|
s.logger.Info("openai responses passthrough dispatch",
|
|
zap.String("run_id", handle.Dispatch().RunID),
|
|
zap.String("node_id", handle.Dispatch().NodeID),
|
|
zap.String("model_group", handle.Dispatch().ModelGroupKey),
|
|
zap.String("adapter", handle.Dispatch().Adapter),
|
|
zap.String("target", handle.Dispatch().Target),
|
|
zap.Bool("stream", env.Stream),
|
|
zap.Int("estimated_input_tokens", handle.Dispatch().EstimatedInputTokens),
|
|
zap.String("context_class", handle.Dispatch().ContextClass),
|
|
zap.String("queue_reason", handle.Dispatch().QueueReason),
|
|
)
|
|
|
|
// requestModel is left empty so the shared tunnel writer relays provider
|
|
// bytes verbatim without rewriting the provider-echoed model back to a
|
|
// caller alias: Responses passthrough prefers provider-original bytes.
|
|
// metricLabels carries endpoint=responses, response_mode=passthrough, and
|
|
// the request model alias so success/error usage is attributed correctly.
|
|
s.writeProviderTunnelResponse(w, r, handle, env.Stream, "", metricLabels)
|
|
}
|
|
|
|
// tunnelResponsesPassthroughSideband serves an explicit /v1/responses
|
|
// passthrough+sideband request. The provider request remains raw passthrough
|
|
// with model rewrite only; the response is extended after the provider returns.
|
|
// Non-streaming JSON object responses receive sideband metadata under the
|
|
// top-level `metadata` field. Streaming responses interleave `event:
|
|
// iop.sideband` events.
|
|
func (s *Server) tunnelResponsesPassthroughSideband(w http.ResponseWriter, r *http.Request, env responsesEnvelope, dispatch routeDispatch, runMeta map[string]string, rawBody []byte, estimate int, contextClass string) {
|
|
providerAuthHeaders, err := s.providerTunnelAuthHeaders(r)
|
|
if err != nil {
|
|
writeError(w, http.StatusBadRequest, "invalid_request_error", "provider auth token is required")
|
|
return
|
|
}
|
|
|
|
metadata := make(map[string]string, len(runMeta)+5)
|
|
for k, v := range runMeta {
|
|
metadata[k] = v
|
|
}
|
|
metadata["openai_model"] = env.Model
|
|
metadata["openai_stream"] = strconv.FormatBool(env.Stream)
|
|
metadata[responseModeMetadataKey] = responseModePassthroughSideband
|
|
metadata["estimated_input_tokens"] = strconv.Itoa(estimate)
|
|
metadata["context_class"] = contextClass
|
|
|
|
tunnelReq := edgeservice.SubmitProviderTunnelRequest{
|
|
NodeRef: dispatch.NodeRef,
|
|
ModelGroupKey: strings.TrimSpace(env.Model),
|
|
Adapter: dispatch.Adapter,
|
|
Target: dispatch.Target,
|
|
SessionID: dispatch.SessionID,
|
|
Method: http.MethodPost,
|
|
Path: "/v1/responses",
|
|
Headers: providerAuthHeaders,
|
|
BuildBody: func(target string) ([]byte, error) {
|
|
return rewriteResponsesModel(rawBody, target)
|
|
},
|
|
Stream: env.Stream,
|
|
TimeoutSec: dispatch.TimeoutSec,
|
|
MaxQueue: dispatch.MaxQueue,
|
|
QueueTimeoutMS: dispatch.QueueTimeoutMS,
|
|
Metadata: metadata,
|
|
EstimatedInputTokens: estimate,
|
|
ContextClass: contextClass,
|
|
ProviderPool: dispatch.ProviderPool,
|
|
}
|
|
|
|
metricLabels := s.usageLabelsFor(r.Context(), strings.TrimSpace(env.Model), usageEndpointResponses, responseModePassthroughSideband)
|
|
handle, err := s.service.SubmitProviderTunnel(r.Context(), tunnelReq)
|
|
if err != nil {
|
|
emitUsageMetrics(metricLabels, usageStatusForError(err), usageObservation{})
|
|
writeError(w, http.StatusBadGateway, "node_dispatch_error", err.Error())
|
|
return
|
|
}
|
|
defer handle.Close()
|
|
|
|
s.logger.Info("openai responses sideband dispatch",
|
|
zap.String("run_id", handle.Dispatch().RunID),
|
|
zap.String("node_id", handle.Dispatch().NodeID),
|
|
zap.String("model_group", handle.Dispatch().ModelGroupKey),
|
|
zap.String("adapter", handle.Dispatch().Adapter),
|
|
zap.String("target", handle.Dispatch().Target),
|
|
zap.Bool("stream", env.Stream),
|
|
zap.Int("estimated_input_tokens", handle.Dispatch().EstimatedInputTokens),
|
|
zap.String("context_class", handle.Dispatch().ContextClass),
|
|
zap.String("queue_reason", handle.Dispatch().QueueReason),
|
|
)
|
|
|
|
if env.Stream {
|
|
s.writeResponsesProviderTunnelSidebandStream(w, r, handle, metricLabels)
|
|
return
|
|
}
|
|
s.writeResponsesProviderTunnelSidebandResponse(w, r, handle, metricLabels)
|
|
}
|
|
|
|
// rewriteResponsesModel replaces only the model field of the caller's original
|
|
// /v1/responses request JSON so the provider receives its served model name.
|
|
// Every other field (input, instructions, tools, max_output_tokens, and any
|
|
// Codex/Responses-specific field) is forwarded without IOP rewriting. An empty
|
|
// target leaves the body untouched; invalid JSON is rejected.
|
|
func rewriteResponsesModel(rawBody []byte, target string) ([]byte, error) {
|
|
if strings.TrimSpace(target) == "" {
|
|
return rawBody, nil
|
|
}
|
|
var raw map[string]json.RawMessage
|
|
if err := json.Unmarshal(rawBody, &raw); err != nil {
|
|
return nil, fmt.Errorf("invalid JSON request")
|
|
}
|
|
modelJSON, err := json.Marshal(target)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
raw["model"] = modelJSON
|
|
return json.Marshal(raw)
|
|
}
|
|
|
|
const responsesSidebandObject = "iop.responses.sideband"
|
|
|
|
type responsesSidebandPayload struct {
|
|
Object string `json:"object"`
|
|
Metadata map[string]any `json:"metadata"`
|
|
}
|
|
|
|
func responsesSidebandMetadata() map[string]any {
|
|
return map[string]any{
|
|
responseModeMetadataKey: responseModePassthroughSideband,
|
|
}
|
|
}
|
|
|
|
func responsesSidebandPayloadBytes() []byte {
|
|
payload, err := json.Marshal(responsesSidebandPayload{
|
|
Object: responsesSidebandObject,
|
|
Metadata: responsesSidebandMetadata(),
|
|
})
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
return payload
|
|
}
|
|
|
|
func injectResponsesSidebandMetadata(body []byte) []byte {
|
|
var obj map[string]json.RawMessage
|
|
if err := json.Unmarshal(body, &obj); err != nil {
|
|
return body
|
|
}
|
|
if obj == nil {
|
|
return body
|
|
}
|
|
|
|
metadata := map[string]any{}
|
|
if raw, ok := obj["metadata"]; ok && len(raw) > 0 && string(raw) != "null" {
|
|
_ = json.Unmarshal(raw, &metadata)
|
|
if metadata == nil {
|
|
metadata = map[string]any{}
|
|
}
|
|
}
|
|
for k, v := range responsesSidebandMetadata() {
|
|
metadata[k] = v
|
|
}
|
|
encodedMetadata, err := json.Marshal(metadata)
|
|
if err != nil {
|
|
return body
|
|
}
|
|
obj["metadata"] = encodedMetadata
|
|
rewritten, err := json.Marshal(obj)
|
|
if err != nil {
|
|
return body
|
|
}
|
|
return rewritten
|
|
}
|
|
|
|
func (s *Server) writeResponsesProviderTunnelSidebandStream(w http.ResponseWriter, r *http.Request, handle edgeservice.ProviderTunnelResult, metricLabels usageLabels) {
|
|
frames := handle.Stream().Frames
|
|
if frames == nil {
|
|
writeError(w, http.StatusBadGateway, "provider_tunnel_error", "tunnel stream unavailable")
|
|
return
|
|
}
|
|
flusher, _ := w.(http.Flusher)
|
|
timer := time.NewTimer(handle.WaitTimeout())
|
|
defer timer.Stop()
|
|
|
|
assembler := &providerChatAssembler{streaming: true}
|
|
wroteHeader := false
|
|
metricStatus := usageStatusError
|
|
var protoObs usageObservation
|
|
tail := ""
|
|
|
|
writeSideband := func() {
|
|
payload := responsesSidebandPayloadBytes()
|
|
if len(payload) == 0 {
|
|
return
|
|
}
|
|
fmt.Fprintf(w, "event: %s\ndata: %s\n\n", sidebandSSEEventName, payload)
|
|
if flusher != nil {
|
|
flusher.Flush()
|
|
}
|
|
}
|
|
|
|
defer func() {
|
|
emitUsageMetrics(metricLabels, metricStatus, mergeUsageObservation(assembler.usageObservation(), protoObs))
|
|
}()
|
|
|
|
for {
|
|
select {
|
|
case <-r.Context().Done():
|
|
s.cancelRunOnHTTPGiveUp(handle.Dispatch(), r.Context().Err())
|
|
metricStatus = usageStatusCancel
|
|
return
|
|
case <-timer.C:
|
|
s.cancelRunOnHTTPGiveUp(handle.Dispatch(), errRunTimedOut)
|
|
metricStatus = usageStatusCancel
|
|
if !wroteHeader {
|
|
writeError(w, http.StatusBadGateway, "run_error", errRunTimedOut.Error())
|
|
}
|
|
return
|
|
case frame, ok := <-frames:
|
|
if !ok {
|
|
if !wroteHeader {
|
|
writeError(w, http.StatusBadGateway, "provider_tunnel_error", "tunnel stream closed before provider response")
|
|
}
|
|
return
|
|
}
|
|
switch frame.GetKind() {
|
|
case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START:
|
|
if wroteHeader {
|
|
continue
|
|
}
|
|
copyProviderResponseHeaders(w.Header(), frame.GetHeaders())
|
|
w.Header().Del("Content-Length")
|
|
w.Header().Set(responseModeHeaderName, responseModePassthroughSideband)
|
|
status := int(frame.GetStatusCode())
|
|
if status == 0 {
|
|
status = http.StatusOK
|
|
}
|
|
w.WriteHeader(status)
|
|
wroteHeader = true
|
|
writeSideband()
|
|
case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY:
|
|
body := frame.GetBody()
|
|
if len(body) == 0 {
|
|
continue
|
|
}
|
|
if !wroteHeader {
|
|
w.Header().Set(responseModeHeaderName, responseModePassthroughSideband)
|
|
w.WriteHeader(http.StatusOK)
|
|
wroteHeader = true
|
|
writeSideband()
|
|
}
|
|
if _, err := w.Write(body); err != nil {
|
|
s.sendCancelRun(handle.Dispatch())
|
|
metricStatus = usageStatusCancel
|
|
return
|
|
}
|
|
assembler.Write(body)
|
|
tail = sseTail(tail, body)
|
|
if flusher != nil {
|
|
flusher.Flush()
|
|
}
|
|
case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_ERROR:
|
|
msg := frame.GetError()
|
|
if msg == "" {
|
|
msg = "provider tunnel failed"
|
|
}
|
|
if !wroteHeader {
|
|
writeError(w, http.StatusBadGateway, "provider_tunnel_error", msg)
|
|
return
|
|
}
|
|
s.logger.Warn("openai responses sideband tunnel error after response start",
|
|
zap.String("run_id", handle.Dispatch().RunID),
|
|
zap.String("error", msg),
|
|
)
|
|
return
|
|
case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_USAGE:
|
|
protoObs = mergeUsageObservation(protoObs, usageObservationFromProtoUsage(frame.GetUsage()))
|
|
case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END:
|
|
if !wroteHeader {
|
|
writeError(w, http.StatusBadGateway, "provider_tunnel_error", "tunnel ended before provider response")
|
|
return
|
|
}
|
|
if tail != "" && !strings.HasSuffix(tail, "\n\n") {
|
|
fmt.Fprint(w, "\n\n")
|
|
if flusher != nil {
|
|
flusher.Flush()
|
|
}
|
|
}
|
|
metricStatus = usageStatusSuccess
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (s *Server) writeResponsesProviderTunnelSidebandResponse(w http.ResponseWriter, r *http.Request, handle edgeservice.ProviderTunnelResult, metricLabels usageLabels) {
|
|
frames := handle.Stream().Frames
|
|
if frames == nil {
|
|
writeError(w, http.StatusBadGateway, "provider_tunnel_error", "tunnel stream unavailable")
|
|
return
|
|
}
|
|
timer := time.NewTimer(handle.WaitTimeout())
|
|
defer timer.Stop()
|
|
|
|
assembler := &providerChatAssembler{}
|
|
var body bytes.Buffer
|
|
providerStatus := 0
|
|
providerHeaders := map[string]string{}
|
|
metricStatus := usageStatusError
|
|
var protoObs usageObservation
|
|
|
|
defer func() {
|
|
// Non-streaming assembler usage is parsed lazily from the buffered JSON
|
|
// body, so force parsing before emitting metrics.
|
|
_ = assembler.observation()
|
|
emitUsageMetrics(metricLabels, metricStatus, mergeUsageObservation(assembler.usageObservation(), protoObs))
|
|
}()
|
|
|
|
for {
|
|
select {
|
|
case <-r.Context().Done():
|
|
s.cancelRunOnHTTPGiveUp(handle.Dispatch(), r.Context().Err())
|
|
metricStatus = usageStatusCancel
|
|
return
|
|
case <-timer.C:
|
|
s.cancelRunOnHTTPGiveUp(handle.Dispatch(), errRunTimedOut)
|
|
metricStatus = usageStatusCancel
|
|
writeError(w, http.StatusBadGateway, "run_error", errRunTimedOut.Error())
|
|
return
|
|
case frame, ok := <-frames:
|
|
if !ok {
|
|
writeError(w, http.StatusBadGateway, "provider_tunnel_error", "tunnel stream closed before provider response")
|
|
return
|
|
}
|
|
switch frame.GetKind() {
|
|
case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START:
|
|
if providerStatus != 0 {
|
|
continue
|
|
}
|
|
providerStatus = int(frame.GetStatusCode())
|
|
if providerStatus == 0 {
|
|
providerStatus = http.StatusOK
|
|
}
|
|
providerHeaders = frame.GetHeaders()
|
|
case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY:
|
|
body.Write(frame.GetBody())
|
|
assembler.Write(frame.GetBody())
|
|
case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_ERROR:
|
|
msg := frame.GetError()
|
|
if msg == "" {
|
|
msg = "provider tunnel failed"
|
|
}
|
|
writeError(w, http.StatusBadGateway, "provider_tunnel_error", msg)
|
|
return
|
|
case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_USAGE:
|
|
protoObs = mergeUsageObservation(protoObs, usageObservationFromProtoUsage(frame.GetUsage()))
|
|
case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END:
|
|
if providerStatus == 0 {
|
|
writeError(w, http.StatusBadGateway, "provider_tunnel_error", "tunnel ended before provider response")
|
|
return
|
|
}
|
|
copyProviderResponseHeaders(w.Header(), providerHeaders)
|
|
w.Header().Del("Content-Length")
|
|
w.Header().Set(responseModeHeaderName, responseModePassthroughSideband)
|
|
w.WriteHeader(providerStatus)
|
|
if _, err := w.Write(injectResponsesSidebandMetadata(body.Bytes())); err != nil {
|
|
s.sendCancelRun(handle.Dispatch())
|
|
metricStatus = usageStatusCancel
|
|
return
|
|
}
|
|
metricStatus = usageStatusSuccess
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (s *Server) completeResponse(w http.ResponseWriter, r *http.Request, req responsesRequest, handle edgeservice.RunResult, outputPolicy strictOutputPolicy) {
|
|
metricLabels := s.usageLabelsFor(r.Context(), strings.TrimSpace(req.Model), usageEndpointResponses, "normalized")
|
|
text, reasoning, _, _, usage, _, err := collectRunResult(r.Context(), handle.Stream(), handle.WaitTimeout())
|
|
if err != nil {
|
|
s.cancelRunOnHTTPGiveUp(handle.Dispatch(), err)
|
|
emitUsageMetrics(metricLabels, usageStatusForError(err), usageObservation{})
|
|
writeError(w, httpStatusForRunError(err), "run_error", err.Error())
|
|
return
|
|
}
|
|
text, reasoning, normalized := normalizeCompletionOutput(outputPolicy, text, reasoning, false)
|
|
s.logger.Info("openai responses output",
|
|
zap.String("run_id", handle.Dispatch().RunID),
|
|
zap.Bool("strict_output", outputPolicy.Strict),
|
|
zap.String("xml_completion_tool", outputPolicy.XMLCompletionTool),
|
|
zap.Bool("normalized", normalized),
|
|
zap.Int("content_len", len(text)),
|
|
zap.Int("reasoning_len", len(reasoning)),
|
|
)
|
|
|
|
var u openAIUsage
|
|
if usage != nil {
|
|
u = *usage
|
|
}
|
|
|
|
emitUsageMetrics(metricLabels, usageStatusSuccess, usageObservationFromOpenAIUsage(usage, len(reasoning)))
|
|
|
|
writeJSON(w, http.StatusOK, responsesResponse{
|
|
ID: "resp-" + handle.Dispatch().RunID,
|
|
Object: "response",
|
|
CreatedAt: time.Now().Unix(),
|
|
Model: responseModel(req.Model, handle.Dispatch().Target),
|
|
OutputText: text,
|
|
Output: []responsesOutputItem{{
|
|
Type: "message",
|
|
Role: "assistant",
|
|
Content: []responsesContentItem{{
|
|
Type: "output_text",
|
|
Text: text,
|
|
}},
|
|
}},
|
|
Usage: u,
|
|
})
|
|
}
|
|
|
|
func parseResponsesInput(raw json.RawMessage) (string, error) {
|
|
if len(raw) == 0 {
|
|
return "", fmt.Errorf("input is required")
|
|
}
|
|
var s string
|
|
if err := json.Unmarshal(raw, &s); err != nil {
|
|
return "", fmt.Errorf("input must be a string")
|
|
}
|
|
if strings.TrimSpace(s) == "" {
|
|
return "", fmt.Errorf("input is required")
|
|
}
|
|
return s, nil
|
|
}
|
|
|
|
func buildResponsesPrompt(instructions, input string) string {
|
|
instructions = strings.TrimSpace(instructions)
|
|
if instructions == "" {
|
|
return input
|
|
}
|
|
return instructions + "\n\n" + input
|
|
}
|
|
|
|
func parseOpenAIMetadata(raw json.RawMessage) (map[string]string, string, error) {
|
|
if len(raw) == 0 || string(raw) == "null" {
|
|
return make(map[string]string), "", nil
|
|
}
|
|
|
|
var rawMap map[string]json.RawMessage
|
|
if err := json.Unmarshal(raw, &rawMap); err != nil {
|
|
return nil, "", fmt.Errorf("metadata must be an object")
|
|
}
|
|
|
|
if len(rawMap) > 16 {
|
|
return nil, "", fmt.Errorf("metadata must contain at most 16 keys")
|
|
}
|
|
|
|
flat := make(map[string]string, len(rawMap))
|
|
var workspace string
|
|
for key, rawValue := range rawMap {
|
|
if len(key) > 64 {
|
|
return nil, "", fmt.Errorf("metadata key %q exceeds 64 characters", key)
|
|
}
|
|
if key == "source" {
|
|
return nil, "", fmt.Errorf("metadata.source is not supported")
|
|
}
|
|
if key == "workspace" {
|
|
workspaceValue, err := metadataStringValue(key, rawValue)
|
|
if err != nil {
|
|
return nil, "", err
|
|
}
|
|
workspace = strings.TrimSpace(workspaceValue)
|
|
continue
|
|
}
|
|
value, err := metadataStringValue(key, rawValue)
|
|
if err != nil {
|
|
return nil, "", err
|
|
}
|
|
flat[key] = value
|
|
}
|
|
|
|
return flat, workspace, nil
|
|
}
|
|
|
|
func metadataStringValue(key string, raw json.RawMessage) (string, error) {
|
|
var value string
|
|
if err := json.Unmarshal(raw, &value); err != nil {
|
|
return "", fmt.Errorf("metadata.%s must be a string", key)
|
|
}
|
|
if len(value) > 512 {
|
|
return "", fmt.Errorf("metadata.%s exceeds 512 characters", key)
|
|
}
|
|
return value, nil
|
|
}
|