package openai import ( "bytes" "encoding/json" "errors" "fmt" "go.uber.org/zap" "io" edgeservice "iop/apps/edge/internal/service" "net/http" "strconv" "strings" ) func (s *Server) handleResponses(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { writeError(w, http.StatusMethodNotAllowed, "method_not_allowed", "method not allowed") return } defer r.Body.Close() // The raw body is preserved for the provider tunnel passthrough path, which // forwards the caller's own Responses payload (model rewritten to the served // target) without strict normalization. rawBody, err := io.ReadAll(r.Body) if err != nil { writeError(w, http.StatusBadRequest, "invalid_request_error", "failed to read request body") return } // The route decision runs before strict normalization: only the // routing-relevant envelope fields are decoded leniently so provider // passthrough can preserve Codex/Responses unknown fields (max_output_tokens, // tools, store, ...). Strict field validation stays on the normalized path. env, err := decodeResponsesEnvelope(rawBody) if err != nil { writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error()) return } dispatch, ok := s.resolveRouteDispatch(env.Model) if !ok { writeError(w, http.StatusBadRequest, "invalid_request_error", "model is required") return } if dispatch.ProviderPool { // Provider-pool path: the one-shot SubmitProviderPool selects a candidate // and dispatches either tunnel passthrough (OpenAI-compatible) or // normalized RunEvent (Ollama/CLI/native) based on the candidate's // executionPath. Raw body is preserved for the tunnel branch; the // normalized branch performs strict decode + prompt build below. requestCtx, ok := s.newResponsesRequestContext(w, r, dispatch, rawBody, env) if !ok { return } s.handleResponsesProviderPool(w, requestCtx) return } // Non-provider routes: tunnel or normalized. Tunnel routes relay the raw // body verbatim; normalized routes require strict decode + prompt build. if routeUsesProviderTunnel(dispatch) { // Provider routes always relay pure passthrough. Caller metadata is // arbitrary context and never selects the route or response shape // (SDD S01/S04); there is no caller-facing response mode selector. requestCtx, ok := s.newResponsesRequestContext(w, r, dispatch, rawBody, env) if !ok { return } s.tunnelResponsesPassthrough(w, requestCtx) return } // Normalized RunEvent path: strict decode, stream/background rejection, // prompt build, and SubmitRun. var req responsesRequest if err := decodeResponsesRequest(json.NewDecoder(bytes.NewReader(rawBody)), &req); err != nil { writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error()) return } // Keep direct normalized validation ahead of identity resolution, matching // the existing strict Responses contract. The dispatch-context constructor // repeats these local checks when it is invoked from provider-pool PrepareRun. if req.Stream { writeError(w, http.StatusBadRequest, "invalid_request_error", "streaming is not supported for /v1/responses") return } if req.Background { writeError(w, http.StatusBadRequest, "invalid_request_error", "background is not supported for /v1/responses") return } if _, err := parseResponsesInput(req.Input); err != nil { writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error()) return } requestCtx, ok := s.newResponsesRequestContext(w, r, dispatch, rawBody, env) if !ok { return } dc, err := s.newResponsesDispatchContext(requestCtx, req) if err != nil { writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error()) return } s.logger.Info("openai responses input", zap.String("model", dc.req.Model), zap.String("target", dc.route.Target), zap.String("adapter", dc.route.Adapter), zap.Bool("strict_output", dc.outputPolicy.Strict), zap.String("xml_completion_tool", dc.outputPolicy.XMLCompletionTool), zap.Bool("contract_instruction", dc.outputPolicy.ContractInstruction), zap.Int("prompt_len", len(dc.prompt)), ) handle, err := s.service.SubmitRun(r.Context(), dc.submitReq) if err != nil { writeError(w, http.StatusBadGateway, "node_dispatch_error", err.Error()) return } defer handle.Close() s.logger.Info("openai responses dispatch", zap.String("run_id", handle.Dispatch().RunID), zap.String("node_id", handle.Dispatch().NodeID), zap.String("provider_id", handle.Dispatch().ProviderID), zap.String("provider_type", handle.Dispatch().ProviderType), zap.String("execution_path", handle.Dispatch().ExecutionPath), zap.String("model_group", handle.Dispatch().ModelGroupKey), zap.String("adapter", handle.Dispatch().Adapter), zap.String("target", handle.Dispatch().Target), zap.Int("estimated_input_tokens", handle.Dispatch().EstimatedInputTokens), zap.String("context_class", handle.Dispatch().ContextClass), zap.String("queue_reason", handle.Dispatch().QueueReason), ) s.completeResponse(w, dc, handle) } // newResponsesRequestContext resolves the identity, estimate, and long-context // classification a /v1/responses request carries into either dispatch path. It // writes the client error and reports false when the caller's identity or // workspace is rejected. func (s *Server) newResponsesRequestContext(w http.ResponseWriter, r *http.Request, dispatch routeDispatch, rawBody []byte, env responsesEnvelope) (*responsesRequestContext, bool) { runMeta, workspace, err := resolveCallerIdentity(r, dispatch, env.Metadata) if err != nil { writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error()) return nil, false } // The Responses estimate is taken from the raw body so both the tunnel and // the normalized branch carry the same observability and long-context // information (SDD S02/S03). estimate := estimateInputTokens(string(rawBody), runMeta, nil, nil) return &responsesRequestContext{ openAIRequestContext: openAIRequestContext{ r: r, route: dispatch, rawBody: rawBody, callerMetadata: runMeta, workspace: workspace, estimate: estimate, contextClass: classifyContext(estimate, s.longContextThreshold()), endpoint: usageEndpointResponses, }, envelope: env, }, true } // newResponsesDispatchContext freezes the strict-decoded request and the // normalized Run request after ingress routing. Raw tunnel data stays on the // parent responsesRequestContext, and the copied metadata below is the only // mutable execution metadata for this normalized branch. func (s *Server) newResponsesDispatchContext(requestCtx *responsesRequestContext, req responsesRequest) (*responsesDispatchContext, error) { if req.Stream { return nil, fmt.Errorf("streaming is not supported for /v1/responses") } if req.Background { return nil, fmt.Errorf("background is not supported for /v1/responses") } inputStr, err := parseResponsesInput(req.Input) if err != nil { return nil, err } prompt := buildResponsesPrompt(req.Instructions, inputStr) outputPolicy := s.resolveOutputPolicy(prompt) if instruction := strictOutputContractInstruction(outputPolicy); instruction != "" { prompt = instruction + "\n" + prompt } defaultThinkingTokenBudget := 0 if catalogEntry := s.findProviderPoolEntry(req.Model); catalogEntry != nil { applyModelCatalogGenerationPolicyToResponses(&req, *catalogEntry) defaultThinkingTokenBudget = catalogEntry.DefaultThinkingTokenBudget } runMetadata := cloneMetadata(requestCtx.callerMetadata) runMetadata["openai_model"] = req.Model runMetadata["openai_stream"] = fmt.Sprintf("%t", req.Stream) runMetadata["strict_output"] = fmt.Sprintf("%t", outputPolicy.Strict) input := map[string]any{"prompt": prompt} if defaultThinkingTokenBudget > 0 { input["think"] = true input["thinking_token_budget"] = defaultThinkingTokenBudget } else if outputPolicy.Strict { input["think"] = false } if options := req.providerOptions(); len(options) > 0 { input["options"] = options } estimate := estimateInputTokens(prompt, runMetadata, nil, nil) contextClass := classifyContext(estimate, s.longContextThreshold()) runMetadata["estimated_input_tokens"] = strconv.Itoa(estimate) runMetadata["context_class"] = contextClass dc := &responsesDispatchContext{ responsesRequestContext: requestCtx, req: req, prompt: prompt, input: input, outputPolicy: outputPolicy, runMetadata: runMetadata, } dc.submitReq = edgeservice.SubmitRunRequest{ NodeRef: dc.route.NodeRef, ModelGroupKey: strings.TrimSpace(dc.req.Model), Adapter: dc.route.Adapter, Target: dc.route.Target, SessionID: dc.route.SessionID, Workspace: dc.workspace, Prompt: dc.prompt, Input: dc.input, TimeoutSec: dc.route.TimeoutSec, MaxQueue: dc.route.MaxQueue, QueueTimeoutMS: dc.route.QueueTimeoutMS, Metadata: dc.runMetadata, EstimatedInputTokens: estimate, ContextClass: contextClass, ProviderPool: dc.route.ProviderPool, } return dc, nil } // handleResponsesProviderPool dispatches a provider-pool /v1/responses request // through the one-shot SubmitProviderPool surface. After a single queue // admission, the selected candidate's executionPath determines whether the // response is delivered via tunnel passthrough (OpenAI-compatible providers) // or the normalized RunEvent path (Ollama/CLI/native). Provider auth is // validated via providerTunnelAuthHeaders for tunnel-path only (SDD D01). // Strict-output policy only applies to normalized dispatch, not to raw tunnel // passthrough (SDD D02). // // Selection-first flow: the provider is selected by SubmitProviderPool before // any path-specific decode, validation, or stream gating. The tunnel branch // only rewrites the model field and relays the raw body (SDD S04). The // normalized branch performs strict decode + stream/background rejection + // prompt build, but only after the candidate is known. This prevents // tunnel-preferring unknown or streaming requests from bypassing the selected // provider's executionPath rule (SDD S02/S03). func (s *Server) handleResponsesProviderPool(w http.ResponseWriter, requestCtx *responsesRequestContext) { // Build a populated base Run for the normalized branch and a Tunnel for // the tunnel branch. Both are sent to SubmitProviderPool so the one-shot // admission selects the provider before any path-specific behavior. r := requestCtx.r dispatch := requestCtx.route rawBody := requestCtx.rawBody workspace := requestCtx.workspace estimate := requestCtx.estimate contextClass := requestCtx.contextClass env := requestCtx.envelope runMeta := cloneMetadata(requestCtx.callerMetadata) runMeta["openai_model"] = env.Model runMeta["openai_stream"] = strconv.FormatBool(env.Stream) runMeta["estimated_input_tokens"] = strconv.Itoa(estimate) runMeta["context_class"] = contextClass // Base Run carries the model-group key that SubmitProviderPool requires // for candidate resolution. Metadata, estimate, and context class are // preserved by PrepareRun on the normalized branch; the tunnel branch // inherits them from the base so tunnel dispatch keeps observability. baseRun := edgeservice.SubmitRunRequest{ NodeRef: dispatch.NodeRef, ModelGroupKey: strings.TrimSpace(env.Model), SessionID: dispatch.SessionID, Workspace: workspace, Metadata: runMeta, EstimatedInputTokens: estimate, ContextClass: contextClass, ProviderPool: true, } baseTunnel := edgeservice.SubmitProviderTunnelRequest{ Metadata: runMeta, EstimatedInputTokens: estimate, ContextClass: contextClass, ModelGroupKey: strings.TrimSpace(env.Model), SessionID: dispatch.SessionID, Method: http.MethodPost, Path: "/v1/responses", Stream: env.Stream, TimeoutSec: dispatch.TimeoutSec, MaxQueue: dispatch.MaxQueue, QueueTimeoutMS: dispatch.QueueTimeoutMS, ProviderPool: true, } poolReq := edgeservice.ProviderPoolDispatchRequest{ Run: baseRun, Tunnel: baseTunnel, } // Pre-dispatch provider auth header injection. Runs inside SubmitProviderPool // BEFORE buildProviderTunnelRequest and the Node Send step, so the auth // header lands on the actual wire request. On failure the slot is released // and no tunnel request is sent (SDD S03). poolReq.PrepareTunnel = func(tunnelReq edgeservice.SubmitProviderTunnelRequest) (edgeservice.SubmitProviderTunnelRequest, error) { headers, err := s.providerTunnelAuthHeaders(r) if err != nil { return tunnelReq, err } tunnelReq.Headers = headers return tunnelReq, nil } // Tunnel branch: rewrite only the model field, preserve all other fields // (tools, max_output_tokens, custom fields). Stream/background gating is // skipped: the provider itself enforces those constraints (SDD S04). poolReq.Tunnel.BuildBody = func(target string) ([]byte, error) { return rewriteResponsesModel(rawBody, target) } // The normalized context is created only after candidate selection. That // preserves raw passthrough for tunnel candidates while giving the // normalized completion one immutable request/policy/metadata snapshot. var preparedDispatch *responsesDispatchContext // Normalized branch: strict decode, stream/background rejection, prompt // build, and generation policy. Only runs AFTER the candidate is selected // by SubmitProviderPool (i.e. only when executionPath is normalized). // Validation errors are wrapped with errProviderRequestValidation so // SubmitProviderPool can relay them as HTTP 400. poolReq.PrepareRun = func(runReq edgeservice.SubmitRunRequest) (edgeservice.SubmitRunRequest, error) { req := responsesRequest{Model: env.Model} if err := decodeResponsesRequest(json.NewDecoder(bytes.NewReader(rawBody)), &req); err != nil { return edgeservice.SubmitRunRequest{}, fmt.Errorf("%w: invalid /v1/responses request format: %s", errProviderRequestValidation, err.Error()) } dc, err := s.newResponsesDispatchContext(requestCtx, req) if err != nil { return edgeservice.SubmitRunRequest{}, fmt.Errorf("%w: %s", errProviderRequestValidation, err.Error()) } // SubmitProviderPool has already filled the common queue-admission // fields in runReq. Keep those and overlay the normalized context's // body, metadata, and execution values. runReq.Prompt = dc.submitReq.Prompt runReq.Input = dc.submitReq.Input runReq.Metadata = dc.submitReq.Metadata runReq.EstimatedInputTokens = dc.submitReq.EstimatedInputTokens runReq.ContextClass = dc.submitReq.ContextClass runReq.TimeoutSec = dc.submitReq.TimeoutSec runReq.MaxQueue = dc.submitReq.MaxQueue runReq.QueueTimeoutMS = dc.submitReq.QueueTimeoutMS preparedDispatch = dc return runReq, nil } result, err := s.service.SubmitProviderPool(r.Context(), poolReq) if err != nil { metricLabels := s.usageLabelsFor(r.Context(), strings.TrimSpace(env.Model), usageEndpointResponses, responseModePassthrough) emitUsageMetrics(metricLabels, usageStatusForError(err), usageObservation{}) // Provider auth failure is a client request error (400), not a // backend dispatch error. The auth check runs inside // SubmitProviderPool (PrepareTunnel) before any tunnel request is sent. // Normalized validation failures (e.g. strict decode, stream) are // also client errors: map to 400. if errors.Is(err, errProviderAuthRequired) { writeError(w, http.StatusBadRequest, "invalid_request_error", "provider auth token is required") return } if isValidationError(err) { writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error()) return } writeError(w, http.StatusBadGateway, "node_dispatch_error", err.Error()) return } s.logger.Info("openai responses provider-pool dispatch", zap.String("run_id", result.DispatchInfo.RunID), zap.String("node_id", result.DispatchInfo.NodeID), zap.String("provider_id", result.DispatchInfo.ProviderID), zap.String("provider_type", result.DispatchInfo.ProviderType), zap.String("execution_path", result.DispatchInfo.ExecutionPath), zap.String("model_group", result.DispatchInfo.ModelGroupKey), zap.String("adapter", result.DispatchInfo.Adapter), zap.String("target", result.DispatchInfo.Target), zap.String("path", string(result.Path)), zap.Int("estimated_input_tokens", result.DispatchInfo.EstimatedInputTokens), zap.String("context_class", result.DispatchInfo.ContextClass), zap.String("queue_reason", result.DispatchInfo.QueueReason), ) switch result.Path { case edgeservice.ProviderPoolPathTunnel: // Tunnel path: provider auth was already validated and injected via // PrepareTunnel before dispatch; on failure SubmitProviderPool returns // an error and no tunnel handle exists. Provider bytes are relayed as // pure passthrough; caller metadata never selects a sideband surface. metricLabels := s.usageLabelsFor(r.Context(), strings.TrimSpace(env.Model), usageEndpointResponses, responseModePassthrough) s.writeProviderTunnelResponse(w, r, result.Tunnel, env.Stream, env.Model, metricLabels) case edgeservice.ProviderPoolPathNormalized: // Normalized path: no auth required, collect from RunEvent stream. handle := result.Run if handle == nil { writeError(w, http.StatusInternalServerError, "run_error", "provider-pool selection returned normalized path but no run result") return } // PrepareRun is only invoked on the normalized path. A normalized // result without a successful PrepareRun is an internal mismatch and // must surface as run_error (SDD D02). if preparedDispatch == nil { writeError(w, http.StatusInternalServerError, "run_error", "provider-pool normalized result received without a successful PrepareRun") return } // Relay the prepared normalized context so strict-output XML wrapping // and the exact derived metadata survive the provider-pool path. s.completeResponse(w, preparedDispatch, handle) } }