package openai import ( "context" "encoding/json" "fmt" "go.uber.org/zap" edgeservice "iop/apps/edge/internal/service" "iop/packages/go/streamgate" "net/http" "os" "strings" "sync" "time" "unicode" ) const ( streamTraceMetadataKey = "iop_trace_stream" streamTraceEnvKey = "IOP_OPENAI_COMPAT_TRACE_STREAM" ) type hotPathChatCodecContextKey struct{} // hotPathChatOuterCodec is the caller-owned Chat wire boundary for one HTTP // turn. Provider protocol decoding stays in the shared Hot Path stage // decoders; this codec sees only the normalized outer-turn accumulator and // renders one Chat response identity, one choice/tool index space, aggregate // usage, and one terminal sequence. type hotPathChatOuterCodec struct { stream bool model string outputCapToken int mu sync.Mutex outer *hotPathOuterTurn rendered bool writer http.ResponseWriter flusher http.Flusher opened bool responseID string created int64 toolIndex map[string]int } func newHotPathChatOuterCodec(stream bool, model string, outputCapToken int) *hotPathChatOuterCodec { if outputCapToken < 0 { outputCapToken = 0 } return &hotPathChatOuterCodec{ stream: stream, model: strings.TrimSpace(model), outputCapToken: outputCapToken, } } func withHotPathChatOuterCodec(r *http.Request, codec *hotPathChatOuterCodec) *http.Request { if r == nil || codec == nil { return r } return r.WithContext(context.WithValue(r.Context(), hotPathChatCodecContextKey{}, codec)) } func hotPathChatOuterCodecFromRequest(r *http.Request) *hotPathChatOuterCodec { if r == nil { return nil } codec, _ := r.Context().Value(hotPathChatCodecContextKey{}).(*hotPathChatOuterCodec) return codec } // callerOuterTurn returns the request-local outer sequencer fixed by the Chat // handler. The first caller supplies the public identity. Later stages in the // same HTTP turn reuse the exact object, so stage changes cannot reset tool // indexes, usage, or the caller cap. func (c *hotPathChatOuterCodec) callerOuterTurn(responseID string, outputCapToken int) *hotPathOuterTurn { if c == nil { return newHotPathCallerCappedOuterTurn(responseID, outputCapToken) } c.mu.Lock() defer c.mu.Unlock() if c.outer == nil { capToken := c.outputCapToken if capToken <= 0 { capToken = outputCapToken } c.outer = newHotPathCallerCappedOuterTurn(responseID, capToken) } return c.outer } func (c *hotPathChatOuterCodec) currentOuterTurn() *hotPathOuterTurn { if c == nil { return nil } c.mu.Lock() defer c.mu.Unlock() return c.outer } // prepareProgressiveWriter attaches the caller writer before an already- // classified Light stage starts. The callback is invoked outside the outer // turn mutex and writes identity-safe text, reasoning, and tool deltas // immediately. The Light outer allocates each caller tool ID before release; // the single finish/usage/[DONE] sequence remains owned by writeResponse. func (c *hotPathChatOuterCodec) prepareProgressiveWriter(w http.ResponseWriter, outer *hotPathOuterTurn) error { if c == nil || !c.stream || outer == nil { return nil } flusher, ok := w.(http.Flusher) if !ok { return fmt.Errorf("response writer does not support flushing") } c.mu.Lock() c.writer = w c.flusher = flusher c.mu.Unlock() return outer.setReleaseCallback(func(delta hotPathReleasedDelta) error { return c.writeProgressiveDelta(outer, delta) }) } func (c *hotPathChatOuterCodec) writeProgressiveDelta(outer *hotPathOuterTurn, delta hotPathReleasedDelta) error { responseID, ok := outer.publicResponseIdentity() if !ok { return fmt.Errorf("Chat outer response is missing provider execution identity") } c.mu.Lock() defer c.mu.Unlock() if c.rendered { return errHotPathTurnTerminal } if err := c.ensureStreamOpenLocked(responseID, 0); err != nil { return err } switch delta.Kind { case streamgate.EventKindReasoningDelta: return c.emitChunkLocked(map[string]any{"reasoning_content": delta.Text}, "", nil) case streamgate.EventKindTextDelta: return c.emitChunkLocked(map[string]any{"content": delta.Text}, "", nil) case streamgate.EventKindToolCallFragment: if c.toolIndex == nil { c.toolIndex = make(map[string]int) } index, exists := c.toolIndex[delta.PublicID] if !exists { index = len(c.toolIndex) c.toolIndex[delta.PublicID] = index } function := map[string]any{"arguments": delta.Args} tool := map[string]any{"index": index, "function": function} if !exists { tool["id"] = delta.PublicID tool["type"] = "function" } if delta.Name != "" { function["name"] = delta.Name } return c.emitChunkLocked(map[string]any{"tool_calls": []any{tool}}, "", nil) default: return fmt.Errorf("unsupported progressive Chat delta kind %q", delta.Kind) } } // hotPathCallerOuterTurn keeps the shared runner endpoint-neutral while each // public endpoint owns its caller codec. func hotPathCallerOuterTurn(r *http.Request, protocol, responseID string, outputCapToken int) *hotPathOuterTurn { if protocol == "openai" { if codec := hotPathChatOuterCodecFromRequest(r); codec != nil { return codec.callerOuterTurn(responseID, outputCapToken) } } if protocol == "anthropic" { if codec := hotPathAnthropicCodecFromRequest(r); codec != nil { return codec.callerOuterTurn(responseID, outputCapToken) } } return newHotPathCallerCappedOuterTurn(responseID, outputCapToken) } // runInitialPresetTurn consumes the result returned by the handler's existing // one-shot provider-pool admission. It never submits or redispatches a selector // attempt. The boolean distinguishes collection failures (no caller response // has been rendered) from shared-turn failures that already own their endpoint // response. func (c *hotPathChatOuterCodec) runInitialPresetTurn( s *Server, w http.ResponseWriter, r *http.Request, dispatch routeDispatch, runMeta map[string]string, result *edgeservice.ProviderPoolDispatchResult, ) (normalizedStageOutput, bool, error) { stage, gate, err := s.collectPresetSelectorResult(r.Context(), dispatch, "openai", result) if err != nil { if contextErr := r.Context().Err(); contextErr != nil { // The active-stage owner already propagated exact cancellation. Mark // the turn as consumed so the handler does not synthesize response // bytes after the caller has gone away. return stage, true, contextErr } return stage, false, err } err = s.dispatchPresetTurn(w, r, dispatch, "openai", c.stream, runMeta, stage, gate) return stage, true, err } func writeHotPathChatOuterResponse(turn *hotPathTurn, output normalizedStageOutput) (bool, error) { if turn == nil { return false, nil } codec := hotPathChatOuterCodecFromRequest(turn.Request) if codec == nil { return false, nil } return true, codec.writeResponse(turn, output) } func writeHotPathChatOuterError( turn *hotPathTurn, status int, errorType, message string, disposition hotPathTerminalDisposition, ) bool { if turn == nil { return false } codec := hotPathChatOuterCodecFromRequest(turn.Request) if codec == nil { return false } _ = codec.writeDisposition(turn.Writer, disposition, status, errorType, message) return true } func (c *hotPathChatOuterCodec) writeResponse(turn *hotPathTurn, output normalizedStageOutput) error { model := c.model if model == "" { model = directPublicModel(turn) } responseID := strings.TrimSpace(output.ResponseID) outer := c.currentOuterTurn() finishReason := openAIDirectFinishReason(output.TerminalReason) if outer != nil { if bound, ok := outer.publicResponseIdentity(); ok { responseID = bound } if disposition, ok := outer.terminalDisposition(); ok { policy := chatHotPathPolicy(disposition) switch { case policy.silent && outer.isTerminalCommitted(): return c.writeDisposition(turn.Writer, disposition, 0, "", "") case policy.errorTerminal && outer.isTerminalCommitted(): return c.writeDisposition( turn.Writer, disposition, policy.status, policy.errorType, disposition.Cause, ) case policy.finishReason != "": finishReason = policy.finishReason } } } if responseID == "" { return fmt.Errorf("Chat outer response is missing provider execution identity") } if finishReason == "" { if len(output.ToolCalls) > 0 { finishReason = "tool_calls" } else { finishReason = "stop" } } if !c.stream { c.mu.Lock() if c.rendered { c.mu.Unlock() return errHotPathTurnTerminal } c.rendered = true c.mu.Unlock() response := map[string]any{ "id": responseID, "object": "chat.completion", "created": output.Created, "model": model, "choices": []any{map[string]any{ "index": 0, "message": openAIDirectMessage(output), "finish_reason": finishReason, }}, } if len(output.Usage) > 0 { response["usage"] = output.Usage } return writeDirectJSON(turn.Writer, http.StatusOK, response) } c.mu.Lock() defer c.mu.Unlock() if c.rendered { return errHotPathTurnTerminal } c.rendered = true c.model = model if c.writer == nil { flusher, ok := turn.Writer.(http.Flusher) if !ok { return fmt.Errorf("response writer does not support flushing") } c.writer = turn.Writer c.flusher = flusher } openedBeforeTerminal := c.opened if err := c.ensureStreamOpenLocked(responseID, output.Created); err != nil { return err } released := []hotPathReleasedDelta(nil) if outer != nil && !output.CallerStageOnly { released = outer.releasedDeltas() } emittedContent, emittedReasoning := false, false emittedTools := make([]bool, len(output.ToolCalls)) toolFragments := hotPathChatToolArgumentFragments(released, output.ToolCalls) toolIndexes := make(map[string]int, len(output.ToolCalls)) nextToolIndex := 0 toolFragmentIndexes := make([]int, len(output.ToolCalls)) for _, delta := range released { switch delta.Kind { case streamgate.EventKindReasoningDelta: if openedBeforeTerminal { continue } emittedReasoning = true if err := c.emitChunkLocked(map[string]any{"reasoning_content": delta.Text}, "", nil); err != nil { return err } case streamgate.EventKindTextDelta: if openedBeforeTerminal { continue } emittedContent = true if err := c.emitChunkLocked(map[string]any{"content": delta.Text}, "", nil); err != nil { return err } case streamgate.EventKindToolCallFragment: if openedBeforeTerminal { continue } index, ok := toolIndexes[delta.PublicID] if !ok { index = nextToolIndex nextToolIndex++ toolIndexes[delta.PublicID] = index } if index >= len(output.ToolCalls) || toolFragments[index] == nil { continue } fragmentIndex := toolFragmentIndexes[index] if fragmentIndex >= len(toolFragments[index]) { continue } call := output.ToolCalls[index] function := map[string]any{"arguments": toolFragments[index][fragmentIndex]} tool := map[string]any{"index": index, "function": function} if fragmentIndex == 0 { tool["id"] = call.ID tool["type"] = "function" function["name"] = call.Name } if err := c.emitChunkLocked(map[string]any{"tool_calls": []any{tool}}, "", nil); err != nil { return err } if c.toolIndex == nil { c.toolIndex = make(map[string]int) } c.toolIndex[call.ID] = index toolFragmentIndexes[index]++ emittedTools[index] = true } } if !openedBeforeTerminal && !emittedReasoning { if output.Reasoning != "" { if err := c.emitChunkLocked(map[string]any{"reasoning_content": output.Reasoning}, "", nil); err != nil { return err } } } if !openedBeforeTerminal && !emittedContent { if output.Content != "" { if err := c.emitChunkLocked(map[string]any{"content": output.Content}, "", nil); err != nil { return err } } } for index, call := range output.ToolCalls { _, progressivelyEmitted := c.toolIndex[call.ID] if emittedTools[index] || progressivelyEmitted { continue } first := map[string]any{ "index": index, "id": call.ID, "type": "function", "function": map[string]any{"name": call.Name, "arguments": directToolArguments(call)}, } if err := c.emitChunkLocked(map[string]any{"tool_calls": []any{first}}, "", nil); err != nil { return err } } if err := c.emitChunkLocked(map[string]any{}, finishReason, output.Usage); err != nil { return err } if _, err := fmt.Fprint(c.writer, "data: [DONE]\n\n"); err != nil { return err } c.flusher.Flush() return nil } func (c *hotPathChatOuterCodec) ensureStreamOpenLocked(responseID string, created int64) error { if c.opened { if c.responseID != responseID { return fmt.Errorf("Chat outer response identity changed after commitment") } return nil } if c.writer == nil || c.flusher == nil { return fmt.Errorf("Chat progressive writer is unavailable") } if created == 0 { created = time.Now().Unix() } c.responseID = responseID c.created = created c.writer.Header().Set("Content-Type", "text/event-stream") c.writer.Header().Set("Cache-Control", "no-cache") c.writer.Header().Set("Connection", "keep-alive") c.writer.WriteHeader(http.StatusOK) c.opened = true return c.emitChunkLocked(map[string]any{"role": "assistant"}, "", nil) } func (c *hotPathChatOuterCodec) emitChunkLocked(delta map[string]any, reason string, usage json.RawMessage) error { choice := map[string]any{"index": 0, "delta": delta, "finish_reason": nil} if reason != "" { choice["finish_reason"] = reason } chunk := map[string]any{ "id": c.responseID, "object": "chat.completion.chunk", "created": c.created, "model": c.model, "choices": []any{choice}, } if len(usage) > 0 { chunk["usage"] = usage } return writeDirectSSEData(c.writer, c.flusher, chunk) } // writeDisposition renders an error or caller cancellation according to the // response commit state. Before commitment, Chat keeps the ordinary JSON // status contract. After the role/delta stream is open, it emits one standard // error envelope as SSE data followed by exactly one [DONE]. Caller // cancellation marks the codec terminal without writing any additional byte. func (c *hotPathChatOuterCodec) writeDisposition( w http.ResponseWriter, disposition hotPathTerminalDisposition, status int, errorType, message string, ) error { if c == nil || w == nil { return fmt.Errorf("Chat Hot Path codec is unavailable") } policy := chatHotPathPolicy(disposition) if policy.status != 0 { status = policy.status } if policy.errorType != "" { errorType = policy.errorType } if strings.TrimSpace(message) == "" { message = hotPathFirstNonEmpty(disposition.Cause, "hot path stage failed") } c.mu.Lock() defer c.mu.Unlock() if c.rendered { return errHotPathTurnTerminal } c.rendered = true if policy.silent { return nil } if !policy.errorTerminal { return fmt.Errorf("Chat disposition %q is not an error terminal", disposition.Kind) } if !c.stream || !c.opened { writeError(w, status, errorType, message) return nil } if c.writer == nil || c.flusher == nil { return fmt.Errorf("Chat progressive writer is unavailable") } if err := writeDirectSSEData(c.writer, c.flusher, errorResponse{ Error: errorBody{Type: errorType, Message: message}, }); err != nil { return err } if _, err := fmt.Fprint(c.writer, "data: [DONE]\n\n"); err != nil { return err } c.flusher.Flush() return nil } // hotPathChatToolArgumentFragments projects the normalized release stream onto // the final mapped tool order. Logical-request mapping may replace public tool // ids after release, so ordering—not an obsolete pre-projection id—is the // stable join key. A mismatch falls back to the final assembled arguments. func hotPathChatToolArgumentFragments(released []hotPathReleasedDelta, calls []normalizedToolCall) [][]string { fragments := make([][]string, len(calls)) if len(calls) == 0 { return fragments } order := make([]string, 0, len(calls)) byID := make(map[string]int, len(calls)) for _, delta := range released { if delta.Kind != streamgate.EventKindToolCallFragment { continue } index, ok := byID[delta.PublicID] if !ok { index = len(order) if index >= len(calls) { continue } byID[delta.PublicID] = index order = append(order, delta.PublicID) } fragments[index] = append(fragments[index], delta.Args) } for index, call := range calls { if strings.Join(fragments[index], "") != directToolArguments(call) { fragments[index] = nil } } return fragments } func (s *Server) streamChatCompletion(w http.ResponseWriter, dc *chatDispatchContext, handle edgeservice.RunResult) { flusher, ok := w.(http.Flusher) if !ok { handle.Close() dc.finishUsageRequest(usageStatusError, responseModeNormalized) writeError(w, http.StatusInternalServerError, "streaming_not_supported", "response writer does not support streaming") return } // Buffered streams collect and validate the full response before emitting // user-visible chunks. Strict buffered output opts in explicitly, and // tool-bearing streams use the same path so malformed tool calls can be // retried or rejected before reaching a client tool runner. if (dc.outputPolicy.Strict && dc.outputPolicy.StreamBuffer) || len(dc.req.Tools) > 0 { s.streamBufferedChatCompletion(w, dc, handle, flusher) return } // The Core request runtime owns response-start/role // staging and commits status/header/role only at first safe release. A // provider-pool dispatch is included: its initial admission result is // handed to the runtime as the initial attempt binding, and every recovery // re-enters SubmitProviderPool through the same request runtime. s.runOpenAIChatStreamGate(w, flusher, dc, handle) } // streamChatCompletionLegacy preserves the stage-level compatibility seam used // by focused tests that construct a dispatch context without the ingress // snapshot required by the request runtime. Production handlers always provide // that snapshot and therefore never enter this helper. func (s *Server) streamChatCompletionLegacy(w http.ResponseWriter, flusher http.Flusher, dc *chatDispatchContext, handle edgeservice.RunResult) { w.Header().Set("Content-Type", "text/event-stream") w.Header().Set("Cache-Control", "no-cache") w.Header().Set("Connection", "keep-alive") defer handle.Close() sess := s.newChatStreamSession(w, flusher, dc.req, dc.submitReq, handle, dc.outputPolicy, dc.usage) sess.writeRole() defer sess.logClosed() stream := handle.Stream() if stream.Events == nil { sess.failStream("run stream unavailable") return } sess.consume(dc.r, stream, handle.WaitTimeout()) } // consume drives the session until it reaches a terminal state: a terminal run // event, a node disconnect, a caller cancel, or the run wait timeout. func (sess *chatStreamSession) consume(r *http.Request, stream edgeservice.RunStream, waitTimeout time.Duration) { for { select { case <-r.Context().Done(): sess.cancel(r.Context().Err(), usageStatusForError(r.Context().Err()), false) return case nodeEvent, ok := <-stream.NodeEvents: if !ok { stream.NodeEvents = nil continue } if edgeservice.IsNodeDisconnected(nodeEvent) { sess.fail("node disconnected") return } case event, ok := <-stream.Events: if !ok { sess.failStream("run stream closed") return } if sess.consumeEvent(event) { return } case <-time.After(waitTimeout): sess.cancel(errRunTimedOut, usageStatusCancel, true) return } } } type streamToolTextFilter struct { pending string } func (f *streamToolTextFilter) Append(delta string) string { f.pending += delta if idx := firstStreamToolTextCandidateIndex(f.pending); idx >= 0 { out := strings.TrimRightFunc(f.pending[:idx], unicode.IsSpace) f.pending = f.pending[idx:] return out } flushLen := streamToolTextSafeFlushLen(f.pending) out := strings.TrimRightFunc(f.pending[:flushLen], unicode.IsSpace) f.pending = f.pending[len(out):] return out } func (f *streamToolTextFilter) Flush() string { out := f.pending f.pending = "" return out } func (f *streamToolTextFilter) FlushNonCandidate() string { if firstStreamToolTextCandidateIndex(f.pending) >= 0 { f.pending = "" return "" } // Even without a full candidate, chunk-boundary protection can leave a // partial candidate suffix (e.g. "