package openai import ( "encoding/json" "fmt" "net/http" "os" "strconv" "strings" "time" "unicode" "go.uber.org/zap" edgeservice "iop/apps/edge/internal/service" iop "iop/proto/gen/iop" ) const ( streamTraceMetadataKey = "iop_trace_stream" streamTraceEnvKey = "IOP_OPENAI_COMPAT_TRACE_STREAM" ) func (s *Server) streamChatCompletion(w http.ResponseWriter, r *http.Request, req chatCompletionRequest, submitReq edgeservice.SubmitRunRequest, handle edgeservice.RunResult, outputPolicy strictOutputPolicy, validation toolValidationContract) { flusher, ok := w.(http.Flusher) if !ok { handle.Close() writeError(w, http.StatusInternalServerError, "streaming_not_supported", "response writer does not support streaming") return } w.Header().Set("Content-Type", "text/event-stream") w.Header().Set("Cache-Control", "no-cache") w.Header().Set("Connection", "keep-alive") // Buffered streams collect and validate the full response before emitting // user-visible chunks. Strict buffered output opts in explicitly, and // tool-bearing streams use the same path so malformed tool calls can be // retried or rejected before reaching a client tool runner. if (outputPolicy.Strict && outputPolicy.StreamBuffer) || len(req.Tools) > 0 { s.streamBufferedChatCompletion(w, r, req, submitReq, handle, flusher, outputPolicy, validation) return } // Live SSE may emit content deltas before the terminal event, so runtime // tool validation is excluded upstream; write the role chunk immediately. defer handle.Close() created := time.Now().Unix() id := "chatcmpl-" + handle.Dispatch().RunID model := responseModel(req.Model, handle.Dispatch().Target) traceStream := openAICompatTraceStreamEnabled(submitReq.Metadata) traceSeq := 0 writeTracedContentDelta := func(content, source string, filtered bool) { if content == "" { return } traceSeq++ if traceStream { logOpenAICompatStreamOutput(s.logger, handle.Dispatch().RunID, traceSeq, "content", content, zap.Int("source_len", len(source)), zap.Bool("filtered", filtered), zap.Bool("source_has_open_think", hasOpenThinkTag(source)), zap.Bool("source_has_close_think", hasCloseThinkTag(source)), ) } writeContentDeltaSSE(w, flusher, id, created, model, content) } writeTracedReasoningDelta := func(reasoning string) { if reasoning == "" { return } traceSeq++ if traceStream { logOpenAICompatStreamOutput(s.logger, handle.Dispatch().RunID, traceSeq, "reasoning", reasoning) } writeSSE(w, flusher, chatCompletionChunk{ ID: id, Object: "chat.completion.chunk", Created: created, Model: model, Choices: []chatCompletionChunkChoice{{ Index: 0, Delta: chatDelta{ReasoningContent: reasoning}, }}, }) } writeSSE(w, flusher, chatCompletionChunk{ ID: id, Object: "chat.completion.chunk", Created: created, Model: model, Choices: []chatCompletionChunkChoice{{ Index: 0, Delta: chatDelta{Role: "assistant"}, }}, }) var contentBuilder strings.Builder var reasoningBuilder strings.Builder var emittedContent strings.Builder var toolTextFilter *streamToolTextFilter contentSentinelFilter := &streamSentinelFilter{} reasoningSentinelFilter := &streamSentinelFilter{} if len(req.Tools) > 0 { toolTextFilter = &streamToolTextFilter{} } defer func() { s.logger.Info("openai chat completion stream closed", zap.String("run_id", handle.Dispatch().RunID), zap.Bool("strict_output", outputPolicy.Strict), zap.Bool("strict_stream_buffer", outputPolicy.StreamBuffer), zap.Int("content_len", contentBuilder.Len()), zap.Int("reasoning_len", reasoningBuilder.Len()), ) }() stream := handle.Stream() if stream.Events == nil { writeSSEError(w, flusher, "run stream unavailable") return } processContentDelta := func(sourceDelta string) { if sourceDelta == "" { return } delta := contentSentinelFilter.Append(sourceDelta) if delta == "" { return } contentBuilder.WriteString(delta) filteredDelta := delta if toolTextFilter != nil { filteredDelta = toolTextFilter.Append(filteredDelta) } if filteredDelta == "" { if traceStream { s.logger.Info("openai chat completion stream output suppressed", zap.String("run_id", handle.Dispatch().RunID), zap.String("type", "content"), zap.Int("source_len", len(sourceDelta)), zap.Bool("source_has_open_think", hasOpenThinkTag(sourceDelta)), zap.Bool("source_has_close_think", hasCloseThinkTag(sourceDelta)), ) } return } emittedContent.WriteString(filteredDelta) writeTracedContentDelta(filteredDelta, sourceDelta, filteredDelta != sourceDelta) } for { select { case <-r.Context().Done(): s.cancelRunOnHTTPGiveUp(handle.Dispatch(), r.Context().Err()) return case nodeEvent, ok := <-stream.NodeEvents: if !ok { stream.NodeEvents = nil continue } if edgeservice.IsNodeDisconnected(nodeEvent) { writeSSEError(w, flusher, "node disconnected") return } case event, ok := <-stream.Events: if !ok { writeSSEError(w, flusher, "run stream closed") return } if event == nil { continue } switch event.GetType() { case "delta": processContentDelta(event.GetDelta()) case "reasoning_delta": if event.GetDelta() == "" { continue } reasoning := reasoningSentinelFilter.Append(event.GetDelta()) if reasoning == "" { continue } reasoningBuilder.WriteString(reasoning) if outputPolicy.Strict || !req.includeReasoning() { continue } writeTracedReasoningDelta(reasoning) case "complete": processContentDelta(contentSentinelFilter.Flush()) if reasoning := reasoningSentinelFilter.Flush(); reasoning != "" { reasoningBuilder.WriteString(reasoning) if !outputPolicy.Strict && req.includeReasoning() { writeTracedReasoningDelta(reasoning) } } metadata := event.GetMetadata() finishReason := metadata["finish_reason"] if finishReason == "" { finishReason = "stop" } nativeToolCalls := toolCallsFromRunMetadata(metadata) if len(nativeToolCalls) > 0 { if toolTextFilter != nil { pending := toolTextFilter.FlushNonCandidate() if pending != "" { emittedContent.WriteString(pending) writeTracedContentDelta(pending, pending, false) } } nativeToolCalls = normalizeNativeToolCallsForResponse(nativeToolCalls, req.Tools) if traceStream { s.logger.Info("openai chat completion stream output chunk", zap.String("run_id", handle.Dispatch().RunID), zap.Int("seq", traceSeq+1), zap.String("type", "tool_calls"), zap.Int("tool_call_count", len(nativeToolCalls)), ) } writeToolCallsDeltaSSE(w, flusher, id, created, model, nativeToolCalls) finishReason = "tool_calls" } else if len(req.Tools) > 0 { res := synthesizeToolCallsFromTextResult(contentBuilder.String(), req.Tools, handle.Dispatch().RunID) if res.validationErr != nil { writeSSEErrorWithType(w, flusher, "tool_validation_error", res.validationErr.Error()) return } if len(res.toolCalls) > 0 { if remaining := unstreamedCleanedText(res.cleaned, emittedContent.String()); remaining != "" { emittedContent.WriteString(remaining) writeTracedContentDelta(remaining, remaining, false) } if traceStream { s.logger.Info("openai chat completion stream output chunk", zap.String("run_id", handle.Dispatch().RunID), zap.Int("seq", traceSeq+1), zap.String("type", "tool_calls"), zap.Int("tool_call_count", len(res.toolCalls)), ) } writeToolCallsDeltaSSE(w, flusher, id, created, model, res.toolCalls) finishReason = "tool_calls" } else if !res.candidateFound { if toolTextFilter != nil { if pending := toolTextFilter.Flush(); pending != "" { emittedContent.WriteString(pending) writeTracedContentDelta(pending, pending, false) } } } } else if toolTextFilter != nil { if pending := toolTextFilter.Flush(); pending != "" { emittedContent.WriteString(pending) writeTracedContentDelta(pending, pending, false) } } if !outputPolicy.Strict && strings.TrimSpace(contentBuilder.String()) == "" && strings.TrimSpace(emittedContent.String()) == "" && strings.TrimSpace(reasoningBuilder.String()) != "" && len(nativeToolCalls) == 0 { fallback := hiddenReasoningFallbackContent(finishReason) if req.includeReasoning() { fallback = reasoningOnlyFallbackContent(reasoningBuilder.String(), finishReason) } emittedContent.WriteString(fallback) writeTracedContentDelta(fallback, reasoningBuilder.String(), false) } if traceStream { s.logger.Info("openai chat completion stream output done", zap.String("run_id", handle.Dispatch().RunID), zap.Int("seq", traceSeq+1), zap.String("finish_reason", finishReason), zap.Int("content_len", contentBuilder.Len()), zap.Int("reasoning_len", reasoningBuilder.Len()), ) } writeSSE(w, flusher, chatCompletionChunk{ ID: id, Object: "chat.completion.chunk", Created: created, Model: model, Choices: []chatCompletionChunkChoice{{ Index: 0, Delta: chatDelta{}, FinishReason: finishReason, }}, }) fmt.Fprint(w, "data: [DONE]\n\n") flusher.Flush() return case "error", "cancelled": msg := event.GetError() if msg == "" { msg = event.GetMessage() } if msg == "" { msg = "run failed" } writeSSEError(w, flusher, msg) return } case <-time.After(handle.WaitTimeout()): s.cancelRunOnHTTPGiveUp(handle.Dispatch(), errRunTimedOut) writeSSEError(w, flusher, errRunTimedOut.Error()) return } } } // tunnelChatCompletionPassthrough serves a Chat Completions request over the // raw provider tunnel (SDD S04/S06): provider status, headers, and body bytes // are written to the caller byte-identically. No IOP sideband fields or events // are added to the response body. func (s *Server) tunnelChatCompletionPassthrough(w http.ResponseWriter, r *http.Request, req chatCompletionRequest, dispatch routeDispatch, runMeta map[string]string, rawBody []byte, estimate int, contextClass string) { metadata := make(map[string]string, len(runMeta)+5) for k, v := range runMeta { metadata[k] = v } metadata["openai_model"] = req.Model metadata["openai_stream"] = strconv.FormatBool(req.Stream) metadata[responseModeMetadataKey] = responseModePassthrough metadata["estimated_input_tokens"] = strconv.Itoa(estimate) metadata["context_class"] = contextClass tunnelReq := edgeservice.SubmitProviderTunnelRequest{ NodeRef: dispatch.NodeRef, ModelGroupKey: strings.TrimSpace(req.Model), Adapter: dispatch.Adapter, Target: dispatch.Target, SessionID: dispatch.SessionID, Method: http.MethodPost, Path: "/v1/chat/completions", BuildBody: func(target string) ([]byte, error) { return rewriteChatCompletionModel(rawBody, target, req) }, Stream: req.Stream, TimeoutSec: dispatch.TimeoutSec, MaxQueue: dispatch.MaxQueue, QueueTimeoutMS: dispatch.QueueTimeoutMS, Metadata: metadata, EstimatedInputTokens: estimate, ContextClass: contextClass, ProviderPool: dispatch.ProviderPool, } handle, err := s.service.SubmitProviderTunnel(r.Context(), tunnelReq) if err != nil { writeError(w, http.StatusBadGateway, "node_dispatch_error", err.Error()) return } defer handle.Close() s.logger.Info("openai chat completion passthrough dispatch", zap.String("run_id", handle.Dispatch().RunID), zap.String("node_id", handle.Dispatch().NodeID), zap.String("model_group", handle.Dispatch().ModelGroupKey), zap.String("adapter", handle.Dispatch().Adapter), zap.String("target", handle.Dispatch().Target), zap.Bool("stream", req.Stream), zap.Int("estimated_input_tokens", handle.Dispatch().EstimatedInputTokens), zap.String("context_class", handle.Dispatch().ContextClass), zap.String("queue_reason", handle.Dispatch().QueueReason), ) s.writeProviderTunnelResponse(w, r, handle) } // writeProviderTunnelResponse relays ordered tunnel frames to the HTTP caller. // The response-start frame sets status/headers, body frames are written and // flushed in order, and END terminates the response. Caller disconnect and // wait timeout propagate cancellation to the Node cancel path. func (s *Server) writeProviderTunnelResponse(w http.ResponseWriter, r *http.Request, handle edgeservice.ProviderTunnelResult) { frames := handle.Stream().Frames if frames == nil { writeError(w, http.StatusBadGateway, "provider_tunnel_error", "tunnel stream unavailable") return } flusher, _ := w.(http.Flusher) timer := time.NewTimer(handle.WaitTimeout()) defer timer.Stop() wroteHeader := false bodyBytes := 0 defer func() { s.logger.Info("openai chat completion passthrough closed", zap.String("run_id", handle.Dispatch().RunID), zap.Bool("wrote_header", wroteHeader), zap.Int("body_bytes", bodyBytes), ) }() for { select { case <-r.Context().Done(): s.cancelRunOnHTTPGiveUp(handle.Dispatch(), r.Context().Err()) return case <-timer.C: s.cancelRunOnHTTPGiveUp(handle.Dispatch(), errRunTimedOut) if !wroteHeader { writeError(w, http.StatusBadGateway, "run_error", errRunTimedOut.Error()) } return case frame, ok := <-frames: if !ok { if !wroteHeader { writeError(w, http.StatusBadGateway, "provider_tunnel_error", "tunnel stream closed before provider response") } return } switch frame.GetKind() { case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START: if wroteHeader { continue } copyProviderResponseHeaders(w.Header(), frame.GetHeaders()) status := int(frame.GetStatusCode()) if status == 0 { status = http.StatusOK } w.WriteHeader(status) wroteHeader = true if flusher != nil { flusher.Flush() } case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY: body := frame.GetBody() if len(body) == 0 { continue } if !wroteHeader { // Defensive: a body frame before response-start still reaches the // caller instead of being dropped. w.WriteHeader(http.StatusOK) wroteHeader = true } if _, err := w.Write(body); err != nil { // The caller is gone mid-stream; propagate cancel to the Node. s.sendCancelRun(handle.Dispatch()) return } bodyBytes += len(body) if flusher != nil { flusher.Flush() } case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_ERROR: msg := frame.GetError() if msg == "" { msg = "provider tunnel failed" } if !wroteHeader { writeError(w, http.StatusBadGateway, "provider_tunnel_error", msg) return } // Status/headers are already committed; the response is truncated // and the caller observes the broken stream. s.logger.Warn("openai passthrough tunnel error after response start", zap.String("run_id", handle.Dispatch().RunID), zap.String("error", msg), ) return case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END: if !wroteHeader { writeError(w, http.StatusBadGateway, "provider_tunnel_error", "tunnel ended before provider response") } return case iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_USAGE: // Usage frames are sideband observation candidates; pure passthrough // never merges them into the response body (SDD D04). continue } } } } // hopByHopResponseHeaders are transport-level headers owned by each hop; they // are not copied from the provider response to the caller response. var hopByHopResponseHeaders = map[string]struct{}{ "Connection": {}, "Keep-Alive": {}, "Proxy-Authenticate": {}, "Proxy-Authorization": {}, "Te": {}, "Trailer": {}, "Transfer-Encoding": {}, "Upgrade": {}, } func copyProviderResponseHeaders(dst http.Header, headers map[string]string) { for k, v := range headers { if _, hop := hopByHopResponseHeaders[http.CanonicalHeaderKey(k)]; hop { continue } dst.Set(k, v) } } type streamToolTextFilter struct { pending string } func (f *streamToolTextFilter) Append(delta string) string { f.pending += delta if idx := firstStreamToolTextCandidateIndex(f.pending); idx >= 0 { out := strings.TrimRightFunc(f.pending[:idx], unicode.IsSpace) f.pending = f.pending[idx:] return out } flushLen := streamToolTextSafeFlushLen(f.pending) out := strings.TrimRightFunc(f.pending[:flushLen], unicode.IsSpace) f.pending = f.pending[len(out):] return out } func (f *streamToolTextFilter) Flush() string { out := f.pending f.pending = "" return out } func (f *streamToolTextFilter) FlushNonCandidate() string { if firstStreamToolTextCandidateIndex(f.pending) >= 0 { f.pending = "" return "" } // Even without a full candidate, chunk-boundary protection can leave a // partial candidate suffix (e.g. " 0 { s.logger.Info("openai chat completion stream output chunk", zap.String("run_id", dispatch.RunID), zap.Int("seq", 2), zap.String("type", "tool_calls"), zap.Int("tool_call_count", len(result.toolCalls)), ) } writeToolCallsDeltaSSE(w, flusher, id, created, model, result.toolCalls) if traceStream { s.logger.Info("openai chat completion stream output done", zap.String("run_id", dispatch.RunID), zap.Int("seq", 3), zap.String("finish_reason", result.finishReason), zap.Int("content_len", len(content)), zap.Int("reasoning_len", result.reasoningLen), ) } writeSSE(w, flusher, chatCompletionChunk{ ID: id, Object: "chat.completion.chunk", Created: created, Model: model, Choices: []chatCompletionChunkChoice{{ Index: 0, Delta: chatDelta{}, FinishReason: result.finishReason, }}, }) fmt.Fprint(w, "data: [DONE]\n\n") flusher.Flush() } func writeSSE(w http.ResponseWriter, flusher http.Flusher, v any) { b, err := json.Marshal(v) if err != nil { return } fmt.Fprintf(w, "data: %s\n\n", b) flusher.Flush() } func writeSSEError(w http.ResponseWriter, flusher http.Flusher, message string) { writeSSEErrorWithType(w, flusher, "run_error", message) } func writeSSEErrorWithType(w http.ResponseWriter, flusher http.Flusher, errType, message string) { writeSSE(w, flusher, errorResponse{Error: errorBody{Type: errType, Message: message}}) fmt.Fprint(w, "data: [DONE]\n\n") flusher.Flush() } func openAICompatTraceStreamEnabled(metadata map[string]string) bool { if traceBool(os.Getenv(streamTraceEnvKey)) { return true } if metadata == nil { return false } return traceBool(metadata[streamTraceMetadataKey]) } func traceBool(v string) bool { switch strings.ToLower(strings.TrimSpace(v)) { case "1", "true", "yes", "on": return true default: return false } } func logOpenAICompatStreamOutput(logger *zap.Logger, runID string, seq int, typ string, delta string, extra ...zap.Field) { fields := []zap.Field{ zap.String("run_id", runID), zap.Int("seq", seq), zap.String("type", typ), zap.Int("delta_len", len(delta)), zap.Bool("delta_has_open_think", hasOpenThinkTag(delta)), zap.Bool("delta_has_close_think", hasCloseThinkTag(delta)), } fields = append(fields, extra...) logger.Info("openai chat completion stream output chunk", fields...) } func hasOpenThinkTag(s string) bool { return strings.Contains(strings.ToLower(s), "