package openai import ( "go.uber.org/zap" edgeservice "iop/apps/edge/internal/service" "net/http" "os" "strings" "time" "unicode" ) const ( streamTraceMetadataKey = "iop_trace_stream" streamTraceEnvKey = "IOP_OPENAI_COMPAT_TRACE_STREAM" ) func (s *Server) streamChatCompletion(w http.ResponseWriter, dc *chatDispatchContext, handle edgeservice.RunResult) { flusher, ok := w.(http.Flusher) if !ok { handle.Close() writeError(w, http.StatusInternalServerError, "streaming_not_supported", "response writer does not support streaming") return } w.Header().Set("Content-Type", "text/event-stream") w.Header().Set("Cache-Control", "no-cache") w.Header().Set("Connection", "keep-alive") // Buffered streams collect and validate the full response before emitting // user-visible chunks. Strict buffered output opts in explicitly, and // tool-bearing streams use the same path so malformed tool calls can be // retried or rejected before reaching a client tool runner. if (dc.outputPolicy.Strict && dc.outputPolicy.StreamBuffer) || len(dc.req.Tools) > 0 { s.streamBufferedChatCompletion(w, dc, handle, flusher) return } // Live SSE may emit content deltas before the terminal event, so runtime // tool validation is excluded upstream; write the role chunk immediately. defer handle.Close() sess := s.newChatStreamSession(w, flusher, dc.req, dc.submitReq, handle, dc.outputPolicy, dc.usageLabels(s, responseModeNormalized)) sess.writeRole() defer sess.logClosed() stream := handle.Stream() if stream.Events == nil { sess.failStream("run stream unavailable") return } sess.consume(dc.r, stream, handle.WaitTimeout()) } // consume drives the session until it reaches a terminal state: a terminal run // event, a node disconnect, a caller cancel, or the run wait timeout. func (sess *chatStreamSession) consume(r *http.Request, stream edgeservice.RunStream, waitTimeout time.Duration) { for { select { case <-r.Context().Done(): sess.cancel(r.Context().Err(), usageStatusForError(r.Context().Err()), false) return case nodeEvent, ok := <-stream.NodeEvents: if !ok { stream.NodeEvents = nil continue } if edgeservice.IsNodeDisconnected(nodeEvent) { sess.fail("node disconnected") return } case event, ok := <-stream.Events: if !ok { sess.failStream("run stream closed") return } if sess.consumeEvent(event) { return } case <-time.After(waitTimeout): sess.cancel(errRunTimedOut, usageStatusCancel, true) return } } } type streamToolTextFilter struct { pending string } func (f *streamToolTextFilter) Append(delta string) string { f.pending += delta if idx := firstStreamToolTextCandidateIndex(f.pending); idx >= 0 { out := strings.TrimRightFunc(f.pending[:idx], unicode.IsSpace) f.pending = f.pending[idx:] return out } flushLen := streamToolTextSafeFlushLen(f.pending) out := strings.TrimRightFunc(f.pending[:flushLen], unicode.IsSpace) f.pending = f.pending[len(out):] return out } func (f *streamToolTextFilter) Flush() string { out := f.pending f.pending = "" return out } func (f *streamToolTextFilter) FlushNonCandidate() string { if firstStreamToolTextCandidateIndex(f.pending) >= 0 { f.pending = "" return "" } // Even without a full candidate, chunk-boundary protection can leave a // partial candidate suffix (e.g. "