753 lines
22 KiB
Go
753 lines
22 KiB
Go
package openai
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"go.uber.org/zap"
|
|
edgeservice "iop/apps/edge/internal/service"
|
|
"iop/packages/go/streamgate"
|
|
"net/http"
|
|
"os"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
"unicode"
|
|
)
|
|
|
|
const (
|
|
streamTraceMetadataKey = "iop_trace_stream"
|
|
streamTraceEnvKey = "IOP_OPENAI_COMPAT_TRACE_STREAM"
|
|
)
|
|
|
|
type hotPathChatCodecContextKey struct{}
|
|
|
|
// hotPathChatOuterCodec is the caller-owned Chat wire boundary for one HTTP
|
|
// turn. Provider protocol decoding stays in the shared Hot Path stage
|
|
// decoders; this codec sees only the normalized outer-turn accumulator and
|
|
// renders one Chat response identity, one choice/tool index space, aggregate
|
|
// usage, and one terminal sequence.
|
|
type hotPathChatOuterCodec struct {
|
|
stream bool
|
|
model string
|
|
outputCapToken int
|
|
|
|
mu sync.Mutex
|
|
outer *hotPathOuterTurn
|
|
rendered bool
|
|
writer http.ResponseWriter
|
|
flusher http.Flusher
|
|
opened bool
|
|
responseID string
|
|
created int64
|
|
toolIndex map[string]int
|
|
}
|
|
|
|
func newHotPathChatOuterCodec(stream bool, model string, outputCapToken int) *hotPathChatOuterCodec {
|
|
if outputCapToken < 0 {
|
|
outputCapToken = 0
|
|
}
|
|
return &hotPathChatOuterCodec{
|
|
stream: stream,
|
|
model: strings.TrimSpace(model),
|
|
outputCapToken: outputCapToken,
|
|
}
|
|
}
|
|
|
|
func withHotPathChatOuterCodec(r *http.Request, codec *hotPathChatOuterCodec) *http.Request {
|
|
if r == nil || codec == nil {
|
|
return r
|
|
}
|
|
return r.WithContext(context.WithValue(r.Context(), hotPathChatCodecContextKey{}, codec))
|
|
}
|
|
|
|
func hotPathChatOuterCodecFromRequest(r *http.Request) *hotPathChatOuterCodec {
|
|
if r == nil {
|
|
return nil
|
|
}
|
|
codec, _ := r.Context().Value(hotPathChatCodecContextKey{}).(*hotPathChatOuterCodec)
|
|
return codec
|
|
}
|
|
|
|
// callerOuterTurn returns the request-local outer sequencer fixed by the Chat
|
|
// handler. The first caller supplies the public identity. Later stages in the
|
|
// same HTTP turn reuse the exact object, so stage changes cannot reset tool
|
|
// indexes, usage, or the caller cap.
|
|
func (c *hotPathChatOuterCodec) callerOuterTurn(responseID string, outputCapToken int) *hotPathOuterTurn {
|
|
if c == nil {
|
|
return newHotPathCallerCappedOuterTurn(responseID, outputCapToken)
|
|
}
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
if c.outer == nil {
|
|
capToken := c.outputCapToken
|
|
if capToken <= 0 {
|
|
capToken = outputCapToken
|
|
}
|
|
c.outer = newHotPathCallerCappedOuterTurn(responseID, capToken)
|
|
}
|
|
return c.outer
|
|
}
|
|
|
|
func (c *hotPathChatOuterCodec) currentOuterTurn() *hotPathOuterTurn {
|
|
if c == nil {
|
|
return nil
|
|
}
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
return c.outer
|
|
}
|
|
|
|
// prepareProgressiveWriter attaches the caller writer before an already-
|
|
// classified Light stage starts. The callback is invoked outside the outer
|
|
// turn mutex and writes identity-safe text, reasoning, and tool deltas
|
|
// immediately. The Light outer allocates each caller tool ID before release;
|
|
// the single finish/usage/[DONE] sequence remains owned by writeResponse.
|
|
func (c *hotPathChatOuterCodec) prepareProgressiveWriter(w http.ResponseWriter, outer *hotPathOuterTurn) error {
|
|
if c == nil || !c.stream || outer == nil {
|
|
return nil
|
|
}
|
|
flusher, ok := w.(http.Flusher)
|
|
if !ok {
|
|
return fmt.Errorf("response writer does not support flushing")
|
|
}
|
|
c.mu.Lock()
|
|
c.writer = w
|
|
c.flusher = flusher
|
|
c.mu.Unlock()
|
|
return outer.setReleaseCallback(func(delta hotPathReleasedDelta) error {
|
|
return c.writeProgressiveDelta(outer, delta)
|
|
})
|
|
}
|
|
|
|
func (c *hotPathChatOuterCodec) writeProgressiveDelta(outer *hotPathOuterTurn, delta hotPathReleasedDelta) error {
|
|
responseID, ok := outer.publicResponseIdentity()
|
|
if !ok {
|
|
return fmt.Errorf("Chat outer response is missing provider execution identity")
|
|
}
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
if c.rendered {
|
|
return errHotPathTurnTerminal
|
|
}
|
|
if err := c.ensureStreamOpenLocked(responseID, 0); err != nil {
|
|
return err
|
|
}
|
|
switch delta.Kind {
|
|
case streamgate.EventKindReasoningDelta:
|
|
return c.emitChunkLocked(map[string]any{"reasoning_content": delta.Text}, "", nil)
|
|
case streamgate.EventKindTextDelta:
|
|
return c.emitChunkLocked(map[string]any{"content": delta.Text}, "", nil)
|
|
case streamgate.EventKindToolCallFragment:
|
|
if c.toolIndex == nil {
|
|
c.toolIndex = make(map[string]int)
|
|
}
|
|
index, exists := c.toolIndex[delta.PublicID]
|
|
if !exists {
|
|
index = len(c.toolIndex)
|
|
c.toolIndex[delta.PublicID] = index
|
|
}
|
|
function := map[string]any{"arguments": delta.Args}
|
|
tool := map[string]any{"index": index, "function": function}
|
|
if !exists {
|
|
tool["id"] = delta.PublicID
|
|
tool["type"] = "function"
|
|
}
|
|
if delta.Name != "" {
|
|
function["name"] = delta.Name
|
|
}
|
|
return c.emitChunkLocked(map[string]any{"tool_calls": []any{tool}}, "", nil)
|
|
default:
|
|
return fmt.Errorf("unsupported progressive Chat delta kind %q", delta.Kind)
|
|
}
|
|
}
|
|
|
|
// hotPathCallerOuterTurn keeps the shared runner endpoint-neutral while each
|
|
// public endpoint owns its caller codec.
|
|
func hotPathCallerOuterTurn(r *http.Request, protocol, responseID string, outputCapToken int) *hotPathOuterTurn {
|
|
if protocol == "openai" {
|
|
if codec := hotPathChatOuterCodecFromRequest(r); codec != nil {
|
|
return codec.callerOuterTurn(responseID, outputCapToken)
|
|
}
|
|
}
|
|
if protocol == "anthropic" {
|
|
if codec := hotPathAnthropicCodecFromRequest(r); codec != nil {
|
|
return codec.callerOuterTurn(responseID, outputCapToken)
|
|
}
|
|
}
|
|
return newHotPathCallerCappedOuterTurn(responseID, outputCapToken)
|
|
}
|
|
|
|
// runInitialPresetTurn consumes the result returned by the handler's existing
|
|
// one-shot provider-pool admission. It never submits or redispatches a selector
|
|
// attempt. The boolean distinguishes collection failures (no caller response
|
|
// has been rendered) from shared-turn failures that already own their endpoint
|
|
// response.
|
|
func (c *hotPathChatOuterCodec) runInitialPresetTurn(
|
|
s *Server,
|
|
w http.ResponseWriter,
|
|
r *http.Request,
|
|
dispatch routeDispatch,
|
|
runMeta map[string]string,
|
|
result *edgeservice.ProviderPoolDispatchResult,
|
|
) (normalizedStageOutput, bool, error) {
|
|
stage, gate, err := s.collectPresetSelectorResult(r.Context(), dispatch, "openai", result)
|
|
if err != nil {
|
|
if contextErr := r.Context().Err(); contextErr != nil {
|
|
// The active-stage owner already propagated exact cancellation. Mark
|
|
// the turn as consumed so the handler does not synthesize response
|
|
// bytes after the caller has gone away.
|
|
return stage, true, contextErr
|
|
}
|
|
return stage, false, err
|
|
}
|
|
err = s.dispatchPresetTurn(w, r, dispatch, "openai", c.stream, runMeta, stage, gate)
|
|
return stage, true, err
|
|
}
|
|
|
|
func writeHotPathChatOuterResponse(turn *hotPathTurn, output normalizedStageOutput) (bool, error) {
|
|
if turn == nil {
|
|
return false, nil
|
|
}
|
|
codec := hotPathChatOuterCodecFromRequest(turn.Request)
|
|
if codec == nil {
|
|
return false, nil
|
|
}
|
|
return true, codec.writeResponse(turn, output)
|
|
}
|
|
|
|
func writeHotPathChatOuterError(
|
|
turn *hotPathTurn,
|
|
status int,
|
|
errorType, message string,
|
|
disposition hotPathTerminalDisposition,
|
|
) bool {
|
|
if turn == nil {
|
|
return false
|
|
}
|
|
codec := hotPathChatOuterCodecFromRequest(turn.Request)
|
|
if codec == nil {
|
|
return false
|
|
}
|
|
_ = codec.writeDisposition(turn.Writer, disposition, status, errorType, message)
|
|
return true
|
|
}
|
|
|
|
func (c *hotPathChatOuterCodec) writeResponse(turn *hotPathTurn, output normalizedStageOutput) error {
|
|
model := c.model
|
|
if model == "" {
|
|
model = directPublicModel(turn)
|
|
}
|
|
responseID := strings.TrimSpace(output.ResponseID)
|
|
outer := c.currentOuterTurn()
|
|
finishReason := openAIDirectFinishReason(output.TerminalReason)
|
|
if outer != nil {
|
|
if bound, ok := outer.publicResponseIdentity(); ok {
|
|
responseID = bound
|
|
}
|
|
if disposition, ok := outer.terminalDisposition(); ok {
|
|
policy := chatHotPathPolicy(disposition)
|
|
switch {
|
|
case policy.silent && outer.isTerminalCommitted():
|
|
return c.writeDisposition(turn.Writer, disposition, 0, "", "")
|
|
case policy.errorTerminal && outer.isTerminalCommitted():
|
|
return c.writeDisposition(
|
|
turn.Writer, disposition, policy.status, policy.errorType, disposition.Cause,
|
|
)
|
|
case policy.finishReason != "":
|
|
finishReason = policy.finishReason
|
|
}
|
|
}
|
|
}
|
|
if responseID == "" {
|
|
return fmt.Errorf("Chat outer response is missing provider execution identity")
|
|
}
|
|
|
|
if finishReason == "" {
|
|
if len(output.ToolCalls) > 0 {
|
|
finishReason = "tool_calls"
|
|
} else {
|
|
finishReason = "stop"
|
|
}
|
|
}
|
|
|
|
if !c.stream {
|
|
c.mu.Lock()
|
|
if c.rendered {
|
|
c.mu.Unlock()
|
|
return errHotPathTurnTerminal
|
|
}
|
|
c.rendered = true
|
|
c.mu.Unlock()
|
|
response := map[string]any{
|
|
"id": responseID, "object": "chat.completion", "created": output.Created, "model": model,
|
|
"choices": []any{map[string]any{
|
|
"index": 0, "message": openAIDirectMessage(output), "finish_reason": finishReason,
|
|
}},
|
|
}
|
|
if len(output.Usage) > 0 {
|
|
response["usage"] = output.Usage
|
|
}
|
|
return writeDirectJSON(turn.Writer, http.StatusOK, response)
|
|
}
|
|
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
if c.rendered {
|
|
return errHotPathTurnTerminal
|
|
}
|
|
c.rendered = true
|
|
c.model = model
|
|
if c.writer == nil {
|
|
flusher, ok := turn.Writer.(http.Flusher)
|
|
if !ok {
|
|
return fmt.Errorf("response writer does not support flushing")
|
|
}
|
|
c.writer = turn.Writer
|
|
c.flusher = flusher
|
|
}
|
|
openedBeforeTerminal := c.opened
|
|
if err := c.ensureStreamOpenLocked(responseID, output.Created); err != nil {
|
|
return err
|
|
}
|
|
released := []hotPathReleasedDelta(nil)
|
|
if outer != nil && !output.CallerStageOnly {
|
|
released = outer.releasedDeltas()
|
|
}
|
|
emittedContent, emittedReasoning := false, false
|
|
emittedTools := make([]bool, len(output.ToolCalls))
|
|
toolFragments := hotPathChatToolArgumentFragments(released, output.ToolCalls)
|
|
toolIndexes := make(map[string]int, len(output.ToolCalls))
|
|
nextToolIndex := 0
|
|
toolFragmentIndexes := make([]int, len(output.ToolCalls))
|
|
for _, delta := range released {
|
|
switch delta.Kind {
|
|
case streamgate.EventKindReasoningDelta:
|
|
if openedBeforeTerminal {
|
|
continue
|
|
}
|
|
emittedReasoning = true
|
|
if err := c.emitChunkLocked(map[string]any{"reasoning_content": delta.Text}, "", nil); err != nil {
|
|
return err
|
|
}
|
|
case streamgate.EventKindTextDelta:
|
|
if openedBeforeTerminal {
|
|
continue
|
|
}
|
|
emittedContent = true
|
|
if err := c.emitChunkLocked(map[string]any{"content": delta.Text}, "", nil); err != nil {
|
|
return err
|
|
}
|
|
case streamgate.EventKindToolCallFragment:
|
|
if openedBeforeTerminal {
|
|
continue
|
|
}
|
|
index, ok := toolIndexes[delta.PublicID]
|
|
if !ok {
|
|
index = nextToolIndex
|
|
nextToolIndex++
|
|
toolIndexes[delta.PublicID] = index
|
|
}
|
|
if index >= len(output.ToolCalls) || toolFragments[index] == nil {
|
|
continue
|
|
}
|
|
fragmentIndex := toolFragmentIndexes[index]
|
|
if fragmentIndex >= len(toolFragments[index]) {
|
|
continue
|
|
}
|
|
call := output.ToolCalls[index]
|
|
function := map[string]any{"arguments": toolFragments[index][fragmentIndex]}
|
|
tool := map[string]any{"index": index, "function": function}
|
|
if fragmentIndex == 0 {
|
|
tool["id"] = call.ID
|
|
tool["type"] = "function"
|
|
function["name"] = call.Name
|
|
}
|
|
if err := c.emitChunkLocked(map[string]any{"tool_calls": []any{tool}}, "", nil); err != nil {
|
|
return err
|
|
}
|
|
if c.toolIndex == nil {
|
|
c.toolIndex = make(map[string]int)
|
|
}
|
|
c.toolIndex[call.ID] = index
|
|
toolFragmentIndexes[index]++
|
|
emittedTools[index] = true
|
|
}
|
|
}
|
|
if !openedBeforeTerminal && !emittedReasoning {
|
|
if output.Reasoning != "" {
|
|
if err := c.emitChunkLocked(map[string]any{"reasoning_content": output.Reasoning}, "", nil); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
if !openedBeforeTerminal && !emittedContent {
|
|
if output.Content != "" {
|
|
if err := c.emitChunkLocked(map[string]any{"content": output.Content}, "", nil); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
|
|
for index, call := range output.ToolCalls {
|
|
_, progressivelyEmitted := c.toolIndex[call.ID]
|
|
if emittedTools[index] || progressivelyEmitted {
|
|
continue
|
|
}
|
|
first := map[string]any{
|
|
"index": index, "id": call.ID, "type": "function",
|
|
"function": map[string]any{"name": call.Name, "arguments": directToolArguments(call)},
|
|
}
|
|
if err := c.emitChunkLocked(map[string]any{"tool_calls": []any{first}}, "", nil); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if err := c.emitChunkLocked(map[string]any{}, finishReason, output.Usage); err != nil {
|
|
return err
|
|
}
|
|
if _, err := fmt.Fprint(c.writer, "data: [DONE]\n\n"); err != nil {
|
|
return err
|
|
}
|
|
c.flusher.Flush()
|
|
return nil
|
|
}
|
|
|
|
func (c *hotPathChatOuterCodec) ensureStreamOpenLocked(responseID string, created int64) error {
|
|
if c.opened {
|
|
if c.responseID != responseID {
|
|
return fmt.Errorf("Chat outer response identity changed after commitment")
|
|
}
|
|
return nil
|
|
}
|
|
if c.writer == nil || c.flusher == nil {
|
|
return fmt.Errorf("Chat progressive writer is unavailable")
|
|
}
|
|
if created == 0 {
|
|
created = time.Now().Unix()
|
|
}
|
|
c.responseID = responseID
|
|
c.created = created
|
|
c.writer.Header().Set("Content-Type", "text/event-stream")
|
|
c.writer.Header().Set("Cache-Control", "no-cache")
|
|
c.writer.Header().Set("Connection", "keep-alive")
|
|
c.writer.WriteHeader(http.StatusOK)
|
|
c.opened = true
|
|
return c.emitChunkLocked(map[string]any{"role": "assistant"}, "", nil)
|
|
}
|
|
|
|
func (c *hotPathChatOuterCodec) emitChunkLocked(delta map[string]any, reason string, usage json.RawMessage) error {
|
|
choice := map[string]any{"index": 0, "delta": delta, "finish_reason": nil}
|
|
if reason != "" {
|
|
choice["finish_reason"] = reason
|
|
}
|
|
chunk := map[string]any{
|
|
"id": c.responseID, "object": "chat.completion.chunk", "created": c.created,
|
|
"model": c.model, "choices": []any{choice},
|
|
}
|
|
if len(usage) > 0 {
|
|
chunk["usage"] = usage
|
|
}
|
|
return writeDirectSSEData(c.writer, c.flusher, chunk)
|
|
}
|
|
|
|
// writeDisposition renders an error or caller cancellation according to the
|
|
// response commit state. Before commitment, Chat keeps the ordinary JSON
|
|
// status contract. After the role/delta stream is open, it emits one standard
|
|
// error envelope as SSE data followed by exactly one [DONE]. Caller
|
|
// cancellation marks the codec terminal without writing any additional byte.
|
|
func (c *hotPathChatOuterCodec) writeDisposition(
|
|
w http.ResponseWriter,
|
|
disposition hotPathTerminalDisposition,
|
|
status int,
|
|
errorType, message string,
|
|
) error {
|
|
if c == nil || w == nil {
|
|
return fmt.Errorf("Chat Hot Path codec is unavailable")
|
|
}
|
|
policy := chatHotPathPolicy(disposition)
|
|
if policy.status != 0 {
|
|
status = policy.status
|
|
}
|
|
if policy.errorType != "" {
|
|
errorType = policy.errorType
|
|
}
|
|
if strings.TrimSpace(message) == "" {
|
|
message = hotPathFirstNonEmpty(disposition.Cause, "hot path stage failed")
|
|
}
|
|
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
if c.rendered {
|
|
return errHotPathTurnTerminal
|
|
}
|
|
c.rendered = true
|
|
if policy.silent {
|
|
return nil
|
|
}
|
|
if !policy.errorTerminal {
|
|
return fmt.Errorf("Chat disposition %q is not an error terminal", disposition.Kind)
|
|
}
|
|
if !c.stream || !c.opened {
|
|
writeError(w, status, errorType, message)
|
|
return nil
|
|
}
|
|
if c.writer == nil || c.flusher == nil {
|
|
return fmt.Errorf("Chat progressive writer is unavailable")
|
|
}
|
|
if err := writeDirectSSEData(c.writer, c.flusher, errorResponse{
|
|
Error: errorBody{Type: errorType, Message: message},
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
if _, err := fmt.Fprint(c.writer, "data: [DONE]\n\n"); err != nil {
|
|
return err
|
|
}
|
|
c.flusher.Flush()
|
|
return nil
|
|
}
|
|
|
|
// hotPathChatToolArgumentFragments projects the normalized release stream onto
|
|
// the final mapped tool order. Logical-request mapping may replace public tool
|
|
// ids after release, so ordering—not an obsolete pre-projection id—is the
|
|
// stable join key. A mismatch falls back to the final assembled arguments.
|
|
func hotPathChatToolArgumentFragments(released []hotPathReleasedDelta, calls []normalizedToolCall) [][]string {
|
|
fragments := make([][]string, len(calls))
|
|
if len(calls) == 0 {
|
|
return fragments
|
|
}
|
|
order := make([]string, 0, len(calls))
|
|
byID := make(map[string]int, len(calls))
|
|
for _, delta := range released {
|
|
if delta.Kind != streamgate.EventKindToolCallFragment {
|
|
continue
|
|
}
|
|
index, ok := byID[delta.PublicID]
|
|
if !ok {
|
|
index = len(order)
|
|
if index >= len(calls) {
|
|
continue
|
|
}
|
|
byID[delta.PublicID] = index
|
|
order = append(order, delta.PublicID)
|
|
}
|
|
fragments[index] = append(fragments[index], delta.Args)
|
|
}
|
|
for index, call := range calls {
|
|
if strings.Join(fragments[index], "") != directToolArguments(call) {
|
|
fragments[index] = nil
|
|
}
|
|
}
|
|
return fragments
|
|
}
|
|
|
|
func (s *Server) streamChatCompletion(w http.ResponseWriter, dc *chatDispatchContext, handle edgeservice.RunResult) {
|
|
flusher, ok := w.(http.Flusher)
|
|
if !ok {
|
|
handle.Close()
|
|
dc.finishUsageRequest(usageStatusError, responseModeNormalized)
|
|
writeError(w, http.StatusInternalServerError, "streaming_not_supported", "response writer does not support streaming")
|
|
return
|
|
}
|
|
|
|
// Buffered streams collect and validate the full response before emitting
|
|
// user-visible chunks. Strict buffered output opts in explicitly, and
|
|
// tool-bearing streams use the same path so malformed tool calls can be
|
|
// retried or rejected before reaching a client tool runner.
|
|
if (dc.outputPolicy.Strict && dc.outputPolicy.StreamBuffer) || len(dc.req.Tools) > 0 {
|
|
s.streamBufferedChatCompletion(w, dc, handle, flusher)
|
|
return
|
|
}
|
|
|
|
// Runtime-enabled: the Core request runtime owns response-start/role
|
|
// staging and commits status/header/role only at first safe release. A
|
|
// provider-pool dispatch is included: its initial admission result is
|
|
// handed to the runtime as the initial attempt binding, and every recovery
|
|
// re-enters SubmitProviderPool through the same request runtime.
|
|
if s.streamGateEnabled() {
|
|
s.runOpenAIChatStreamGate(w, flusher, dc, handle)
|
|
return
|
|
}
|
|
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
w.Header().Set("Cache-Control", "no-cache")
|
|
w.Header().Set("Connection", "keep-alive")
|
|
|
|
// Live SSE may emit content deltas before the terminal event, so runtime
|
|
// tool validation is excluded upstream; write the role chunk immediately.
|
|
defer handle.Close()
|
|
sess := s.newChatStreamSession(w, flusher, dc.req, dc.submitReq, handle, dc.outputPolicy, dc.usage)
|
|
sess.writeRole()
|
|
defer sess.logClosed()
|
|
|
|
stream := handle.Stream()
|
|
if stream.Events == nil {
|
|
sess.failStream("run stream unavailable")
|
|
return
|
|
}
|
|
sess.consume(dc.r, stream, handle.WaitTimeout())
|
|
}
|
|
|
|
// consume drives the session until it reaches a terminal state: a terminal run
|
|
// event, a node disconnect, a caller cancel, or the run wait timeout.
|
|
func (sess *chatStreamSession) consume(r *http.Request, stream edgeservice.RunStream, waitTimeout time.Duration) {
|
|
for {
|
|
select {
|
|
case <-r.Context().Done():
|
|
sess.cancel(r.Context().Err(), usageStatusForError(r.Context().Err()), false)
|
|
return
|
|
case nodeEvent, ok := <-stream.NodeEvents:
|
|
if !ok {
|
|
stream.NodeEvents = nil
|
|
continue
|
|
}
|
|
if edgeservice.IsNodeDisconnected(nodeEvent) {
|
|
sess.fail("node disconnected")
|
|
return
|
|
}
|
|
case event, ok := <-stream.Events:
|
|
if !ok {
|
|
sess.failStream("run stream closed")
|
|
return
|
|
}
|
|
if sess.consumeEvent(event) {
|
|
return
|
|
}
|
|
case <-time.After(waitTimeout):
|
|
sess.cancel(errRunTimedOut, usageStatusCancel, true)
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
type streamToolTextFilter struct {
|
|
pending string
|
|
}
|
|
|
|
func (f *streamToolTextFilter) Append(delta string) string {
|
|
f.pending += delta
|
|
if idx := firstStreamToolTextCandidateIndex(f.pending); idx >= 0 {
|
|
out := strings.TrimRightFunc(f.pending[:idx], unicode.IsSpace)
|
|
f.pending = f.pending[idx:]
|
|
return out
|
|
}
|
|
flushLen := streamToolTextSafeFlushLen(f.pending)
|
|
out := strings.TrimRightFunc(f.pending[:flushLen], unicode.IsSpace)
|
|
f.pending = f.pending[len(out):]
|
|
return out
|
|
}
|
|
|
|
func (f *streamToolTextFilter) Flush() string {
|
|
out := f.pending
|
|
f.pending = ""
|
|
return out
|
|
}
|
|
|
|
func (f *streamToolTextFilter) FlushNonCandidate() string {
|
|
if firstStreamToolTextCandidateIndex(f.pending) >= 0 {
|
|
f.pending = ""
|
|
return ""
|
|
}
|
|
// Even without a full candidate, chunk-boundary protection can leave a
|
|
// partial candidate suffix (e.g. "<tool_ca", trailing "{") in pending.
|
|
// Return only the safe non-candidate prefix and drop the partial suffix so
|
|
// a native tool_calls completion never flushes a raw candidate fragment.
|
|
flushLen := streamToolTextSafeFlushLen(f.pending)
|
|
out := strings.TrimRightFunc(f.pending[:flushLen], unicode.IsSpace)
|
|
f.pending = ""
|
|
return out
|
|
}
|
|
|
|
func firstStreamToolTextCandidateIndex(s string) int {
|
|
xml := strings.Index(strings.ToLower(s), "<tool_call")
|
|
mustache := strings.Index(s, textMustacheToolCallOpenHint)
|
|
switch {
|
|
case xml < 0:
|
|
return mustache
|
|
case mustache < 0:
|
|
return xml
|
|
case xml < mustache:
|
|
return xml
|
|
default:
|
|
return mustache
|
|
}
|
|
}
|
|
|
|
func streamToolTextSafeFlushLen(s string) int {
|
|
if s == "" {
|
|
return 0
|
|
}
|
|
const xmlHint = "<tool_call"
|
|
lower := strings.ToLower(s)
|
|
start := len(s) - len(xmlHint) + 1
|
|
if start < 0 {
|
|
start = 0
|
|
}
|
|
for i := start; i < len(s); i++ {
|
|
if strings.HasPrefix(xmlHint, lower[i:]) ||
|
|
strings.HasPrefix(textMustacheToolCallOpenHint, s[i:]) {
|
|
return i
|
|
}
|
|
}
|
|
return len(s)
|
|
}
|
|
|
|
func unstreamedCleanedText(cleaned, emitted string) string {
|
|
if cleaned == "" {
|
|
return ""
|
|
}
|
|
if emitted == "" {
|
|
return cleaned
|
|
}
|
|
if strings.HasPrefix(cleaned, emitted) {
|
|
return cleaned[len(emitted):]
|
|
}
|
|
cleaned = strings.TrimSpace(cleaned)
|
|
emitted = strings.TrimSpace(emitted)
|
|
if cleaned == "" || cleaned == emitted {
|
|
return ""
|
|
}
|
|
if emitted != "" && strings.HasPrefix(cleaned, emitted) {
|
|
return strings.TrimSpace(strings.TrimPrefix(cleaned, emitted))
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func openAICompatTraceStreamEnabled(metadata map[string]string) bool {
|
|
if traceBool(os.Getenv(streamTraceEnvKey)) {
|
|
return true
|
|
}
|
|
if metadata == nil {
|
|
return false
|
|
}
|
|
return traceBool(metadata[streamTraceMetadataKey])
|
|
}
|
|
|
|
func traceBool(v string) bool {
|
|
switch strings.ToLower(strings.TrimSpace(v)) {
|
|
case "1", "true", "yes", "on":
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func logOpenAICompatStreamOutput(logger *zap.Logger, runID string, seq int, typ string, delta string, extra ...zap.Field) {
|
|
fields := []zap.Field{
|
|
zap.String("run_id", runID),
|
|
zap.Int("seq", seq),
|
|
zap.String("type", typ),
|
|
zap.Int("delta_len", len(delta)),
|
|
zap.Bool("delta_has_open_think", hasOpenThinkTag(delta)),
|
|
zap.Bool("delta_has_close_think", hasCloseThinkTag(delta)),
|
|
}
|
|
fields = append(fields, extra...)
|
|
logger.Info("openai chat completion stream output chunk", fields...)
|
|
}
|
|
|
|
func hasOpenThinkTag(s string) bool {
|
|
return strings.Contains(strings.ToLower(s), "<think")
|
|
}
|
|
|
|
func hasCloseThinkTag(s string) bool {
|
|
return strings.Contains(strings.ToLower(s), "</think")
|
|
}
|