iop/apps/edge/internal/openai/normalized_sse.go
toki 495996fee4 feat(openai): 핫패스 에이전트 실행 경로를 확장한다
Anthropic·Chat 게이트와 관찰·종료 제어를 통합하고 관련 계약·검증 산출물을 반영한다.
2026-08-06 00:09:24 +09:00

753 lines
22 KiB
Go

package openai
import (
"context"
"encoding/json"
"fmt"
"go.uber.org/zap"
edgeservice "iop/apps/edge/internal/service"
"iop/packages/go/streamgate"
"net/http"
"os"
"strings"
"sync"
"time"
"unicode"
)
const (
streamTraceMetadataKey = "iop_trace_stream"
streamTraceEnvKey = "IOP_OPENAI_COMPAT_TRACE_STREAM"
)
type hotPathChatCodecContextKey struct{}
// hotPathChatOuterCodec is the caller-owned Chat wire boundary for one HTTP
// turn. Provider protocol decoding stays in the shared Hot Path stage
// decoders; this codec sees only the normalized outer-turn accumulator and
// renders one Chat response identity, one choice/tool index space, aggregate
// usage, and one terminal sequence.
type hotPathChatOuterCodec struct {
stream bool
model string
outputCapToken int
mu sync.Mutex
outer *hotPathOuterTurn
rendered bool
writer http.ResponseWriter
flusher http.Flusher
opened bool
responseID string
created int64
toolIndex map[string]int
}
func newHotPathChatOuterCodec(stream bool, model string, outputCapToken int) *hotPathChatOuterCodec {
if outputCapToken < 0 {
outputCapToken = 0
}
return &hotPathChatOuterCodec{
stream: stream,
model: strings.TrimSpace(model),
outputCapToken: outputCapToken,
}
}
func withHotPathChatOuterCodec(r *http.Request, codec *hotPathChatOuterCodec) *http.Request {
if r == nil || codec == nil {
return r
}
return r.WithContext(context.WithValue(r.Context(), hotPathChatCodecContextKey{}, codec))
}
func hotPathChatOuterCodecFromRequest(r *http.Request) *hotPathChatOuterCodec {
if r == nil {
return nil
}
codec, _ := r.Context().Value(hotPathChatCodecContextKey{}).(*hotPathChatOuterCodec)
return codec
}
// callerOuterTurn returns the request-local outer sequencer fixed by the Chat
// handler. The first caller supplies the public identity. Later stages in the
// same HTTP turn reuse the exact object, so stage changes cannot reset tool
// indexes, usage, or the caller cap.
func (c *hotPathChatOuterCodec) callerOuterTurn(responseID string, outputCapToken int) *hotPathOuterTurn {
if c == nil {
return newHotPathCallerCappedOuterTurn(responseID, outputCapToken)
}
c.mu.Lock()
defer c.mu.Unlock()
if c.outer == nil {
capToken := c.outputCapToken
if capToken <= 0 {
capToken = outputCapToken
}
c.outer = newHotPathCallerCappedOuterTurn(responseID, capToken)
}
return c.outer
}
func (c *hotPathChatOuterCodec) currentOuterTurn() *hotPathOuterTurn {
if c == nil {
return nil
}
c.mu.Lock()
defer c.mu.Unlock()
return c.outer
}
// prepareProgressiveWriter attaches the caller writer before an already-
// classified Light stage starts. The callback is invoked outside the outer
// turn mutex and writes identity-safe text, reasoning, and tool deltas
// immediately. The Light outer allocates each caller tool ID before release;
// the single finish/usage/[DONE] sequence remains owned by writeResponse.
func (c *hotPathChatOuterCodec) prepareProgressiveWriter(w http.ResponseWriter, outer *hotPathOuterTurn) error {
if c == nil || !c.stream || outer == nil {
return nil
}
flusher, ok := w.(http.Flusher)
if !ok {
return fmt.Errorf("response writer does not support flushing")
}
c.mu.Lock()
c.writer = w
c.flusher = flusher
c.mu.Unlock()
return outer.setReleaseCallback(func(delta hotPathReleasedDelta) error {
return c.writeProgressiveDelta(outer, delta)
})
}
func (c *hotPathChatOuterCodec) writeProgressiveDelta(outer *hotPathOuterTurn, delta hotPathReleasedDelta) error {
responseID, ok := outer.publicResponseIdentity()
if !ok {
return fmt.Errorf("Chat outer response is missing provider execution identity")
}
c.mu.Lock()
defer c.mu.Unlock()
if c.rendered {
return errHotPathTurnTerminal
}
if err := c.ensureStreamOpenLocked(responseID, 0); err != nil {
return err
}
switch delta.Kind {
case streamgate.EventKindReasoningDelta:
return c.emitChunkLocked(map[string]any{"reasoning_content": delta.Text}, "", nil)
case streamgate.EventKindTextDelta:
return c.emitChunkLocked(map[string]any{"content": delta.Text}, "", nil)
case streamgate.EventKindToolCallFragment:
if c.toolIndex == nil {
c.toolIndex = make(map[string]int)
}
index, exists := c.toolIndex[delta.PublicID]
if !exists {
index = len(c.toolIndex)
c.toolIndex[delta.PublicID] = index
}
function := map[string]any{"arguments": delta.Args}
tool := map[string]any{"index": index, "function": function}
if !exists {
tool["id"] = delta.PublicID
tool["type"] = "function"
}
if delta.Name != "" {
function["name"] = delta.Name
}
return c.emitChunkLocked(map[string]any{"tool_calls": []any{tool}}, "", nil)
default:
return fmt.Errorf("unsupported progressive Chat delta kind %q", delta.Kind)
}
}
// hotPathCallerOuterTurn keeps the shared runner endpoint-neutral while each
// public endpoint owns its caller codec.
func hotPathCallerOuterTurn(r *http.Request, protocol, responseID string, outputCapToken int) *hotPathOuterTurn {
if protocol == "openai" {
if codec := hotPathChatOuterCodecFromRequest(r); codec != nil {
return codec.callerOuterTurn(responseID, outputCapToken)
}
}
if protocol == "anthropic" {
if codec := hotPathAnthropicCodecFromRequest(r); codec != nil {
return codec.callerOuterTurn(responseID, outputCapToken)
}
}
return newHotPathCallerCappedOuterTurn(responseID, outputCapToken)
}
// runInitialPresetTurn consumes the result returned by the handler's existing
// one-shot provider-pool admission. It never submits or redispatches a selector
// attempt. The boolean distinguishes collection failures (no caller response
// has been rendered) from shared-turn failures that already own their endpoint
// response.
func (c *hotPathChatOuterCodec) runInitialPresetTurn(
s *Server,
w http.ResponseWriter,
r *http.Request,
dispatch routeDispatch,
runMeta map[string]string,
result *edgeservice.ProviderPoolDispatchResult,
) (normalizedStageOutput, bool, error) {
stage, gate, err := s.collectPresetSelectorResult(r.Context(), dispatch, "openai", result)
if err != nil {
if contextErr := r.Context().Err(); contextErr != nil {
// The active-stage owner already propagated exact cancellation. Mark
// the turn as consumed so the handler does not synthesize response
// bytes after the caller has gone away.
return stage, true, contextErr
}
return stage, false, err
}
err = s.dispatchPresetTurn(w, r, dispatch, "openai", c.stream, runMeta, stage, gate)
return stage, true, err
}
func writeHotPathChatOuterResponse(turn *hotPathTurn, output normalizedStageOutput) (bool, error) {
if turn == nil {
return false, nil
}
codec := hotPathChatOuterCodecFromRequest(turn.Request)
if codec == nil {
return false, nil
}
return true, codec.writeResponse(turn, output)
}
func writeHotPathChatOuterError(
turn *hotPathTurn,
status int,
errorType, message string,
disposition hotPathTerminalDisposition,
) bool {
if turn == nil {
return false
}
codec := hotPathChatOuterCodecFromRequest(turn.Request)
if codec == nil {
return false
}
_ = codec.writeDisposition(turn.Writer, disposition, status, errorType, message)
return true
}
func (c *hotPathChatOuterCodec) writeResponse(turn *hotPathTurn, output normalizedStageOutput) error {
model := c.model
if model == "" {
model = directPublicModel(turn)
}
responseID := strings.TrimSpace(output.ResponseID)
outer := c.currentOuterTurn()
finishReason := openAIDirectFinishReason(output.TerminalReason)
if outer != nil {
if bound, ok := outer.publicResponseIdentity(); ok {
responseID = bound
}
if disposition, ok := outer.terminalDisposition(); ok {
policy := chatHotPathPolicy(disposition)
switch {
case policy.silent && outer.isTerminalCommitted():
return c.writeDisposition(turn.Writer, disposition, 0, "", "")
case policy.errorTerminal && outer.isTerminalCommitted():
return c.writeDisposition(
turn.Writer, disposition, policy.status, policy.errorType, disposition.Cause,
)
case policy.finishReason != "":
finishReason = policy.finishReason
}
}
}
if responseID == "" {
return fmt.Errorf("Chat outer response is missing provider execution identity")
}
if finishReason == "" {
if len(output.ToolCalls) > 0 {
finishReason = "tool_calls"
} else {
finishReason = "stop"
}
}
if !c.stream {
c.mu.Lock()
if c.rendered {
c.mu.Unlock()
return errHotPathTurnTerminal
}
c.rendered = true
c.mu.Unlock()
response := map[string]any{
"id": responseID, "object": "chat.completion", "created": output.Created, "model": model,
"choices": []any{map[string]any{
"index": 0, "message": openAIDirectMessage(output), "finish_reason": finishReason,
}},
}
if len(output.Usage) > 0 {
response["usage"] = output.Usage
}
return writeDirectJSON(turn.Writer, http.StatusOK, response)
}
c.mu.Lock()
defer c.mu.Unlock()
if c.rendered {
return errHotPathTurnTerminal
}
c.rendered = true
c.model = model
if c.writer == nil {
flusher, ok := turn.Writer.(http.Flusher)
if !ok {
return fmt.Errorf("response writer does not support flushing")
}
c.writer = turn.Writer
c.flusher = flusher
}
openedBeforeTerminal := c.opened
if err := c.ensureStreamOpenLocked(responseID, output.Created); err != nil {
return err
}
released := []hotPathReleasedDelta(nil)
if outer != nil && !output.CallerStageOnly {
released = outer.releasedDeltas()
}
emittedContent, emittedReasoning := false, false
emittedTools := make([]bool, len(output.ToolCalls))
toolFragments := hotPathChatToolArgumentFragments(released, output.ToolCalls)
toolIndexes := make(map[string]int, len(output.ToolCalls))
nextToolIndex := 0
toolFragmentIndexes := make([]int, len(output.ToolCalls))
for _, delta := range released {
switch delta.Kind {
case streamgate.EventKindReasoningDelta:
if openedBeforeTerminal {
continue
}
emittedReasoning = true
if err := c.emitChunkLocked(map[string]any{"reasoning_content": delta.Text}, "", nil); err != nil {
return err
}
case streamgate.EventKindTextDelta:
if openedBeforeTerminal {
continue
}
emittedContent = true
if err := c.emitChunkLocked(map[string]any{"content": delta.Text}, "", nil); err != nil {
return err
}
case streamgate.EventKindToolCallFragment:
if openedBeforeTerminal {
continue
}
index, ok := toolIndexes[delta.PublicID]
if !ok {
index = nextToolIndex
nextToolIndex++
toolIndexes[delta.PublicID] = index
}
if index >= len(output.ToolCalls) || toolFragments[index] == nil {
continue
}
fragmentIndex := toolFragmentIndexes[index]
if fragmentIndex >= len(toolFragments[index]) {
continue
}
call := output.ToolCalls[index]
function := map[string]any{"arguments": toolFragments[index][fragmentIndex]}
tool := map[string]any{"index": index, "function": function}
if fragmentIndex == 0 {
tool["id"] = call.ID
tool["type"] = "function"
function["name"] = call.Name
}
if err := c.emitChunkLocked(map[string]any{"tool_calls": []any{tool}}, "", nil); err != nil {
return err
}
if c.toolIndex == nil {
c.toolIndex = make(map[string]int)
}
c.toolIndex[call.ID] = index
toolFragmentIndexes[index]++
emittedTools[index] = true
}
}
if !openedBeforeTerminal && !emittedReasoning {
if output.Reasoning != "" {
if err := c.emitChunkLocked(map[string]any{"reasoning_content": output.Reasoning}, "", nil); err != nil {
return err
}
}
}
if !openedBeforeTerminal && !emittedContent {
if output.Content != "" {
if err := c.emitChunkLocked(map[string]any{"content": output.Content}, "", nil); err != nil {
return err
}
}
}
for index, call := range output.ToolCalls {
_, progressivelyEmitted := c.toolIndex[call.ID]
if emittedTools[index] || progressivelyEmitted {
continue
}
first := map[string]any{
"index": index, "id": call.ID, "type": "function",
"function": map[string]any{"name": call.Name, "arguments": directToolArguments(call)},
}
if err := c.emitChunkLocked(map[string]any{"tool_calls": []any{first}}, "", nil); err != nil {
return err
}
}
if err := c.emitChunkLocked(map[string]any{}, finishReason, output.Usage); err != nil {
return err
}
if _, err := fmt.Fprint(c.writer, "data: [DONE]\n\n"); err != nil {
return err
}
c.flusher.Flush()
return nil
}
func (c *hotPathChatOuterCodec) ensureStreamOpenLocked(responseID string, created int64) error {
if c.opened {
if c.responseID != responseID {
return fmt.Errorf("Chat outer response identity changed after commitment")
}
return nil
}
if c.writer == nil || c.flusher == nil {
return fmt.Errorf("Chat progressive writer is unavailable")
}
if created == 0 {
created = time.Now().Unix()
}
c.responseID = responseID
c.created = created
c.writer.Header().Set("Content-Type", "text/event-stream")
c.writer.Header().Set("Cache-Control", "no-cache")
c.writer.Header().Set("Connection", "keep-alive")
c.writer.WriteHeader(http.StatusOK)
c.opened = true
return c.emitChunkLocked(map[string]any{"role": "assistant"}, "", nil)
}
func (c *hotPathChatOuterCodec) emitChunkLocked(delta map[string]any, reason string, usage json.RawMessage) error {
choice := map[string]any{"index": 0, "delta": delta, "finish_reason": nil}
if reason != "" {
choice["finish_reason"] = reason
}
chunk := map[string]any{
"id": c.responseID, "object": "chat.completion.chunk", "created": c.created,
"model": c.model, "choices": []any{choice},
}
if len(usage) > 0 {
chunk["usage"] = usage
}
return writeDirectSSEData(c.writer, c.flusher, chunk)
}
// writeDisposition renders an error or caller cancellation according to the
// response commit state. Before commitment, Chat keeps the ordinary JSON
// status contract. After the role/delta stream is open, it emits one standard
// error envelope as SSE data followed by exactly one [DONE]. Caller
// cancellation marks the codec terminal without writing any additional byte.
func (c *hotPathChatOuterCodec) writeDisposition(
w http.ResponseWriter,
disposition hotPathTerminalDisposition,
status int,
errorType, message string,
) error {
if c == nil || w == nil {
return fmt.Errorf("Chat Hot Path codec is unavailable")
}
policy := chatHotPathPolicy(disposition)
if policy.status != 0 {
status = policy.status
}
if policy.errorType != "" {
errorType = policy.errorType
}
if strings.TrimSpace(message) == "" {
message = hotPathFirstNonEmpty(disposition.Cause, "hot path stage failed")
}
c.mu.Lock()
defer c.mu.Unlock()
if c.rendered {
return errHotPathTurnTerminal
}
c.rendered = true
if policy.silent {
return nil
}
if !policy.errorTerminal {
return fmt.Errorf("Chat disposition %q is not an error terminal", disposition.Kind)
}
if !c.stream || !c.opened {
writeError(w, status, errorType, message)
return nil
}
if c.writer == nil || c.flusher == nil {
return fmt.Errorf("Chat progressive writer is unavailable")
}
if err := writeDirectSSEData(c.writer, c.flusher, errorResponse{
Error: errorBody{Type: errorType, Message: message},
}); err != nil {
return err
}
if _, err := fmt.Fprint(c.writer, "data: [DONE]\n\n"); err != nil {
return err
}
c.flusher.Flush()
return nil
}
// hotPathChatToolArgumentFragments projects the normalized release stream onto
// the final mapped tool order. Logical-request mapping may replace public tool
// ids after release, so ordering—not an obsolete pre-projection id—is the
// stable join key. A mismatch falls back to the final assembled arguments.
func hotPathChatToolArgumentFragments(released []hotPathReleasedDelta, calls []normalizedToolCall) [][]string {
fragments := make([][]string, len(calls))
if len(calls) == 0 {
return fragments
}
order := make([]string, 0, len(calls))
byID := make(map[string]int, len(calls))
for _, delta := range released {
if delta.Kind != streamgate.EventKindToolCallFragment {
continue
}
index, ok := byID[delta.PublicID]
if !ok {
index = len(order)
if index >= len(calls) {
continue
}
byID[delta.PublicID] = index
order = append(order, delta.PublicID)
}
fragments[index] = append(fragments[index], delta.Args)
}
for index, call := range calls {
if strings.Join(fragments[index], "") != directToolArguments(call) {
fragments[index] = nil
}
}
return fragments
}
func (s *Server) streamChatCompletion(w http.ResponseWriter, dc *chatDispatchContext, handle edgeservice.RunResult) {
flusher, ok := w.(http.Flusher)
if !ok {
handle.Close()
dc.finishUsageRequest(usageStatusError, responseModeNormalized)
writeError(w, http.StatusInternalServerError, "streaming_not_supported", "response writer does not support streaming")
return
}
// Buffered streams collect and validate the full response before emitting
// user-visible chunks. Strict buffered output opts in explicitly, and
// tool-bearing streams use the same path so malformed tool calls can be
// retried or rejected before reaching a client tool runner.
if (dc.outputPolicy.Strict && dc.outputPolicy.StreamBuffer) || len(dc.req.Tools) > 0 {
s.streamBufferedChatCompletion(w, dc, handle, flusher)
return
}
// Runtime-enabled: the Core request runtime owns response-start/role
// staging and commits status/header/role only at first safe release. A
// provider-pool dispatch is included: its initial admission result is
// handed to the runtime as the initial attempt binding, and every recovery
// re-enters SubmitProviderPool through the same request runtime.
if s.streamGateEnabled() {
s.runOpenAIChatStreamGate(w, flusher, dc, handle)
return
}
w.Header().Set("Content-Type", "text/event-stream")
w.Header().Set("Cache-Control", "no-cache")
w.Header().Set("Connection", "keep-alive")
// Live SSE may emit content deltas before the terminal event, so runtime
// tool validation is excluded upstream; write the role chunk immediately.
defer handle.Close()
sess := s.newChatStreamSession(w, flusher, dc.req, dc.submitReq, handle, dc.outputPolicy, dc.usage)
sess.writeRole()
defer sess.logClosed()
stream := handle.Stream()
if stream.Events == nil {
sess.failStream("run stream unavailable")
return
}
sess.consume(dc.r, stream, handle.WaitTimeout())
}
// consume drives the session until it reaches a terminal state: a terminal run
// event, a node disconnect, a caller cancel, or the run wait timeout.
func (sess *chatStreamSession) consume(r *http.Request, stream edgeservice.RunStream, waitTimeout time.Duration) {
for {
select {
case <-r.Context().Done():
sess.cancel(r.Context().Err(), usageStatusForError(r.Context().Err()), false)
return
case nodeEvent, ok := <-stream.NodeEvents:
if !ok {
stream.NodeEvents = nil
continue
}
if edgeservice.IsNodeDisconnected(nodeEvent) {
sess.fail("node disconnected")
return
}
case event, ok := <-stream.Events:
if !ok {
sess.failStream("run stream closed")
return
}
if sess.consumeEvent(event) {
return
}
case <-time.After(waitTimeout):
sess.cancel(errRunTimedOut, usageStatusCancel, true)
return
}
}
}
type streamToolTextFilter struct {
pending string
}
func (f *streamToolTextFilter) Append(delta string) string {
f.pending += delta
if idx := firstStreamToolTextCandidateIndex(f.pending); idx >= 0 {
out := strings.TrimRightFunc(f.pending[:idx], unicode.IsSpace)
f.pending = f.pending[idx:]
return out
}
flushLen := streamToolTextSafeFlushLen(f.pending)
out := strings.TrimRightFunc(f.pending[:flushLen], unicode.IsSpace)
f.pending = f.pending[len(out):]
return out
}
func (f *streamToolTextFilter) Flush() string {
out := f.pending
f.pending = ""
return out
}
func (f *streamToolTextFilter) FlushNonCandidate() string {
if firstStreamToolTextCandidateIndex(f.pending) >= 0 {
f.pending = ""
return ""
}
// Even without a full candidate, chunk-boundary protection can leave a
// partial candidate suffix (e.g. "<tool_ca", trailing "{") in pending.
// Return only the safe non-candidate prefix and drop the partial suffix so
// a native tool_calls completion never flushes a raw candidate fragment.
flushLen := streamToolTextSafeFlushLen(f.pending)
out := strings.TrimRightFunc(f.pending[:flushLen], unicode.IsSpace)
f.pending = ""
return out
}
func firstStreamToolTextCandidateIndex(s string) int {
xml := strings.Index(strings.ToLower(s), "<tool_call")
mustache := strings.Index(s, textMustacheToolCallOpenHint)
switch {
case xml < 0:
return mustache
case mustache < 0:
return xml
case xml < mustache:
return xml
default:
return mustache
}
}
func streamToolTextSafeFlushLen(s string) int {
if s == "" {
return 0
}
const xmlHint = "<tool_call"
lower := strings.ToLower(s)
start := len(s) - len(xmlHint) + 1
if start < 0 {
start = 0
}
for i := start; i < len(s); i++ {
if strings.HasPrefix(xmlHint, lower[i:]) ||
strings.HasPrefix(textMustacheToolCallOpenHint, s[i:]) {
return i
}
}
return len(s)
}
func unstreamedCleanedText(cleaned, emitted string) string {
if cleaned == "" {
return ""
}
if emitted == "" {
return cleaned
}
if strings.HasPrefix(cleaned, emitted) {
return cleaned[len(emitted):]
}
cleaned = strings.TrimSpace(cleaned)
emitted = strings.TrimSpace(emitted)
if cleaned == "" || cleaned == emitted {
return ""
}
if emitted != "" && strings.HasPrefix(cleaned, emitted) {
return strings.TrimSpace(strings.TrimPrefix(cleaned, emitted))
}
return ""
}
func openAICompatTraceStreamEnabled(metadata map[string]string) bool {
if traceBool(os.Getenv(streamTraceEnvKey)) {
return true
}
if metadata == nil {
return false
}
return traceBool(metadata[streamTraceMetadataKey])
}
func traceBool(v string) bool {
switch strings.ToLower(strings.TrimSpace(v)) {
case "1", "true", "yes", "on":
return true
default:
return false
}
}
func logOpenAICompatStreamOutput(logger *zap.Logger, runID string, seq int, typ string, delta string, extra ...zap.Field) {
fields := []zap.Field{
zap.String("run_id", runID),
zap.Int("seq", seq),
zap.String("type", typ),
zap.Int("delta_len", len(delta)),
zap.Bool("delta_has_open_think", hasOpenThinkTag(delta)),
zap.Bool("delta_has_close_think", hasCloseThinkTag(delta)),
}
fields = append(fields, extra...)
logger.Info("openai chat completion stream output chunk", fields...)
}
func hasOpenThinkTag(s string) bool {
return strings.Contains(strings.ToLower(s), "<think")
}
func hasCloseThinkTag(s string) bool {
return strings.Contains(strings.ToLower(s), "</think")
}