package streamgate import ( "context" "errors" "fmt" "sync" "time" ) const ( // DefaultMaxEvidenceRunes is the default Unicode rune threshold for evidence holding. DefaultMaxEvidenceRunes = 500 // DefaultMaxBufferRunes is the default maximum rune buffer size. DefaultMaxBufferRunes = 4096 // DefaultMaxIngressSnapshotBytes is the default byte limit for ingress body snapshots (16 MiB). DefaultMaxIngressSnapshotBytes int64 = 16 * 1024 * 1024 // DefaultMaxRecoveryAttemptsTotal is the default total fault recovery attempt limit. DefaultMaxRecoveryAttemptsTotal = 3 ) // RuntimeOptions captures immutable configuration parameters for a streamgate request runtime. type RuntimeOptions struct { maxEvidenceRunes int maxBufferRunes int maxIngressSnapshotBytes int64 maxRecoveryAttemptsTotal int // maxRecoveryAttemptsByStrategy is the immutable per-fault-strategy recovery // cap snapshot. A nil/empty map means every fault strategy inherits the // request-total cap; a non-empty map disables any fault strategy it omits and // caps each listed strategy (an explicit 0 forbids that strategy entirely). maxRecoveryAttemptsByStrategy map[RecoveryStrategy]int gateOptions GateCoordinatorOptions recoveryOptions RecoveryCoordinatorOptions } // DefaultRuntimeOptions returns RuntimeOptions populated with default settings. func DefaultRuntimeOptions() RuntimeOptions { opts, err := NewRuntimeOptions( DefaultMaxEvidenceRunes, DefaultMaxBufferRunes, DefaultMaxIngressSnapshotBytes, DefaultMaxRecoveryAttemptsTotal, GateCoordinatorOptions{}, RecoveryCoordinatorOptions{}, ) if err != nil { panic(err) } return opts } // NewRuntimeOptions constructs validated RuntimeOptions. func NewRuntimeOptions( maxEvidenceRunes, maxBufferRunes int, maxIngressBytes int64, maxRecoveryAttempts int, gateOpts GateCoordinatorOptions, recoveryOpts RecoveryCoordinatorOptions, ) (RuntimeOptions, error) { if maxEvidenceRunes <= 0 { return RuntimeOptions{}, errors.New("streamgate: runtime options max evidence runes must be positive") } if maxBufferRunes < maxEvidenceRunes { return RuntimeOptions{}, errors.New("streamgate: runtime options max buffer runes must be >= max evidence runes") } if maxIngressBytes <= 0 { return RuntimeOptions{}, errors.New("streamgate: runtime options max ingress snapshot bytes must be positive") } if maxRecoveryAttempts < 0 || maxRecoveryAttempts > DefaultMaxRecoveryAttemptsTotal { return RuntimeOptions{}, errors.New("streamgate: runtime options max recovery attempts total must be between 0 and 3") } opts := RuntimeOptions{ maxEvidenceRunes: maxEvidenceRunes, maxBufferRunes: maxBufferRunes, maxIngressSnapshotBytes: maxIngressBytes, maxRecoveryAttemptsTotal: maxRecoveryAttempts, gateOptions: gateOpts, recoveryOptions: recoveryOpts, } if err := opts.Validate(); err != nil { return RuntimeOptions{}, err } return opts, nil } // Validate returns nil when RuntimeOptions is in a consistent state. func (o RuntimeOptions) Validate() error { if o.maxEvidenceRunes <= 0 { return errors.New("streamgate: runtime options max evidence runes must be positive") } if o.maxBufferRunes < o.maxEvidenceRunes { return errors.New("streamgate: runtime options max buffer runes must be >= max evidence runes") } if o.maxIngressSnapshotBytes <= 0 || o.maxIngressSnapshotBytes > DefaultMaxIngressSnapshotBytes { return errors.New("streamgate: runtime options max ingress snapshot bytes must be between 1 and 16777216") } if o.maxRecoveryAttemptsTotal < 0 || o.maxRecoveryAttemptsTotal > DefaultMaxRecoveryAttemptsTotal { return errors.New("streamgate: runtime options max recovery attempts total must be between 0 and 3") } for strategy, limit := range o.maxRecoveryAttemptsByStrategy { if !isFaultRecoveryStrategy(strategy) { return errors.New("streamgate: runtime options strategy cap contains a non-fault strategy") } if limit < 0 || limit > o.maxRecoveryAttemptsTotal { return errors.New("streamgate: runtime options strategy cap must be between zero and max recovery attempts total") } } return nil } // WithRecoveryStrategyLimits returns a validated copy of the options carrying // an effective per-fault-strategy recovery cap. A nil/empty map inherits the // request-total cap for every strategy; a non-empty map disables any strategy // it omits and caps each listed strategy at its value (0 forbids the strategy). func (o RuntimeOptions) WithRecoveryStrategyLimits(limits map[RecoveryStrategy]int) (RuntimeOptions, error) { next := o next.maxRecoveryAttemptsByStrategy = copyRecoveryStrategyCounts(limits) if err := next.Validate(); err != nil { return RuntimeOptions{}, err } return next, nil } // MaxEvidenceRunes returns the max evidence runes setting. func (o RuntimeOptions) MaxEvidenceRunes() int { return o.maxEvidenceRunes } // MaxBufferRunes returns the max buffer runes setting. func (o RuntimeOptions) MaxBufferRunes() int { return o.maxBufferRunes } // MaxIngressSnapshotBytes returns the max ingress snapshot bytes setting. func (o RuntimeOptions) MaxIngressSnapshotBytes() int64 { return o.maxIngressSnapshotBytes } // MaxRecoveryAttemptsTotal returns the total recovery attempts limit. func (o RuntimeOptions) MaxRecoveryAttemptsTotal() int { return o.maxRecoveryAttemptsTotal } // MaxRecoveryAttemptsByStrategy returns a defensive copy of the effective // per-fault-strategy recovery cap. A nil result means every fault strategy // inherits the request-total cap. func (o RuntimeOptions) MaxRecoveryAttemptsByStrategy() map[RecoveryStrategy]int { return copyRecoveryStrategyCounts(o.maxRecoveryAttemptsByStrategy) } // GateOptions returns the gate coordinator options. func (o RuntimeOptions) GateOptions() GateCoordinatorOptions { return o.gateOptions } // RecoveryOptions returns the recovery coordinator options. func (o RuntimeOptions) RecoveryOptions() RecoveryCoordinatorOptions { return o.recoveryOptions } // RequestRuntimeSnapshot binds request-local state, host seams, filter registry snapshot, // and runtime options into an immutable contract to be consumed by request-local owners. type RequestRuntimeSnapshot struct { requestID StableToken configGen StableToken environment string endpoint string family string options RuntimeOptions registrySnapshot FilterRegistrySnapshot ingressSnapshot *IngressSnapshot requestSnapshotRef RecoveryRequestSnapshotRef dispatcher AttemptDispatcher rebuilder RequestRebuilder preparer RecoveryPlanPreparer prepFactory RecoveryPreparationSnapshotFactory sink ReleaseSink obsSink ObservationSink } // RecoveryPreparationSnapshotFactory creates a request-local RecoveryPreparationSnapshot // bound to the given request snapshot and attempt binding. type RecoveryPreparationSnapshotFactory interface { CreatePreparationSnapshot(ctx context.Context, snapshot RequestRuntimeSnapshot, binding AttemptBinding) (RecoveryPreparationSnapshot, error) } // NewRequestRuntimeSnapshot constructs a validated RequestRuntimeSnapshot. func NewRequestRuntimeSnapshot( requestID, configGen, environment, endpoint, family string, options RuntimeOptions, registrySnapshot FilterRegistrySnapshot, ingressSnapshot *IngressSnapshot, requestSnapshotRef RecoveryRequestSnapshotRef, dispatcher AttemptDispatcher, rebuilder RequestRebuilder, preparer RecoveryPlanPreparer, prepFactory RecoveryPreparationSnapshotFactory, sink ReleaseSink, ) (RequestRuntimeSnapshot, error) { reqIDToken, err := NewStableTokenRequired("requestID", requestID) if err != nil { return RequestRuntimeSnapshot{}, err } cfgGenToken, err := NewStableTokenRequired("configGen", configGen) if err != nil { return RequestRuntimeSnapshot{}, err } if err := options.Validate(); err != nil { return RequestRuntimeSnapshot{}, err } if registrySnapshot.Generation() == "" { return RequestRuntimeSnapshot{}, errors.New("streamgate: request runtime snapshot registry snapshot generation is required") } if dispatcher == nil { return RequestRuntimeSnapshot{}, errors.New("streamgate: request runtime snapshot dispatcher is required") } if rebuilder == nil { return RequestRuntimeSnapshot{}, errors.New("streamgate: request runtime snapshot rebuilder is required") } if sink == nil { return RequestRuntimeSnapshot{}, errors.New("streamgate: request runtime snapshot release sink is required") } if err := requestSnapshotRef.Validate(); err != nil { return RequestRuntimeSnapshot{}, err } snap := RequestRuntimeSnapshot{ requestID: reqIDToken, configGen: cfgGenToken, environment: environment, endpoint: endpoint, family: family, options: options, registrySnapshot: registrySnapshot, ingressSnapshot: ingressSnapshot, requestSnapshotRef: requestSnapshotRef, dispatcher: dispatcher, rebuilder: rebuilder, preparer: preparer, prepFactory: prepFactory, sink: sink, } if err := snap.Validate(); err != nil { return RequestRuntimeSnapshot{}, err } return snap, nil } // Validate returns nil when the RequestRuntimeSnapshot is consistent. func (r RequestRuntimeSnapshot) Validate() error { if err := validateStableTokenRequired("requestID", r.requestID.value); err != nil { return err } if err := validateStableTokenRequired("configGen", r.configGen.value); err != nil { return err } if err := r.options.Validate(); err != nil { return err } registryGeneration := r.registrySnapshot.Generation() if registryGeneration == "" { return errors.New("streamgate: request runtime snapshot registry snapshot generation is required") } if registryGeneration != r.configGen.String() { return errors.New("streamgate: request runtime snapshot config and registry generations must match") } if r.dispatcher == nil { return errors.New("streamgate: request runtime snapshot dispatcher is required") } if r.rebuilder == nil { return errors.New("streamgate: request runtime snapshot rebuilder is required") } if r.sink == nil { return errors.New("streamgate: request runtime snapshot release sink is required") } if err := r.requestSnapshotRef.Validate(); err != nil { return err } if (r.preparer == nil && r.prepFactory != nil) || (r.preparer != nil && r.prepFactory == nil) { return errors.New("streamgate: request runtime snapshot preparer and preparation snapshot factory must both be nil or both non-nil") } return nil } // RequestID returns the stable request identifier string. func (r RequestRuntimeSnapshot) RequestID() string { return r.requestID.value } // ConfigGeneration returns the configuration generation string. func (r RequestRuntimeSnapshot) ConfigGeneration() string { return r.configGen.value } // Environment returns the environment label. func (r RequestRuntimeSnapshot) Environment() string { return r.environment } // Endpoint returns the endpoint label. func (r RequestRuntimeSnapshot) Endpoint() string { return r.endpoint } // Family returns the model family label. func (r RequestRuntimeSnapshot) Family() string { return r.family } // Options returns the runtime options. func (r RequestRuntimeSnapshot) Options() RuntimeOptions { return r.options } // RegistrySnapshot returns the filter registry snapshot. func (r RequestRuntimeSnapshot) RegistrySnapshot() FilterRegistrySnapshot { return r.registrySnapshot } // IngressSnapshot returns the ingress snapshot pointer, or nil if unset. func (r RequestRuntimeSnapshot) IngressSnapshot() *IngressSnapshot { return r.ingressSnapshot } // RequestSnapshotRef returns the recovery request snapshot reference. func (r RequestRuntimeSnapshot) RequestSnapshotRef() RecoveryRequestSnapshotRef { return r.requestSnapshotRef } // Dispatcher returns the host attempt dispatcher. func (r RequestRuntimeSnapshot) Dispatcher() AttemptDispatcher { return r.dispatcher } // Rebuilder returns the host request rebuilder. func (r RequestRuntimeSnapshot) Rebuilder() RequestRebuilder { return r.rebuilder } // Preparer returns the optional host recovery plan preparer. func (r RequestRuntimeSnapshot) Preparer() RecoveryPlanPreparer { return r.preparer } // PreparationSnapshotFactory returns the optional recovery preparation snapshot factory. func (r RequestRuntimeSnapshot) PreparationSnapshotFactory() RecoveryPreparationSnapshotFactory { return r.prepFactory } // Sink returns the host release sink. func (r RequestRuntimeSnapshot) Sink() ReleaseSink { return r.sink } // ObservationSink returns the observation sink, defaulting to NoopObservationSink if unset. func (r RequestRuntimeSnapshot) ObservationSink() ObservationSink { if r.obsSink == nil { return NoopObservationSink{} } return r.obsSink } // WithObservationSink returns a copy of RequestRuntimeSnapshot with the given ObservationSink. func (r RequestRuntimeSnapshot) WithObservationSink(sink ObservationSink) RequestRuntimeSnapshot { cp := r cp.obsSink = sink return cp } // BuildAttemptFilterContext constructs a FilterContext bound to this request snapshot. func (r RequestRuntimeSnapshot) BuildAttemptFilterContext( attemptID string, target AttemptTarget, epochID uint64, commitState CommitState, ) (FilterContext, error) { builder := NewFilterContextBuilder(r.configGen.value, attemptID). SetEnvironment(r.environment). SetEndpoint(r.endpoint). SetFamily(r.family). SetModelGroup(target.ModelGroup()). SetActualModel(target.Model()). SetActualProvider(target.Provider()). SetExecutionPath(target.ExecutionPath()). SetCommitState(commitState). SetStableCorrelation(r.requestID.value) if epochID > 0 { var err error builder, err = builder.SetEpoch(epochID) if err != nil { return FilterContext{}, err } } return builder.Build() } // attemptCancellerAdapter wraps RequestRuntime to satisfy StreamReleaser's AttemptCanceller seam. type attemptCancellerAdapter struct { r *RequestRuntime } func (a *attemptCancellerAdapter) CancelAttempt(ctx context.Context, attemptID string) error { if a.r != nil { a.r.mu.Lock() ctrl := a.r.currentBinding.Controller() a.r.mu.Unlock() if ctrl != nil { return ctrl.AbortAttempt(ctx) } } return nil } // RequestRuntime drives the request-local lifecycle loop, managing attempt bindings, // evidence collection, parallel filter evaluation through GateCoordinator, decision arbitration, // stream release, and fault recovery. type RequestRuntime struct { snapshot RequestRuntimeSnapshot modelGroup string currentBinding AttemptBinding tail *EvidenceTail boundary *CommitBoundary releaser *StreamReleaser gate *GateCoordinator arbiter *DecisionArbiter recoveryCoord *RecoveryCoordinator recoveryPolicy RecoveryPolicySnapshot recoveryUsage RecoveryUsageSnapshot resolvedFilters []ResolvedFilter stagedStart *ResponseStart terminalCommitted bool running bool suppressContinuationStart bool resourcesClosed bool planIDCounter int sequencer *ObservationSequencer mu sync.Mutex } func (r *RequestRuntime) emitObservation(kind ObservationKind, epochID uint64, mutate func(*FilterObservationInput)) { if r == nil || r.sequencer == nil { return } r.mu.Lock() binding := r.currentBinding commitState := r.boundary.State() r.mu.Unlock() attemptTarget, err := NewObservationAttemptTarget(r.modelGroup, binding.Model(), binding.Provider(), binding.ExecutionPath()) if err != nil { return } input := FilterObservationInput{ Kind: kind, StableCorrelation: r.snapshot.RequestID(), ConfigGeneration: r.snapshot.ConfigGeneration(), AttemptID: binding.AttemptID(), AttemptTarget: attemptTarget, EpochID: epochID, CommitState: commitState, OccurredAt: time.Now(), } if mutate != nil { mutate(&input) } _, _ = r.sequencer.Emit(context.Background(), input) } // gracefulAttemptController is an optional AttemptController extension a host // controller may implement to release provider ownership after a normal // terminal without issuing a provider-side cancel. CloseRequestResources uses // it on a graceful (successful-terminal) close and falls back to AbortAttempt // for every non-graceful outcome. type gracefulAttemptController interface { AttemptController CloseAttempt(ctx context.Context) error } // requestResourceCloser is an optional RequestRebuilder extension: a // request-local host rebuilder that must be closed exactly once when the // request runtime finishes so its rebuilt-request store and reserved snapshot // bytes are released on success, error, and caller-cancel alike. type requestResourceCloser interface { Close() } // CloseRequestResources performs idempotent end-of-request cleanup of the host // resources this runtime owns. It aborts (or, when graceful, gracefully closes) // the current attempt binding's controller — releasing the latest provider // transport and any rebuilt lease, and canceling the latest provider run on a // non-graceful outcome — and closes the request rebuilder. It is safe to call // exactly once after Run returns for any terminal, error, or caller-cancel // outcome. graceful must be true only when Run committed a successful terminal. func (r *RequestRuntime) CloseRequestResources(ctx context.Context, graceful bool) error { if r == nil { return nil } r.mu.Lock() if r.resourcesClosed { r.mu.Unlock() return nil } r.resourcesClosed = true ctrl := r.currentBinding.Controller() r.mu.Unlock() var err error if ctrl != nil { if graceful { if gc, ok := ctrl.(gracefulAttemptController); ok { err = gc.CloseAttempt(ctx) } else { err = ctrl.AbortAttempt(ctx) } } else { err = ctrl.AbortAttempt(ctx) } } if closer, ok := r.snapshot.Rebuilder().(requestResourceCloser); ok { closer.Close() } return err } func providerErrorToTerminalResult(ev NormalizedEvent) (TerminalResult, error) { return ev.AsProviderError() } func normalizedTerminalToResult(ev NormalizedEvent) (TerminalResult, error) { if ev.Kind() == EventKindTerminal { return NewSuccessTerminalResult(ev.Channel(), ev.Timestamp()) } return providerErrorToTerminalResult(ev) } // NewRequestRuntime constructs a new RequestRuntime instance. func NewRequestRuntime(snapshot RequestRuntimeSnapshot, modelGroup string, initial AttemptBinding) (*RequestRuntime, error) { if err := snapshot.Validate(); err != nil { return nil, fmt.Errorf("streamgate: new request runtime snapshot invalid: %w", err) } if modelGroup == "" { return nil, errors.New("streamgate: new request runtime model group is required") } if err := initial.Validate(); err != nil { return nil, fmt.Errorf("streamgate: new request runtime initial binding invalid: %w", err) } boundary, err := NewCommitBoundary(snapshot.Sink()) if err != nil { return nil, err } arbiter := NewDecisionArbiter() gate, err := NewGateCoordinator(context.Background(), snapshot.Options().GateOptions(), arbiter) if err != nil { return nil, err } seq := NewObservationSequencer(snapshot.ObservationSink(), nil) gate.SetObservationSequencer(seq) recPolicy, err := NewRecoveryPolicySnapshot( snapshot.Options().MaxRecoveryAttemptsTotal(), snapshot.Options().MaxRecoveryAttemptsByStrategy(), ) if err != nil { return nil, err } recUsage, err := NewRecoveryUsageSnapshot(0, nil) if err != nil { return nil, err } preparers := make(map[string]RecoveryPlanPreparer) if snapshot.Preparer() != nil { preparers["default"] = snapshot.Preparer() } recOpts := RecoveryCoordinatorOptions{ Policy: recPolicy, Usage: recUsage, RequestSnapshot: snapshot.RequestSnapshotRef(), CurrentBinding: initial, Rebuilder: snapshot.Rebuilder(), Dispatcher: snapshot.Dispatcher(), Preparers: preparers, PreparationTimeout: 5 * time.Second, ObservationSink: snapshot.ObservationSink(), } recCoord, err := NewRecoveryCoordinator(recOpts) if err != nil { return nil, err } recCoord.SetObservationSequencer(seq) rt := &RequestRuntime{ snapshot: snapshot, modelGroup: modelGroup, boundary: boundary, gate: gate, arbiter: arbiter, recoveryCoord: recCoord, recoveryPolicy: recPolicy, recoveryUsage: recUsage, sequencer: seq, } cand, err := rt.prepareAttemptInstall(initial, RecoveryResumeModeReplaceAttempt) if err != nil { return nil, err } if err := rt.commitAttemptInstall(context.Background(), cand); err != nil { return nil, err } return rt, nil } type attemptInstallCandidate struct { binding AttemptBinding resumeMode RecoveryResumeMode oldAttemptID string target AttemptTarget resolvedFilters []ResolvedFilter plan EvidencePlan tail *EvidenceTail releaser *StreamReleaser } func (r *RequestRuntime) prepareAttemptInstall(binding AttemptBinding, resumeMode RecoveryResumeMode) (attemptInstallCandidate, error) { if err := binding.Validate(); err != nil { return attemptInstallCandidate{}, err } if err := resumeMode.Validate(); err != nil { return attemptInstallCandidate{}, err } target, err := NewAttemptTarget(r.modelGroup, binding.Model(), binding.Provider(), binding.ExecutionPath(), nil) if err != nil { return attemptInstallCandidate{}, err } rfc, err := NewRequestFilterContext( r.snapshot.ConfigGeneration(), binding.AttemptID(), r.snapshot.Environment(), r.snapshot.Endpoint(), r.snapshot.Family(), "", r.boundary.State(), false, false, r.snapshot.RequestID(), ) if err != nil { return attemptInstallCandidate{}, err } rfSnap, err := r.snapshot.RegistrySnapshot().BeginRequest(rfc) if err != nil { return attemptInstallCandidate{}, err } resolved, err := rfSnap.ResolveAttempt(target) if err != nil { return attemptInstallCandidate{}, err } plan, err := NewEvidencePlanFromResolvedFilters(resolved) if err != nil { return attemptInstallCandidate{}, err } if r.tail == nil { tail, err := NewEvidenceTail(plan) if err != nil { return attemptInstallCandidate{}, err } releaser, err := NewStreamReleaser(tail, r.boundary, &attemptCancellerAdapter{r: r}) if err != nil { return attemptInstallCandidate{}, err } return attemptInstallCandidate{ binding: binding, resumeMode: resumeMode, oldAttemptID: "", target: target, resolvedFilters: resolved, plan: plan, tail: tail, releaser: releaser, }, nil } if resumeMode == RecoveryResumeModeReplaceAttempt { tail, err := NewEvidenceTail(plan) if err != nil { return attemptInstallCandidate{}, err } releaser, err := NewStreamReleaser(tail, r.boundary, &attemptCancellerAdapter{r: r}) if err != nil { return attemptInstallCandidate{}, err } return attemptInstallCandidate{ binding: binding, resumeMode: resumeMode, oldAttemptID: r.currentBinding.AttemptID(), target: target, resolvedFilters: resolved, plan: plan, tail: tail, releaser: releaser, }, nil } if resumeMode == RecoveryResumeModeContinueStream { oldID := "" if r.currentBinding.AttemptID() != "" { oldID = r.currentBinding.AttemptID() } return attemptInstallCandidate{ binding: binding, resumeMode: resumeMode, oldAttemptID: oldID, target: target, resolvedFilters: resolved, plan: plan, tail: nil, releaser: nil, }, nil } return attemptInstallCandidate{}, fmt.Errorf("streamgate: unsupported resume mode: %v", resumeMode) } func (r *RequestRuntime) commitAttemptInstall(ctx context.Context, candidate attemptInstallCandidate) error { r.mu.Lock() hasTail := r.tail != nil rel := r.releaser r.mu.Unlock() if !hasTail { if err := r.boundary.BeginAttempt(candidate.binding.AttemptID()); err != nil { return err } r.mu.Lock() r.currentBinding = candidate.binding r.resolvedFilters = candidate.resolvedFilters r.tail = candidate.tail r.releaser = candidate.releaser r.mu.Unlock() return nil } if candidate.resumeMode == RecoveryResumeModeReplaceAttempt { if candidate.oldAttemptID != "" { if err := candidate.releaser.ReplaceUncommittedAttempt(ctx, candidate.oldAttemptID, candidate.binding.AttemptID()); err != nil { return err } } else { if err := r.boundary.BeginAttempt(candidate.binding.AttemptID()); err != nil { return err } candidate.tail.ResetForReplace() } r.mu.Lock() r.currentBinding = candidate.binding r.resolvedFilters = candidate.resolvedFilters r.tail = candidate.tail r.releaser = candidate.releaser // The staged response start is attempt-local: the boundary already // dropped it during handoff, so the runtime mirror must not leak the // previous attempt's status/header into the new attempt's batches. r.stagedStart = nil r.mu.Unlock() return nil } if candidate.resumeMode == RecoveryResumeModeContinueStream { if candidate.oldAttemptID != "" && rel != nil { if err := rel.ContinueOpenAttempt(ctx, candidate.oldAttemptID, candidate.binding.AttemptID()); err != nil { return err } } r.mu.Lock() r.tail.PrepareContinuationWithPlan(candidate.plan) r.currentBinding = candidate.binding r.resolvedFilters = candidate.resolvedFilters r.suppressContinuationStart = true r.mu.Unlock() return nil } return fmt.Errorf("streamgate: unsupported resume mode: %v", candidate.resumeMode) } // Run executes the normalized event owner loop for the request lifetime. func (r *RequestRuntime) Run(ctx context.Context) error { r.mu.Lock() if r.running || r.terminalCommitted { r.mu.Unlock() return errors.New("streamgate: runtime already running or terminal") } r.running = true r.mu.Unlock() defer func() { if r.gate != nil { _ = r.gate.Close() } }() for { r.mu.Lock() if r.terminalCommitted { r.mu.Unlock() return nil } binding := r.currentBinding tail := r.tail releaser := r.releaser resolvedFilters := r.resolvedFilters suppressStart := r.suppressContinuationStart if suppressStart { r.suppressContinuationStart = false } r.mu.Unlock() if ctx.Err() != nil { return ctx.Err() } ev, err := binding.EventSource().NextEvent(ctx) if err != nil { if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) || ctx.Err() != nil { return ctx.Err() } desc, errDesc := NewExternalDescriptor("error", "source_failure", "event_source_read_error", "") if errDesc != nil { return errDesc } cause, errCause := NewFailureCause("event_source", "read_error", "", "", "") if errCause != nil { return errCause } causes, errCauses := NewFailureCauseChain([]FailureCause{cause}) if errCauses != nil { return errCauses } termRes, errTerm := NewErrorTerminalResult("default", desc, causes, time.Now()) if errTerm != nil { return errTerm } if _, errRel := releaser.FailPending(ctx, binding.AttemptID(), termRes); errRel != nil { return errRel } r.emitObservation(ObservationKindTerminalCommitted, 0, func(in *FilterObservationInput) { in.CommitState = CommitStateTerminalCommitted in.Causes = causes }) r.mu.Lock() r.terminalCommitted = true r.mu.Unlock() return nil } if ev.Kind() == EventKindResponseStart { rs, errStart := ev.AsResponseStart() if errStart != nil { return errStart } if suppressStart { continue } r.mu.Lock() r.stagedStart = &rs r.mu.Unlock() if errStage := r.boundary.StageResponseStart(binding.AttemptID(), rs); errStage != nil { return errStage } r.emitObservation(ObservationKindResponseStaged, 0, func(in *FilterObservationInput) { in.CommitState = CommitStateTransportUncommitted hold, _ := NewObservationHoldInfo(FilterHoldModeRolling, EventKindResponseStart, r.snapshot.Options().MaxEvidenceRunes(), 0) in.Hold = &hold }) continue } epoch, signal, errAppend := tail.Append(ev) if errAppend != nil { return errAppend } if signal == EvidenceTailSignalBufferOverflow { desc, errDesc := NewExternalDescriptor("error", "buffer_overflow", "buffer_overflow_occurred", "") if errDesc != nil { return errDesc } cause, errCause := NewFailureCause("evidence_tail", "buffer_overflow", "", "", "") if errCause != nil { return errCause } causes, errCauses := NewFailureCauseChain([]FailureCause{cause}) if errCauses != nil { return errCauses } termRes, errTerm := NewErrorTerminalResult(ev.Channel(), desc, causes, time.Now()) if errTerm != nil { return errTerm } if _, errRel := releaser.FailPending(ctx, binding.AttemptID(), termRes); errRel != nil { return errRel } r.emitObservation(ObservationKindTerminalCommitted, epoch.ID(), func(in *FilterObservationInput) { in.CommitState = CommitStateTerminalCommitted in.Causes = causes }) r.mu.Lock() r.terminalCommitted = true r.mu.Unlock() return nil } terminalFlag := (ev.Kind() == EventKindTerminal || ev.Kind() == EventKindProviderError) isKindSubscribed := func(kind EventKind, kinds []EventKind) bool { for _, k := range kinds { if k == kind { return true } } return false } passThrough := signal == EvidenceTailSignalReady || (signal == EvidenceTailSignalNone && !isKindSubscribed(ev.Kind(), tail.Plan().BlockingKinds(ev.Channel()))) shouldEvaluate := terminalFlag || passThrough || signal == EvidenceTailSignalThreshold || signal == EvidenceTailSignalTrigger if shouldEvaluate { epochFilters, errBind := tail.Plan().BindEpochFilters(epoch, resolvedFilters) if errBind != nil { return errBind } r.mu.Lock() stagedStartPtr := r.stagedStart r.mu.Unlock() if r.boundary.State() != CommitStateTransportUncommitted { stagedStartPtr = nil } batch, errBatch := NewEvidenceBatch( []NormalizedEvent{ev}, tail.PendingEventsByChannel(), tail.CommittedLookBehindByChannel(), stagedStartPtr, terminalFlag, r.boundary.State(), time.Now(), ) if errBatch != nil { return errBatch } attemptTarget, errTarget := NewObservationAttemptTarget(r.modelGroup, binding.Model(), binding.Provider(), binding.ExecutionPath()) if errTarget != nil { return errTarget } obsCtx := ObservationContext{ Sequencer: r.sequencer, StableCorrelation: r.snapshot.RequestID(), ConfigGeneration: r.snapshot.ConfigGeneration(), AttemptID: binding.AttemptID(), AttemptTarget: attemptTarget, EpochID: epoch.ID(), } evalCtx := WithObservationContext(ctx, obsCtx) arbResult, errGate := r.gate.Submit(evalCtx, batch, epochFilters) if errGate != nil { if ctx.Err() != nil { return ctx.Err() } return errGate } arbInfo, _ := NewObservationArbitrationInfo(arbResult.Action(), arbResult.BaseDisposition()) r.emitObservation(ObservationKindArbitrationDecided, epoch.ID(), func(in *FilterObservationInput) { in.Arbitration = &arbInfo }) switch arbResult.Action() { case ArbitrationActionReplacement: proposal := arbResult.ReplacementProposal() if proposal == nil { return errors.New("streamgate: replacement action missing replacement proposal") } if err := proposal.Validate(); err != nil { return err } if _, errRel := r.releaser.releaseReplacementEvents( ctx, binding.AttemptID(), epoch.ID(), proposal.Events(), ); errRel != nil { return errRel } r.emitObservation(ObservationKindReleaseCommitted, epoch.ID(), func(in *FilterObservationInput) { in.CommitState = CommitStateStreamOpen }) if terminalFlag { // The substituted payload replaces the held content only; the // terminal disposition of the original event still closes the // stream exactly once. termRes, errRes := normalizedTerminalToResult(ev) if errRes != nil { return errRes } if errRel := r.releaser.CommitTerminalWithoutEpoch(ctx, binding.AttemptID(), termRes); errRel != nil { return errRel } r.emitObservation(ObservationKindTerminalCommitted, epoch.ID(), func(in *FilterObservationInput) { in.CommitState = CommitStateTerminalCommitted if termRes.Success() { in.TerminalReason = TerminalReasonCompleted } else { in.Causes = termRes.FailureCauses() } }) r.mu.Lock() r.terminalCommitted = true r.mu.Unlock() return nil } case ArbitrationActionRelease: useUnbuffered := passThrough || !r.tail.HasUnconfirmedEvents(epoch.ID()) if terminalFlag { termRes, errRes := normalizedTerminalToResult(ev) if errRes != nil { return errRes } if useUnbuffered { if errRel := r.releaser.CommitTerminalWithoutEpoch(ctx, binding.AttemptID(), termRes); errRel != nil { return errRel } } else { if _, errRel := r.releaser.ReleaseTerminalEpoch(ctx, binding.AttemptID(), epoch.ID(), termRes); errRel != nil { return errRel } } r.emitObservation(ObservationKindReleaseCommitted, epoch.ID(), func(in *FilterObservationInput) { in.CommitState = CommitStateStreamOpen }) r.emitObservation(ObservationKindTerminalCommitted, epoch.ID(), func(in *FilterObservationInput) { in.CommitState = CommitStateTerminalCommitted if termRes.Success() { in.TerminalReason = TerminalReasonCompleted } else { in.Causes = termRes.FailureCauses() } }) r.mu.Lock() r.terminalCommitted = true r.mu.Unlock() return nil } if useUnbuffered { if _, errRel := r.releaser.ReleasePassThroughEvent(ctx, binding.AttemptID(), ev); errRel != nil { return errRel } } else { if _, errRel := r.releaser.ReleaseEpoch(ctx, binding.AttemptID(), epoch.ID()); errRel != nil { return errRel } } r.emitObservation(ObservationKindReleaseCommitted, epoch.ID(), func(in *FilterObservationInput) { in.CommitState = CommitStateStreamOpen }) case ArbitrationActionHold: // Hold downstream; events stay buffered in tail case ArbitrationActionRecover: r.planIDCounter++ planID := fmt.Sprintf("plan-%s-%d", r.snapshot.RequestID(), r.planIDCounter) idempotencyKey := fmt.Sprintf("idemp-%s-%d", r.snapshot.RequestID(), r.planIDCounter) preparerID := "" var prepSnapshot RecoveryPreparationSnapshot if r.snapshot.Preparer() != nil && r.snapshot.PreparationSnapshotFactory() != nil { preparerID = "default" ps, errPrep := r.snapshot.PreparationSnapshotFactory().CreatePreparationSnapshot(ctx, r.snapshot, binding) if errPrep != nil { desc, errD := NewExternalDescriptor("error", "preparation_snapshot_failed", "preparation_snapshot_factory_failed", "") if errD != nil { return errD } cause, errC := NewFailureCause("preparer_factory", "snapshot_creation_failed", "", "", "") if errC != nil { return errC } causes, errCh := NewFailureCauseChain([]FailureCause{cause}) if errCh != nil { return errCh } termRes, errTerm := NewErrorTerminalResult(ev.Channel(), desc, causes, time.Now()) if errTerm != nil { return errTerm } if _, errRel := r.releaser.FailPending(ctx, binding.AttemptID(), termRes); errRel != nil { return errRel } r.emitObservation(ObservationKindTerminalCommitted, epoch.ID(), func(in *FilterObservationInput) { in.CommitState = CommitStateTerminalCommitted in.Causes = causes }) r.mu.Lock() r.terminalCommitted = true r.mu.Unlock() return nil } prepSnapshot = ps } cycleInput := RecoveryCycleInput{ Arbitration: arbResult, PlanID: planID, IdempotencyKey: idempotencyKey, ConsumerID: arbResult.FilterID(), CommitState: r.boundary.State(), CallerCanceled: ctx.Err() != nil, PreparerID: preparerID, PreparationSnapshot: prepSnapshot, } cycleRes, errRec := r.recoveryCoord.Execute(evalCtx, cycleInput) newBinding, hasBinding := cycleRes.Binding() plan, hasPlan := cycleRes.Plan() if errRec != nil || !hasBinding || !hasPlan { desc, errD := NewExternalDescriptor("error", "recovery_failed", "recovery_failed", "") if errD != nil { return errD } causes := cycleRes.FailureCauses() if causes.Len() == 0 { c, errC := NewFailureCause("recovery", "recovery_failed", "", "", "") if errC != nil { return errC } causesChain, errCh := NewFailureCauseChain([]FailureCause{c}) if errCh != nil { return errCh } causes = causesChain } termRes, errTerm := NewErrorTerminalResult(ev.Channel(), desc, causes, time.Now()) if errTerm != nil { return errTerm } if cycleRes.PreviousAttemptClosed() { if errRel := r.releaser.DiscardPendingAndCommitTerminal(ctx, binding.AttemptID(), termRes); errRel != nil { return errRel } } else { if _, errRel := r.releaser.FailPending(ctx, binding.AttemptID(), termRes); errRel != nil { return errRel } } r.emitObservation(ObservationKindTerminalCommitted, epoch.ID(), func(in *FilterObservationInput) { in.CommitState = CommitStateTerminalCommitted in.Causes = causes }) r.mu.Lock() r.terminalCommitted = true r.mu.Unlock() return nil } cand, prepErr := r.prepareAttemptInstall(newBinding, plan.ResumeMode()) if prepErr != nil { abortFailed := false if newCtrl := newBinding.Controller(); newCtrl != nil { if abortErr := newCtrl.AbortAttempt(ctx); abortErr != nil { abortFailed = true } } desc, errD := NewExternalDescriptor("error", "attempt_install_failed", "failed_to_prepare_attempt_install", "") if errD != nil { return errD } cause, errC := NewFailureCause("runtime", "install_prepare_failed", "", "", "") if errC != nil { return errC } causeList := []FailureCause{cause} if abortFailed { abortCause, errAbortCause := NewFailureCause("runtime", "attempt_abort_failed", "", "", "") if errAbortCause != nil { return errAbortCause } causeList = append(causeList, abortCause) } causes, errCh := NewFailureCauseChain(causeList) if errCh != nil { return errCh } termRes, errTerm := NewErrorTerminalResult(ev.Channel(), desc, causes, time.Now()) if errTerm != nil { return errTerm } if errRel := r.releaser.DiscardPendingAndCommitTerminal(ctx, binding.AttemptID(), termRes); errRel != nil { return errRel } r.emitObservation(ObservationKindTerminalCommitted, epoch.ID(), func(in *FilterObservationInput) { in.CommitState = CommitStateTerminalCommitted in.Causes = causes }) r.mu.Lock() r.terminalCommitted = true r.mu.Unlock() return nil } if commitErr := r.commitAttemptInstall(ctx, cand); commitErr != nil { abortCauseAdded := false if newCtrl := newBinding.Controller(); newCtrl != nil { if aErr := newCtrl.AbortAttempt(ctx); aErr != nil { abortCauseAdded = true } } desc, errD := NewExternalDescriptor("error", "attempt_install_failed", "failed_to_commit_attempt_install", "") if errD != nil { return errD } commitCause, errCommitCause := NewFailureCause("runtime", "install_commit_failed", "", "", "") if errCommitCause != nil { return errCommitCause } causeList := []FailureCause{commitCause} if abortCauseAdded { abortCause, errAbortCause := NewFailureCause("runtime", "attempt_abort_failed", "", "", "") if errAbortCause != nil { return errAbortCause } causeList = append(causeList, abortCause) } causes, errCh := NewFailureCauseChain(causeList) if errCh != nil { return errCh } termRes, errTerm := NewErrorTerminalResult(ev.Channel(), desc, causes, time.Now()) if errTerm != nil { return errTerm } if errRel := r.releaser.DiscardPendingAndCommitTerminal(ctx, binding.AttemptID(), termRes); errRel != nil { return errRel } r.emitObservation(ObservationKindTerminalCommitted, epoch.ID(), func(in *FilterObservationInput) { in.CommitState = CommitStateTerminalCommitted in.Causes = causes }) r.mu.Lock() r.terminalCommitted = true r.mu.Unlock() return nil } case ArbitrationActionTerminal: useUnbuffered := passThrough || !r.tail.HasUnconfirmedEvents(epoch.ID()) var termCauses FailureCauseChain if arbResult.BaseDisposition() == BaseDispositionTerminalSuccessCandidate { termRes, errRes := ev.AsTerminal() if errRes != nil { return errRes } if useUnbuffered { if errRel := r.releaser.CommitTerminalWithoutEpoch(ctx, binding.AttemptID(), termRes); errRel != nil { return errRel } } else { if _, errRel := r.releaser.ReleaseTerminalEpoch(ctx, binding.AttemptID(), epoch.ID(), termRes); errRel != nil { return errRel } } } else if arbResult.BaseDisposition() == BaseDispositionTerminalErrorCandidate { termRes, errRes := providerErrorToTerminalResult(ev) if errRes != nil { return errRes } termCauses = termRes.FailureCauses() if termCauses.Len() == 0 { c, _ := NewFailureCause("provider", "error", "", "", "") termCauses, _ = NewFailureCauseChain([]FailureCause{c}) } if useUnbuffered { if errRel := r.releaser.CommitTerminalWithoutEpoch(ctx, binding.AttemptID(), termRes); errRel != nil { return errRel } } else { if _, errRel := r.releaser.ReleaseTerminalEpoch(ctx, binding.AttemptID(), epoch.ID(), termRes); errRel != nil { return errRel } } } else { desc, errD := NewExternalDescriptor("error", "fatal_violation", "fatal_filter_violation", "") if errD != nil { return errD } cause, errC := NewFailureCause("arbiter", "fatal_violation", "", arbResult.FilterID(), arbResult.RuleID()) if errC != nil { return errC } causes, errCh := NewFailureCauseChain([]FailureCause{cause}) if errCh != nil { return errCh } termCauses = causes termRes, errTerm := NewErrorTerminalResult(ev.Channel(), desc, causes, time.Now()) if errTerm != nil { return errTerm } if useUnbuffered { if errRel := r.releaser.CommitTerminalWithoutEpoch(ctx, binding.AttemptID(), termRes); errRel != nil { return errRel } } else { if _, errRel := r.releaser.FailPending(ctx, binding.AttemptID(), termRes); errRel != nil { return errRel } } } if arbResult.BaseDisposition() == BaseDispositionTerminalSuccessCandidate { r.emitObservation(ObservationKindReleaseCommitted, epoch.ID(), func(in *FilterObservationInput) { in.CommitState = CommitStateStreamOpen }) } r.emitObservation(ObservationKindTerminalCommitted, epoch.ID(), func(in *FilterObservationInput) { in.CommitState = CommitStateTerminalCommitted if arbResult.BaseDisposition() == BaseDispositionTerminalSuccessCandidate { in.TerminalReason = TerminalReasonCompleted } else { in.Causes = termCauses } }) r.mu.Lock() r.terminalCommitted = true r.mu.Unlock() return nil } } } } // PendingEventsByChannel returns a defensive map of pending events per channel. func (t *EvidenceTail) PendingEventsByChannel() map[string][]NormalizedEvent { if t == nil || len(t.channelState) == 0 { return nil } out := make(map[string][]NormalizedEvent) for ch, cs := range t.channelState { var pe []NormalizedEvent for _, e := range cs.pendingEntries { if e.event.Kind() != EventKindTerminal && e.event.Kind() != EventKindProviderError { pe = append(pe, cloneNormalizedEvent(e.event)) } } if len(pe) > 0 { out[ch] = pe } } if len(out) == 0 { return nil } return out } // CommittedLookBehindByChannel returns a defensive map of committed look-behind events per channel. func (t *EvidenceTail) CommittedLookBehindByChannel() map[string][]NormalizedEvent { if t == nil || len(t.channelState) == 0 { return nil } out := make(map[string][]NormalizedEvent) for ch, cs := range t.channelState { if len(cs.committedLookBehind) > 0 { lb := make([]NormalizedEvent, len(cs.committedLookBehind)) for i, ev := range cs.committedLookBehind { lb[i] = cloneNormalizedEvent(ev) } out[ch] = lb } } if len(out) == 0 { return nil } return out } // PrepareContinuationWithPlan updates the evidence plan and prepares continuation. func (t *EvidenceTail) PrepareContinuationWithPlan(plan EvidencePlan) { t.plan = plan t.PrepareContinuation() } // Plan returns the current EvidencePlan. func (t *EvidenceTail) Plan() EvidencePlan { if t == nil { return EvidencePlan{} } return t.plan }