iop/packages/go/streamgate/runtime.go
toki 7634ca8962 feat(stream-evidence-gate-core): complete stream evidence gate core implementation
- OpenAI request rebuilder with tool validation and provider tunnel
- Edge config runtime refresh for stream evidence gate
- Filter observation contract and runtime with sink/correlation
- Stream gate dispatcher, release sink, and vertical slice
- Recovery coordinator for evidence tail
- Parallel evaluation and commit boundary
- E2E test script for OpenAI vLLM
- Archive completed task groups to archive/2026/07
2026-07-28 04:08:12 +09:00

1381 lines
45 KiB
Go

package streamgate
import (
"context"
"errors"
"fmt"
"sync"
"time"
)
const (
// DefaultMaxEvidenceRunes is the default Unicode rune threshold for evidence holding.
DefaultMaxEvidenceRunes = 500
// DefaultMaxBufferRunes is the default maximum rune buffer size.
DefaultMaxBufferRunes = 4096
// DefaultMaxIngressSnapshotBytes is the default byte limit for ingress body snapshots (16 MiB).
DefaultMaxIngressSnapshotBytes int64 = 16 * 1024 * 1024
// DefaultMaxRecoveryAttemptsTotal is the default total fault recovery attempt limit.
DefaultMaxRecoveryAttemptsTotal = 3
)
// RuntimeOptions captures immutable configuration parameters for a streamgate request runtime.
type RuntimeOptions struct {
maxEvidenceRunes int
maxBufferRunes int
maxIngressSnapshotBytes int64
maxRecoveryAttemptsTotal int
// maxRecoveryAttemptsByStrategy is the immutable per-fault-strategy recovery
// cap snapshot. A nil/empty map means every fault strategy inherits the
// request-total cap; a non-empty map disables any fault strategy it omits and
// caps each listed strategy (an explicit 0 forbids that strategy entirely).
maxRecoveryAttemptsByStrategy map[RecoveryStrategy]int
gateOptions GateCoordinatorOptions
recoveryOptions RecoveryCoordinatorOptions
}
// DefaultRuntimeOptions returns RuntimeOptions populated with default settings.
func DefaultRuntimeOptions() RuntimeOptions {
opts, err := NewRuntimeOptions(
DefaultMaxEvidenceRunes,
DefaultMaxBufferRunes,
DefaultMaxIngressSnapshotBytes,
DefaultMaxRecoveryAttemptsTotal,
GateCoordinatorOptions{},
RecoveryCoordinatorOptions{},
)
if err != nil {
panic(err)
}
return opts
}
// NewRuntimeOptions constructs validated RuntimeOptions.
func NewRuntimeOptions(
maxEvidenceRunes, maxBufferRunes int,
maxIngressBytes int64,
maxRecoveryAttempts int,
gateOpts GateCoordinatorOptions,
recoveryOpts RecoveryCoordinatorOptions,
) (RuntimeOptions, error) {
if maxEvidenceRunes <= 0 {
return RuntimeOptions{}, errors.New("streamgate: runtime options max evidence runes must be positive")
}
if maxBufferRunes < maxEvidenceRunes {
return RuntimeOptions{}, errors.New("streamgate: runtime options max buffer runes must be >= max evidence runes")
}
if maxIngressBytes <= 0 {
return RuntimeOptions{}, errors.New("streamgate: runtime options max ingress snapshot bytes must be positive")
}
if maxRecoveryAttempts < 0 || maxRecoveryAttempts > DefaultMaxRecoveryAttemptsTotal {
return RuntimeOptions{}, errors.New("streamgate: runtime options max recovery attempts total must be between 0 and 3")
}
opts := RuntimeOptions{
maxEvidenceRunes: maxEvidenceRunes,
maxBufferRunes: maxBufferRunes,
maxIngressSnapshotBytes: maxIngressBytes,
maxRecoveryAttemptsTotal: maxRecoveryAttempts,
gateOptions: gateOpts,
recoveryOptions: recoveryOpts,
}
if err := opts.Validate(); err != nil {
return RuntimeOptions{}, err
}
return opts, nil
}
// Validate returns nil when RuntimeOptions is in a consistent state.
func (o RuntimeOptions) Validate() error {
if o.maxEvidenceRunes <= 0 {
return errors.New("streamgate: runtime options max evidence runes must be positive")
}
if o.maxBufferRunes < o.maxEvidenceRunes {
return errors.New("streamgate: runtime options max buffer runes must be >= max evidence runes")
}
if o.maxIngressSnapshotBytes <= 0 || o.maxIngressSnapshotBytes > DefaultMaxIngressSnapshotBytes {
return errors.New("streamgate: runtime options max ingress snapshot bytes must be between 1 and 16777216")
}
if o.maxRecoveryAttemptsTotal < 0 || o.maxRecoveryAttemptsTotal > DefaultMaxRecoveryAttemptsTotal {
return errors.New("streamgate: runtime options max recovery attempts total must be between 0 and 3")
}
for strategy, limit := range o.maxRecoveryAttemptsByStrategy {
if !isFaultRecoveryStrategy(strategy) {
return errors.New("streamgate: runtime options strategy cap contains a non-fault strategy")
}
if limit < 0 || limit > o.maxRecoveryAttemptsTotal {
return errors.New("streamgate: runtime options strategy cap must be between zero and max recovery attempts total")
}
}
return nil
}
// WithRecoveryStrategyLimits returns a validated copy of the options carrying
// an effective per-fault-strategy recovery cap. A nil/empty map inherits the
// request-total cap for every strategy; a non-empty map disables any strategy
// it omits and caps each listed strategy at its value (0 forbids the strategy).
func (o RuntimeOptions) WithRecoveryStrategyLimits(limits map[RecoveryStrategy]int) (RuntimeOptions, error) {
next := o
next.maxRecoveryAttemptsByStrategy = copyRecoveryStrategyCounts(limits)
if err := next.Validate(); err != nil {
return RuntimeOptions{}, err
}
return next, nil
}
// MaxEvidenceRunes returns the max evidence runes setting.
func (o RuntimeOptions) MaxEvidenceRunes() int { return o.maxEvidenceRunes }
// MaxBufferRunes returns the max buffer runes setting.
func (o RuntimeOptions) MaxBufferRunes() int { return o.maxBufferRunes }
// MaxIngressSnapshotBytes returns the max ingress snapshot bytes setting.
func (o RuntimeOptions) MaxIngressSnapshotBytes() int64 { return o.maxIngressSnapshotBytes }
// MaxRecoveryAttemptsTotal returns the total recovery attempts limit.
func (o RuntimeOptions) MaxRecoveryAttemptsTotal() int { return o.maxRecoveryAttemptsTotal }
// MaxRecoveryAttemptsByStrategy returns a defensive copy of the effective
// per-fault-strategy recovery cap. A nil result means every fault strategy
// inherits the request-total cap.
func (o RuntimeOptions) MaxRecoveryAttemptsByStrategy() map[RecoveryStrategy]int {
return copyRecoveryStrategyCounts(o.maxRecoveryAttemptsByStrategy)
}
// GateOptions returns the gate coordinator options.
func (o RuntimeOptions) GateOptions() GateCoordinatorOptions { return o.gateOptions }
// RecoveryOptions returns the recovery coordinator options.
func (o RuntimeOptions) RecoveryOptions() RecoveryCoordinatorOptions { return o.recoveryOptions }
// RequestRuntimeSnapshot binds request-local state, host seams, filter registry snapshot,
// and runtime options into an immutable contract to be consumed by request-local owners.
type RequestRuntimeSnapshot struct {
requestID StableToken
configGen StableToken
environment string
endpoint string
family string
options RuntimeOptions
registrySnapshot FilterRegistrySnapshot
ingressSnapshot *IngressSnapshot
requestSnapshotRef RecoveryRequestSnapshotRef
dispatcher AttemptDispatcher
rebuilder RequestRebuilder
preparer RecoveryPlanPreparer
prepFactory RecoveryPreparationSnapshotFactory
sink ReleaseSink
obsSink ObservationSink
}
// RecoveryPreparationSnapshotFactory creates a request-local RecoveryPreparationSnapshot
// bound to the given request snapshot and attempt binding.
type RecoveryPreparationSnapshotFactory interface {
CreatePreparationSnapshot(ctx context.Context, snapshot RequestRuntimeSnapshot, binding AttemptBinding) (RecoveryPreparationSnapshot, error)
}
// NewRequestRuntimeSnapshot constructs a validated RequestRuntimeSnapshot.
func NewRequestRuntimeSnapshot(
requestID, configGen, environment, endpoint, family string,
options RuntimeOptions,
registrySnapshot FilterRegistrySnapshot,
ingressSnapshot *IngressSnapshot,
requestSnapshotRef RecoveryRequestSnapshotRef,
dispatcher AttemptDispatcher,
rebuilder RequestRebuilder,
preparer RecoveryPlanPreparer,
prepFactory RecoveryPreparationSnapshotFactory,
sink ReleaseSink,
) (RequestRuntimeSnapshot, error) {
reqIDToken, err := NewStableTokenRequired("requestID", requestID)
if err != nil {
return RequestRuntimeSnapshot{}, err
}
cfgGenToken, err := NewStableTokenRequired("configGen", configGen)
if err != nil {
return RequestRuntimeSnapshot{}, err
}
if err := options.Validate(); err != nil {
return RequestRuntimeSnapshot{}, err
}
if registrySnapshot.Generation() == "" {
return RequestRuntimeSnapshot{}, errors.New("streamgate: request runtime snapshot registry snapshot generation is required")
}
if dispatcher == nil {
return RequestRuntimeSnapshot{}, errors.New("streamgate: request runtime snapshot dispatcher is required")
}
if rebuilder == nil {
return RequestRuntimeSnapshot{}, errors.New("streamgate: request runtime snapshot rebuilder is required")
}
if sink == nil {
return RequestRuntimeSnapshot{}, errors.New("streamgate: request runtime snapshot release sink is required")
}
if err := requestSnapshotRef.Validate(); err != nil {
return RequestRuntimeSnapshot{}, err
}
snap := RequestRuntimeSnapshot{
requestID: reqIDToken,
configGen: cfgGenToken,
environment: environment,
endpoint: endpoint,
family: family,
options: options,
registrySnapshot: registrySnapshot,
ingressSnapshot: ingressSnapshot,
requestSnapshotRef: requestSnapshotRef,
dispatcher: dispatcher,
rebuilder: rebuilder,
preparer: preparer,
prepFactory: prepFactory,
sink: sink,
}
if err := snap.Validate(); err != nil {
return RequestRuntimeSnapshot{}, err
}
return snap, nil
}
// Validate returns nil when the RequestRuntimeSnapshot is consistent.
func (r RequestRuntimeSnapshot) Validate() error {
if err := validateStableTokenRequired("requestID", r.requestID.value); err != nil {
return err
}
if err := validateStableTokenRequired("configGen", r.configGen.value); err != nil {
return err
}
if err := r.options.Validate(); err != nil {
return err
}
registryGeneration := r.registrySnapshot.Generation()
if registryGeneration == "" {
return errors.New("streamgate: request runtime snapshot registry snapshot generation is required")
}
if registryGeneration != r.configGen.String() {
return errors.New("streamgate: request runtime snapshot config and registry generations must match")
}
if r.dispatcher == nil {
return errors.New("streamgate: request runtime snapshot dispatcher is required")
}
if r.rebuilder == nil {
return errors.New("streamgate: request runtime snapshot rebuilder is required")
}
if r.sink == nil {
return errors.New("streamgate: request runtime snapshot release sink is required")
}
if err := r.requestSnapshotRef.Validate(); err != nil {
return err
}
if (r.preparer == nil && r.prepFactory != nil) || (r.preparer != nil && r.prepFactory == nil) {
return errors.New("streamgate: request runtime snapshot preparer and preparation snapshot factory must both be nil or both non-nil")
}
return nil
}
// RequestID returns the stable request identifier string.
func (r RequestRuntimeSnapshot) RequestID() string { return r.requestID.value }
// ConfigGeneration returns the configuration generation string.
func (r RequestRuntimeSnapshot) ConfigGeneration() string { return r.configGen.value }
// Environment returns the environment label.
func (r RequestRuntimeSnapshot) Environment() string { return r.environment }
// Endpoint returns the endpoint label.
func (r RequestRuntimeSnapshot) Endpoint() string { return r.endpoint }
// Family returns the model family label.
func (r RequestRuntimeSnapshot) Family() string { return r.family }
// Options returns the runtime options.
func (r RequestRuntimeSnapshot) Options() RuntimeOptions { return r.options }
// RegistrySnapshot returns the filter registry snapshot.
func (r RequestRuntimeSnapshot) RegistrySnapshot() FilterRegistrySnapshot {
return r.registrySnapshot
}
// IngressSnapshot returns the ingress snapshot pointer, or nil if unset.
func (r RequestRuntimeSnapshot) IngressSnapshot() *IngressSnapshot {
return r.ingressSnapshot
}
// RequestSnapshotRef returns the recovery request snapshot reference.
func (r RequestRuntimeSnapshot) RequestSnapshotRef() RecoveryRequestSnapshotRef {
return r.requestSnapshotRef
}
// Dispatcher returns the host attempt dispatcher.
func (r RequestRuntimeSnapshot) Dispatcher() AttemptDispatcher { return r.dispatcher }
// Rebuilder returns the host request rebuilder.
func (r RequestRuntimeSnapshot) Rebuilder() RequestRebuilder { return r.rebuilder }
// Preparer returns the optional host recovery plan preparer.
func (r RequestRuntimeSnapshot) Preparer() RecoveryPlanPreparer { return r.preparer }
// PreparationSnapshotFactory returns the optional recovery preparation snapshot factory.
func (r RequestRuntimeSnapshot) PreparationSnapshotFactory() RecoveryPreparationSnapshotFactory {
return r.prepFactory
}
// Sink returns the host release sink.
func (r RequestRuntimeSnapshot) Sink() ReleaseSink { return r.sink }
// ObservationSink returns the observation sink, defaulting to NoopObservationSink if unset.
func (r RequestRuntimeSnapshot) ObservationSink() ObservationSink {
if r.obsSink == nil {
return NoopObservationSink{}
}
return r.obsSink
}
// WithObservationSink returns a copy of RequestRuntimeSnapshot with the given ObservationSink.
func (r RequestRuntimeSnapshot) WithObservationSink(sink ObservationSink) RequestRuntimeSnapshot {
cp := r
cp.obsSink = sink
return cp
}
// BuildAttemptFilterContext constructs a FilterContext bound to this request snapshot.
func (r RequestRuntimeSnapshot) BuildAttemptFilterContext(
attemptID string,
target AttemptTarget,
epochID uint64,
commitState CommitState,
) (FilterContext, error) {
builder := NewFilterContextBuilder(r.configGen.value, attemptID).
SetEnvironment(r.environment).
SetEndpoint(r.endpoint).
SetFamily(r.family).
SetModelGroup(target.ModelGroup()).
SetActualModel(target.Model()).
SetActualProvider(target.Provider()).
SetExecutionPath(target.ExecutionPath()).
SetCommitState(commitState).
SetStableCorrelation(r.requestID.value)
if epochID > 0 {
var err error
builder, err = builder.SetEpoch(epochID)
if err != nil {
return FilterContext{}, err
}
}
return builder.Build()
}
// attemptCancellerAdapter wraps RequestRuntime to satisfy StreamReleaser's AttemptCanceller seam.
type attemptCancellerAdapter struct {
r *RequestRuntime
}
func (a *attemptCancellerAdapter) CancelAttempt(ctx context.Context, attemptID string) error {
if a.r != nil {
a.r.mu.Lock()
ctrl := a.r.currentBinding.Controller()
a.r.mu.Unlock()
if ctrl != nil {
return ctrl.AbortAttempt(ctx)
}
}
return nil
}
// RequestRuntime drives the request-local lifecycle loop, managing attempt bindings,
// evidence collection, parallel filter evaluation through GateCoordinator, decision arbitration,
// stream release, and fault recovery.
type RequestRuntime struct {
snapshot RequestRuntimeSnapshot
modelGroup string
currentBinding AttemptBinding
tail *EvidenceTail
boundary *CommitBoundary
releaser *StreamReleaser
gate *GateCoordinator
arbiter *DecisionArbiter
recoveryCoord *RecoveryCoordinator
recoveryPolicy RecoveryPolicySnapshot
recoveryUsage RecoveryUsageSnapshot
resolvedFilters []ResolvedFilter
stagedStart *ResponseStart
terminalCommitted bool
running bool
suppressContinuationStart bool
resourcesClosed bool
planIDCounter int
sequencer *ObservationSequencer
mu sync.Mutex
}
func (r *RequestRuntime) emitObservation(kind ObservationKind, epochID uint64, mutate func(*FilterObservationInput)) {
if r == nil || r.sequencer == nil {
return
}
r.mu.Lock()
binding := r.currentBinding
commitState := r.boundary.State()
r.mu.Unlock()
attemptTarget, err := NewObservationAttemptTarget(r.modelGroup, binding.Model(), binding.Provider(), binding.ExecutionPath())
if err != nil {
return
}
input := FilterObservationInput{
Kind: kind,
StableCorrelation: r.snapshot.RequestID(),
ConfigGeneration: r.snapshot.ConfigGeneration(),
AttemptID: binding.AttemptID(),
AttemptTarget: attemptTarget,
EpochID: epochID,
CommitState: commitState,
OccurredAt: time.Now(),
}
if mutate != nil {
mutate(&input)
}
_, _ = r.sequencer.Emit(context.Background(), input)
}
// gracefulAttemptController is an optional AttemptController extension a host
// controller may implement to release provider ownership after a normal
// terminal without issuing a provider-side cancel. CloseRequestResources uses
// it on a graceful (successful-terminal) close and falls back to AbortAttempt
// for every non-graceful outcome.
type gracefulAttemptController interface {
AttemptController
CloseAttempt(ctx context.Context) error
}
// requestResourceCloser is an optional RequestRebuilder extension: a
// request-local host rebuilder that must be closed exactly once when the
// request runtime finishes so its rebuilt-request store and reserved snapshot
// bytes are released on success, error, and caller-cancel alike.
type requestResourceCloser interface {
Close()
}
// CloseRequestResources performs idempotent end-of-request cleanup of the host
// resources this runtime owns. It aborts (or, when graceful, gracefully closes)
// the current attempt binding's controller — releasing the latest provider
// transport and any rebuilt lease, and canceling the latest provider run on a
// non-graceful outcome — and closes the request rebuilder. It is safe to call
// exactly once after Run returns for any terminal, error, or caller-cancel
// outcome. graceful must be true only when Run committed a successful terminal.
func (r *RequestRuntime) CloseRequestResources(ctx context.Context, graceful bool) error {
if r == nil {
return nil
}
r.mu.Lock()
if r.resourcesClosed {
r.mu.Unlock()
return nil
}
r.resourcesClosed = true
ctrl := r.currentBinding.Controller()
r.mu.Unlock()
var err error
if ctrl != nil {
if graceful {
if gc, ok := ctrl.(gracefulAttemptController); ok {
err = gc.CloseAttempt(ctx)
} else {
err = ctrl.AbortAttempt(ctx)
}
} else {
err = ctrl.AbortAttempt(ctx)
}
}
if closer, ok := r.snapshot.Rebuilder().(requestResourceCloser); ok {
closer.Close()
}
return err
}
func providerErrorToTerminalResult(ev NormalizedEvent) (TerminalResult, error) {
return ev.AsProviderError()
}
func normalizedTerminalToResult(ev NormalizedEvent) (TerminalResult, error) {
if ev.Kind() == EventKindTerminal {
return NewSuccessTerminalResult(ev.Channel(), ev.Timestamp())
}
return providerErrorToTerminalResult(ev)
}
// NewRequestRuntime constructs a new RequestRuntime instance.
func NewRequestRuntime(snapshot RequestRuntimeSnapshot, modelGroup string, initial AttemptBinding) (*RequestRuntime, error) {
if err := snapshot.Validate(); err != nil {
return nil, fmt.Errorf("streamgate: new request runtime snapshot invalid: %w", err)
}
if modelGroup == "" {
return nil, errors.New("streamgate: new request runtime model group is required")
}
if err := initial.Validate(); err != nil {
return nil, fmt.Errorf("streamgate: new request runtime initial binding invalid: %w", err)
}
boundary, err := NewCommitBoundary(snapshot.Sink())
if err != nil {
return nil, err
}
arbiter := NewDecisionArbiter()
gate, err := NewGateCoordinator(context.Background(), snapshot.Options().GateOptions(), arbiter)
if err != nil {
return nil, err
}
seq := NewObservationSequencer(snapshot.ObservationSink(), nil)
gate.SetObservationSequencer(seq)
recPolicy, err := NewRecoveryPolicySnapshot(
snapshot.Options().MaxRecoveryAttemptsTotal(),
snapshot.Options().MaxRecoveryAttemptsByStrategy(),
)
if err != nil {
return nil, err
}
recUsage, err := NewRecoveryUsageSnapshot(0, nil)
if err != nil {
return nil, err
}
preparers := make(map[string]RecoveryPlanPreparer)
if snapshot.Preparer() != nil {
preparers["default"] = snapshot.Preparer()
}
recOpts := RecoveryCoordinatorOptions{
Policy: recPolicy,
Usage: recUsage,
RequestSnapshot: snapshot.RequestSnapshotRef(),
CurrentBinding: initial,
Rebuilder: snapshot.Rebuilder(),
Dispatcher: snapshot.Dispatcher(),
Preparers: preparers,
PreparationTimeout: 5 * time.Second,
ObservationSink: snapshot.ObservationSink(),
}
recCoord, err := NewRecoveryCoordinator(recOpts)
if err != nil {
return nil, err
}
recCoord.SetObservationSequencer(seq)
rt := &RequestRuntime{
snapshot: snapshot,
modelGroup: modelGroup,
boundary: boundary,
gate: gate,
arbiter: arbiter,
recoveryCoord: recCoord,
recoveryPolicy: recPolicy,
recoveryUsage: recUsage,
sequencer: seq,
}
cand, err := rt.prepareAttemptInstall(initial, RecoveryResumeModeReplaceAttempt)
if err != nil {
return nil, err
}
if err := rt.commitAttemptInstall(context.Background(), cand); err != nil {
return nil, err
}
return rt, nil
}
type attemptInstallCandidate struct {
binding AttemptBinding
resumeMode RecoveryResumeMode
oldAttemptID string
target AttemptTarget
resolvedFilters []ResolvedFilter
plan EvidencePlan
tail *EvidenceTail
releaser *StreamReleaser
}
func (r *RequestRuntime) prepareAttemptInstall(binding AttemptBinding, resumeMode RecoveryResumeMode) (attemptInstallCandidate, error) {
if err := binding.Validate(); err != nil {
return attemptInstallCandidate{}, err
}
if err := resumeMode.Validate(); err != nil {
return attemptInstallCandidate{}, err
}
target, err := NewAttemptTarget(r.modelGroup, binding.Model(), binding.Provider(), binding.ExecutionPath(), nil)
if err != nil {
return attemptInstallCandidate{}, err
}
rfc, err := NewRequestFilterContext(
r.snapshot.ConfigGeneration(),
binding.AttemptID(),
r.snapshot.Environment(),
r.snapshot.Endpoint(),
r.snapshot.Family(),
"",
r.boundary.State(),
false,
false,
r.snapshot.RequestID(),
)
if err != nil {
return attemptInstallCandidate{}, err
}
rfSnap, err := r.snapshot.RegistrySnapshot().BeginRequest(rfc)
if err != nil {
return attemptInstallCandidate{}, err
}
resolved, err := rfSnap.ResolveAttempt(target)
if err != nil {
return attemptInstallCandidate{}, err
}
plan, err := NewEvidencePlanFromResolvedFilters(resolved)
if err != nil {
return attemptInstallCandidate{}, err
}
if r.tail == nil {
tail, err := NewEvidenceTail(plan)
if err != nil {
return attemptInstallCandidate{}, err
}
releaser, err := NewStreamReleaser(tail, r.boundary, &attemptCancellerAdapter{r: r})
if err != nil {
return attemptInstallCandidate{}, err
}
return attemptInstallCandidate{
binding: binding,
resumeMode: resumeMode,
oldAttemptID: "",
target: target,
resolvedFilters: resolved,
plan: plan,
tail: tail,
releaser: releaser,
}, nil
}
if resumeMode == RecoveryResumeModeReplaceAttempt {
tail, err := NewEvidenceTail(plan)
if err != nil {
return attemptInstallCandidate{}, err
}
releaser, err := NewStreamReleaser(tail, r.boundary, &attemptCancellerAdapter{r: r})
if err != nil {
return attemptInstallCandidate{}, err
}
return attemptInstallCandidate{
binding: binding,
resumeMode: resumeMode,
oldAttemptID: r.currentBinding.AttemptID(),
target: target,
resolvedFilters: resolved,
plan: plan,
tail: tail,
releaser: releaser,
}, nil
}
if resumeMode == RecoveryResumeModeContinueStream {
oldID := ""
if r.currentBinding.AttemptID() != "" {
oldID = r.currentBinding.AttemptID()
}
return attemptInstallCandidate{
binding: binding,
resumeMode: resumeMode,
oldAttemptID: oldID,
target: target,
resolvedFilters: resolved,
plan: plan,
tail: nil,
releaser: nil,
}, nil
}
return attemptInstallCandidate{}, fmt.Errorf("streamgate: unsupported resume mode: %v", resumeMode)
}
func (r *RequestRuntime) commitAttemptInstall(ctx context.Context, candidate attemptInstallCandidate) error {
r.mu.Lock()
hasTail := r.tail != nil
rel := r.releaser
r.mu.Unlock()
if !hasTail {
if err := r.boundary.BeginAttempt(candidate.binding.AttemptID()); err != nil {
return err
}
r.mu.Lock()
r.currentBinding = candidate.binding
r.resolvedFilters = candidate.resolvedFilters
r.tail = candidate.tail
r.releaser = candidate.releaser
r.mu.Unlock()
return nil
}
if candidate.resumeMode == RecoveryResumeModeReplaceAttempt {
if candidate.oldAttemptID != "" {
if err := candidate.releaser.ReplaceUncommittedAttempt(ctx, candidate.oldAttemptID, candidate.binding.AttemptID()); err != nil {
return err
}
} else {
if err := r.boundary.BeginAttempt(candidate.binding.AttemptID()); err != nil {
return err
}
candidate.tail.ResetForReplace()
}
r.mu.Lock()
r.currentBinding = candidate.binding
r.resolvedFilters = candidate.resolvedFilters
r.tail = candidate.tail
r.releaser = candidate.releaser
// The staged response start is attempt-local: the boundary already
// dropped it during handoff, so the runtime mirror must not leak the
// previous attempt's status/header into the new attempt's batches.
r.stagedStart = nil
r.mu.Unlock()
return nil
}
if candidate.resumeMode == RecoveryResumeModeContinueStream {
if candidate.oldAttemptID != "" && rel != nil {
if err := rel.ContinueOpenAttempt(ctx, candidate.oldAttemptID, candidate.binding.AttemptID()); err != nil {
return err
}
}
r.mu.Lock()
r.tail.PrepareContinuationWithPlan(candidate.plan)
r.currentBinding = candidate.binding
r.resolvedFilters = candidate.resolvedFilters
r.suppressContinuationStart = true
r.mu.Unlock()
return nil
}
return fmt.Errorf("streamgate: unsupported resume mode: %v", candidate.resumeMode)
}
// Run executes the normalized event owner loop for the request lifetime.
func (r *RequestRuntime) Run(ctx context.Context) error {
r.mu.Lock()
if r.running || r.terminalCommitted {
r.mu.Unlock()
return errors.New("streamgate: runtime already running or terminal")
}
r.running = true
r.mu.Unlock()
defer func() {
if r.gate != nil {
_ = r.gate.Close()
}
}()
for {
r.mu.Lock()
if r.terminalCommitted {
r.mu.Unlock()
return nil
}
binding := r.currentBinding
tail := r.tail
releaser := r.releaser
resolvedFilters := r.resolvedFilters
suppressStart := r.suppressContinuationStart
if suppressStart {
r.suppressContinuationStart = false
}
r.mu.Unlock()
if ctx.Err() != nil {
return ctx.Err()
}
ev, err := binding.EventSource().NextEvent(ctx)
if err != nil {
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) || ctx.Err() != nil {
return ctx.Err()
}
desc, errDesc := NewExternalDescriptor("error", "source_failure", "event_source_read_error", "")
if errDesc != nil {
return errDesc
}
cause, errCause := NewFailureCause("event_source", "read_error", "", "", "")
if errCause != nil {
return errCause
}
causes, errCauses := NewFailureCauseChain([]FailureCause{cause})
if errCauses != nil {
return errCauses
}
termRes, errTerm := NewErrorTerminalResult("default", desc, causes, time.Now())
if errTerm != nil {
return errTerm
}
if _, errRel := releaser.FailPending(ctx, binding.AttemptID(), termRes); errRel != nil {
return errRel
}
r.emitObservation(ObservationKindTerminalCommitted, 0, func(in *FilterObservationInput) {
in.CommitState = CommitStateTerminalCommitted
in.Causes = causes
})
r.mu.Lock()
r.terminalCommitted = true
r.mu.Unlock()
return nil
}
if ev.Kind() == EventKindResponseStart {
rs, errStart := ev.AsResponseStart()
if errStart != nil {
return errStart
}
if suppressStart {
continue
}
r.mu.Lock()
r.stagedStart = &rs
r.mu.Unlock()
if errStage := r.boundary.StageResponseStart(binding.AttemptID(), rs); errStage != nil {
return errStage
}
r.emitObservation(ObservationKindResponseStaged, 0, func(in *FilterObservationInput) {
in.CommitState = CommitStateTransportUncommitted
hold, _ := NewObservationHoldInfo(FilterHoldModeRolling, EventKindResponseStart, r.snapshot.Options().MaxEvidenceRunes(), 0)
in.Hold = &hold
})
continue
}
epoch, signal, errAppend := tail.Append(ev)
if errAppend != nil {
return errAppend
}
if signal == EvidenceTailSignalBufferOverflow {
desc, errDesc := NewExternalDescriptor("error", "buffer_overflow", "buffer_overflow_occurred", "")
if errDesc != nil {
return errDesc
}
cause, errCause := NewFailureCause("evidence_tail", "buffer_overflow", "", "", "")
if errCause != nil {
return errCause
}
causes, errCauses := NewFailureCauseChain([]FailureCause{cause})
if errCauses != nil {
return errCauses
}
termRes, errTerm := NewErrorTerminalResult(ev.Channel(), desc, causes, time.Now())
if errTerm != nil {
return errTerm
}
if _, errRel := releaser.FailPending(ctx, binding.AttemptID(), termRes); errRel != nil {
return errRel
}
r.emitObservation(ObservationKindTerminalCommitted, epoch.ID(), func(in *FilterObservationInput) {
in.CommitState = CommitStateTerminalCommitted
in.Causes = causes
})
r.mu.Lock()
r.terminalCommitted = true
r.mu.Unlock()
return nil
}
terminalFlag := (ev.Kind() == EventKindTerminal || ev.Kind() == EventKindProviderError)
isKindSubscribed := func(kind EventKind, kinds []EventKind) bool {
for _, k := range kinds {
if k == kind {
return true
}
}
return false
}
passThrough := signal == EvidenceTailSignalReady ||
(signal == EvidenceTailSignalNone && !isKindSubscribed(ev.Kind(), tail.Plan().BlockingKinds(ev.Channel())))
shouldEvaluate := terminalFlag || passThrough || signal == EvidenceTailSignalThreshold || signal == EvidenceTailSignalTrigger
if shouldEvaluate {
epochFilters, errBind := tail.Plan().BindEpochFilters(epoch, resolvedFilters)
if errBind != nil {
return errBind
}
r.mu.Lock()
stagedStartPtr := r.stagedStart
r.mu.Unlock()
if r.boundary.State() != CommitStateTransportUncommitted {
stagedStartPtr = nil
}
batch, errBatch := NewEvidenceBatch(
[]NormalizedEvent{ev},
tail.PendingEventsByChannel(),
tail.CommittedLookBehindByChannel(),
stagedStartPtr,
terminalFlag,
r.boundary.State(),
time.Now(),
)
if errBatch != nil {
return errBatch
}
attemptTarget, errTarget := NewObservationAttemptTarget(r.modelGroup, binding.Model(), binding.Provider(), binding.ExecutionPath())
if errTarget != nil {
return errTarget
}
obsCtx := ObservationContext{
Sequencer: r.sequencer,
StableCorrelation: r.snapshot.RequestID(),
ConfigGeneration: r.snapshot.ConfigGeneration(),
AttemptID: binding.AttemptID(),
AttemptTarget: attemptTarget,
EpochID: epoch.ID(),
}
evalCtx := WithObservationContext(ctx, obsCtx)
arbResult, errGate := r.gate.Submit(evalCtx, batch, epochFilters)
if errGate != nil {
if ctx.Err() != nil {
return ctx.Err()
}
return errGate
}
arbInfo, _ := NewObservationArbitrationInfo(arbResult.Action(), arbResult.BaseDisposition())
r.emitObservation(ObservationKindArbitrationDecided, epoch.ID(), func(in *FilterObservationInput) {
in.Arbitration = &arbInfo
})
switch arbResult.Action() {
case ArbitrationActionReplacement:
proposal := arbResult.ReplacementProposal()
if proposal == nil {
return errors.New("streamgate: replacement action missing replacement proposal")
}
if err := proposal.Validate(); err != nil {
return err
}
if _, errRel := r.releaser.releaseReplacementEvents(
ctx, binding.AttemptID(), epoch.ID(), proposal.Events(),
); errRel != nil {
return errRel
}
r.emitObservation(ObservationKindReleaseCommitted, epoch.ID(), func(in *FilterObservationInput) {
in.CommitState = CommitStateStreamOpen
})
if terminalFlag {
// The substituted payload replaces the held content only; the
// terminal disposition of the original event still closes the
// stream exactly once.
termRes, errRes := normalizedTerminalToResult(ev)
if errRes != nil {
return errRes
}
if errRel := r.releaser.CommitTerminalWithoutEpoch(ctx, binding.AttemptID(), termRes); errRel != nil {
return errRel
}
r.emitObservation(ObservationKindTerminalCommitted, epoch.ID(), func(in *FilterObservationInput) {
in.CommitState = CommitStateTerminalCommitted
if termRes.Success() {
in.TerminalReason = TerminalReasonCompleted
} else {
in.Causes = termRes.FailureCauses()
}
})
r.mu.Lock()
r.terminalCommitted = true
r.mu.Unlock()
return nil
}
case ArbitrationActionRelease:
useUnbuffered := passThrough || !r.tail.HasUnconfirmedEvents(epoch.ID())
if terminalFlag {
termRes, errRes := normalizedTerminalToResult(ev)
if errRes != nil {
return errRes
}
if useUnbuffered {
if errRel := r.releaser.CommitTerminalWithoutEpoch(ctx, binding.AttemptID(), termRes); errRel != nil {
return errRel
}
} else {
if _, errRel := r.releaser.ReleaseTerminalEpoch(ctx, binding.AttemptID(), epoch.ID(), termRes); errRel != nil {
return errRel
}
}
r.emitObservation(ObservationKindReleaseCommitted, epoch.ID(), func(in *FilterObservationInput) {
in.CommitState = CommitStateStreamOpen
})
r.emitObservation(ObservationKindTerminalCommitted, epoch.ID(), func(in *FilterObservationInput) {
in.CommitState = CommitStateTerminalCommitted
if termRes.Success() {
in.TerminalReason = TerminalReasonCompleted
} else {
in.Causes = termRes.FailureCauses()
}
})
r.mu.Lock()
r.terminalCommitted = true
r.mu.Unlock()
return nil
}
if useUnbuffered {
if _, errRel := r.releaser.ReleasePassThroughEvent(ctx, binding.AttemptID(), ev); errRel != nil {
return errRel
}
} else {
if _, errRel := r.releaser.ReleaseEpoch(ctx, binding.AttemptID(), epoch.ID()); errRel != nil {
return errRel
}
}
r.emitObservation(ObservationKindReleaseCommitted, epoch.ID(), func(in *FilterObservationInput) {
in.CommitState = CommitStateStreamOpen
})
case ArbitrationActionHold:
// Hold downstream; events stay buffered in tail
case ArbitrationActionRecover:
r.planIDCounter++
planID := fmt.Sprintf("plan-%s-%d", r.snapshot.RequestID(), r.planIDCounter)
idempotencyKey := fmt.Sprintf("idemp-%s-%d", r.snapshot.RequestID(), r.planIDCounter)
preparerID := ""
var prepSnapshot RecoveryPreparationSnapshot
if r.snapshot.Preparer() != nil && r.snapshot.PreparationSnapshotFactory() != nil {
preparerID = "default"
ps, errPrep := r.snapshot.PreparationSnapshotFactory().CreatePreparationSnapshot(ctx, r.snapshot, binding)
if errPrep != nil {
desc, errD := NewExternalDescriptor("error", "preparation_snapshot_failed", "preparation_snapshot_factory_failed", "")
if errD != nil {
return errD
}
cause, errC := NewFailureCause("preparer_factory", "snapshot_creation_failed", "", "", "")
if errC != nil {
return errC
}
causes, errCh := NewFailureCauseChain([]FailureCause{cause})
if errCh != nil {
return errCh
}
termRes, errTerm := NewErrorTerminalResult(ev.Channel(), desc, causes, time.Now())
if errTerm != nil {
return errTerm
}
if _, errRel := r.releaser.FailPending(ctx, binding.AttemptID(), termRes); errRel != nil {
return errRel
}
r.emitObservation(ObservationKindTerminalCommitted, epoch.ID(), func(in *FilterObservationInput) {
in.CommitState = CommitStateTerminalCommitted
in.Causes = causes
})
r.mu.Lock()
r.terminalCommitted = true
r.mu.Unlock()
return nil
}
prepSnapshot = ps
}
cycleInput := RecoveryCycleInput{
Arbitration: arbResult,
PlanID: planID,
IdempotencyKey: idempotencyKey,
ConsumerID: arbResult.FilterID(),
CommitState: r.boundary.State(),
CallerCanceled: ctx.Err() != nil,
PreparerID: preparerID,
PreparationSnapshot: prepSnapshot,
}
cycleRes, errRec := r.recoveryCoord.Execute(evalCtx, cycleInput)
newBinding, hasBinding := cycleRes.Binding()
plan, hasPlan := cycleRes.Plan()
if errRec != nil || !hasBinding || !hasPlan {
desc, errD := NewExternalDescriptor("error", "recovery_failed", "recovery_failed", "")
if errD != nil {
return errD
}
causes := cycleRes.FailureCauses()
if causes.Len() == 0 {
c, errC := NewFailureCause("recovery", "recovery_failed", "", "", "")
if errC != nil {
return errC
}
causesChain, errCh := NewFailureCauseChain([]FailureCause{c})
if errCh != nil {
return errCh
}
causes = causesChain
}
termRes, errTerm := NewErrorTerminalResult(ev.Channel(), desc, causes, time.Now())
if errTerm != nil {
return errTerm
}
if cycleRes.PreviousAttemptClosed() {
if errRel := r.releaser.DiscardPendingAndCommitTerminal(ctx, binding.AttemptID(), termRes); errRel != nil {
return errRel
}
} else {
if _, errRel := r.releaser.FailPending(ctx, binding.AttemptID(), termRes); errRel != nil {
return errRel
}
}
r.emitObservation(ObservationKindTerminalCommitted, epoch.ID(), func(in *FilterObservationInput) {
in.CommitState = CommitStateTerminalCommitted
in.Causes = causes
})
r.mu.Lock()
r.terminalCommitted = true
r.mu.Unlock()
return nil
}
cand, prepErr := r.prepareAttemptInstall(newBinding, plan.ResumeMode())
if prepErr != nil {
abortFailed := false
if newCtrl := newBinding.Controller(); newCtrl != nil {
if abortErr := newCtrl.AbortAttempt(ctx); abortErr != nil {
abortFailed = true
}
}
desc, errD := NewExternalDescriptor("error", "attempt_install_failed", "failed_to_prepare_attempt_install", "")
if errD != nil {
return errD
}
cause, errC := NewFailureCause("runtime", "install_prepare_failed", "", "", "")
if errC != nil {
return errC
}
causeList := []FailureCause{cause}
if abortFailed {
abortCause, errAbortCause := NewFailureCause("runtime", "attempt_abort_failed", "", "", "")
if errAbortCause != nil {
return errAbortCause
}
causeList = append(causeList, abortCause)
}
causes, errCh := NewFailureCauseChain(causeList)
if errCh != nil {
return errCh
}
termRes, errTerm := NewErrorTerminalResult(ev.Channel(), desc, causes, time.Now())
if errTerm != nil {
return errTerm
}
if errRel := r.releaser.DiscardPendingAndCommitTerminal(ctx, binding.AttemptID(), termRes); errRel != nil {
return errRel
}
r.emitObservation(ObservationKindTerminalCommitted, epoch.ID(), func(in *FilterObservationInput) {
in.CommitState = CommitStateTerminalCommitted
in.Causes = causes
})
r.mu.Lock()
r.terminalCommitted = true
r.mu.Unlock()
return nil
}
if commitErr := r.commitAttemptInstall(ctx, cand); commitErr != nil {
abortCauseAdded := false
if newCtrl := newBinding.Controller(); newCtrl != nil {
if aErr := newCtrl.AbortAttempt(ctx); aErr != nil {
abortCauseAdded = true
}
}
desc, errD := NewExternalDescriptor("error", "attempt_install_failed", "failed_to_commit_attempt_install", "")
if errD != nil {
return errD
}
commitCause, errCommitCause := NewFailureCause("runtime", "install_commit_failed", "", "", "")
if errCommitCause != nil {
return errCommitCause
}
causeList := []FailureCause{commitCause}
if abortCauseAdded {
abortCause, errAbortCause := NewFailureCause("runtime", "attempt_abort_failed", "", "", "")
if errAbortCause != nil {
return errAbortCause
}
causeList = append(causeList, abortCause)
}
causes, errCh := NewFailureCauseChain(causeList)
if errCh != nil {
return errCh
}
termRes, errTerm := NewErrorTerminalResult(ev.Channel(), desc, causes, time.Now())
if errTerm != nil {
return errTerm
}
if errRel := r.releaser.DiscardPendingAndCommitTerminal(ctx, binding.AttemptID(), termRes); errRel != nil {
return errRel
}
r.emitObservation(ObservationKindTerminalCommitted, epoch.ID(), func(in *FilterObservationInput) {
in.CommitState = CommitStateTerminalCommitted
in.Causes = causes
})
r.mu.Lock()
r.terminalCommitted = true
r.mu.Unlock()
return nil
}
case ArbitrationActionTerminal:
useUnbuffered := passThrough || !r.tail.HasUnconfirmedEvents(epoch.ID())
var termCauses FailureCauseChain
if arbResult.BaseDisposition() == BaseDispositionTerminalSuccessCandidate {
termRes, errRes := ev.AsTerminal()
if errRes != nil {
return errRes
}
if useUnbuffered {
if errRel := r.releaser.CommitTerminalWithoutEpoch(ctx, binding.AttemptID(), termRes); errRel != nil {
return errRel
}
} else {
if _, errRel := r.releaser.ReleaseTerminalEpoch(ctx, binding.AttemptID(), epoch.ID(), termRes); errRel != nil {
return errRel
}
}
} else if arbResult.BaseDisposition() == BaseDispositionTerminalErrorCandidate {
termRes, errRes := providerErrorToTerminalResult(ev)
if errRes != nil {
return errRes
}
termCauses = termRes.FailureCauses()
if termCauses.Len() == 0 {
c, _ := NewFailureCause("provider", "error", "", "", "")
termCauses, _ = NewFailureCauseChain([]FailureCause{c})
}
if useUnbuffered {
if errRel := r.releaser.CommitTerminalWithoutEpoch(ctx, binding.AttemptID(), termRes); errRel != nil {
return errRel
}
} else {
if _, errRel := r.releaser.ReleaseTerminalEpoch(ctx, binding.AttemptID(), epoch.ID(), termRes); errRel != nil {
return errRel
}
}
} else {
desc, errD := NewExternalDescriptor("error", "fatal_violation", "fatal_filter_violation", "")
if errD != nil {
return errD
}
cause, errC := NewFailureCause("arbiter", "fatal_violation", "", arbResult.FilterID(), arbResult.RuleID())
if errC != nil {
return errC
}
causes, errCh := NewFailureCauseChain([]FailureCause{cause})
if errCh != nil {
return errCh
}
termCauses = causes
termRes, errTerm := NewErrorTerminalResult(ev.Channel(), desc, causes, time.Now())
if errTerm != nil {
return errTerm
}
if useUnbuffered {
if errRel := r.releaser.CommitTerminalWithoutEpoch(ctx, binding.AttemptID(), termRes); errRel != nil {
return errRel
}
} else {
if _, errRel := r.releaser.FailPending(ctx, binding.AttemptID(), termRes); errRel != nil {
return errRel
}
}
}
if arbResult.BaseDisposition() == BaseDispositionTerminalSuccessCandidate {
r.emitObservation(ObservationKindReleaseCommitted, epoch.ID(), func(in *FilterObservationInput) {
in.CommitState = CommitStateStreamOpen
})
}
r.emitObservation(ObservationKindTerminalCommitted, epoch.ID(), func(in *FilterObservationInput) {
in.CommitState = CommitStateTerminalCommitted
if arbResult.BaseDisposition() == BaseDispositionTerminalSuccessCandidate {
in.TerminalReason = TerminalReasonCompleted
} else {
in.Causes = termCauses
}
})
r.mu.Lock()
r.terminalCommitted = true
r.mu.Unlock()
return nil
}
}
}
}
// PendingEventsByChannel returns a defensive map of pending events per channel.
func (t *EvidenceTail) PendingEventsByChannel() map[string][]NormalizedEvent {
if t == nil || len(t.channelState) == 0 {
return nil
}
out := make(map[string][]NormalizedEvent)
for ch, cs := range t.channelState {
var pe []NormalizedEvent
for _, e := range cs.pendingEntries {
if e.event.Kind() != EventKindTerminal && e.event.Kind() != EventKindProviderError {
pe = append(pe, cloneNormalizedEvent(e.event))
}
}
if len(pe) > 0 {
out[ch] = pe
}
}
if len(out) == 0 {
return nil
}
return out
}
// CommittedLookBehindByChannel returns a defensive map of committed look-behind events per channel.
func (t *EvidenceTail) CommittedLookBehindByChannel() map[string][]NormalizedEvent {
if t == nil || len(t.channelState) == 0 {
return nil
}
out := make(map[string][]NormalizedEvent)
for ch, cs := range t.channelState {
if len(cs.committedLookBehind) > 0 {
lb := make([]NormalizedEvent, len(cs.committedLookBehind))
for i, ev := range cs.committedLookBehind {
lb[i] = cloneNormalizedEvent(ev)
}
out[ch] = lb
}
}
if len(out) == 0 {
return nil
}
return out
}
// PrepareContinuationWithPlan updates the evidence plan and prepares continuation.
func (t *EvidenceTail) PrepareContinuationWithPlan(plan EvidencePlan) {
t.plan = plan
t.PrepareContinuation()
}
// Plan returns the current EvidencePlan.
func (t *EvidenceTail) Plan() EvidencePlan {
if t == nil {
return EvidencePlan{}
}
return t.plan
}