- OpenAI request rebuilder with tool validation and provider tunnel - Edge config runtime refresh for stream evidence gate - Filter observation contract and runtime with sink/correlation - Stream gate dispatcher, release sink, and vertical slice - Recovery coordinator for evidence tail - Parallel evaluation and commit boundary - E2E test script for OpenAI vLLM - Archive completed task groups to archive/2026/07
1381 lines
45 KiB
Go
1381 lines
45 KiB
Go
package streamgate
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
const (
|
|
// DefaultMaxEvidenceRunes is the default Unicode rune threshold for evidence holding.
|
|
DefaultMaxEvidenceRunes = 500
|
|
|
|
// DefaultMaxBufferRunes is the default maximum rune buffer size.
|
|
DefaultMaxBufferRunes = 4096
|
|
|
|
// DefaultMaxIngressSnapshotBytes is the default byte limit for ingress body snapshots (16 MiB).
|
|
DefaultMaxIngressSnapshotBytes int64 = 16 * 1024 * 1024
|
|
|
|
// DefaultMaxRecoveryAttemptsTotal is the default total fault recovery attempt limit.
|
|
DefaultMaxRecoveryAttemptsTotal = 3
|
|
)
|
|
|
|
// RuntimeOptions captures immutable configuration parameters for a streamgate request runtime.
|
|
type RuntimeOptions struct {
|
|
maxEvidenceRunes int
|
|
maxBufferRunes int
|
|
maxIngressSnapshotBytes int64
|
|
maxRecoveryAttemptsTotal int
|
|
// maxRecoveryAttemptsByStrategy is the immutable per-fault-strategy recovery
|
|
// cap snapshot. A nil/empty map means every fault strategy inherits the
|
|
// request-total cap; a non-empty map disables any fault strategy it omits and
|
|
// caps each listed strategy (an explicit 0 forbids that strategy entirely).
|
|
maxRecoveryAttemptsByStrategy map[RecoveryStrategy]int
|
|
gateOptions GateCoordinatorOptions
|
|
recoveryOptions RecoveryCoordinatorOptions
|
|
}
|
|
|
|
// DefaultRuntimeOptions returns RuntimeOptions populated with default settings.
|
|
func DefaultRuntimeOptions() RuntimeOptions {
|
|
opts, err := NewRuntimeOptions(
|
|
DefaultMaxEvidenceRunes,
|
|
DefaultMaxBufferRunes,
|
|
DefaultMaxIngressSnapshotBytes,
|
|
DefaultMaxRecoveryAttemptsTotal,
|
|
GateCoordinatorOptions{},
|
|
RecoveryCoordinatorOptions{},
|
|
)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
return opts
|
|
}
|
|
|
|
// NewRuntimeOptions constructs validated RuntimeOptions.
|
|
func NewRuntimeOptions(
|
|
maxEvidenceRunes, maxBufferRunes int,
|
|
maxIngressBytes int64,
|
|
maxRecoveryAttempts int,
|
|
gateOpts GateCoordinatorOptions,
|
|
recoveryOpts RecoveryCoordinatorOptions,
|
|
) (RuntimeOptions, error) {
|
|
if maxEvidenceRunes <= 0 {
|
|
return RuntimeOptions{}, errors.New("streamgate: runtime options max evidence runes must be positive")
|
|
}
|
|
if maxBufferRunes < maxEvidenceRunes {
|
|
return RuntimeOptions{}, errors.New("streamgate: runtime options max buffer runes must be >= max evidence runes")
|
|
}
|
|
if maxIngressBytes <= 0 {
|
|
return RuntimeOptions{}, errors.New("streamgate: runtime options max ingress snapshot bytes must be positive")
|
|
}
|
|
if maxRecoveryAttempts < 0 || maxRecoveryAttempts > DefaultMaxRecoveryAttemptsTotal {
|
|
return RuntimeOptions{}, errors.New("streamgate: runtime options max recovery attempts total must be between 0 and 3")
|
|
}
|
|
opts := RuntimeOptions{
|
|
maxEvidenceRunes: maxEvidenceRunes,
|
|
maxBufferRunes: maxBufferRunes,
|
|
maxIngressSnapshotBytes: maxIngressBytes,
|
|
maxRecoveryAttemptsTotal: maxRecoveryAttempts,
|
|
gateOptions: gateOpts,
|
|
recoveryOptions: recoveryOpts,
|
|
}
|
|
if err := opts.Validate(); err != nil {
|
|
return RuntimeOptions{}, err
|
|
}
|
|
return opts, nil
|
|
}
|
|
|
|
// Validate returns nil when RuntimeOptions is in a consistent state.
|
|
func (o RuntimeOptions) Validate() error {
|
|
if o.maxEvidenceRunes <= 0 {
|
|
return errors.New("streamgate: runtime options max evidence runes must be positive")
|
|
}
|
|
if o.maxBufferRunes < o.maxEvidenceRunes {
|
|
return errors.New("streamgate: runtime options max buffer runes must be >= max evidence runes")
|
|
}
|
|
if o.maxIngressSnapshotBytes <= 0 || o.maxIngressSnapshotBytes > DefaultMaxIngressSnapshotBytes {
|
|
return errors.New("streamgate: runtime options max ingress snapshot bytes must be between 1 and 16777216")
|
|
}
|
|
if o.maxRecoveryAttemptsTotal < 0 || o.maxRecoveryAttemptsTotal > DefaultMaxRecoveryAttemptsTotal {
|
|
return errors.New("streamgate: runtime options max recovery attempts total must be between 0 and 3")
|
|
}
|
|
for strategy, limit := range o.maxRecoveryAttemptsByStrategy {
|
|
if !isFaultRecoveryStrategy(strategy) {
|
|
return errors.New("streamgate: runtime options strategy cap contains a non-fault strategy")
|
|
}
|
|
if limit < 0 || limit > o.maxRecoveryAttemptsTotal {
|
|
return errors.New("streamgate: runtime options strategy cap must be between zero and max recovery attempts total")
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// WithRecoveryStrategyLimits returns a validated copy of the options carrying
|
|
// an effective per-fault-strategy recovery cap. A nil/empty map inherits the
|
|
// request-total cap for every strategy; a non-empty map disables any strategy
|
|
// it omits and caps each listed strategy at its value (0 forbids the strategy).
|
|
func (o RuntimeOptions) WithRecoveryStrategyLimits(limits map[RecoveryStrategy]int) (RuntimeOptions, error) {
|
|
next := o
|
|
next.maxRecoveryAttemptsByStrategy = copyRecoveryStrategyCounts(limits)
|
|
if err := next.Validate(); err != nil {
|
|
return RuntimeOptions{}, err
|
|
}
|
|
return next, nil
|
|
}
|
|
|
|
// MaxEvidenceRunes returns the max evidence runes setting.
|
|
func (o RuntimeOptions) MaxEvidenceRunes() int { return o.maxEvidenceRunes }
|
|
|
|
// MaxBufferRunes returns the max buffer runes setting.
|
|
func (o RuntimeOptions) MaxBufferRunes() int { return o.maxBufferRunes }
|
|
|
|
// MaxIngressSnapshotBytes returns the max ingress snapshot bytes setting.
|
|
func (o RuntimeOptions) MaxIngressSnapshotBytes() int64 { return o.maxIngressSnapshotBytes }
|
|
|
|
// MaxRecoveryAttemptsTotal returns the total recovery attempts limit.
|
|
func (o RuntimeOptions) MaxRecoveryAttemptsTotal() int { return o.maxRecoveryAttemptsTotal }
|
|
|
|
// MaxRecoveryAttemptsByStrategy returns a defensive copy of the effective
|
|
// per-fault-strategy recovery cap. A nil result means every fault strategy
|
|
// inherits the request-total cap.
|
|
func (o RuntimeOptions) MaxRecoveryAttemptsByStrategy() map[RecoveryStrategy]int {
|
|
return copyRecoveryStrategyCounts(o.maxRecoveryAttemptsByStrategy)
|
|
}
|
|
|
|
// GateOptions returns the gate coordinator options.
|
|
func (o RuntimeOptions) GateOptions() GateCoordinatorOptions { return o.gateOptions }
|
|
|
|
// RecoveryOptions returns the recovery coordinator options.
|
|
func (o RuntimeOptions) RecoveryOptions() RecoveryCoordinatorOptions { return o.recoveryOptions }
|
|
|
|
// RequestRuntimeSnapshot binds request-local state, host seams, filter registry snapshot,
|
|
// and runtime options into an immutable contract to be consumed by request-local owners.
|
|
type RequestRuntimeSnapshot struct {
|
|
requestID StableToken
|
|
configGen StableToken
|
|
environment string
|
|
endpoint string
|
|
family string
|
|
options RuntimeOptions
|
|
registrySnapshot FilterRegistrySnapshot
|
|
ingressSnapshot *IngressSnapshot
|
|
requestSnapshotRef RecoveryRequestSnapshotRef
|
|
dispatcher AttemptDispatcher
|
|
rebuilder RequestRebuilder
|
|
preparer RecoveryPlanPreparer
|
|
prepFactory RecoveryPreparationSnapshotFactory
|
|
sink ReleaseSink
|
|
obsSink ObservationSink
|
|
}
|
|
|
|
// RecoveryPreparationSnapshotFactory creates a request-local RecoveryPreparationSnapshot
|
|
// bound to the given request snapshot and attempt binding.
|
|
type RecoveryPreparationSnapshotFactory interface {
|
|
CreatePreparationSnapshot(ctx context.Context, snapshot RequestRuntimeSnapshot, binding AttemptBinding) (RecoveryPreparationSnapshot, error)
|
|
}
|
|
|
|
// NewRequestRuntimeSnapshot constructs a validated RequestRuntimeSnapshot.
|
|
func NewRequestRuntimeSnapshot(
|
|
requestID, configGen, environment, endpoint, family string,
|
|
options RuntimeOptions,
|
|
registrySnapshot FilterRegistrySnapshot,
|
|
ingressSnapshot *IngressSnapshot,
|
|
requestSnapshotRef RecoveryRequestSnapshotRef,
|
|
dispatcher AttemptDispatcher,
|
|
rebuilder RequestRebuilder,
|
|
preparer RecoveryPlanPreparer,
|
|
prepFactory RecoveryPreparationSnapshotFactory,
|
|
sink ReleaseSink,
|
|
) (RequestRuntimeSnapshot, error) {
|
|
reqIDToken, err := NewStableTokenRequired("requestID", requestID)
|
|
if err != nil {
|
|
return RequestRuntimeSnapshot{}, err
|
|
}
|
|
cfgGenToken, err := NewStableTokenRequired("configGen", configGen)
|
|
if err != nil {
|
|
return RequestRuntimeSnapshot{}, err
|
|
}
|
|
if err := options.Validate(); err != nil {
|
|
return RequestRuntimeSnapshot{}, err
|
|
}
|
|
if registrySnapshot.Generation() == "" {
|
|
return RequestRuntimeSnapshot{}, errors.New("streamgate: request runtime snapshot registry snapshot generation is required")
|
|
}
|
|
if dispatcher == nil {
|
|
return RequestRuntimeSnapshot{}, errors.New("streamgate: request runtime snapshot dispatcher is required")
|
|
}
|
|
if rebuilder == nil {
|
|
return RequestRuntimeSnapshot{}, errors.New("streamgate: request runtime snapshot rebuilder is required")
|
|
}
|
|
if sink == nil {
|
|
return RequestRuntimeSnapshot{}, errors.New("streamgate: request runtime snapshot release sink is required")
|
|
}
|
|
if err := requestSnapshotRef.Validate(); err != nil {
|
|
return RequestRuntimeSnapshot{}, err
|
|
}
|
|
|
|
snap := RequestRuntimeSnapshot{
|
|
requestID: reqIDToken,
|
|
configGen: cfgGenToken,
|
|
environment: environment,
|
|
endpoint: endpoint,
|
|
family: family,
|
|
options: options,
|
|
registrySnapshot: registrySnapshot,
|
|
ingressSnapshot: ingressSnapshot,
|
|
requestSnapshotRef: requestSnapshotRef,
|
|
dispatcher: dispatcher,
|
|
rebuilder: rebuilder,
|
|
preparer: preparer,
|
|
prepFactory: prepFactory,
|
|
sink: sink,
|
|
}
|
|
|
|
if err := snap.Validate(); err != nil {
|
|
return RequestRuntimeSnapshot{}, err
|
|
}
|
|
return snap, nil
|
|
}
|
|
|
|
// Validate returns nil when the RequestRuntimeSnapshot is consistent.
|
|
func (r RequestRuntimeSnapshot) Validate() error {
|
|
if err := validateStableTokenRequired("requestID", r.requestID.value); err != nil {
|
|
return err
|
|
}
|
|
if err := validateStableTokenRequired("configGen", r.configGen.value); err != nil {
|
|
return err
|
|
}
|
|
if err := r.options.Validate(); err != nil {
|
|
return err
|
|
}
|
|
registryGeneration := r.registrySnapshot.Generation()
|
|
if registryGeneration == "" {
|
|
return errors.New("streamgate: request runtime snapshot registry snapshot generation is required")
|
|
}
|
|
if registryGeneration != r.configGen.String() {
|
|
return errors.New("streamgate: request runtime snapshot config and registry generations must match")
|
|
}
|
|
if r.dispatcher == nil {
|
|
return errors.New("streamgate: request runtime snapshot dispatcher is required")
|
|
}
|
|
if r.rebuilder == nil {
|
|
return errors.New("streamgate: request runtime snapshot rebuilder is required")
|
|
}
|
|
if r.sink == nil {
|
|
return errors.New("streamgate: request runtime snapshot release sink is required")
|
|
}
|
|
if err := r.requestSnapshotRef.Validate(); err != nil {
|
|
return err
|
|
}
|
|
if (r.preparer == nil && r.prepFactory != nil) || (r.preparer != nil && r.prepFactory == nil) {
|
|
return errors.New("streamgate: request runtime snapshot preparer and preparation snapshot factory must both be nil or both non-nil")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// RequestID returns the stable request identifier string.
|
|
func (r RequestRuntimeSnapshot) RequestID() string { return r.requestID.value }
|
|
|
|
// ConfigGeneration returns the configuration generation string.
|
|
func (r RequestRuntimeSnapshot) ConfigGeneration() string { return r.configGen.value }
|
|
|
|
// Environment returns the environment label.
|
|
func (r RequestRuntimeSnapshot) Environment() string { return r.environment }
|
|
|
|
// Endpoint returns the endpoint label.
|
|
func (r RequestRuntimeSnapshot) Endpoint() string { return r.endpoint }
|
|
|
|
// Family returns the model family label.
|
|
func (r RequestRuntimeSnapshot) Family() string { return r.family }
|
|
|
|
// Options returns the runtime options.
|
|
func (r RequestRuntimeSnapshot) Options() RuntimeOptions { return r.options }
|
|
|
|
// RegistrySnapshot returns the filter registry snapshot.
|
|
func (r RequestRuntimeSnapshot) RegistrySnapshot() FilterRegistrySnapshot {
|
|
return r.registrySnapshot
|
|
}
|
|
|
|
// IngressSnapshot returns the ingress snapshot pointer, or nil if unset.
|
|
func (r RequestRuntimeSnapshot) IngressSnapshot() *IngressSnapshot {
|
|
return r.ingressSnapshot
|
|
}
|
|
|
|
// RequestSnapshotRef returns the recovery request snapshot reference.
|
|
func (r RequestRuntimeSnapshot) RequestSnapshotRef() RecoveryRequestSnapshotRef {
|
|
return r.requestSnapshotRef
|
|
}
|
|
|
|
// Dispatcher returns the host attempt dispatcher.
|
|
func (r RequestRuntimeSnapshot) Dispatcher() AttemptDispatcher { return r.dispatcher }
|
|
|
|
// Rebuilder returns the host request rebuilder.
|
|
func (r RequestRuntimeSnapshot) Rebuilder() RequestRebuilder { return r.rebuilder }
|
|
|
|
// Preparer returns the optional host recovery plan preparer.
|
|
func (r RequestRuntimeSnapshot) Preparer() RecoveryPlanPreparer { return r.preparer }
|
|
|
|
// PreparationSnapshotFactory returns the optional recovery preparation snapshot factory.
|
|
func (r RequestRuntimeSnapshot) PreparationSnapshotFactory() RecoveryPreparationSnapshotFactory {
|
|
return r.prepFactory
|
|
}
|
|
|
|
// Sink returns the host release sink.
|
|
func (r RequestRuntimeSnapshot) Sink() ReleaseSink { return r.sink }
|
|
|
|
// ObservationSink returns the observation sink, defaulting to NoopObservationSink if unset.
|
|
func (r RequestRuntimeSnapshot) ObservationSink() ObservationSink {
|
|
if r.obsSink == nil {
|
|
return NoopObservationSink{}
|
|
}
|
|
return r.obsSink
|
|
}
|
|
|
|
// WithObservationSink returns a copy of RequestRuntimeSnapshot with the given ObservationSink.
|
|
func (r RequestRuntimeSnapshot) WithObservationSink(sink ObservationSink) RequestRuntimeSnapshot {
|
|
cp := r
|
|
cp.obsSink = sink
|
|
return cp
|
|
}
|
|
|
|
// BuildAttemptFilterContext constructs a FilterContext bound to this request snapshot.
|
|
func (r RequestRuntimeSnapshot) BuildAttemptFilterContext(
|
|
attemptID string,
|
|
target AttemptTarget,
|
|
epochID uint64,
|
|
commitState CommitState,
|
|
) (FilterContext, error) {
|
|
builder := NewFilterContextBuilder(r.configGen.value, attemptID).
|
|
SetEnvironment(r.environment).
|
|
SetEndpoint(r.endpoint).
|
|
SetFamily(r.family).
|
|
SetModelGroup(target.ModelGroup()).
|
|
SetActualModel(target.Model()).
|
|
SetActualProvider(target.Provider()).
|
|
SetExecutionPath(target.ExecutionPath()).
|
|
SetCommitState(commitState).
|
|
SetStableCorrelation(r.requestID.value)
|
|
|
|
if epochID > 0 {
|
|
var err error
|
|
builder, err = builder.SetEpoch(epochID)
|
|
if err != nil {
|
|
return FilterContext{}, err
|
|
}
|
|
}
|
|
return builder.Build()
|
|
}
|
|
|
|
// attemptCancellerAdapter wraps RequestRuntime to satisfy StreamReleaser's AttemptCanceller seam.
|
|
type attemptCancellerAdapter struct {
|
|
r *RequestRuntime
|
|
}
|
|
|
|
func (a *attemptCancellerAdapter) CancelAttempt(ctx context.Context, attemptID string) error {
|
|
if a.r != nil {
|
|
a.r.mu.Lock()
|
|
ctrl := a.r.currentBinding.Controller()
|
|
a.r.mu.Unlock()
|
|
if ctrl != nil {
|
|
return ctrl.AbortAttempt(ctx)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// RequestRuntime drives the request-local lifecycle loop, managing attempt bindings,
|
|
// evidence collection, parallel filter evaluation through GateCoordinator, decision arbitration,
|
|
// stream release, and fault recovery.
|
|
type RequestRuntime struct {
|
|
snapshot RequestRuntimeSnapshot
|
|
modelGroup string
|
|
currentBinding AttemptBinding
|
|
tail *EvidenceTail
|
|
boundary *CommitBoundary
|
|
releaser *StreamReleaser
|
|
gate *GateCoordinator
|
|
arbiter *DecisionArbiter
|
|
recoveryCoord *RecoveryCoordinator
|
|
recoveryPolicy RecoveryPolicySnapshot
|
|
recoveryUsage RecoveryUsageSnapshot
|
|
resolvedFilters []ResolvedFilter
|
|
stagedStart *ResponseStart
|
|
terminalCommitted bool
|
|
running bool
|
|
suppressContinuationStart bool
|
|
resourcesClosed bool
|
|
planIDCounter int
|
|
sequencer *ObservationSequencer
|
|
mu sync.Mutex
|
|
}
|
|
|
|
func (r *RequestRuntime) emitObservation(kind ObservationKind, epochID uint64, mutate func(*FilterObservationInput)) {
|
|
if r == nil || r.sequencer == nil {
|
|
return
|
|
}
|
|
r.mu.Lock()
|
|
binding := r.currentBinding
|
|
commitState := r.boundary.State()
|
|
r.mu.Unlock()
|
|
attemptTarget, err := NewObservationAttemptTarget(r.modelGroup, binding.Model(), binding.Provider(), binding.ExecutionPath())
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
input := FilterObservationInput{
|
|
Kind: kind,
|
|
StableCorrelation: r.snapshot.RequestID(),
|
|
ConfigGeneration: r.snapshot.ConfigGeneration(),
|
|
AttemptID: binding.AttemptID(),
|
|
AttemptTarget: attemptTarget,
|
|
EpochID: epochID,
|
|
CommitState: commitState,
|
|
OccurredAt: time.Now(),
|
|
}
|
|
if mutate != nil {
|
|
mutate(&input)
|
|
}
|
|
_, _ = r.sequencer.Emit(context.Background(), input)
|
|
}
|
|
|
|
// gracefulAttemptController is an optional AttemptController extension a host
|
|
// controller may implement to release provider ownership after a normal
|
|
// terminal without issuing a provider-side cancel. CloseRequestResources uses
|
|
// it on a graceful (successful-terminal) close and falls back to AbortAttempt
|
|
// for every non-graceful outcome.
|
|
type gracefulAttemptController interface {
|
|
AttemptController
|
|
CloseAttempt(ctx context.Context) error
|
|
}
|
|
|
|
// requestResourceCloser is an optional RequestRebuilder extension: a
|
|
// request-local host rebuilder that must be closed exactly once when the
|
|
// request runtime finishes so its rebuilt-request store and reserved snapshot
|
|
// bytes are released on success, error, and caller-cancel alike.
|
|
type requestResourceCloser interface {
|
|
Close()
|
|
}
|
|
|
|
// CloseRequestResources performs idempotent end-of-request cleanup of the host
|
|
// resources this runtime owns. It aborts (or, when graceful, gracefully closes)
|
|
// the current attempt binding's controller — releasing the latest provider
|
|
// transport and any rebuilt lease, and canceling the latest provider run on a
|
|
// non-graceful outcome — and closes the request rebuilder. It is safe to call
|
|
// exactly once after Run returns for any terminal, error, or caller-cancel
|
|
// outcome. graceful must be true only when Run committed a successful terminal.
|
|
func (r *RequestRuntime) CloseRequestResources(ctx context.Context, graceful bool) error {
|
|
if r == nil {
|
|
return nil
|
|
}
|
|
r.mu.Lock()
|
|
if r.resourcesClosed {
|
|
r.mu.Unlock()
|
|
return nil
|
|
}
|
|
r.resourcesClosed = true
|
|
ctrl := r.currentBinding.Controller()
|
|
r.mu.Unlock()
|
|
|
|
var err error
|
|
if ctrl != nil {
|
|
if graceful {
|
|
if gc, ok := ctrl.(gracefulAttemptController); ok {
|
|
err = gc.CloseAttempt(ctx)
|
|
} else {
|
|
err = ctrl.AbortAttempt(ctx)
|
|
}
|
|
} else {
|
|
err = ctrl.AbortAttempt(ctx)
|
|
}
|
|
}
|
|
if closer, ok := r.snapshot.Rebuilder().(requestResourceCloser); ok {
|
|
closer.Close()
|
|
}
|
|
return err
|
|
}
|
|
|
|
func providerErrorToTerminalResult(ev NormalizedEvent) (TerminalResult, error) {
|
|
return ev.AsProviderError()
|
|
}
|
|
|
|
func normalizedTerminalToResult(ev NormalizedEvent) (TerminalResult, error) {
|
|
if ev.Kind() == EventKindTerminal {
|
|
return NewSuccessTerminalResult(ev.Channel(), ev.Timestamp())
|
|
}
|
|
return providerErrorToTerminalResult(ev)
|
|
}
|
|
|
|
// NewRequestRuntime constructs a new RequestRuntime instance.
|
|
func NewRequestRuntime(snapshot RequestRuntimeSnapshot, modelGroup string, initial AttemptBinding) (*RequestRuntime, error) {
|
|
if err := snapshot.Validate(); err != nil {
|
|
return nil, fmt.Errorf("streamgate: new request runtime snapshot invalid: %w", err)
|
|
}
|
|
if modelGroup == "" {
|
|
return nil, errors.New("streamgate: new request runtime model group is required")
|
|
}
|
|
if err := initial.Validate(); err != nil {
|
|
return nil, fmt.Errorf("streamgate: new request runtime initial binding invalid: %w", err)
|
|
}
|
|
|
|
boundary, err := NewCommitBoundary(snapshot.Sink())
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
arbiter := NewDecisionArbiter()
|
|
gate, err := NewGateCoordinator(context.Background(), snapshot.Options().GateOptions(), arbiter)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
seq := NewObservationSequencer(snapshot.ObservationSink(), nil)
|
|
gate.SetObservationSequencer(seq)
|
|
|
|
recPolicy, err := NewRecoveryPolicySnapshot(
|
|
snapshot.Options().MaxRecoveryAttemptsTotal(),
|
|
snapshot.Options().MaxRecoveryAttemptsByStrategy(),
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
recUsage, err := NewRecoveryUsageSnapshot(0, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
preparers := make(map[string]RecoveryPlanPreparer)
|
|
if snapshot.Preparer() != nil {
|
|
preparers["default"] = snapshot.Preparer()
|
|
}
|
|
|
|
recOpts := RecoveryCoordinatorOptions{
|
|
Policy: recPolicy,
|
|
Usage: recUsage,
|
|
RequestSnapshot: snapshot.RequestSnapshotRef(),
|
|
CurrentBinding: initial,
|
|
Rebuilder: snapshot.Rebuilder(),
|
|
Dispatcher: snapshot.Dispatcher(),
|
|
Preparers: preparers,
|
|
PreparationTimeout: 5 * time.Second,
|
|
ObservationSink: snapshot.ObservationSink(),
|
|
}
|
|
|
|
recCoord, err := NewRecoveryCoordinator(recOpts)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
recCoord.SetObservationSequencer(seq)
|
|
|
|
rt := &RequestRuntime{
|
|
snapshot: snapshot,
|
|
modelGroup: modelGroup,
|
|
boundary: boundary,
|
|
gate: gate,
|
|
arbiter: arbiter,
|
|
recoveryCoord: recCoord,
|
|
recoveryPolicy: recPolicy,
|
|
recoveryUsage: recUsage,
|
|
sequencer: seq,
|
|
}
|
|
|
|
cand, err := rt.prepareAttemptInstall(initial, RecoveryResumeModeReplaceAttempt)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if err := rt.commitAttemptInstall(context.Background(), cand); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return rt, nil
|
|
}
|
|
|
|
type attemptInstallCandidate struct {
|
|
binding AttemptBinding
|
|
resumeMode RecoveryResumeMode
|
|
oldAttemptID string
|
|
target AttemptTarget
|
|
resolvedFilters []ResolvedFilter
|
|
plan EvidencePlan
|
|
tail *EvidenceTail
|
|
releaser *StreamReleaser
|
|
}
|
|
|
|
func (r *RequestRuntime) prepareAttemptInstall(binding AttemptBinding, resumeMode RecoveryResumeMode) (attemptInstallCandidate, error) {
|
|
if err := binding.Validate(); err != nil {
|
|
return attemptInstallCandidate{}, err
|
|
}
|
|
if err := resumeMode.Validate(); err != nil {
|
|
return attemptInstallCandidate{}, err
|
|
}
|
|
|
|
target, err := NewAttemptTarget(r.modelGroup, binding.Model(), binding.Provider(), binding.ExecutionPath(), nil)
|
|
if err != nil {
|
|
return attemptInstallCandidate{}, err
|
|
}
|
|
|
|
rfc, err := NewRequestFilterContext(
|
|
r.snapshot.ConfigGeneration(),
|
|
binding.AttemptID(),
|
|
r.snapshot.Environment(),
|
|
r.snapshot.Endpoint(),
|
|
r.snapshot.Family(),
|
|
"",
|
|
r.boundary.State(),
|
|
false,
|
|
false,
|
|
r.snapshot.RequestID(),
|
|
)
|
|
if err != nil {
|
|
return attemptInstallCandidate{}, err
|
|
}
|
|
|
|
rfSnap, err := r.snapshot.RegistrySnapshot().BeginRequest(rfc)
|
|
if err != nil {
|
|
return attemptInstallCandidate{}, err
|
|
}
|
|
|
|
resolved, err := rfSnap.ResolveAttempt(target)
|
|
if err != nil {
|
|
return attemptInstallCandidate{}, err
|
|
}
|
|
|
|
plan, err := NewEvidencePlanFromResolvedFilters(resolved)
|
|
if err != nil {
|
|
return attemptInstallCandidate{}, err
|
|
}
|
|
|
|
if r.tail == nil {
|
|
tail, err := NewEvidenceTail(plan)
|
|
if err != nil {
|
|
return attemptInstallCandidate{}, err
|
|
}
|
|
releaser, err := NewStreamReleaser(tail, r.boundary, &attemptCancellerAdapter{r: r})
|
|
if err != nil {
|
|
return attemptInstallCandidate{}, err
|
|
}
|
|
return attemptInstallCandidate{
|
|
binding: binding,
|
|
resumeMode: resumeMode,
|
|
oldAttemptID: "",
|
|
target: target,
|
|
resolvedFilters: resolved,
|
|
plan: plan,
|
|
tail: tail,
|
|
releaser: releaser,
|
|
}, nil
|
|
}
|
|
|
|
if resumeMode == RecoveryResumeModeReplaceAttempt {
|
|
tail, err := NewEvidenceTail(plan)
|
|
if err != nil {
|
|
return attemptInstallCandidate{}, err
|
|
}
|
|
releaser, err := NewStreamReleaser(tail, r.boundary, &attemptCancellerAdapter{r: r})
|
|
if err != nil {
|
|
return attemptInstallCandidate{}, err
|
|
}
|
|
return attemptInstallCandidate{
|
|
binding: binding,
|
|
resumeMode: resumeMode,
|
|
oldAttemptID: r.currentBinding.AttemptID(),
|
|
target: target,
|
|
resolvedFilters: resolved,
|
|
plan: plan,
|
|
tail: tail,
|
|
releaser: releaser,
|
|
}, nil
|
|
}
|
|
|
|
if resumeMode == RecoveryResumeModeContinueStream {
|
|
oldID := ""
|
|
if r.currentBinding.AttemptID() != "" {
|
|
oldID = r.currentBinding.AttemptID()
|
|
}
|
|
return attemptInstallCandidate{
|
|
binding: binding,
|
|
resumeMode: resumeMode,
|
|
oldAttemptID: oldID,
|
|
target: target,
|
|
resolvedFilters: resolved,
|
|
plan: plan,
|
|
tail: nil,
|
|
releaser: nil,
|
|
}, nil
|
|
}
|
|
|
|
return attemptInstallCandidate{}, fmt.Errorf("streamgate: unsupported resume mode: %v", resumeMode)
|
|
}
|
|
|
|
func (r *RequestRuntime) commitAttemptInstall(ctx context.Context, candidate attemptInstallCandidate) error {
|
|
r.mu.Lock()
|
|
hasTail := r.tail != nil
|
|
rel := r.releaser
|
|
r.mu.Unlock()
|
|
|
|
if !hasTail {
|
|
if err := r.boundary.BeginAttempt(candidate.binding.AttemptID()); err != nil {
|
|
return err
|
|
}
|
|
r.mu.Lock()
|
|
r.currentBinding = candidate.binding
|
|
r.resolvedFilters = candidate.resolvedFilters
|
|
r.tail = candidate.tail
|
|
r.releaser = candidate.releaser
|
|
r.mu.Unlock()
|
|
return nil
|
|
}
|
|
|
|
if candidate.resumeMode == RecoveryResumeModeReplaceAttempt {
|
|
if candidate.oldAttemptID != "" {
|
|
if err := candidate.releaser.ReplaceUncommittedAttempt(ctx, candidate.oldAttemptID, candidate.binding.AttemptID()); err != nil {
|
|
return err
|
|
}
|
|
} else {
|
|
if err := r.boundary.BeginAttempt(candidate.binding.AttemptID()); err != nil {
|
|
return err
|
|
}
|
|
candidate.tail.ResetForReplace()
|
|
}
|
|
r.mu.Lock()
|
|
r.currentBinding = candidate.binding
|
|
r.resolvedFilters = candidate.resolvedFilters
|
|
r.tail = candidate.tail
|
|
r.releaser = candidate.releaser
|
|
// The staged response start is attempt-local: the boundary already
|
|
// dropped it during handoff, so the runtime mirror must not leak the
|
|
// previous attempt's status/header into the new attempt's batches.
|
|
r.stagedStart = nil
|
|
r.mu.Unlock()
|
|
return nil
|
|
}
|
|
|
|
if candidate.resumeMode == RecoveryResumeModeContinueStream {
|
|
if candidate.oldAttemptID != "" && rel != nil {
|
|
if err := rel.ContinueOpenAttempt(ctx, candidate.oldAttemptID, candidate.binding.AttemptID()); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
r.mu.Lock()
|
|
r.tail.PrepareContinuationWithPlan(candidate.plan)
|
|
r.currentBinding = candidate.binding
|
|
r.resolvedFilters = candidate.resolvedFilters
|
|
r.suppressContinuationStart = true
|
|
r.mu.Unlock()
|
|
return nil
|
|
}
|
|
|
|
return fmt.Errorf("streamgate: unsupported resume mode: %v", candidate.resumeMode)
|
|
}
|
|
|
|
// Run executes the normalized event owner loop for the request lifetime.
|
|
func (r *RequestRuntime) Run(ctx context.Context) error {
|
|
r.mu.Lock()
|
|
if r.running || r.terminalCommitted {
|
|
r.mu.Unlock()
|
|
return errors.New("streamgate: runtime already running or terminal")
|
|
}
|
|
r.running = true
|
|
r.mu.Unlock()
|
|
|
|
defer func() {
|
|
if r.gate != nil {
|
|
_ = r.gate.Close()
|
|
}
|
|
}()
|
|
|
|
for {
|
|
r.mu.Lock()
|
|
if r.terminalCommitted {
|
|
r.mu.Unlock()
|
|
return nil
|
|
}
|
|
binding := r.currentBinding
|
|
tail := r.tail
|
|
releaser := r.releaser
|
|
resolvedFilters := r.resolvedFilters
|
|
suppressStart := r.suppressContinuationStart
|
|
if suppressStart {
|
|
r.suppressContinuationStart = false
|
|
}
|
|
r.mu.Unlock()
|
|
|
|
if ctx.Err() != nil {
|
|
return ctx.Err()
|
|
}
|
|
|
|
ev, err := binding.EventSource().NextEvent(ctx)
|
|
if err != nil {
|
|
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) || ctx.Err() != nil {
|
|
return ctx.Err()
|
|
}
|
|
desc, errDesc := NewExternalDescriptor("error", "source_failure", "event_source_read_error", "")
|
|
if errDesc != nil {
|
|
return errDesc
|
|
}
|
|
cause, errCause := NewFailureCause("event_source", "read_error", "", "", "")
|
|
if errCause != nil {
|
|
return errCause
|
|
}
|
|
causes, errCauses := NewFailureCauseChain([]FailureCause{cause})
|
|
if errCauses != nil {
|
|
return errCauses
|
|
}
|
|
termRes, errTerm := NewErrorTerminalResult("default", desc, causes, time.Now())
|
|
if errTerm != nil {
|
|
return errTerm
|
|
}
|
|
if _, errRel := releaser.FailPending(ctx, binding.AttemptID(), termRes); errRel != nil {
|
|
return errRel
|
|
}
|
|
r.emitObservation(ObservationKindTerminalCommitted, 0, func(in *FilterObservationInput) {
|
|
in.CommitState = CommitStateTerminalCommitted
|
|
in.Causes = causes
|
|
})
|
|
r.mu.Lock()
|
|
r.terminalCommitted = true
|
|
r.mu.Unlock()
|
|
return nil
|
|
}
|
|
|
|
if ev.Kind() == EventKindResponseStart {
|
|
rs, errStart := ev.AsResponseStart()
|
|
if errStart != nil {
|
|
return errStart
|
|
}
|
|
if suppressStart {
|
|
continue
|
|
}
|
|
r.mu.Lock()
|
|
r.stagedStart = &rs
|
|
r.mu.Unlock()
|
|
if errStage := r.boundary.StageResponseStart(binding.AttemptID(), rs); errStage != nil {
|
|
return errStage
|
|
}
|
|
r.emitObservation(ObservationKindResponseStaged, 0, func(in *FilterObservationInput) {
|
|
in.CommitState = CommitStateTransportUncommitted
|
|
hold, _ := NewObservationHoldInfo(FilterHoldModeRolling, EventKindResponseStart, r.snapshot.Options().MaxEvidenceRunes(), 0)
|
|
in.Hold = &hold
|
|
})
|
|
continue
|
|
}
|
|
|
|
epoch, signal, errAppend := tail.Append(ev)
|
|
if errAppend != nil {
|
|
return errAppend
|
|
}
|
|
|
|
if signal == EvidenceTailSignalBufferOverflow {
|
|
desc, errDesc := NewExternalDescriptor("error", "buffer_overflow", "buffer_overflow_occurred", "")
|
|
if errDesc != nil {
|
|
return errDesc
|
|
}
|
|
cause, errCause := NewFailureCause("evidence_tail", "buffer_overflow", "", "", "")
|
|
if errCause != nil {
|
|
return errCause
|
|
}
|
|
causes, errCauses := NewFailureCauseChain([]FailureCause{cause})
|
|
if errCauses != nil {
|
|
return errCauses
|
|
}
|
|
termRes, errTerm := NewErrorTerminalResult(ev.Channel(), desc, causes, time.Now())
|
|
if errTerm != nil {
|
|
return errTerm
|
|
}
|
|
if _, errRel := releaser.FailPending(ctx, binding.AttemptID(), termRes); errRel != nil {
|
|
return errRel
|
|
}
|
|
r.emitObservation(ObservationKindTerminalCommitted, epoch.ID(), func(in *FilterObservationInput) {
|
|
in.CommitState = CommitStateTerminalCommitted
|
|
in.Causes = causes
|
|
})
|
|
r.mu.Lock()
|
|
r.terminalCommitted = true
|
|
r.mu.Unlock()
|
|
return nil
|
|
}
|
|
|
|
terminalFlag := (ev.Kind() == EventKindTerminal || ev.Kind() == EventKindProviderError)
|
|
isKindSubscribed := func(kind EventKind, kinds []EventKind) bool {
|
|
for _, k := range kinds {
|
|
if k == kind {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
passThrough := signal == EvidenceTailSignalReady ||
|
|
(signal == EvidenceTailSignalNone && !isKindSubscribed(ev.Kind(), tail.Plan().BlockingKinds(ev.Channel())))
|
|
shouldEvaluate := terminalFlag || passThrough || signal == EvidenceTailSignalThreshold || signal == EvidenceTailSignalTrigger
|
|
|
|
if shouldEvaluate {
|
|
epochFilters, errBind := tail.Plan().BindEpochFilters(epoch, resolvedFilters)
|
|
if errBind != nil {
|
|
return errBind
|
|
}
|
|
|
|
r.mu.Lock()
|
|
stagedStartPtr := r.stagedStart
|
|
r.mu.Unlock()
|
|
if r.boundary.State() != CommitStateTransportUncommitted {
|
|
stagedStartPtr = nil
|
|
}
|
|
|
|
batch, errBatch := NewEvidenceBatch(
|
|
[]NormalizedEvent{ev},
|
|
tail.PendingEventsByChannel(),
|
|
tail.CommittedLookBehindByChannel(),
|
|
stagedStartPtr,
|
|
terminalFlag,
|
|
r.boundary.State(),
|
|
time.Now(),
|
|
)
|
|
if errBatch != nil {
|
|
return errBatch
|
|
}
|
|
|
|
attemptTarget, errTarget := NewObservationAttemptTarget(r.modelGroup, binding.Model(), binding.Provider(), binding.ExecutionPath())
|
|
if errTarget != nil {
|
|
return errTarget
|
|
}
|
|
obsCtx := ObservationContext{
|
|
Sequencer: r.sequencer,
|
|
StableCorrelation: r.snapshot.RequestID(),
|
|
ConfigGeneration: r.snapshot.ConfigGeneration(),
|
|
AttemptID: binding.AttemptID(),
|
|
AttemptTarget: attemptTarget,
|
|
EpochID: epoch.ID(),
|
|
}
|
|
evalCtx := WithObservationContext(ctx, obsCtx)
|
|
|
|
arbResult, errGate := r.gate.Submit(evalCtx, batch, epochFilters)
|
|
if errGate != nil {
|
|
if ctx.Err() != nil {
|
|
return ctx.Err()
|
|
}
|
|
return errGate
|
|
}
|
|
|
|
arbInfo, _ := NewObservationArbitrationInfo(arbResult.Action(), arbResult.BaseDisposition())
|
|
r.emitObservation(ObservationKindArbitrationDecided, epoch.ID(), func(in *FilterObservationInput) {
|
|
in.Arbitration = &arbInfo
|
|
})
|
|
|
|
switch arbResult.Action() {
|
|
case ArbitrationActionReplacement:
|
|
proposal := arbResult.ReplacementProposal()
|
|
if proposal == nil {
|
|
return errors.New("streamgate: replacement action missing replacement proposal")
|
|
}
|
|
if err := proposal.Validate(); err != nil {
|
|
return err
|
|
}
|
|
if _, errRel := r.releaser.releaseReplacementEvents(
|
|
ctx, binding.AttemptID(), epoch.ID(), proposal.Events(),
|
|
); errRel != nil {
|
|
return errRel
|
|
}
|
|
r.emitObservation(ObservationKindReleaseCommitted, epoch.ID(), func(in *FilterObservationInput) {
|
|
in.CommitState = CommitStateStreamOpen
|
|
})
|
|
if terminalFlag {
|
|
// The substituted payload replaces the held content only; the
|
|
// terminal disposition of the original event still closes the
|
|
// stream exactly once.
|
|
termRes, errRes := normalizedTerminalToResult(ev)
|
|
if errRes != nil {
|
|
return errRes
|
|
}
|
|
if errRel := r.releaser.CommitTerminalWithoutEpoch(ctx, binding.AttemptID(), termRes); errRel != nil {
|
|
return errRel
|
|
}
|
|
r.emitObservation(ObservationKindTerminalCommitted, epoch.ID(), func(in *FilterObservationInput) {
|
|
in.CommitState = CommitStateTerminalCommitted
|
|
if termRes.Success() {
|
|
in.TerminalReason = TerminalReasonCompleted
|
|
} else {
|
|
in.Causes = termRes.FailureCauses()
|
|
}
|
|
})
|
|
r.mu.Lock()
|
|
r.terminalCommitted = true
|
|
r.mu.Unlock()
|
|
return nil
|
|
}
|
|
|
|
case ArbitrationActionRelease:
|
|
useUnbuffered := passThrough || !r.tail.HasUnconfirmedEvents(epoch.ID())
|
|
if terminalFlag {
|
|
termRes, errRes := normalizedTerminalToResult(ev)
|
|
if errRes != nil {
|
|
return errRes
|
|
}
|
|
if useUnbuffered {
|
|
if errRel := r.releaser.CommitTerminalWithoutEpoch(ctx, binding.AttemptID(), termRes); errRel != nil {
|
|
return errRel
|
|
}
|
|
} else {
|
|
if _, errRel := r.releaser.ReleaseTerminalEpoch(ctx, binding.AttemptID(), epoch.ID(), termRes); errRel != nil {
|
|
return errRel
|
|
}
|
|
}
|
|
r.emitObservation(ObservationKindReleaseCommitted, epoch.ID(), func(in *FilterObservationInput) {
|
|
in.CommitState = CommitStateStreamOpen
|
|
})
|
|
r.emitObservation(ObservationKindTerminalCommitted, epoch.ID(), func(in *FilterObservationInput) {
|
|
in.CommitState = CommitStateTerminalCommitted
|
|
if termRes.Success() {
|
|
in.TerminalReason = TerminalReasonCompleted
|
|
} else {
|
|
in.Causes = termRes.FailureCauses()
|
|
}
|
|
})
|
|
r.mu.Lock()
|
|
r.terminalCommitted = true
|
|
r.mu.Unlock()
|
|
return nil
|
|
}
|
|
if useUnbuffered {
|
|
if _, errRel := r.releaser.ReleasePassThroughEvent(ctx, binding.AttemptID(), ev); errRel != nil {
|
|
return errRel
|
|
}
|
|
} else {
|
|
if _, errRel := r.releaser.ReleaseEpoch(ctx, binding.AttemptID(), epoch.ID()); errRel != nil {
|
|
return errRel
|
|
}
|
|
}
|
|
r.emitObservation(ObservationKindReleaseCommitted, epoch.ID(), func(in *FilterObservationInput) {
|
|
in.CommitState = CommitStateStreamOpen
|
|
})
|
|
|
|
case ArbitrationActionHold:
|
|
// Hold downstream; events stay buffered in tail
|
|
|
|
case ArbitrationActionRecover:
|
|
r.planIDCounter++
|
|
planID := fmt.Sprintf("plan-%s-%d", r.snapshot.RequestID(), r.planIDCounter)
|
|
idempotencyKey := fmt.Sprintf("idemp-%s-%d", r.snapshot.RequestID(), r.planIDCounter)
|
|
preparerID := ""
|
|
var prepSnapshot RecoveryPreparationSnapshot
|
|
if r.snapshot.Preparer() != nil && r.snapshot.PreparationSnapshotFactory() != nil {
|
|
preparerID = "default"
|
|
ps, errPrep := r.snapshot.PreparationSnapshotFactory().CreatePreparationSnapshot(ctx, r.snapshot, binding)
|
|
if errPrep != nil {
|
|
desc, errD := NewExternalDescriptor("error", "preparation_snapshot_failed", "preparation_snapshot_factory_failed", "")
|
|
if errD != nil {
|
|
return errD
|
|
}
|
|
cause, errC := NewFailureCause("preparer_factory", "snapshot_creation_failed", "", "", "")
|
|
if errC != nil {
|
|
return errC
|
|
}
|
|
causes, errCh := NewFailureCauseChain([]FailureCause{cause})
|
|
if errCh != nil {
|
|
return errCh
|
|
}
|
|
termRes, errTerm := NewErrorTerminalResult(ev.Channel(), desc, causes, time.Now())
|
|
if errTerm != nil {
|
|
return errTerm
|
|
}
|
|
if _, errRel := r.releaser.FailPending(ctx, binding.AttemptID(), termRes); errRel != nil {
|
|
return errRel
|
|
}
|
|
r.emitObservation(ObservationKindTerminalCommitted, epoch.ID(), func(in *FilterObservationInput) {
|
|
in.CommitState = CommitStateTerminalCommitted
|
|
in.Causes = causes
|
|
})
|
|
r.mu.Lock()
|
|
r.terminalCommitted = true
|
|
r.mu.Unlock()
|
|
return nil
|
|
}
|
|
prepSnapshot = ps
|
|
}
|
|
|
|
cycleInput := RecoveryCycleInput{
|
|
Arbitration: arbResult,
|
|
PlanID: planID,
|
|
IdempotencyKey: idempotencyKey,
|
|
ConsumerID: arbResult.FilterID(),
|
|
CommitState: r.boundary.State(),
|
|
CallerCanceled: ctx.Err() != nil,
|
|
PreparerID: preparerID,
|
|
PreparationSnapshot: prepSnapshot,
|
|
}
|
|
|
|
cycleRes, errRec := r.recoveryCoord.Execute(evalCtx, cycleInput)
|
|
newBinding, hasBinding := cycleRes.Binding()
|
|
plan, hasPlan := cycleRes.Plan()
|
|
if errRec != nil || !hasBinding || !hasPlan {
|
|
desc, errD := NewExternalDescriptor("error", "recovery_failed", "recovery_failed", "")
|
|
if errD != nil {
|
|
return errD
|
|
}
|
|
causes := cycleRes.FailureCauses()
|
|
if causes.Len() == 0 {
|
|
c, errC := NewFailureCause("recovery", "recovery_failed", "", "", "")
|
|
if errC != nil {
|
|
return errC
|
|
}
|
|
causesChain, errCh := NewFailureCauseChain([]FailureCause{c})
|
|
if errCh != nil {
|
|
return errCh
|
|
}
|
|
causes = causesChain
|
|
}
|
|
termRes, errTerm := NewErrorTerminalResult(ev.Channel(), desc, causes, time.Now())
|
|
if errTerm != nil {
|
|
return errTerm
|
|
}
|
|
if cycleRes.PreviousAttemptClosed() {
|
|
if errRel := r.releaser.DiscardPendingAndCommitTerminal(ctx, binding.AttemptID(), termRes); errRel != nil {
|
|
return errRel
|
|
}
|
|
} else {
|
|
if _, errRel := r.releaser.FailPending(ctx, binding.AttemptID(), termRes); errRel != nil {
|
|
return errRel
|
|
}
|
|
}
|
|
r.emitObservation(ObservationKindTerminalCommitted, epoch.ID(), func(in *FilterObservationInput) {
|
|
in.CommitState = CommitStateTerminalCommitted
|
|
in.Causes = causes
|
|
})
|
|
r.mu.Lock()
|
|
r.terminalCommitted = true
|
|
r.mu.Unlock()
|
|
return nil
|
|
}
|
|
|
|
cand, prepErr := r.prepareAttemptInstall(newBinding, plan.ResumeMode())
|
|
if prepErr != nil {
|
|
abortFailed := false
|
|
if newCtrl := newBinding.Controller(); newCtrl != nil {
|
|
if abortErr := newCtrl.AbortAttempt(ctx); abortErr != nil {
|
|
abortFailed = true
|
|
}
|
|
}
|
|
desc, errD := NewExternalDescriptor("error", "attempt_install_failed", "failed_to_prepare_attempt_install", "")
|
|
if errD != nil {
|
|
return errD
|
|
}
|
|
cause, errC := NewFailureCause("runtime", "install_prepare_failed", "", "", "")
|
|
if errC != nil {
|
|
return errC
|
|
}
|
|
causeList := []FailureCause{cause}
|
|
if abortFailed {
|
|
abortCause, errAbortCause := NewFailureCause("runtime", "attempt_abort_failed", "", "", "")
|
|
if errAbortCause != nil {
|
|
return errAbortCause
|
|
}
|
|
causeList = append(causeList, abortCause)
|
|
}
|
|
causes, errCh := NewFailureCauseChain(causeList)
|
|
if errCh != nil {
|
|
return errCh
|
|
}
|
|
termRes, errTerm := NewErrorTerminalResult(ev.Channel(), desc, causes, time.Now())
|
|
if errTerm != nil {
|
|
return errTerm
|
|
}
|
|
if errRel := r.releaser.DiscardPendingAndCommitTerminal(ctx, binding.AttemptID(), termRes); errRel != nil {
|
|
return errRel
|
|
}
|
|
r.emitObservation(ObservationKindTerminalCommitted, epoch.ID(), func(in *FilterObservationInput) {
|
|
in.CommitState = CommitStateTerminalCommitted
|
|
in.Causes = causes
|
|
})
|
|
r.mu.Lock()
|
|
r.terminalCommitted = true
|
|
r.mu.Unlock()
|
|
return nil
|
|
}
|
|
|
|
if commitErr := r.commitAttemptInstall(ctx, cand); commitErr != nil {
|
|
abortCauseAdded := false
|
|
if newCtrl := newBinding.Controller(); newCtrl != nil {
|
|
if aErr := newCtrl.AbortAttempt(ctx); aErr != nil {
|
|
abortCauseAdded = true
|
|
}
|
|
}
|
|
desc, errD := NewExternalDescriptor("error", "attempt_install_failed", "failed_to_commit_attempt_install", "")
|
|
if errD != nil {
|
|
return errD
|
|
}
|
|
commitCause, errCommitCause := NewFailureCause("runtime", "install_commit_failed", "", "", "")
|
|
if errCommitCause != nil {
|
|
return errCommitCause
|
|
}
|
|
causeList := []FailureCause{commitCause}
|
|
if abortCauseAdded {
|
|
abortCause, errAbortCause := NewFailureCause("runtime", "attempt_abort_failed", "", "", "")
|
|
if errAbortCause != nil {
|
|
return errAbortCause
|
|
}
|
|
causeList = append(causeList, abortCause)
|
|
}
|
|
causes, errCh := NewFailureCauseChain(causeList)
|
|
if errCh != nil {
|
|
return errCh
|
|
}
|
|
termRes, errTerm := NewErrorTerminalResult(ev.Channel(), desc, causes, time.Now())
|
|
if errTerm != nil {
|
|
return errTerm
|
|
}
|
|
if errRel := r.releaser.DiscardPendingAndCommitTerminal(ctx, binding.AttemptID(), termRes); errRel != nil {
|
|
return errRel
|
|
}
|
|
r.emitObservation(ObservationKindTerminalCommitted, epoch.ID(), func(in *FilterObservationInput) {
|
|
in.CommitState = CommitStateTerminalCommitted
|
|
in.Causes = causes
|
|
})
|
|
r.mu.Lock()
|
|
r.terminalCommitted = true
|
|
r.mu.Unlock()
|
|
return nil
|
|
}
|
|
|
|
case ArbitrationActionTerminal:
|
|
useUnbuffered := passThrough || !r.tail.HasUnconfirmedEvents(epoch.ID())
|
|
var termCauses FailureCauseChain
|
|
if arbResult.BaseDisposition() == BaseDispositionTerminalSuccessCandidate {
|
|
termRes, errRes := ev.AsTerminal()
|
|
if errRes != nil {
|
|
return errRes
|
|
}
|
|
if useUnbuffered {
|
|
if errRel := r.releaser.CommitTerminalWithoutEpoch(ctx, binding.AttemptID(), termRes); errRel != nil {
|
|
return errRel
|
|
}
|
|
} else {
|
|
if _, errRel := r.releaser.ReleaseTerminalEpoch(ctx, binding.AttemptID(), epoch.ID(), termRes); errRel != nil {
|
|
return errRel
|
|
}
|
|
}
|
|
} else if arbResult.BaseDisposition() == BaseDispositionTerminalErrorCandidate {
|
|
termRes, errRes := providerErrorToTerminalResult(ev)
|
|
if errRes != nil {
|
|
return errRes
|
|
}
|
|
termCauses = termRes.FailureCauses()
|
|
if termCauses.Len() == 0 {
|
|
c, _ := NewFailureCause("provider", "error", "", "", "")
|
|
termCauses, _ = NewFailureCauseChain([]FailureCause{c})
|
|
}
|
|
if useUnbuffered {
|
|
if errRel := r.releaser.CommitTerminalWithoutEpoch(ctx, binding.AttemptID(), termRes); errRel != nil {
|
|
return errRel
|
|
}
|
|
} else {
|
|
if _, errRel := r.releaser.ReleaseTerminalEpoch(ctx, binding.AttemptID(), epoch.ID(), termRes); errRel != nil {
|
|
return errRel
|
|
}
|
|
}
|
|
} else {
|
|
desc, errD := NewExternalDescriptor("error", "fatal_violation", "fatal_filter_violation", "")
|
|
if errD != nil {
|
|
return errD
|
|
}
|
|
cause, errC := NewFailureCause("arbiter", "fatal_violation", "", arbResult.FilterID(), arbResult.RuleID())
|
|
if errC != nil {
|
|
return errC
|
|
}
|
|
causes, errCh := NewFailureCauseChain([]FailureCause{cause})
|
|
if errCh != nil {
|
|
return errCh
|
|
}
|
|
termCauses = causes
|
|
termRes, errTerm := NewErrorTerminalResult(ev.Channel(), desc, causes, time.Now())
|
|
if errTerm != nil {
|
|
return errTerm
|
|
}
|
|
if useUnbuffered {
|
|
if errRel := r.releaser.CommitTerminalWithoutEpoch(ctx, binding.AttemptID(), termRes); errRel != nil {
|
|
return errRel
|
|
}
|
|
} else {
|
|
if _, errRel := r.releaser.FailPending(ctx, binding.AttemptID(), termRes); errRel != nil {
|
|
return errRel
|
|
}
|
|
}
|
|
}
|
|
if arbResult.BaseDisposition() == BaseDispositionTerminalSuccessCandidate {
|
|
r.emitObservation(ObservationKindReleaseCommitted, epoch.ID(), func(in *FilterObservationInput) {
|
|
in.CommitState = CommitStateStreamOpen
|
|
})
|
|
}
|
|
r.emitObservation(ObservationKindTerminalCommitted, epoch.ID(), func(in *FilterObservationInput) {
|
|
in.CommitState = CommitStateTerminalCommitted
|
|
if arbResult.BaseDisposition() == BaseDispositionTerminalSuccessCandidate {
|
|
in.TerminalReason = TerminalReasonCompleted
|
|
} else {
|
|
in.Causes = termCauses
|
|
}
|
|
})
|
|
r.mu.Lock()
|
|
r.terminalCommitted = true
|
|
r.mu.Unlock()
|
|
return nil
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// PendingEventsByChannel returns a defensive map of pending events per channel.
|
|
func (t *EvidenceTail) PendingEventsByChannel() map[string][]NormalizedEvent {
|
|
if t == nil || len(t.channelState) == 0 {
|
|
return nil
|
|
}
|
|
out := make(map[string][]NormalizedEvent)
|
|
for ch, cs := range t.channelState {
|
|
var pe []NormalizedEvent
|
|
for _, e := range cs.pendingEntries {
|
|
if e.event.Kind() != EventKindTerminal && e.event.Kind() != EventKindProviderError {
|
|
pe = append(pe, cloneNormalizedEvent(e.event))
|
|
}
|
|
}
|
|
if len(pe) > 0 {
|
|
out[ch] = pe
|
|
}
|
|
}
|
|
if len(out) == 0 {
|
|
return nil
|
|
}
|
|
return out
|
|
}
|
|
|
|
// CommittedLookBehindByChannel returns a defensive map of committed look-behind events per channel.
|
|
func (t *EvidenceTail) CommittedLookBehindByChannel() map[string][]NormalizedEvent {
|
|
if t == nil || len(t.channelState) == 0 {
|
|
return nil
|
|
}
|
|
out := make(map[string][]NormalizedEvent)
|
|
for ch, cs := range t.channelState {
|
|
if len(cs.committedLookBehind) > 0 {
|
|
lb := make([]NormalizedEvent, len(cs.committedLookBehind))
|
|
for i, ev := range cs.committedLookBehind {
|
|
lb[i] = cloneNormalizedEvent(ev)
|
|
}
|
|
out[ch] = lb
|
|
}
|
|
}
|
|
if len(out) == 0 {
|
|
return nil
|
|
}
|
|
return out
|
|
}
|
|
|
|
// PrepareContinuationWithPlan updates the evidence plan and prepares continuation.
|
|
func (t *EvidenceTail) PrepareContinuationWithPlan(plan EvidencePlan) {
|
|
t.plan = plan
|
|
t.PrepareContinuation()
|
|
}
|
|
|
|
// Plan returns the current EvidencePlan.
|
|
func (t *EvidenceTail) Plan() EvidencePlan {
|
|
if t == nil {
|
|
return EvidencePlan{}
|
|
}
|
|
return t.plan
|
|
}
|