812 lines
27 KiB
Go
812 lines
27 KiB
Go
package openai
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"strings"
|
|
"sync"
|
|
|
|
"go.uber.org/zap"
|
|
)
|
|
|
|
// hotPathEventClass is the closed top-level event class for every Hot Path
|
|
// observation. It scopes the lifecycle without exposing request, stage, or
|
|
// attempt identity (SDD S15).
|
|
type hotPathEventClass string
|
|
|
|
const (
|
|
hotPathEventClassDispatch hotPathEventClass = "dispatch"
|
|
hotPathEventClassStage hotPathEventClass = "stage"
|
|
hotPathEventClassLight hotPathEventClass = "light"
|
|
hotPathEventClassTerminal hotPathEventClass = "terminal"
|
|
hotPathEventClassCleanup hotPathEventClass = "cleanup"
|
|
hotPathEventClassOrphan hotPathEventClass = "orphan"
|
|
)
|
|
|
|
// hotPathMode is the closed execution mode observed on dispatch events.
|
|
type hotPathMode string
|
|
|
|
const (
|
|
hotPathModeDirect hotPathMode = "direct"
|
|
hotPathModeLight hotPathMode = "light"
|
|
)
|
|
|
|
// hotPathStageKind is the closed stage role observed on stage events.
|
|
type hotPathStageKind string
|
|
|
|
const (
|
|
hotPathStageKindSelector hotPathStageKind = "selector"
|
|
hotPathStageKindLocal hotPathStageKind = "local"
|
|
hotPathStageKindReview hotPathStageKind = "review"
|
|
hotPathStageKindCleanup hotPathStageKind = "cleanup"
|
|
)
|
|
|
|
// hotPathAttemptBucket is the closed attempt-order bucket observed on stage
|
|
// events. It is deliberately coarse: first vs retry, never an absolute count.
|
|
type hotPathAttemptBucket string
|
|
|
|
const (
|
|
hotPathAttemptFirst hotPathAttemptBucket = "first"
|
|
hotPathAttemptRetry hotPathAttemptBucket = "retry"
|
|
)
|
|
|
|
// hotPathRouteReason is the closed reason emitted on dispatch events when
|
|
// admission fails. It is never a raw error string.
|
|
type hotPathRouteReason string
|
|
|
|
const (
|
|
hotPathRouteReasonModeDisabled hotPathRouteReason = "mode_disabled"
|
|
hotPathRouteReasonArtifactReq hotPathRouteReason = "artifact_required"
|
|
hotPathRouteReasonInvalidInput hotPathRouteReason = "invalid_input"
|
|
hotPathRouteReasonProviderError hotPathRouteReason = "provider_error"
|
|
hotPathRouteReasonTimeout hotPathRouteReason = "timeout"
|
|
hotPathRouteReasonCallerCancel hotPathRouteReason = "caller_cancel"
|
|
)
|
|
|
|
// hotPathDispositionKind is the closed terminal disposition observed on
|
|
// terminal events. It reuses the vocabulary of hotPathTerminalDisposition
|
|
// without depending on its struct shape so projection can run from the
|
|
// string value alone.
|
|
type hotPathTerminalDispositionKind string
|
|
|
|
const (
|
|
hotPathTerminalDispositionSuccess hotPathTerminalDispositionKind = "success"
|
|
hotPathTerminalDispositionToolTurn hotPathTerminalDispositionKind = "tool_turn"
|
|
hotPathTerminalDispositionLength hotPathTerminalDispositionKind = "length"
|
|
hotPathTerminalDispositionProviderError hotPathTerminalDispositionKind = "provider_error"
|
|
hotPathTerminalDispositionValidationError hotPathTerminalDispositionKind = "validation_error"
|
|
hotPathTerminalDispositionTimeout hotPathTerminalDispositionKind = "timeout"
|
|
hotPathTerminalDispositionCallerCancel hotPathTerminalDispositionKind = "caller_cancel"
|
|
)
|
|
|
|
// hotPathCleanupOutcome is the closed cleanup result observed on cleanup
|
|
// events.
|
|
type hotPathCleanupOutcome string
|
|
|
|
const (
|
|
hotPathCleanupOutcomeSuccess hotPathCleanupOutcome = "success"
|
|
hotPathCleanupOutcomePrimaryError hotPathCleanupOutcome = "primary_error"
|
|
hotPathCleanupOutcomeTTLExpired hotPathCleanupOutcome = "ttl_expired"
|
|
)
|
|
|
|
// hotPathOrphanOutcome is the closed orphan outcome observed on orphan
|
|
// events.
|
|
type hotPathOrphanOutcome string
|
|
|
|
const (
|
|
hotPathOrphanOutcomeTTLExpired hotPathOrphanOutcome = "ttl_expired"
|
|
hotPathOrphanOutcomeCleanupFailed hotPathOrphanOutcome = "cleanup_failed"
|
|
)
|
|
|
|
// hotPathTerminalDispositionIsValid reports whether d is a known disposition
|
|
// value. Unknown values normalize to empty string in projection.
|
|
func hotPathTerminalDispositionIsValid(d hotPathTerminalDispositionKind) bool {
|
|
switch d {
|
|
case hotPathTerminalDispositionSuccess,
|
|
hotPathTerminalDispositionToolTurn,
|
|
hotPathTerminalDispositionLength,
|
|
hotPathTerminalDispositionProviderError,
|
|
hotPathTerminalDispositionValidationError,
|
|
hotPathTerminalDispositionTimeout,
|
|
hotPathTerminalDispositionCallerCancel:
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
// hotPathEventClassIsValid reports whether c is a known event class.
|
|
func hotPathEventClassIsValid(c hotPathEventClass) bool {
|
|
switch c {
|
|
case hotPathEventClassDispatch, hotPathEventClassStage, hotPathEventClassLight,
|
|
hotPathEventClassTerminal, hotPathEventClassCleanup, hotPathEventClassOrphan:
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
// hotPathModeIsValid reports whether m is a known execution mode.
|
|
func hotPathModeIsValid(m hotPathMode) bool {
|
|
switch m {
|
|
case hotPathModeDirect, hotPathModeLight:
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
// hotPathStageKindIsValid reports whether k is a known stage role.
|
|
func hotPathStageKindIsValid(k hotPathStageKind) bool {
|
|
switch k {
|
|
case hotPathStageKindSelector, hotPathStageKindLocal, hotPathStageKindReview, hotPathStageKindCleanup:
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
// hotPathAttemptBucketIsValid reports whether b is a known attempt bucket.
|
|
func hotPathAttemptBucketIsValid(b hotPathAttemptBucket) bool {
|
|
switch b {
|
|
case hotPathAttemptFirst, hotPathAttemptRetry:
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
// hotPathRouteReasonIsValid reports whether r is a known route reason.
|
|
func hotPathRouteReasonIsValid(r hotPathRouteReason) bool {
|
|
switch r {
|
|
case hotPathRouteReasonModeDisabled, hotPathRouteReasonArtifactReq,
|
|
hotPathRouteReasonInvalidInput, hotPathRouteReasonProviderError,
|
|
hotPathRouteReasonTimeout, hotPathRouteReasonCallerCancel:
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
// hotPathCleanupOutcomeIsValid reports whether o is a known cleanup outcome.
|
|
func hotPathCleanupOutcomeIsValid(o hotPathCleanupOutcome) bool {
|
|
switch o {
|
|
case hotPathCleanupOutcomeSuccess, hotPathCleanupOutcomePrimaryError, hotPathCleanupOutcomeTTLExpired:
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
// hotPathOrphanOutcomeIsValid reports whether o is a known orphan outcome.
|
|
func hotPathOrphanOutcomeIsValid(o hotPathOrphanOutcome) bool {
|
|
switch o {
|
|
case hotPathOrphanOutcomeTTLExpired, hotPathOrphanOutcomeCleanupFailed:
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
// hotPathNormalizeAttemptBucket converts a raw attempt bucket string to its
|
|
// closed form. Unknown values become empty so callers cannot smuggle arbitrary
|
|
// text into metrics labels or log fields.
|
|
func hotPathNormalizeAttemptBucket(raw string) hotPathAttemptBucket {
|
|
switch hotPathAttemptBucket(raw) {
|
|
case hotPathAttemptFirst, hotPathAttemptRetry:
|
|
return hotPathAttemptBucket(raw)
|
|
default:
|
|
return ""
|
|
}
|
|
}
|
|
|
|
// hotPathNormalizeDisposition converts a raw disposition string to its closed
|
|
// form. Unknown values become empty so callers cannot smuggle arbitrary text
|
|
// into metrics labels or log fields.
|
|
func hotPathNormalizeDisposition(raw string) hotPathTerminalDispositionKind {
|
|
switch hotPathTerminalDispositionKind(raw) {
|
|
case hotPathTerminalDispositionSuccess,
|
|
hotPathTerminalDispositionToolTurn,
|
|
hotPathTerminalDispositionLength,
|
|
hotPathTerminalDispositionProviderError,
|
|
hotPathTerminalDispositionValidationError,
|
|
hotPathTerminalDispositionTimeout,
|
|
hotPathTerminalDispositionCallerCancel:
|
|
return hotPathTerminalDispositionKind(raw)
|
|
default:
|
|
return ""
|
|
}
|
|
}
|
|
|
|
// hotPathNormalizeEventClass converts a raw event class string to its closed
|
|
// form. Unknown values become empty so callers cannot smuggle arbitrary text
|
|
// into metrics labels or log fields.
|
|
func hotPathNormalizeEventClass(raw string) hotPathEventClass {
|
|
switch hotPathEventClass(raw) {
|
|
case hotPathEventClassDispatch, hotPathEventClassStage, hotPathEventClassLight,
|
|
hotPathEventClassTerminal, hotPathEventClassCleanup, hotPathEventClassOrphan:
|
|
return hotPathEventClass(raw)
|
|
default:
|
|
return ""
|
|
}
|
|
}
|
|
|
|
// hotPathNormalizeMode converts a raw mode string to its closed form. Unknown
|
|
// values become empty so callers cannot smuggle arbitrary text into metrics
|
|
// labels or log fields.
|
|
func hotPathNormalizeMode(raw string) hotPathMode {
|
|
switch hotPathMode(raw) {
|
|
case hotPathModeDirect, hotPathModeLight:
|
|
return hotPathMode(raw)
|
|
default:
|
|
return ""
|
|
}
|
|
}
|
|
|
|
// hotPathNormalizeStageKind converts a raw stage kind string to its closed
|
|
// form. Unknown values become empty so callers cannot smuggle arbitrary text
|
|
// into metrics labels or log fields.
|
|
func hotPathNormalizeStageKind(raw string) hotPathStageKind {
|
|
switch hotPathStageKind(raw) {
|
|
case hotPathStageKindSelector, hotPathStageKindLocal, hotPathStageKindReview, hotPathStageKindCleanup:
|
|
return hotPathStageKind(raw)
|
|
default:
|
|
return ""
|
|
}
|
|
}
|
|
|
|
// hotPathNormalizeRouteReason converts a raw route reason string to its closed
|
|
// form. Unknown values become empty so callers cannot smuggle arbitrary text
|
|
// into metrics labels or log fields.
|
|
func hotPathNormalizeRouteReason(raw string) hotPathRouteReason {
|
|
switch hotPathRouteReason(raw) {
|
|
case hotPathRouteReasonModeDisabled, hotPathRouteReasonArtifactReq,
|
|
hotPathRouteReasonInvalidInput, hotPathRouteReasonProviderError,
|
|
hotPathRouteReasonTimeout, hotPathRouteReasonCallerCancel:
|
|
return hotPathRouteReason(raw)
|
|
default:
|
|
return ""
|
|
}
|
|
}
|
|
|
|
// hotPathNormalizeCleanupOutcome converts a raw cleanup outcome string to its
|
|
// closed form. Unknown values become empty so callers cannot smuggle arbitrary
|
|
// text into metrics labels or log fields.
|
|
func hotPathNormalizeCleanupOutcome(raw string) hotPathCleanupOutcome {
|
|
switch hotPathCleanupOutcome(raw) {
|
|
case hotPathCleanupOutcomeSuccess, hotPathCleanupOutcomePrimaryError, hotPathCleanupOutcomeTTLExpired:
|
|
return hotPathCleanupOutcome(raw)
|
|
default:
|
|
return ""
|
|
}
|
|
}
|
|
|
|
// hotPathNormalizeOrphanOutcome converts a raw orphan outcome string to its
|
|
// closed form. Unknown values become empty so callers cannot smuggle arbitrary
|
|
// text into metrics labels or log fields.
|
|
func hotPathNormalizeOrphanOutcome(raw string) hotPathOrphanOutcome {
|
|
switch hotPathOrphanOutcome(raw) {
|
|
case hotPathOrphanOutcomeTTLExpired, hotPathOrphanOutcomeCleanupFailed:
|
|
return hotPathOrphanOutcome(raw)
|
|
default:
|
|
return ""
|
|
}
|
|
}
|
|
|
|
// hotPathLogProjection is the closed set of keys emitted on Hot Path log
|
|
// events. The projection is deliberately separate from metric labels so log
|
|
// correlation ids can be included while metric cardinality stays bounded
|
|
// (SDD S15).
|
|
type hotPathLogProjection struct {
|
|
EventClass hotPathEventClass
|
|
Mode hotPathMode
|
|
StageKind hotPathStageKind
|
|
Disposition hotPathTerminalDispositionKind
|
|
Correlation string
|
|
StageID string
|
|
RequestID string
|
|
CallID string
|
|
OwnerEdgeID string
|
|
Reason hotPathRouteReason
|
|
PresetID string
|
|
AttemptBucket hotPathAttemptBucket
|
|
CleanupOutcome hotPathCleanupOutcome
|
|
OrphanOutcome hotPathOrphanOutcome
|
|
}
|
|
|
|
// logProjectionKeys returns the ordered, allowlisted set of keys that every
|
|
// Hot Path log projection emits. Tests assert on this exact slice.
|
|
func logProjectionKeys() []string {
|
|
return []string{
|
|
"hot_path_event_class",
|
|
"hot_path_mode",
|
|
"hot_path_stage_kind",
|
|
"hot_path_disposition",
|
|
"hot_path_correlation",
|
|
"hot_path_stage_id",
|
|
"hot_path_request_id",
|
|
"hot_path_call_id",
|
|
"hot_path_owner_edge_id",
|
|
"hot_path_reason",
|
|
"hot_path_preset_id",
|
|
"hot_path_attempt_bucket",
|
|
"hot_path_cleanup_outcome",
|
|
"hot_path_orphan_outcome",
|
|
}
|
|
}
|
|
|
|
// logProjectionAllowlist returns the log projection key set as a map for O(1)
|
|
// membership checks. Tests use this to reject non-allowlisted keys.
|
|
func logProjectionAllowlist() map[string]struct{} {
|
|
out := make(map[string]struct{}, len(logProjectionKeys()))
|
|
for _, k := range logProjectionKeys() {
|
|
out[k] = struct{}{}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func containsSecretSentinel(s string) bool {
|
|
lower := strings.ToLower(s)
|
|
return strings.Contains(lower, "secret") ||
|
|
strings.Contains(lower, "bearer") ||
|
|
strings.Contains(lower, "api_key") ||
|
|
strings.Contains(lower, "token") ||
|
|
strings.Contains(s, "\x00")
|
|
}
|
|
|
|
func sanitizeLogString(s string) string {
|
|
if containsSecretSentinel(s) {
|
|
return ""
|
|
}
|
|
if len(s) > 64 {
|
|
return s[:64]
|
|
}
|
|
return s
|
|
}
|
|
|
|
// hotPathValidateLogProjection checks all typed enum fields and string metadata.
|
|
// Unknown enums or secret sentinels cause validation failure (return false).
|
|
func hotPathValidateLogProjection(p hotPathLogProjection) (hotPathLogProjection, bool) {
|
|
if !hotPathEventClassIsValid(p.EventClass) {
|
|
return hotPathLogProjection{}, false
|
|
}
|
|
if p.Mode != "" && !hotPathModeIsValid(p.Mode) {
|
|
return hotPathLogProjection{}, false
|
|
}
|
|
if p.StageKind != "" && !hotPathStageKindIsValid(p.StageKind) {
|
|
return hotPathLogProjection{}, false
|
|
}
|
|
if p.Disposition != "" && !hotPathTerminalDispositionIsValid(p.Disposition) {
|
|
return hotPathLogProjection{}, false
|
|
}
|
|
if p.Reason != "" && !hotPathRouteReasonIsValid(p.Reason) {
|
|
return hotPathLogProjection{}, false
|
|
}
|
|
if p.AttemptBucket != "" && !hotPathAttemptBucketIsValid(p.AttemptBucket) {
|
|
return hotPathLogProjection{}, false
|
|
}
|
|
if p.CleanupOutcome != "" && !hotPathCleanupOutcomeIsValid(p.CleanupOutcome) {
|
|
return hotPathLogProjection{}, false
|
|
}
|
|
if p.OrphanOutcome != "" && !hotPathOrphanOutcomeIsValid(p.OrphanOutcome) {
|
|
return hotPathLogProjection{}, false
|
|
}
|
|
|
|
if containsSecretSentinel(p.PresetID) ||
|
|
containsSecretSentinel(p.StageID) ||
|
|
containsSecretSentinel(p.RequestID) ||
|
|
containsSecretSentinel(p.CallID) ||
|
|
containsSecretSentinel(p.OwnerEdgeID) ||
|
|
containsSecretSentinel(p.Correlation) {
|
|
return hotPathLogProjection{}, false
|
|
}
|
|
|
|
p.PresetID = sanitizeLogString(p.PresetID)
|
|
p.StageID = sanitizeLogString(p.StageID)
|
|
p.RequestID = sanitizeLogString(p.RequestID)
|
|
p.CallID = sanitizeLogString(p.CallID)
|
|
p.OwnerEdgeID = sanitizeLogString(p.OwnerEdgeID)
|
|
|
|
if p.Correlation == "" && (p.RequestID != "" || p.StageID != "" || p.CallID != "") {
|
|
p.Correlation = string(newHotPathCorrelationID(p.RequestID, p.StageID, p.CallID))
|
|
} else {
|
|
p.Correlation = sanitizeLogString(p.Correlation)
|
|
}
|
|
|
|
return p, true
|
|
}
|
|
|
|
// hotPathCorrelationID is a path-safe, bounded correlation id emitted on log
|
|
// events. It is never used as an auth secret or metric label (SDD S15).
|
|
type hotPathCorrelationID string
|
|
|
|
// newHotPathCorrelationID builds a bounded correlation id from request, stage,
|
|
// and call identifiers. Empty segments are skipped so the id never carries
|
|
// raw caller input.
|
|
func newHotPathCorrelationID(requestID, stageID, callID string) hotPathCorrelationID {
|
|
var parts []string
|
|
if strings.TrimSpace(requestID) != "" {
|
|
parts = append(parts, sanitizeCorrelationToken("req", requestID))
|
|
}
|
|
if strings.TrimSpace(stageID) != "" {
|
|
parts = append(parts, sanitizeCorrelationToken("stage", stageID))
|
|
}
|
|
if strings.TrimSpace(callID) != "" {
|
|
parts = append(parts, sanitizeCorrelationToken("call", callID))
|
|
}
|
|
if len(parts) == 0 {
|
|
return ""
|
|
}
|
|
return hotPathCorrelationID(strings.Join(parts, ":"))
|
|
}
|
|
|
|
// sanitizeCorrelationToken normalizes a raw id segment into a path-safe token
|
|
// suitable for log correlation ids. Spaces, slashes, and control characters
|
|
// are stripped and the result is capped to 64 runes so the overall id stays
|
|
// bounded.
|
|
func sanitizeCorrelationToken(prefix, raw string) string {
|
|
var b strings.Builder
|
|
b.Grow(len(raw))
|
|
for _, r := range raw {
|
|
switch {
|
|
case r == '/' || r == '\\':
|
|
b.WriteByte('_')
|
|
case r == ' ' || r == '\t' || r == '\n' || r == '\r':
|
|
b.WriteByte('_')
|
|
case (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '-':
|
|
b.WriteRune(r)
|
|
default:
|
|
b.WriteByte('_')
|
|
}
|
|
}
|
|
s := "hot_path." + prefix + "." + b.String()
|
|
if len(s) > 64 {
|
|
s = s[:64]
|
|
}
|
|
return s
|
|
}
|
|
|
|
// hotPathObserver is the internal Hot Path observation contract. Implementations
|
|
// own storage and retention; callers only own the bounded projection inputs.
|
|
// Emit must not block indefinitely — sinks that need bounded work should apply
|
|
// their own timeout internally.
|
|
type hotPathObserver interface {
|
|
Emit(ctx context.Context, projection hotPathLogProjection) error
|
|
}
|
|
|
|
// hotPathNoopObserver discards every observation. It is the default observer
|
|
// for hosts that have not wired a logging backend yet.
|
|
type hotPathNoopObserver struct{}
|
|
|
|
// Emit discards the observation and always returns nil.
|
|
func (hotPathNoopObserver) Emit(ctx context.Context, projection hotPathLogProjection) error {
|
|
return nil
|
|
}
|
|
|
|
const hotPathObservationMessage = "hot_path_observation"
|
|
|
|
// zapHotPathObserver is the production projection sink. It writes one fixed
|
|
// message and exactly the fields returned by logProjectionKeys; raw errors,
|
|
// request bodies, provider data, credentials, and dynamic keys have no input
|
|
// seam here.
|
|
type zapHotPathObserver struct {
|
|
logger *zap.Logger
|
|
}
|
|
|
|
func newZapHotPathObserver(logger *zap.Logger) hotPathObserver {
|
|
if logger == nil {
|
|
logger = zap.NewNop()
|
|
}
|
|
return &zapHotPathObserver{logger: logger}
|
|
}
|
|
|
|
func (o *zapHotPathObserver) Emit(_ context.Context, p hotPathLogProjection) error {
|
|
if o == nil || o.logger == nil {
|
|
return nil
|
|
}
|
|
o.logger.Info(hotPathObservationMessage,
|
|
zap.String("hot_path_event_class", string(p.EventClass)),
|
|
zap.String("hot_path_mode", string(p.Mode)),
|
|
zap.String("hot_path_stage_kind", string(p.StageKind)),
|
|
zap.String("hot_path_disposition", string(p.Disposition)),
|
|
zap.String("hot_path_correlation", p.Correlation),
|
|
zap.String("hot_path_stage_id", p.StageID),
|
|
zap.String("hot_path_request_id", p.RequestID),
|
|
zap.String("hot_path_call_id", p.CallID),
|
|
zap.String("hot_path_owner_edge_id", p.OwnerEdgeID),
|
|
zap.String("hot_path_reason", string(p.Reason)),
|
|
zap.String("hot_path_preset_id", p.PresetID),
|
|
zap.String("hot_path_attempt_bucket", string(p.AttemptBucket)),
|
|
zap.String("hot_path_cleanup_outcome", string(p.CleanupOutcome)),
|
|
zap.String("hot_path_orphan_outcome", string(p.OrphanOutcome)),
|
|
)
|
|
return nil
|
|
}
|
|
|
|
// hotPathBoundedObserver validates every projection before delegating to the
|
|
// configured sink. Failure isolation is provided by hotPathSafeObserver at the
|
|
// server seam.
|
|
type hotPathBoundedObserver struct {
|
|
inner hotPathObserver
|
|
}
|
|
|
|
// Emit validates the projection and delegates to the inner observer if valid.
|
|
// A nil inner is treated as a noop.
|
|
func (b *hotPathBoundedObserver) Emit(ctx context.Context, projection hotPathLogProjection) error {
|
|
if b == nil || b.inner == nil {
|
|
return nil
|
|
}
|
|
validated, ok := hotPathValidateLogProjection(projection)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
return b.inner.Emit(ctx, validated)
|
|
}
|
|
|
|
// hotPathObserverFailureHook is called when an observer failure occurs. It is
|
|
// optional; the observer isolates failures so they never affect request
|
|
// results.
|
|
type hotPathObserverFailureHook func(projection hotPathLogProjection, err error)
|
|
|
|
func invokeHotPathObserverFailureHookSafely(hook hotPathObserverFailureHook, projection hotPathLogProjection, err error) {
|
|
if hook == nil {
|
|
return
|
|
}
|
|
defer func() {
|
|
_ = recover()
|
|
}()
|
|
hook(projection, err)
|
|
}
|
|
|
|
// hotPathSafeObserver wraps an inner observer with failure isolation. If the
|
|
// inner observer panics or returns an error, the failure is reported through
|
|
// the hook (if set) and the call returns nil. Both observer and hook panics
|
|
// are completely isolated so the request path is never interrupted.
|
|
type hotPathSafeObserver struct {
|
|
inner hotPathObserver
|
|
onFailure hotPathObserverFailureHook
|
|
failures int64
|
|
mu sync.Mutex
|
|
}
|
|
|
|
// Emit forwards the projection to the inner observer with failure isolation.
|
|
// If the inner observer returns an error or panics, the failure is reported
|
|
// through the hook (which is also panic-isolated) and Emit returns nil.
|
|
func (s *hotPathSafeObserver) Emit(ctx context.Context, projection hotPathLogProjection) error {
|
|
if s == nil || s.inner == nil {
|
|
return nil
|
|
}
|
|
func() {
|
|
defer func() {
|
|
if r := recover(); r != nil {
|
|
s.mu.Lock()
|
|
s.failures++
|
|
s.mu.Unlock()
|
|
if s.onFailure != nil {
|
|
func() {
|
|
defer func() {
|
|
_ = recover()
|
|
}()
|
|
s.onFailure(projection, fmt.Errorf("observer panic: %v", r))
|
|
}()
|
|
}
|
|
}
|
|
}()
|
|
if err := s.inner.Emit(ctx, projection); err != nil {
|
|
s.mu.Lock()
|
|
s.failures++
|
|
s.mu.Unlock()
|
|
if s.onFailure != nil {
|
|
func() {
|
|
defer func() {
|
|
_ = recover()
|
|
}()
|
|
s.onFailure(projection, err)
|
|
}()
|
|
}
|
|
return
|
|
}
|
|
}()
|
|
return nil
|
|
}
|
|
|
|
// failureCount returns the number of isolated failures observed so far. It is
|
|
// safe for concurrent reads from tests.
|
|
func (s *hotPathSafeObserver) failureCount() int64 {
|
|
if s == nil {
|
|
return 0
|
|
}
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
return s.failures
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Lifecycle emission boundary helpers (API-1).
|
|
//
|
|
// Each helper is the single owner of one Hot Path observation class for a
|
|
// request. They emit the closed log projection through emitHotPathObservation
|
|
// (which validates, sanitizes, and isolates observer failures) and record the
|
|
// matching bounded metric. Cause normalization happens before projection so
|
|
// raw error strings never reach logs or labels (SDD S15). All emission is best
|
|
// effort: an observer error or panic cannot alter the response, cancellation,
|
|
// or cleanup semantics.
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// hotPathRouteReasonForDecision maps a selector/planner decision reason to its
|
|
// closed observation route reason. Unknown reasons collapse to invalid_input so
|
|
// the rejection is still observable without leaking raw reason text.
|
|
func hotPathRouteReasonForDecision(reason string) hotPathRouteReason {
|
|
switch reason {
|
|
case reasonModeDisabled:
|
|
return hotPathRouteReasonModeDisabled
|
|
case reasonUnhealthyRoute:
|
|
return hotPathRouteReasonProviderError
|
|
case reasonArtifactRequired:
|
|
return hotPathRouteReasonArtifactReq
|
|
default:
|
|
return hotPathRouteReasonInvalidInput
|
|
}
|
|
}
|
|
|
|
// hotPathStageKindForPhase maps a light-flow phase to its closed observation
|
|
// stage kind. Phases that do not own a provider dispatch map to empty so the
|
|
// bounded observer skips them.
|
|
func hotPathStageKindForPhase(phase hotPathLightPhase) hotPathStageKind {
|
|
switch phase {
|
|
case hotPathPhaseLocalActive:
|
|
return hotPathStageKindLocal
|
|
case hotPathPhaseReviewActive, hotPathPhaseReviewAwaitRead, hotPathPhaseReviewResolution, hotPathPhaseReviewRepair:
|
|
return hotPathStageKindReview
|
|
case hotPathPhaseCleanupPending:
|
|
return hotPathStageKindCleanup
|
|
default:
|
|
return ""
|
|
}
|
|
}
|
|
|
|
// hotPathAttemptBucketForTranscript returns the closed attempt bucket for a
|
|
// stage dispatch: "first" for the initial dispatch in a stage and "retry" for
|
|
// any re-dispatch after a tool round-trip.
|
|
func hotPathAttemptBucketForTranscript(transcript []hotPathStageExchange) hotPathAttemptBucket {
|
|
if len(transcript) == 0 {
|
|
return hotPathAttemptFirst
|
|
}
|
|
return hotPathAttemptRetry
|
|
}
|
|
|
|
// hotPathTerminalDispositionFromKind converts the internal hotPathDispositionKind
|
|
// to its closed observation terminal disposition kind. Both enums share the same
|
|
// string vocabulary, so the value is validated through the normalizer.
|
|
func hotPathTerminalDispositionFromKind(kind hotPathDispositionKind) hotPathTerminalDispositionKind {
|
|
return hotPathNormalizeDisposition(string(kind))
|
|
}
|
|
|
|
// observeHotPathDispatch emits the admission/route selection observation. It is
|
|
// the single owner of the dispatch log event for a request. A non-empty reason
|
|
// records the bounded dispatch metric; a successful admission records the log
|
|
// projection only.
|
|
func (s *Server) observeHotPathDispatch(ctx context.Context, mode hotPathMode, reason hotPathRouteReason, requestID, stageID, presetID string) {
|
|
if s == nil {
|
|
return
|
|
}
|
|
ownerEdgeID := s.edgeIDValue()
|
|
s.emitHotPathObservation(ctx, hotPathLogProjection{
|
|
EventClass: hotPathEventClassDispatch,
|
|
Mode: mode,
|
|
Reason: reason,
|
|
RequestID: requestID,
|
|
StageID: stageID,
|
|
PresetID: presetID,
|
|
OwnerEdgeID: ownerEdgeID,
|
|
})
|
|
if reason != "" {
|
|
initHotPathMetrics().recordDispatch(ownerEdgeID, mode, reason)
|
|
}
|
|
}
|
|
|
|
// observeHotPathStage emits a stage dispatch observation and records the bounded
|
|
// stage duration. It is the single owner of stage events for light provider
|
|
// dispatches.
|
|
func (s *Server) observeHotPathStage(ctx context.Context, mode hotPathMode, stageKind hotPathStageKind, attempt hotPathAttemptBucket, disposition hotPathTerminalDispositionKind, requestID, stageID, presetID string, durationSeconds float64) {
|
|
if s == nil {
|
|
return
|
|
}
|
|
ownerEdgeID := s.edgeIDValue()
|
|
s.emitHotPathObservation(ctx, hotPathLogProjection{
|
|
EventClass: hotPathEventClassStage,
|
|
Mode: mode,
|
|
StageKind: stageKind,
|
|
Disposition: disposition,
|
|
AttemptBucket: attempt,
|
|
RequestID: requestID,
|
|
StageID: stageID,
|
|
PresetID: presetID,
|
|
OwnerEdgeID: ownerEdgeID,
|
|
})
|
|
if durationSeconds > 0 {
|
|
initHotPathMetrics().recordStageDuration(ownerEdgeID, mode, stageKind, attempt, durationSeconds)
|
|
}
|
|
}
|
|
|
|
// observeHotPathLightTransition emits a light-mode stage transition observation
|
|
// (e.g. local completion promoting to the review stage). It carries the joined
|
|
// lifecycle through the log projection and records no metric of its own.
|
|
func (s *Server) observeHotPathLightTransition(ctx context.Context, stageKind hotPathStageKind, attempt hotPathAttemptBucket, requestID, stageID, presetID string) {
|
|
if s == nil {
|
|
return
|
|
}
|
|
s.emitHotPathObservation(ctx, hotPathLogProjection{
|
|
EventClass: hotPathEventClassLight,
|
|
Mode: hotPathModeLight,
|
|
StageKind: stageKind,
|
|
AttemptBucket: attempt,
|
|
RequestID: requestID,
|
|
StageID: stageID,
|
|
PresetID: presetID,
|
|
OwnerEdgeID: s.edgeIDValue(),
|
|
})
|
|
}
|
|
|
|
func (s *Server) observeHotPathCleanupTransition(ctx context.Context, requestID, presetID string) {
|
|
stageID := ""
|
|
if s != nil && s.lightFlows != nil {
|
|
stageID = s.lightFlows.cleanupStage(requestID, s.edgeIDValue())
|
|
}
|
|
s.observeHotPathLightTransition(ctx, hotPathStageKindCleanup, hotPathAttemptFirst, requestID, stageID, presetID)
|
|
}
|
|
|
|
// observeHotPathTerminal emits the single outer terminal observation for a
|
|
// request and records the bounded terminal metric. The caller passes the
|
|
// already-normalized disposition so raw error text never reaches the projection.
|
|
func (s *Server) observeHotPathTerminal(ctx context.Context, mode hotPathMode, disposition hotPathTerminalDispositionKind, requestID, stageID, presetID string) {
|
|
if s == nil {
|
|
return
|
|
}
|
|
ownerEdgeID := s.edgeIDValue()
|
|
s.emitHotPathObservation(ctx, hotPathLogProjection{
|
|
EventClass: hotPathEventClassTerminal,
|
|
Mode: mode,
|
|
Disposition: disposition,
|
|
RequestID: requestID,
|
|
StageID: stageID,
|
|
PresetID: presetID,
|
|
OwnerEdgeID: ownerEdgeID,
|
|
})
|
|
initHotPathMetrics().recordTerminal(ownerEdgeID, mode, disposition)
|
|
}
|
|
|
|
// observeHotPathCleanup emits the single cleanup-result observation for a
|
|
// request and records the bounded cleanup metric.
|
|
func (s *Server) observeHotPathCleanup(ctx context.Context, outcome hotPathCleanupOutcome, requestID, stageID string) {
|
|
if s == nil {
|
|
return
|
|
}
|
|
ownerEdgeID := s.edgeIDValue()
|
|
s.emitHotPathObservation(ctx, hotPathLogProjection{
|
|
EventClass: hotPathEventClassCleanup,
|
|
CleanupOutcome: outcome,
|
|
RequestID: requestID,
|
|
StageID: stageID,
|
|
OwnerEdgeID: ownerEdgeID,
|
|
})
|
|
initHotPathMetrics().recordCleanup(ownerEdgeID, outcome)
|
|
}
|
|
|
|
// observeHotPathOrphan emits the orphan/TTL observation for a request whose
|
|
// server-side state expired while workspace artifacts may still exist, and
|
|
// records the bounded orphan metric.
|
|
func (s *Server) observeHotPathOrphan(ctx context.Context, outcome hotPathOrphanOutcome, requestID, stageID string) {
|
|
if s == nil {
|
|
return
|
|
}
|
|
ownerEdgeID := s.edgeIDValue()
|
|
s.emitHotPathObservation(ctx, hotPathLogProjection{
|
|
EventClass: hotPathEventClassOrphan,
|
|
OrphanOutcome: outcome,
|
|
RequestID: requestID,
|
|
StageID: stageID,
|
|
OwnerEdgeID: ownerEdgeID,
|
|
})
|
|
initHotPathMetrics().recordOrphan(ownerEdgeID, outcome)
|
|
}
|