366 lines
12 KiB
Go
366 lines
12 KiB
Go
package openai
|
|
|
|
import (
|
|
"fmt"
|
|
"sync"
|
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
"github.com/prometheus/client_golang/prometheus/promauto"
|
|
)
|
|
|
|
// hotPathDurationBucket is the closed duration bucket observed on stage and
|
|
// terminal metrics. Buckets are deliberately coarse so label cardinality stays
|
|
// bounded (SDD S15).
|
|
type hotPathDurationBucket string
|
|
|
|
const (
|
|
hotPathDurationSubMS hotPathDurationBucket = "sub_ms"
|
|
hotPathDuration1to10MS hotPathDurationBucket = "1_to_10ms"
|
|
hotPathDuration10to100MS hotPathDurationBucket = "10_to_100ms"
|
|
hotPathDuration100to1S hotPathDurationBucket = "100ms_to_1s"
|
|
hotPathDuration1to10S hotPathDurationBucket = "1_to_10s"
|
|
hotPathDuration10to60S hotPathDurationBucket = "10_to_60s"
|
|
hotPathDurationOver60S hotPathDurationBucket = "over_60s"
|
|
)
|
|
|
|
// hotPathDurationBucketIsValid reports whether b is a known duration bucket.
|
|
func hotPathDurationBucketIsValid(b hotPathDurationBucket) bool {
|
|
switch b {
|
|
case hotPathDurationSubMS, hotPathDuration1to10MS, hotPathDuration10to100MS,
|
|
hotPathDuration100to1S, hotPathDuration1to10S, hotPathDuration10to60S, hotPathDurationOver60S:
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
// hotPathNormalizeDurationBucket converts a raw duration string to its closed
|
|
// form. Unknown values become empty so callers cannot smuggle arbitrary text
|
|
// into metric labels.
|
|
func hotPathNormalizeDurationBucket(raw string) hotPathDurationBucket {
|
|
switch hotPathDurationBucket(raw) {
|
|
case hotPathDurationSubMS, hotPathDuration1to10MS, hotPathDuration10to100MS,
|
|
hotPathDuration100to1S, hotPathDuration1to10S, hotPathDuration10to60S, hotPathDurationOver60S:
|
|
return hotPathDurationBucket(raw)
|
|
default:
|
|
return ""
|
|
}
|
|
}
|
|
|
|
// hotPathUsageBucket is the closed token usage type observed on usage metrics.
|
|
type hotPathUsageBucket string
|
|
|
|
const (
|
|
hotPathUsagePrompt hotPathUsageBucket = "prompt"
|
|
hotPathUsageCompletion hotPathUsageBucket = "completion"
|
|
hotPathUsageReasoning hotPathUsageBucket = "reasoning"
|
|
hotPathUsageCachedInput hotPathUsageBucket = "cached_input"
|
|
)
|
|
|
|
// hotPathUsageBucketIsValid reports whether b is a known usage bucket.
|
|
func hotPathUsageBucketIsValid(b hotPathUsageBucket) bool {
|
|
switch b {
|
|
case hotPathUsagePrompt, hotPathUsageCompletion, hotPathUsageReasoning, hotPathUsageCachedInput:
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
// hotPathNormalizeUsageBucket converts a raw usage bucket string to its closed
|
|
// form. Unknown values become empty so callers cannot smuggle arbitrary text
|
|
// into metric labels.
|
|
func hotPathNormalizeUsageBucket(raw string) hotPathUsageBucket {
|
|
switch hotPathUsageBucket(raw) {
|
|
case hotPathUsagePrompt, hotPathUsageCompletion, hotPathUsageReasoning, hotPathUsageCachedInput:
|
|
return hotPathUsageBucket(raw)
|
|
default:
|
|
return ""
|
|
}
|
|
}
|
|
|
|
// hotPathMetricLabelNames is the fixed, low-cardinality label set for every
|
|
// Hot Path metric. It deliberately excludes request_id, stage_id, attempt_id,
|
|
// run_id, provider_id, content, headers, error strings, and credentials
|
|
// (SDD S15).
|
|
var hotPathMetricLabelNames = []string{
|
|
"edge_id",
|
|
"hot_path_event_class",
|
|
"hot_path_mode",
|
|
"hot_path_stage_kind",
|
|
"hot_path_disposition",
|
|
"hot_path_duration_bucket",
|
|
"hot_path_usage_bucket",
|
|
"hot_path_attempt_bucket",
|
|
"hot_path_reason",
|
|
"hot_path_cleanup_outcome",
|
|
"hot_path_orphan_outcome",
|
|
}
|
|
|
|
// hotPathMetricLabelCardinality is the fixed label cardinality budget map.
|
|
var hotPathMetricLabelCardinality = map[string]int{
|
|
"edge_id": 64,
|
|
"hot_path_event_class": 6,
|
|
"hot_path_mode": 2,
|
|
"hot_path_stage_kind": 4,
|
|
"hot_path_disposition": 7,
|
|
"hot_path_duration_bucket": 7,
|
|
"hot_path_usage_bucket": 4,
|
|
"hot_path_attempt_bucket": 2,
|
|
"hot_path_reason": 6,
|
|
"hot_path_cleanup_outcome": 3,
|
|
"hot_path_orphan_outcome": 2,
|
|
}
|
|
|
|
// hotPathMetrics is the owner of every Hot Path prometheus collector. It is
|
|
// safe for concurrent use and is initialized once at package load.
|
|
type hotPathMetrics struct {
|
|
// stageDuration is the per-stage duration histogram.
|
|
stageDuration *prometheus.HistogramVec
|
|
|
|
// terminalCounter is the per-terminal disposition counter.
|
|
terminalCounter *prometheus.CounterVec
|
|
|
|
// usageCounter is the per-token-type usage counter.
|
|
usageCounter *prometheus.CounterVec
|
|
|
|
// dispatchCounter is the per-mode dispatch counter.
|
|
dispatchCounter *prometheus.CounterVec
|
|
|
|
// cleanupCounter is the per-cleanup-outcome counter.
|
|
cleanupCounter *prometheus.CounterVec
|
|
|
|
// orphanCounter is the per-orphan-outcome counter.
|
|
orphanCounter *prometheus.CounterVec
|
|
|
|
// observerFailures is the per-observer-failure counter.
|
|
observerFailures *prometheus.CounterVec
|
|
|
|
mu sync.Mutex
|
|
}
|
|
|
|
var hotPathMetricsOnce sync.Once
|
|
var hotPathMetricsInstance *hotPathMetrics
|
|
|
|
func initHotPathMetrics() *hotPathMetrics {
|
|
hotPathMetricsOnce.Do(func() {
|
|
hotPathMetricsInstance = &hotPathMetrics{
|
|
stageDuration: promauto.NewHistogramVec(prometheus.HistogramOpts{
|
|
Name: "iop_hot_path_stage_duration_seconds",
|
|
Help: "Hot Path stage duration by stage kind and duration bucket.",
|
|
Buckets: prometheus.DefBuckets,
|
|
}, []string{"edge_id", "hot_path_mode", "hot_path_stage_kind", "hot_path_attempt_bucket", "hot_path_duration_bucket"}),
|
|
|
|
terminalCounter: promauto.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "iop_hot_path_terminal_total",
|
|
Help: "Hot Path terminal events by disposition.",
|
|
}, []string{"edge_id", "hot_path_mode", "hot_path_disposition"}),
|
|
|
|
usageCounter: promauto.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "iop_hot_path_usage_tokens_total",
|
|
Help: "Hot Path provider-reported token usage by token type.",
|
|
}, []string{"edge_id", "hot_path_mode", "hot_path_usage_bucket"}),
|
|
|
|
dispatchCounter: promauto.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "iop_hot_path_dispatch_total",
|
|
Help: "Hot Path dispatch events by mode and route reason.",
|
|
}, []string{"edge_id", "hot_path_mode", "hot_path_reason"}),
|
|
|
|
cleanupCounter: promauto.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "iop_hot_path_cleanup_total",
|
|
Help: "Hot Path cleanup events by outcome.",
|
|
}, []string{"edge_id", "hot_path_cleanup_outcome"}),
|
|
|
|
orphanCounter: promauto.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "iop_hot_path_orphan_total",
|
|
Help: "Hot Path orphan events by outcome.",
|
|
}, []string{"edge_id", "hot_path_orphan_outcome"}),
|
|
|
|
observerFailures: promauto.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "iop_hot_path_observer_failures_total",
|
|
Help: "Hot Path observer emission failures, isolated from request results.",
|
|
}, []string{"edge_id"}),
|
|
}
|
|
})
|
|
return hotPathMetricsInstance
|
|
}
|
|
|
|
func hotPathNormalizeEdgeID(raw string) string {
|
|
if raw == "" {
|
|
return "edge-local"
|
|
}
|
|
if containsSecretSentinel(raw) {
|
|
return "edge-local"
|
|
}
|
|
if len(raw) > 64 {
|
|
return raw[:64]
|
|
}
|
|
return raw
|
|
}
|
|
|
|
// hotPathRecordStageDuration records a stage duration in the bounded histogram.
|
|
func (m *hotPathMetrics) recordStageDuration(edgeID string, mode hotPathMode, stageKind hotPathStageKind, attempt hotPathAttemptBucket, durationSeconds float64) {
|
|
mode = hotPathNormalizeMode(string(mode))
|
|
stageKind = hotPathNormalizeStageKind(string(stageKind))
|
|
attempt = hotPathNormalizeAttemptBucket(string(attempt))
|
|
bucket := hotPathDurationBucketFromSeconds(durationSeconds)
|
|
if m == nil || mode == "" || stageKind == "" || attempt == "" || bucket == "" {
|
|
return
|
|
}
|
|
edgeID = hotPathNormalizeEdgeID(edgeID)
|
|
m.stageDuration.WithLabelValues(
|
|
edgeID,
|
|
string(mode),
|
|
string(stageKind),
|
|
string(attempt),
|
|
string(bucket),
|
|
).Observe(durationSeconds)
|
|
}
|
|
|
|
// hotPathRecordTerminal records a terminal disposition event in the bounded counter.
|
|
func (m *hotPathMetrics) recordTerminal(edgeID string, mode hotPathMode, disposition hotPathTerminalDispositionKind) {
|
|
mode = hotPathNormalizeMode(string(mode))
|
|
disposition = hotPathNormalizeDisposition(string(disposition))
|
|
if m == nil || mode == "" || disposition == "" {
|
|
return
|
|
}
|
|
edgeID = hotPathNormalizeEdgeID(edgeID)
|
|
m.terminalCounter.WithLabelValues(
|
|
edgeID,
|
|
string(mode),
|
|
string(disposition),
|
|
).Inc()
|
|
}
|
|
|
|
// hotPathRecordUsage records a token usage count in the bounded counter.
|
|
func (m *hotPathMetrics) recordUsage(edgeID string, mode hotPathMode, usageBucket hotPathUsageBucket, count int64) {
|
|
mode = hotPathNormalizeMode(string(mode))
|
|
usageBucket = hotPathNormalizeUsageBucket(string(usageBucket))
|
|
if m == nil || count <= 0 || mode == "" || usageBucket == "" {
|
|
return
|
|
}
|
|
edgeID = hotPathNormalizeEdgeID(edgeID)
|
|
m.usageCounter.WithLabelValues(
|
|
edgeID,
|
|
string(mode),
|
|
string(usageBucket),
|
|
).Add(float64(count))
|
|
}
|
|
|
|
// hotPathRecordDispatch records a dispatch event in the bounded counter.
|
|
func (m *hotPathMetrics) recordDispatch(edgeID string, mode hotPathMode, reason hotPathRouteReason) {
|
|
mode = hotPathNormalizeMode(string(mode))
|
|
reason = hotPathNormalizeRouteReason(string(reason))
|
|
if m == nil || mode == "" || reason == "" {
|
|
return
|
|
}
|
|
edgeID = hotPathNormalizeEdgeID(edgeID)
|
|
m.dispatchCounter.WithLabelValues(
|
|
edgeID,
|
|
string(mode),
|
|
string(reason),
|
|
).Inc()
|
|
}
|
|
|
|
// hotPathRecordCleanup records a cleanup event in the bounded counter.
|
|
func (m *hotPathMetrics) recordCleanup(edgeID string, outcome hotPathCleanupOutcome) {
|
|
outcome = hotPathNormalizeCleanupOutcome(string(outcome))
|
|
if m == nil || outcome == "" {
|
|
return
|
|
}
|
|
edgeID = hotPathNormalizeEdgeID(edgeID)
|
|
m.cleanupCounter.WithLabelValues(
|
|
edgeID,
|
|
string(outcome),
|
|
).Inc()
|
|
}
|
|
|
|
// hotPathRecordOrphan records an orphan event in the bounded counter.
|
|
func (m *hotPathMetrics) recordOrphan(edgeID string, outcome hotPathOrphanOutcome) {
|
|
outcome = hotPathNormalizeOrphanOutcome(string(outcome))
|
|
if m == nil || outcome == "" {
|
|
return
|
|
}
|
|
edgeID = hotPathNormalizeEdgeID(edgeID)
|
|
m.orphanCounter.WithLabelValues(
|
|
edgeID,
|
|
string(outcome),
|
|
).Inc()
|
|
}
|
|
|
|
// hotPathRecordObserverFailure records an observer failure in the bounded counter.
|
|
func (m *hotPathMetrics) recordObserverFailure(edgeID string) {
|
|
if m == nil {
|
|
return
|
|
}
|
|
edgeID = hotPathNormalizeEdgeID(edgeID)
|
|
m.observerFailures.WithLabelValues(
|
|
edgeID,
|
|
).Inc()
|
|
}
|
|
|
|
// hotPathDurationBucketFromSeconds converts a raw duration in seconds to the
|
|
// closed duration bucket.
|
|
func hotPathDurationBucketFromSeconds(seconds float64) hotPathDurationBucket {
|
|
switch {
|
|
case seconds < 0.001:
|
|
return hotPathDurationSubMS
|
|
case seconds < 0.01:
|
|
return hotPathDuration1to10MS
|
|
case seconds < 0.1:
|
|
return hotPathDuration10to100MS
|
|
case seconds < 1.0:
|
|
return hotPathDuration100to1S
|
|
case seconds < 10.0:
|
|
return hotPathDuration1to10S
|
|
case seconds < 60.0:
|
|
return hotPathDuration10to60S
|
|
default:
|
|
return hotPathDurationOver60S
|
|
}
|
|
}
|
|
|
|
// hotPathMetricLabelCardinalityTotal returns the sum of max metric vector time series.
|
|
func hotPathMetricLabelCardinalityTotal() int {
|
|
stageDur := 64 * 2 * 4 * 2 * 7
|
|
term := 64 * 2 * 7
|
|
usage := 64 * 2 * 4
|
|
disp := 64 * 2 * 6
|
|
clean := 64 * 3
|
|
orph := 64 * 2
|
|
fail := 64
|
|
return stageDur + term + usage + disp + clean + orph + fail
|
|
}
|
|
|
|
// hotPathMetricLabelCardinalityBudget is the maximum allowed product of all
|
|
// per-label cardinalities. It is exported so tests can assert against it
|
|
// directly.
|
|
const hotPathMetricLabelCardinalityBudget = 1_000_000
|
|
|
|
// hotPathMetricLabelNamesSnapshot returns a copy of the fixed label names.
|
|
// Tests use this to assert the allowlist exactly.
|
|
func hotPathMetricLabelNamesSnapshot() []string {
|
|
out := make([]string, len(hotPathMetricLabelNames))
|
|
copy(out, hotPathMetricLabelNames)
|
|
return out
|
|
}
|
|
|
|
// hotPathMetricLabelCardinalitySnapshot returns a copy of the per-label
|
|
// cardinality map. Tests use this to assert the budget exactly.
|
|
func hotPathMetricLabelCardinalitySnapshot() map[string]int {
|
|
out := make(map[string]int, len(hotPathMetricLabelCardinality))
|
|
for k, v := range hotPathMetricLabelCardinality {
|
|
out[k] = v
|
|
}
|
|
return out
|
|
}
|
|
|
|
// hotPathMetricLabelCardinalityCheck validates the cardinality budget and
|
|
// returns an error if exceeded. It is exported for tests.
|
|
func hotPathMetricLabelCardinalityCheck() error {
|
|
total := hotPathMetricLabelCardinalityTotal()
|
|
if total > hotPathMetricLabelCardinalityBudget {
|
|
return fmt.Errorf("hot path metric label cardinality budget exceeded: %d > %d", total, hotPathMetricLabelCardinalityBudget)
|
|
}
|
|
return nil
|
|
}
|