toki
9b7b4e044f
- Move usage metrics aggregation to stream layer in edge openai handler - Refactor stream.go to relay usage metrics from node responses - Update grafana doc with new metric structure and dashboard notes - Remove completed G05 plan/code-review artifacts - Archive closed task artifacts
671 lines
29 KiB
Go
671 lines
29 KiB
Go
package openai
|
|
|
|
import (
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/prometheus/client_golang/prometheus/testutil"
|
|
|
|
"iop/packages/go/config"
|
|
iop "iop/proto/gen/iop"
|
|
)
|
|
|
|
// principalTokenCfg builds an EdgeOpenAIConf whose bearer token maps to a fixed
|
|
// principal so usage-metric label values are populated.
|
|
func principalTokenCfg(rawToken, adapter string) config.EdgeOpenAIConf {
|
|
return config.EdgeOpenAIConf{
|
|
Adapter: adapter,
|
|
PrincipalTokens: []config.OpenAIPrincipalTokenConf{
|
|
{TokenRef: "iop-tok-alice", TokenHashSHA256: sha256Hex(rawToken), PrincipalRef: "user:alice", PrincipalAlias: "alice"},
|
|
},
|
|
}
|
|
}
|
|
|
|
func providerRouteUsageMetricServer(rawToken, edgeID, model, target string, fake *fakeRunService) *Server {
|
|
conf := principalTokenCfg(rawToken, "ollama")
|
|
conf.ModelRoutes = []config.OpenAIRouteEntry{
|
|
{Model: model, Adapter: "openai_compat", Target: target},
|
|
}
|
|
srv := NewServer(conf, fake, nil)
|
|
srv.SetEdgeID(edgeID)
|
|
return srv
|
|
}
|
|
|
|
func requestTokenValue(t *testing.T, l usageLabels, tokenType string) float64 {
|
|
t.Helper()
|
|
return testutil.ToFloat64(openAIUsageTokensTotal.WithLabelValues(
|
|
l.edgeID, l.principalRef, l.principalAlias, l.tokenRef,
|
|
l.modelGroup, l.endpoint, l.responseMode, tokenType,
|
|
))
|
|
}
|
|
|
|
func requestReasoningEstimateValue(t *testing.T, l usageLabels, method string) float64 {
|
|
t.Helper()
|
|
return testutil.ToFloat64(openAIReasoningEstimatedTokensTotal.WithLabelValues(
|
|
l.edgeID, l.principalRef, l.principalAlias, l.tokenRef,
|
|
l.modelGroup, l.endpoint, l.responseMode, method,
|
|
))
|
|
}
|
|
|
|
// TestOpenAIUsageMetricsLabelsExcludeSecretsAndHighCardinality asserts the label
|
|
// allowlists never carry raw tokens, prompt/response text, or per-request
|
|
// high-cardinality identifiers (SDD S08).
|
|
func TestOpenAIUsageMetricsLabelsExcludeSecretsAndHighCardinality(t *testing.T) {
|
|
// Substrings that would indicate a per-request identifier, a raw secret, or
|
|
// raw payload text. response_mode / model_group are low-cardinality enums and
|
|
// are intentionally not caught here.
|
|
forbidden := []string{
|
|
"request_id", "session_id", "run_id", "token_hash", "bearer",
|
|
"authorization", "api_key", "prompt", "secret", "raw",
|
|
}
|
|
// token_ref is an allowlisted stable alias, not a raw token; guard against a
|
|
// bare "token" label that could carry the secret itself.
|
|
for _, set := range [][]string{
|
|
usageRequestLabelNames, usageTokenLabelNames,
|
|
usageReasoningLabelNames, usageReasoningEstimateLabelNames,
|
|
} {
|
|
for _, name := range set {
|
|
if name == "token" {
|
|
t.Fatalf("label %q may carry a raw token value; use token_ref", name)
|
|
}
|
|
for _, bad := range forbidden {
|
|
if strings.Contains(name, bad) {
|
|
t.Fatalf("label %q contains forbidden fragment %q", name, bad)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
// The allowlist must still carry the intended identity/attribution labels.
|
|
for _, want := range []string{"edge_id", "principal_ref", "principal_alias", "token_ref", "model_group", "endpoint", "response_mode"} {
|
|
if !containsString(usageRequestLabelNames, want) {
|
|
t.Fatalf("request labels missing %q", want)
|
|
}
|
|
}
|
|
if !containsString(usageRequestLabelNames, "usage_source") || !containsString(usageRequestLabelNames, "status") {
|
|
t.Fatalf("request labels must include status and usage_source: %v", usageRequestLabelNames)
|
|
}
|
|
if !containsString(usageTokenLabelNames, "token_type") {
|
|
t.Fatalf("token labels must include token_type: %v", usageTokenLabelNames)
|
|
}
|
|
// estimated reasoning labels: only estimation_method on top of the base set.
|
|
if !containsString(usageReasoningEstimateLabelNames, "estimation_method") {
|
|
t.Fatalf("reasoning estimate labels must include estimation_method: %v", usageReasoningEstimateLabelNames)
|
|
}
|
|
for _, want := range []string{"edge_id", "principal_ref", "principal_alias", "token_ref", "model_group", "endpoint", "response_mode"} {
|
|
if !containsString(usageReasoningEstimateLabelNames, want) {
|
|
t.Fatalf("reasoning estimate labels missing %q", want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestOpenAIUsageMetricsCountTokenTypes drives a normalized chat completion whose
|
|
// run reports input/output/reasoning/cached usage and asserts each token type is
|
|
// counted under a provider_reported request (SDD S06).
|
|
func TestOpenAIUsageMetricsCountTokenTypes(t *testing.T) {
|
|
const rawToken = "sk-count-raw-token"
|
|
const edgeID = "edge-count-token-types"
|
|
const model = "count-model"
|
|
|
|
fake := &fakeRunService{events: make(chan *iop.RunEvent, 2)}
|
|
fake.events <- &iop.RunEvent{Type: "delta", Delta: "hi"}
|
|
fake.events <- &iop.RunEvent{Type: "complete", Usage: &iop.Usage{
|
|
InputTokens: 10, OutputTokens: 5, ReasoningTokens: 3, CachedInputTokens: 2,
|
|
}}
|
|
|
|
srv := NewServer(principalTokenCfg(rawToken, "ollama"), fake, nil)
|
|
srv.SetEdgeID(edgeID)
|
|
|
|
labels := usageLabels{
|
|
edgeID: edgeID, principalRef: "user:alice", principalAlias: "alice", tokenRef: "iop-tok-alice",
|
|
modelGroup: model, endpoint: usageEndpointChatCompletions, responseMode: "normalized",
|
|
}
|
|
before := map[string]float64{
|
|
tokenTypeInput: requestTokenValue(t, labels, tokenTypeInput),
|
|
tokenTypeOutput: requestTokenValue(t, labels, tokenTypeOutput),
|
|
tokenTypeReasoning: requestTokenValue(t, labels, tokenTypeReasoning),
|
|
tokenTypeCachedInput: requestTokenValue(t, labels, tokenTypeCachedInput),
|
|
}
|
|
reqBefore := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues(
|
|
edgeID, "user:alice", "alice", "iop-tok-alice", model,
|
|
usageEndpointChatCompletions, "normalized", usageStatusSuccess, usageSourceProviderReported,
|
|
))
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"count-model",
|
|
"messages":[{"role":"user","content":"hi"}]
|
|
}`))
|
|
req.Header.Set("Authorization", "Bearer "+rawToken)
|
|
w := httptest.NewRecorder()
|
|
srv.routes().ServeHTTP(w, req)
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
|
|
for tokenType, want := range map[string]float64{
|
|
tokenTypeInput: 10, tokenTypeOutput: 5, tokenTypeReasoning: 3, tokenTypeCachedInput: 2,
|
|
} {
|
|
got := requestTokenValue(t, labels, tokenType) - before[tokenType]
|
|
if got != want {
|
|
t.Fatalf("token_type %s: got delta %v, want %v", tokenType, got, want)
|
|
}
|
|
}
|
|
reqAfter := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues(
|
|
edgeID, "user:alice", "alice", "iop-tok-alice", model,
|
|
usageEndpointChatCompletions, "normalized", usageStatusSuccess, usageSourceProviderReported,
|
|
))
|
|
if reqAfter-reqBefore != 1 {
|
|
t.Fatalf("requests_total success/provider_reported: got delta %v, want 1", reqAfter-reqBefore)
|
|
}
|
|
}
|
|
|
|
// TestOpenAIReasoningObservedEstimatesTokens verifies that reasoning text
|
|
// observed without a provider-reported reasoning token count increments the
|
|
// auxiliary reasoning counters and a separate estimated-token counter based on
|
|
// chars/4 rounding up, while leaving the provider-reported reasoning token
|
|
// counter untouched (SDD S07).
|
|
func TestOpenAIReasoningObservedEstimatesTokens(t *testing.T) {
|
|
const rawToken = "sk-reasoning-raw-token"
|
|
const edgeID = "edge-reasoning-observed"
|
|
const model = "reasoning-model"
|
|
const reasoning = "abcde" // 5 chars
|
|
|
|
fake := &fakeRunService{events: make(chan *iop.RunEvent, 3)}
|
|
fake.events <- &iop.RunEvent{Type: "reasoning_delta", Delta: reasoning}
|
|
fake.events <- &iop.RunEvent{Type: "delta", Delta: "answer"}
|
|
fake.events <- &iop.RunEvent{Type: "complete", Usage: &iop.Usage{InputTokens: 8, OutputTokens: 4}}
|
|
|
|
srv := NewServer(principalTokenCfg(rawToken, "ollama"), fake, nil)
|
|
srv.SetEdgeID(edgeID)
|
|
|
|
labels := usageLabels{
|
|
edgeID: edgeID, principalRef: "user:alice", principalAlias: "alice", tokenRef: "iop-tok-alice",
|
|
modelGroup: model, endpoint: usageEndpointChatCompletions, responseMode: "normalized",
|
|
}
|
|
reasoningBefore := requestTokenValue(t, labels, tokenTypeReasoning)
|
|
estimatedBefore := requestReasoningEstimateValue(t, labels, "chars_div_4")
|
|
observedBefore := testutil.ToFloat64(openAIReasoningObservedTotal.WithLabelValues(labels.reasoningValues()...))
|
|
charsBefore := testutil.ToFloat64(openAIReasoningCharsTotal.WithLabelValues(labels.reasoningValues()...))
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"reasoning-model",
|
|
"messages":[{"role":"user","content":"hi"}]
|
|
}`))
|
|
req.Header.Set("Authorization", "Bearer "+rawToken)
|
|
w := httptest.NewRecorder()
|
|
srv.routes().ServeHTTP(w, req)
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
|
|
if got := requestTokenValue(t, labels, tokenTypeReasoning) - reasoningBefore; got != 0 {
|
|
t.Fatalf("provider-reported reasoning token counter must not move for estimated reasoning: got delta %v", got)
|
|
}
|
|
if got := requestReasoningEstimateValue(t, labels, "chars_div_4") - estimatedBefore; got != 2 {
|
|
t.Fatalf("reasoning estimated tokens: got delta %v, want 2", got)
|
|
}
|
|
if got := testutil.ToFloat64(openAIReasoningObservedTotal.WithLabelValues(labels.reasoningValues()...)) - observedBefore; got != 1 {
|
|
t.Fatalf("reasoning_observed_total: got delta %v, want 1", got)
|
|
}
|
|
if got := testutil.ToFloat64(openAIReasoningCharsTotal.WithLabelValues(labels.reasoningValues()...)) - charsBefore; got != float64(len(reasoning)) {
|
|
t.Fatalf("reasoning_chars_total: got delta %v, want %d", got, len(reasoning))
|
|
}
|
|
}
|
|
|
|
// TestOpenAIReasoningProviderReportedDoesNotEstimate verifies that when the
|
|
// provider reports reasoningTokens > 0 alongside observed reasoning text, the
|
|
// estimated-token counter stays untouched and the provider-reported reasoning
|
|
// counter advances by the reported count (SDD S07).
|
|
func TestOpenAIReasoningProviderReportedDoesNotEstimate(t *testing.T) {
|
|
const rawToken = "sk-reasoning-provider-token"
|
|
const edgeID = "edge-reasoning-provider"
|
|
const model = "reasoning-model"
|
|
const reasoning = "abcdef" // 6 chars; irrelevant for the no-estimate path
|
|
|
|
fake := &fakeRunService{events: make(chan *iop.RunEvent, 3)}
|
|
fake.events <- &iop.RunEvent{Type: "reasoning_delta", Delta: reasoning}
|
|
fake.events <- &iop.RunEvent{Type: "delta", Delta: "answer"}
|
|
fake.events <- &iop.RunEvent{Type: "complete", Usage: &iop.Usage{
|
|
InputTokens: 8, OutputTokens: 4, ReasoningTokens: 3,
|
|
}}
|
|
|
|
srv := NewServer(principalTokenCfg(rawToken, "ollama"), fake, nil)
|
|
srv.SetEdgeID(edgeID)
|
|
|
|
labels := usageLabels{
|
|
edgeID: edgeID, principalRef: "user:alice", principalAlias: "alice", tokenRef: "iop-tok-alice",
|
|
modelGroup: model, endpoint: usageEndpointChatCompletions, responseMode: "normalized",
|
|
}
|
|
reasoningBefore := requestTokenValue(t, labels, tokenTypeReasoning)
|
|
estimatedBefore := requestReasoningEstimateValue(t, labels, "chars_div_4")
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"reasoning-model",
|
|
"messages":[{"role":"user","content":"hi"}]
|
|
}`))
|
|
req.Header.Set("Authorization", "Bearer "+rawToken)
|
|
w := httptest.NewRecorder()
|
|
srv.routes().ServeHTTP(w, req)
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
|
|
if got := requestTokenValue(t, labels, tokenTypeReasoning) - reasoningBefore; got != 3 {
|
|
t.Fatalf("provider-reported reasoning token: got delta %v, want 3", got)
|
|
}
|
|
if got := requestReasoningEstimateValue(t, labels, "chars_div_4") - estimatedBefore; got != 0 {
|
|
t.Fatalf("estimated reasoning must be zero when provider reports reasoning tokens: got delta %v", got)
|
|
}
|
|
}
|
|
|
|
// TestProviderTunnelPassthroughPreservesBodyAndObservesUsage verifies that a pure
|
|
// passthrough provider response is relayed byte-for-byte while its usage is
|
|
// observed into metrics (SDD S05/S06).
|
|
func TestProviderTunnelPassthroughPreservesBodyAndObservesUsage(t *testing.T) {
|
|
const rawToken = "sk-passthrough-raw-token"
|
|
const edgeID = "edge-passthrough-usage"
|
|
const model = "ornith:35b"
|
|
providerBody := `{"choices":[{"message":{"role":"assistant","content":"hi there"}}],"usage":{"prompt_tokens":12,"completion_tokens":7,"prompt_tokens_details":{"cached_tokens":3},"completion_tokens_details":{"reasoning_tokens":4}}}`
|
|
|
|
fake := &fakeRunService{tunnelFrames: staticProviderTunnelFrames(providerBody)}
|
|
srv := NewServer(config.EdgeOpenAIConf{
|
|
PrincipalTokens: []config.OpenAIPrincipalTokenConf{
|
|
{TokenRef: "iop-tok-alice", TokenHashSHA256: sha256Hex(rawToken), PrincipalRef: "user:alice", PrincipalAlias: "alice"},
|
|
},
|
|
}, fake, nil)
|
|
srv.SetEdgeID(edgeID)
|
|
srv.SetModelCatalog([]config.ModelCatalogEntry{{ID: model, Providers: map[string]string{"prov-vllm": "Ornith-1.0-35B"}}})
|
|
|
|
labels := usageLabels{
|
|
edgeID: edgeID, principalRef: "user:alice", principalAlias: "alice", tokenRef: "iop-tok-alice",
|
|
modelGroup: model, endpoint: usageEndpointChatCompletions, responseMode: responseModePassthrough,
|
|
}
|
|
before := map[string]float64{
|
|
tokenTypeInput: requestTokenValue(t, labels, tokenTypeInput),
|
|
tokenTypeOutput: requestTokenValue(t, labels, tokenTypeOutput),
|
|
tokenTypeReasoning: requestTokenValue(t, labels, tokenTypeReasoning),
|
|
tokenTypeCachedInput: requestTokenValue(t, labels, tokenTypeCachedInput),
|
|
}
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"ornith:35b",
|
|
"messages":[{"role":"user","content":"hello"}]
|
|
}`))
|
|
req.Header.Set("Authorization", "Bearer "+rawToken)
|
|
w := httptest.NewRecorder()
|
|
srv.routes().ServeHTTP(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
if w.Body.String() != providerBody {
|
|
t.Fatalf("passthrough body must be preserved byte-for-byte.\n got: %s\nwant: %s", w.Body.String(), providerBody)
|
|
}
|
|
if len(fake.tunnelReqsSnapshot()) != 1 {
|
|
t.Fatalf("expected exactly one provider tunnel dispatch, got %d", len(fake.tunnelReqsSnapshot()))
|
|
}
|
|
for tokenType, want := range map[string]float64{
|
|
tokenTypeInput: 12, tokenTypeOutput: 7, tokenTypeReasoning: 4, tokenTypeCachedInput: 3,
|
|
} {
|
|
got := requestTokenValue(t, labels, tokenType) - before[tokenType]
|
|
if got != want {
|
|
t.Fatalf("passthrough token_type %s: got delta %v, want %v", tokenType, got, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestResponsesProviderTunnelPassthroughObservesUsageMetrics verifies that a
|
|
// successful /v1/responses provider passthrough attributes its usage metrics to
|
|
// endpoint=responses, response_mode=passthrough, and model_group=request alias,
|
|
// while the response body stays provider-original. It is a regression test for
|
|
// the shared tunnel writer previously recomputing chat.completions labels with an
|
|
// empty model_group for the Responses path (REVIEW_SEULGI_RESPONSES-1).
|
|
func TestResponsesProviderTunnelPassthroughObservesUsageMetrics(t *testing.T) {
|
|
const rawToken = "sk-responses-passthrough-token"
|
|
const edgeID = "edge-responses-passthrough-usage"
|
|
const model = "pool-model"
|
|
// Body-only Responses JSON: no separate USAGE frame. This matches how the
|
|
// real Node tunnel relay returns usage as part of the body JSON.
|
|
providerBody := `{"id":"resp-1","object":"response","output_text":"hi there","usage":{"input_tokens":9,"output_tokens":6,"input_tokens_details":{"cached_tokens":3},"output_tokens_details":{"reasoning_tokens":4}}}`
|
|
|
|
frames := make(chan *iop.ProviderTunnelFrame, 5)
|
|
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START, StatusCode: 200}
|
|
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(providerBody)}
|
|
// NOTE: No proto USAGE frame — usage is only in the body JSON.
|
|
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true}
|
|
close(frames)
|
|
|
|
srv := NewServer(config.EdgeOpenAIConf{
|
|
PrincipalTokens: []config.OpenAIPrincipalTokenConf{
|
|
{TokenRef: "iop-tok-alice", TokenHashSHA256: sha256Hex(rawToken), PrincipalRef: "user:alice", PrincipalAlias: "alice"},
|
|
},
|
|
}, &fakeRunService{tunnelFrames: frames, tunnelServedTarget: "served-model"}, nil)
|
|
srv.SetEdgeID(edgeID)
|
|
srv.SetModelCatalog([]config.ModelCatalogEntry{{ID: model, Providers: map[string]string{"prov-1": "served-model"}}})
|
|
|
|
labels := usageLabels{
|
|
edgeID: edgeID, principalRef: "user:alice", principalAlias: "alice", tokenRef: "iop-tok-alice",
|
|
modelGroup: model, endpoint: usageEndpointResponses, responseMode: responseModePassthrough,
|
|
}
|
|
reqBefore := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues(
|
|
edgeID, "user:alice", "alice", "iop-tok-alice", model,
|
|
usageEndpointResponses, responseModePassthrough, usageStatusSuccess, usageSourceProviderReported,
|
|
))
|
|
// The pre-fix bug recorded success under endpoint=chat.completions with an
|
|
// empty model_group; assert that mislabeled counter does not move.
|
|
wrongBefore := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues(
|
|
edgeID, "user:alice", "alice", "iop-tok-alice", "",
|
|
usageEndpointChatCompletions, responseModePassthrough, usageStatusSuccess, usageSourceProviderReported,
|
|
))
|
|
before := map[string]float64{
|
|
tokenTypeInput: requestTokenValue(t, labels, tokenTypeInput),
|
|
tokenTypeOutput: requestTokenValue(t, labels, tokenTypeOutput),
|
|
tokenTypeReasoning: requestTokenValue(t, labels, tokenTypeReasoning),
|
|
tokenTypeCachedInput: requestTokenValue(t, labels, tokenTypeCachedInput),
|
|
}
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(`{
|
|
"model":"pool-model",
|
|
"input":"hello"
|
|
}`))
|
|
req.Header.Set("Authorization", "Bearer "+rawToken)
|
|
w := httptest.NewRecorder()
|
|
srv.routes().ServeHTTP(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
if w.Body.String() != providerBody {
|
|
t.Fatalf("responses passthrough body must be preserved byte-for-byte.\n got: %s\nwant: %s", w.Body.String(), providerBody)
|
|
}
|
|
|
|
reqAfter := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues(
|
|
edgeID, "user:alice", "alice", "iop-tok-alice", model,
|
|
usageEndpointResponses, responseModePassthrough, usageStatusSuccess, usageSourceProviderReported,
|
|
))
|
|
if reqAfter-reqBefore != 1 {
|
|
t.Fatalf("requests_total responses/success: got delta %v, want 1", reqAfter-reqBefore)
|
|
}
|
|
wrongAfter := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues(
|
|
edgeID, "user:alice", "alice", "iop-tok-alice", "",
|
|
usageEndpointChatCompletions, responseModePassthrough, usageStatusSuccess, usageSourceProviderReported,
|
|
))
|
|
if wrongAfter-wrongBefore != 0 {
|
|
t.Fatalf("mislabeled chat.completions/empty-model counter must not move: got delta %v, want 0", wrongAfter-wrongBefore)
|
|
}
|
|
// Body-only Responses usage (no proto USAGE frame): input, output, reasoning,
|
|
// cached_input all come from the body JSON (REVIEW_REVIEW_SEULGI_RESPONSES-1).
|
|
for tokenType, want := range map[string]float64{
|
|
tokenTypeInput: 9,
|
|
tokenTypeOutput: 6,
|
|
tokenTypeReasoning: 4,
|
|
tokenTypeCachedInput: 3,
|
|
} {
|
|
got := requestTokenValue(t, labels, tokenType) - before[tokenType]
|
|
if got != want {
|
|
t.Fatalf("responses token_type %s: got delta %v, want %v", tokenType, got, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestResponsesProviderTunnelPassthroughStreamingObservesUsageMetrics(t *testing.T) {
|
|
const rawToken = "sk-responses-streaming-token"
|
|
const edgeID = "edge-responses-streaming-usage"
|
|
const model = "pool-model"
|
|
// Responses streaming SSE: delta event followed by a usage-bearing event
|
|
// (response.completed with nested usage), matching the real provider tunnel.
|
|
providerBody := `data: {"type":"response.created","response":{"id":"resp-1"}}
|
|
|
|
data: {"type":"response.output_item.added","item":{"id":"msg-1","type":"message","role":"assistant"}}
|
|
|
|
data: {"type":"response.message.delta","delta":"hello world"}
|
|
|
|
data: {"type":"response.message.completed","item":{"id":"msg-1","role":"assistant","content":[{"type":"output_text","text":"hello world"}],"status":"completed"}}
|
|
|
|
data: {"type":"response.completed","response":{"id":"resp-1","usage":{"input_tokens":15,"output_tokens":10,"input_tokens_details":{"cached_tokens":5},"output_tokens_details":{"reasoning_tokens":2}}}}
|
|
|
|
data: [DONE]
|
|
|
|
`
|
|
|
|
frames := make(chan *iop.ProviderTunnelFrame, 5)
|
|
frames <- &iop.ProviderTunnelFrame{
|
|
Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_RESPONSE_START,
|
|
StatusCode: 200,
|
|
Headers: map[string]string{"Content-Type": "text/event-stream"},
|
|
}
|
|
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_BODY, Body: []byte(providerBody)}
|
|
// NOTE: No proto USAGE frame — usage is only in the SSE body.
|
|
frames <- &iop.ProviderTunnelFrame{Kind: iop.ProviderTunnelFrameKind_PROVIDER_TUNNEL_FRAME_KIND_END, End: true}
|
|
close(frames)
|
|
|
|
srv := NewServer(config.EdgeOpenAIConf{
|
|
PrincipalTokens: []config.OpenAIPrincipalTokenConf{
|
|
{TokenRef: "iop-tok-alice", TokenHashSHA256: sha256Hex(rawToken), PrincipalRef: "user:alice", PrincipalAlias: "alice"},
|
|
},
|
|
}, &fakeRunService{tunnelFrames: frames, tunnelServedTarget: "served-model"}, nil)
|
|
srv.SetEdgeID(edgeID)
|
|
srv.SetModelCatalog([]config.ModelCatalogEntry{{ID: model, Providers: map[string]string{"prov-1": "served-model"}}})
|
|
|
|
labels := usageLabels{
|
|
edgeID: edgeID, principalRef: "user:alice", principalAlias: "alice", tokenRef: "iop-tok-alice",
|
|
modelGroup: model, endpoint: usageEndpointResponses, responseMode: responseModePassthrough,
|
|
}
|
|
reqBefore := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues(
|
|
edgeID, "user:alice", "alice", "iop-tok-alice", model,
|
|
usageEndpointResponses, responseModePassthrough, usageStatusSuccess, usageSourceProviderReported,
|
|
))
|
|
before := map[string]float64{
|
|
tokenTypeInput: requestTokenValue(t, labels, tokenTypeInput),
|
|
tokenTypeOutput: requestTokenValue(t, labels, tokenTypeOutput),
|
|
tokenTypeReasoning: requestTokenValue(t, labels, tokenTypeReasoning),
|
|
tokenTypeCachedInput: requestTokenValue(t, labels, tokenTypeCachedInput),
|
|
}
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(`{
|
|
"model":"pool-model",
|
|
"input":"hello",
|
|
"stream":true
|
|
}`))
|
|
req.Header.Set("Authorization", "Bearer "+rawToken)
|
|
w := httptest.NewRecorder()
|
|
srv.routes().ServeHTTP(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("status: got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
// Raw SSE body must be preserved byte-for-byte: no model echo rewrite on
|
|
// passthrough because requestModel is empty (Responses passthrough prefers
|
|
// provider-original bytes). Streaming body is written directly to the
|
|
// response writer without modification.
|
|
if w.Body.String() != providerBody {
|
|
t.Fatalf("streaming SSE body must be preserved byte-for-byte.\n got: %s\nwant: %s", w.Body.String(), providerBody)
|
|
}
|
|
|
|
reqAfter := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues(
|
|
edgeID, "user:alice", "alice", "iop-tok-alice", model,
|
|
usageEndpointResponses, responseModePassthrough, usageStatusSuccess, usageSourceProviderReported,
|
|
))
|
|
if reqAfter-reqBefore != 1 {
|
|
t.Fatalf("requests_total responses/success: got delta %v, want 1", reqAfter-reqBefore)
|
|
}
|
|
// Body-only Responses streaming usage: all token types from SSE body.
|
|
for tokenType, want := range map[string]float64{
|
|
tokenTypeInput: 15,
|
|
tokenTypeOutput: 10,
|
|
tokenTypeReasoning: 2,
|
|
tokenTypeCachedInput: 5,
|
|
} {
|
|
got := requestTokenValue(t, labels, tokenType) - before[tokenType]
|
|
if got != want {
|
|
t.Fatalf("responses streaming token_type %s: got delta %v, want %v", tokenType, got, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestOpenAIScopeExcludesA2AAndNonOpenAISurfaces documents that usage metering is
|
|
// scoped to the OpenAI-compatible input surface only (SDD S04). The endpoint
|
|
// label allowlist carries the OpenAI routes and no A2A/CLI endpoint value, and
|
|
// this plan does not modify the A2A surface.
|
|
func TestOpenAIScopeExcludesA2AAndNonOpenAISurfaces(t *testing.T) {
|
|
endpoints := map[string]struct{}{
|
|
usageEndpointChatCompletions: {},
|
|
usageEndpointResponses: {},
|
|
}
|
|
for _, forbidden := range []string{"a2a", "cli", "ollama", "message/send", "tasks"} {
|
|
if _, ok := endpoints[forbidden]; ok {
|
|
t.Fatalf("usage endpoint label must not include non-OpenAI surface %q", forbidden)
|
|
}
|
|
}
|
|
if !containsString(usageTokenLabelNames, "endpoint") {
|
|
t.Fatalf("token metrics must carry an endpoint label to keep OpenAI scope explicit")
|
|
}
|
|
}
|
|
|
|
func TestChatCompletionsDispatchFailureEmitsErrorMetric(t *testing.T) {
|
|
const rawToken = "sk-dispatch-fail-token"
|
|
const edgeID = "edge-dispatch-fail"
|
|
const model = "pool-model"
|
|
|
|
fake := &fakeRunService{tunnelErr: fmt.Errorf("node unavailable")}
|
|
srv := NewServer(principalTokenCfg(rawToken, "ollama"), fake, nil)
|
|
srv.SetEdgeID(edgeID)
|
|
srv.SetModelCatalog([]config.ModelCatalogEntry{{ID: model, Providers: map[string]string{"prov-1": "served-model"}}})
|
|
|
|
reqBefore := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues(
|
|
edgeID, "user:alice", "alice", "iop-tok-alice", model,
|
|
usageEndpointChatCompletions, responseModePassthrough, usageStatusError, usageSourceUnavailable,
|
|
))
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"pool-model",
|
|
"messages":[{"role":"user","content":"hello"}]
|
|
}`))
|
|
req.Header.Set("Authorization", "Bearer "+rawToken)
|
|
w := httptest.NewRecorder()
|
|
srv.routes().ServeHTTP(w, req)
|
|
|
|
if w.Code != http.StatusBadGateway {
|
|
t.Fatalf("expected 502, got %d body=%s", w.Code, w.Body.String())
|
|
}
|
|
|
|
if len(fake.tunnelReqsSnapshot()) == 0 {
|
|
t.Fatal("expected tunnel dispatch attempt, got none")
|
|
}
|
|
|
|
reqAfter := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues(
|
|
edgeID, "user:alice", "alice", "iop-tok-alice", model,
|
|
usageEndpointChatCompletions, responseModePassthrough, usageStatusError, usageSourceUnavailable,
|
|
))
|
|
if reqAfter-reqBefore != 1 {
|
|
t.Fatalf("requests_total error: got delta %v, want 1", reqAfter-reqBefore)
|
|
}
|
|
}
|
|
|
|
// TestToolValidationRetryDispatchFailureEmitsErrorMetric verifies that a
|
|
// non-stream Chat tool-validation retry SubmitRun failure emits the error
|
|
// request metric exactly once and returns tool_validation_retry_error (REVIEW_USAGE_METRIC_RETRY-1).
|
|
func TestToolValidationRetryDispatchFailureEmitsErrorMetric(t *testing.T) {
|
|
const rawToken = "sk-retry-dispatch-fail-token"
|
|
const edgeID = "edge-retry-dispatch-fail"
|
|
const model = "retry-model"
|
|
|
|
fake := &fakeRunService{
|
|
eventRuns: []chan *iop.RunEvent{
|
|
bufferedRunEvents(&iop.RunEvent{
|
|
Type: "complete",
|
|
Metadata: map[string]string{
|
|
"finish_reason": "tool_calls",
|
|
runtimeMetadataOpenAIToolCalls: `[{"id":"call_bad","type":"function","function":{"name":"cmd","arguments":"{\"unexpected\":true}"}}]`,
|
|
},
|
|
}),
|
|
},
|
|
runIDs: []string{"run-bad"},
|
|
submitErrAfter: 2,
|
|
}
|
|
srv := NewServer(principalTokenCfg(rawToken, "ollama"), fake, nil)
|
|
srv.SetEdgeID(edgeID)
|
|
|
|
reqBefore := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues(
|
|
edgeID, "user:alice", "alice", "iop-tok-alice", model,
|
|
usageEndpointChatCompletions, "normalized", usageStatusError, usageSourceUnavailable,
|
|
))
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"retry-model",
|
|
"messages":[{"role":"user","content":"hello"}],
|
|
"tools":[{"type":"function","function":{"name":"cmd","parameters":{"type":"object","properties":{"commands":{"type":"array","items":{"type":"string"}}},"required":["commands"]}}}]}
|
|
}`))
|
|
req.Header.Set("Authorization", "Bearer "+rawToken)
|
|
|
|
w := httptest.NewRecorder()
|
|
srv.routes().ServeHTTP(w, req)
|
|
|
|
if !strings.Contains(w.Body.String(), `"type":"tool_validation_retry_error"`) {
|
|
t.Fatalf("expected tool_validation_retry_error, got: %s", w.Body.String())
|
|
}
|
|
|
|
reqAfter := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues(
|
|
edgeID, "user:alice", "alice", "iop-tok-alice", model,
|
|
usageEndpointChatCompletions, "normalized", usageStatusError, usageSourceUnavailable,
|
|
))
|
|
if reqAfter-reqBefore != 1 {
|
|
t.Fatalf("requests_total error: got delta %v, want 1", reqAfter-reqBefore)
|
|
}
|
|
}
|
|
|
|
// TestBufferedStreamToolValidationRetryDispatchFailureEmitsErrorMetric verifies
|
|
// that a buffered stream Chat tool-validation retry SubmitRun failure emits
|
|
// the error request metric exactly once (REVIEW_USAGE_METRIC_RETRY-1).
|
|
func TestBufferedStreamToolValidationRetryDispatchFailureEmitsErrorMetric(t *testing.T) {
|
|
const rawToken = "sk-buf-retry-dispatch-fail-token"
|
|
const edgeID = "edge-buf-retry-dispatch"
|
|
const model = "buf-retry-model"
|
|
|
|
fake := &fakeRunService{
|
|
eventRuns: []chan *iop.RunEvent{
|
|
bufferedRunEvents(&iop.RunEvent{
|
|
Type: "complete",
|
|
Metadata: map[string]string{
|
|
"finish_reason": "tool_calls",
|
|
runtimeMetadataOpenAIToolCalls: `[{"id":"call_bad","type":"function","function":{"name":"cmd","arguments":"{\"unexpected\":true}"}}]`,
|
|
},
|
|
}),
|
|
},
|
|
runIDs: []string{"run-bad"},
|
|
submitErrAfter: 2,
|
|
}
|
|
srv := NewServer(principalTokenCfg(rawToken, ""), fake, nil)
|
|
srv.SetEdgeID(edgeID)
|
|
|
|
reqBefore := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues(
|
|
edgeID, "user:alice", "alice", "iop-tok-alice", model,
|
|
usageEndpointChatCompletions, "normalized", usageStatusError, usageSourceUnavailable,
|
|
))
|
|
|
|
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{
|
|
"model":"buf-retry-model",
|
|
"stream":true,
|
|
"messages":[{"role":"user","content":"hello"}],
|
|
"tools":[{"type":"function","function":{"name":"cmd","parameters":{"type":"object","properties":{"commands":{"type":"array","items":{"type":"string"}}},"required":["commands"]}}}]}
|
|
}`))
|
|
req.Header.Set("Authorization", "Bearer "+rawToken)
|
|
|
|
w := httptest.NewRecorder()
|
|
srv.routes().ServeHTTP(w, req)
|
|
|
|
body := w.Body.String()
|
|
if !strings.Contains(body, `tool_validation_retry_error`) {
|
|
t.Fatalf("expected tool_validation_retry_error in stream, got: %s", body)
|
|
}
|
|
|
|
reqAfter := testutil.ToFloat64(openAIRequestsTotal.WithLabelValues(
|
|
edgeID, "user:alice", "alice", "iop-tok-alice", model,
|
|
usageEndpointChatCompletions, "normalized", usageStatusError, usageSourceUnavailable,
|
|
))
|
|
if reqAfter-reqBefore != 1 {
|
|
t.Fatalf("requests_total error: got delta %v, want 1", reqAfter-reqBefore)
|
|
}
|
|
}
|