toki
d1e32b6e06
출력 반복을 요청 단위로 감지하고 안전한 continuation lifecycle을 보장하기 위해 Chat/Responses codec과 stream gate evidence를 함께 정렬한다.
579 lines
25 KiB
Go
579 lines
25 KiB
Go
package openai
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"testing"
|
|
|
|
edgeservice "iop/apps/edge/internal/service"
|
|
"iop/packages/go/config"
|
|
"iop/packages/go/streamgate"
|
|
)
|
|
|
|
func TestOpenAIRepeatHistoryChatAliases(t *testing.T) {
|
|
const anchor = "assistant-only anchor"
|
|
body := []byte(`{
|
|
"model":"m",
|
|
"messages":[
|
|
{"role":"assistant","content":"assistant-only anchor"},
|
|
{"role":"assistant","reasoning_content":"assistant-only anchor"},
|
|
{"role":"assistant","reasoning":"assistant-only anchor"},
|
|
{"role":"assistant","reasoning_text":"assistant-only anchor"},
|
|
{"role":"assistant","encrypted_reasoning":"RAW_SECRET","signed_reasoning":"RAW_SIGNATURE"}
|
|
]
|
|
}`)
|
|
history, err := decodeOpenAIChatRepeatHistory(body)
|
|
if err != nil {
|
|
t.Fatalf("decodeOpenAIChatRepeatHistory: %v", err)
|
|
}
|
|
fingerprint, ok := openAIRepeatTextFingerprint(anchor)
|
|
if !ok {
|
|
t.Fatal("anchor fingerprint was empty")
|
|
}
|
|
if count, ok := history.assistantCandidate(openAIRepeatChannelContent, fingerprint); !ok || count != 4 {
|
|
t.Fatalf("content candidate = (%d, %v), want (4, true)", count, ok)
|
|
}
|
|
if count, ok := history.assistantCandidate(openAIRepeatChannelReasoning, fingerprint); !ok || count != 4 {
|
|
t.Fatalf("reasoning candidate = (%d, %v), want (4, true)", count, ok)
|
|
}
|
|
for _, sentinel := range []string{"RAW_SECRET", "RAW_SIGNATURE"} {
|
|
sentinelFingerprint, _ := openAIRepeatTextFingerprint(sentinel)
|
|
if _, ok := history.assistantCandidate(openAIRepeatChannelReasoning, sentinelFingerprint); ok {
|
|
t.Fatalf("unknown protected field %q became a reasoning candidate", sentinel)
|
|
}
|
|
}
|
|
|
|
userBody := []byte(`{"messages":[
|
|
{"role":"user","content":"assistant-only anchor"},
|
|
{"role":"assistant","reasoning_content":"assistant-only anchor"},
|
|
{"role":"assistant","reasoning":"assistant-only anchor"}
|
|
]}`)
|
|
userHistory, err := decodeOpenAIChatRepeatHistory(userBody)
|
|
if err != nil {
|
|
t.Fatalf("decode user-exclusion history: %v", err)
|
|
}
|
|
if _, ok := userHistory.assistantCandidate(openAIRepeatChannelReasoning, fingerprint); ok {
|
|
t.Fatal("user occurrence did not exclude the assistant anchor")
|
|
}
|
|
}
|
|
|
|
func TestOpenAIRepeatHistoryAssistantCandidateMetadata(t *testing.T) {
|
|
const anchor = "assistant\tanchor 한글"
|
|
fingerprint, ok := openAIRepeatTextFingerprint(anchor)
|
|
if !ok {
|
|
t.Fatal("anchor fingerprint was empty")
|
|
}
|
|
|
|
builder := newOpenAIRepeatHistoryBuilder()
|
|
builder.recordText("assistant", openAIRepeatChannelContent, anchor)
|
|
builder.recordText("assistant", openAIRepeatChannelReasoning, "assistant anchor 한글")
|
|
history := builder.snapshot()
|
|
for _, channel := range []openAIRepeatChannel{
|
|
openAIRepeatChannelContent,
|
|
openAIRepeatChannelReasoning,
|
|
} {
|
|
metadata, found := history.assistantCandidateMetadata(channel, fingerprint)
|
|
if !found {
|
|
t.Fatalf("%s candidate metadata is missing", channel)
|
|
}
|
|
if metadata.count != 2 || metadata.normalizedRunes != len([]rune("assistant anchor 한글")) {
|
|
t.Fatalf("%s metadata = %+v", channel, metadata)
|
|
}
|
|
}
|
|
|
|
builder.recordText("user", openAIRepeatChannelContent, " assistant anchor\n한글 ")
|
|
if _, found := builder.snapshot().assistantCandidateMetadata(openAIRepeatChannelContent, fingerprint); found {
|
|
t.Fatal("normalized user occurrence did not exclude the assistant candidate")
|
|
}
|
|
|
|
capped := newOpenAIRepeatHistoryBuilder()
|
|
var firstFingerprint streamgate.FixedFingerprint
|
|
for i := 0; i < openAIRepeatHistoryMaxFingerprints; i++ {
|
|
value := fmt.Sprintf("bounded assistant anchor %04d", i)
|
|
capped.recordText("assistant", openAIRepeatChannelContent, value)
|
|
capped.recordText("assistant", openAIRepeatChannelContent, value)
|
|
if i == 0 {
|
|
firstFingerprint, _ = openAIRepeatTextFingerprint(value)
|
|
}
|
|
}
|
|
capped.recordText("assistant", openAIRepeatChannelContent, "candidate beyond cap")
|
|
capped.recordText("assistant", openAIRepeatChannelContent, "candidate beyond cap")
|
|
cappedSnapshot := capped.snapshot()
|
|
if got := len(cappedSnapshot.assistantCandidates[openAIRepeatChannelContent]); got != openAIRepeatHistoryMaxFingerprints {
|
|
t.Fatalf("candidate cap = %d, want %d", got, openAIRepeatHistoryMaxFingerprints)
|
|
}
|
|
beyondFingerprint, _ := openAIRepeatTextFingerprint("candidate beyond cap")
|
|
if _, found := cappedSnapshot.assistantCandidateMetadata(openAIRepeatChannelContent, beyondFingerprint); found {
|
|
t.Fatal("candidate beyond the raw-free fingerprint cap was retained")
|
|
}
|
|
|
|
// A user occurrence of an already admitted fingerprint must still be
|
|
// recorded after the cap is full so provenance exclusion cannot be bypassed.
|
|
capped.recordText("user", openAIRepeatChannelContent, "bounded assistant anchor 0000")
|
|
if _, found := capped.snapshot().assistantCandidateMetadata(openAIRepeatChannelContent, firstFingerprint); found {
|
|
t.Fatal("user exclusion was dropped after the fingerprint cap filled")
|
|
}
|
|
}
|
|
|
|
func TestOpenAIRepeatHistoryResponsesProvenance(t *testing.T) {
|
|
body := []byte(`{"model":"m","input":[
|
|
{"type":"reasoning","content":[{"type":"reasoning_text","text":"reasoning anchor"},{"type":"encrypted_reasoning","text":"RAW_SECRET"}]},
|
|
{"type":"reasoning","content":[{"type":"reasoning_text","text":"reasoning anchor"}]},
|
|
{"type":"function_call","call_id":"call-1","name":"lookup","arguments":"{\"id\":1}"},
|
|
{"type":"function_call_output","call_id":"call-1","output":"same result"},
|
|
{"type":"function_call","call_id":"call-2","name":"lookup","arguments":"{\"id\":1}"},
|
|
{"type":"function_call_output","call_id":"call-2","output":"same result"},
|
|
{"type":"unknown","payload":"RAW_UNKNOWN"}
|
|
]}`)
|
|
history, err := decodeOpenAIResponsesRepeatHistory(body)
|
|
if err != nil {
|
|
t.Fatalf("decodeOpenAIResponsesRepeatHistory: %v", err)
|
|
}
|
|
anchorFingerprint, _ := openAIRepeatTextFingerprint("reasoning anchor")
|
|
if count, ok := history.assistantCandidate(openAIRepeatChannelReasoning, anchorFingerprint); !ok || count != 2 {
|
|
t.Fatalf("reasoning candidate = (%d, %v), want (2, true)", count, ok)
|
|
}
|
|
actionFingerprint, _ := openAIRepeatActionFingerprint("lookup", json.RawMessage(`{"id":1}`))
|
|
if count, ok := history.repeatedAction(actionFingerprint); !ok || count != 1 {
|
|
t.Fatalf("repeated action = (%d, %v), want (1, true)", count, ok)
|
|
}
|
|
for _, sentinel := range []string{"RAW_SECRET", "RAW_UNKNOWN"} {
|
|
fingerprint, _ := openAIRepeatTextFingerprint(sentinel)
|
|
if _, ok := history.assistantCandidate(openAIRepeatChannelReasoning, fingerprint); ok {
|
|
t.Fatalf("protected Responses value %q became a candidate", sentinel)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestOpenAIRepeatHistoryDoesNotInferLineage(t *testing.T) {
|
|
for name, decode := range map[string]func([]byte) (openAIRepeatHistorySnapshot, error){
|
|
"chat": decodeOpenAIChatRepeatHistory,
|
|
"responses": decodeOpenAIResponsesRepeatHistory,
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
history, err := decode([]byte(`{"model":"m","metadata":{"session_id":"caller-session"},"messages":[],"input":"current user input"}`))
|
|
if err != nil {
|
|
t.Fatalf("decode history: %v", err)
|
|
}
|
|
if len(history.assistantCandidates[openAIRepeatChannelContent]) != 0 ||
|
|
len(history.assistantCandidates[openAIRepeatChannelReasoning]) != 0 ||
|
|
len(history.repeatedActions) != 0 {
|
|
t.Fatalf("omitted history inferred lineage: %+v", history)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestOpenAIRepeatHistoryProgressBoundary(t *testing.T) {
|
|
noProgressBody := []byte(`{"messages":[
|
|
{"role":"assistant","tool_calls":[{"id":"a","type":"function","function":{"name":"lookup","arguments":"{\"id\":1}"}}]},
|
|
{"role":"tool","tool_call_id":"a","content":"same"},
|
|
{"role":"assistant","tool_calls":[{"id":"b","type":"function","function":{"name":"lookup","arguments":"{\"id\":1}"}}]},
|
|
{"role":"tool","tool_call_id":"b","content":"same"}
|
|
]}`)
|
|
noProgress, err := decodeOpenAIChatRepeatHistory(noProgressBody)
|
|
if err != nil {
|
|
t.Fatalf("decode no-progress history: %v", err)
|
|
}
|
|
actionFingerprint, _ := openAIRepeatActionFingerprint("lookup", json.RawMessage(`{"id":1}`))
|
|
if _, ok := noProgress.repeatedAction(actionFingerprint); !ok {
|
|
t.Fatal("identical completed action/result was not marked no-progress")
|
|
}
|
|
if noProgress.completedProgress() {
|
|
t.Fatal("identical completed action/result was marked progress")
|
|
}
|
|
|
|
progressBody := []byte(`{"messages":[
|
|
{"role":"assistant","tool_calls":[{"id":"a","type":"function","function":{"name":"lookup","arguments":"{\"id\":1}"}}]},
|
|
{"role":"tool","tool_call_id":"a","content":"first"},
|
|
{"role":"assistant","tool_calls":[{"id":"b","type":"function","function":{"name":"other","arguments":"{\"id\":2}"}}]},
|
|
{"role":"tool","tool_call_id":"b","content":"changed"}
|
|
]}`)
|
|
progress, err := decodeOpenAIChatRepeatHistory(progressBody)
|
|
if err != nil {
|
|
t.Fatalf("decode progress history: %v", err)
|
|
}
|
|
if !progress.completedProgress() {
|
|
t.Fatal("changed completed result hash was not marked progress")
|
|
}
|
|
if len(progress.repeatedActions) != 0 {
|
|
t.Fatalf("distinct completed action/result was marked repeated: %+v", progress.repeatedActions)
|
|
}
|
|
}
|
|
|
|
func TestOpenAIRepeatHistoryLatestPairWins(t *testing.T) {
|
|
body := []byte(`{"messages":[
|
|
{"role":"assistant","tool_calls":[{"id":"older-a","type":"function","function":{"name":"lookup","arguments":"{\"id\":0}"}}]},
|
|
{"role":"tool","tool_call_id":"older-a","content":"older result"},
|
|
{"role":"assistant","tool_calls":[{"id":"older-b","type":"function","function":{"name":"lookup","arguments":"{\"id\":1}"}}]},
|
|
{"role":"tool","tool_call_id":"older-b","content":"changed result"},
|
|
{"role":"assistant","tool_calls":[{"id":"latest-a","type":"function","function":{"name":"lookup","arguments":"{\"id\":1}"}}]},
|
|
{"role":"tool","tool_call_id":"latest-a","content":"same latest failure"},
|
|
{"role":"assistant","tool_calls":[{"id":"latest-b","type":"function","function":{"name":"lookup","arguments":"{\"id\":1}"}}]},
|
|
{"role":"tool","tool_call_id":"latest-b","content":"same latest failure"}
|
|
]}`)
|
|
history, err := decodeOpenAIChatRepeatHistory(body)
|
|
if err != nil {
|
|
t.Fatalf("decode history: %v", err)
|
|
}
|
|
if history.completedProgress() {
|
|
t.Fatal("latest identical action/result churn was hidden by older progress")
|
|
}
|
|
fingerprint, ok := openAIRepeatActionFingerprint("lookup", json.RawMessage(`{"id":1}`))
|
|
if !ok {
|
|
t.Fatal("build latest action fingerprint")
|
|
}
|
|
if _, repeated := history.repeatedAction(fingerprint); !repeated {
|
|
t.Fatal("latest identical action/result pair is not blockable")
|
|
}
|
|
|
|
changedLatest := []byte(`{"messages":[
|
|
{"role":"assistant","tool_calls":[{"id":"a","type":"function","function":{"name":"lookup","arguments":"{\"id\":1}"}}]},
|
|
{"role":"tool","tool_call_id":"a","content":"first"},
|
|
{"role":"assistant","tool_calls":[{"id":"b","type":"function","function":{"name":"lookup","arguments":"{\"id\":1}"}}]},
|
|
{"role":"tool","tool_call_id":"b","content":"changed"}
|
|
]}`)
|
|
progress, err := decodeOpenAIChatRepeatHistory(changedLatest)
|
|
if err != nil {
|
|
t.Fatalf("decode latest progress history: %v", err)
|
|
}
|
|
if !progress.completedProgress() {
|
|
t.Fatal("latest changed result did not release the progress guard")
|
|
}
|
|
}
|
|
|
|
func resolvedFilterIDs(resolved []streamgate.ResolvedFilter) map[string]bool {
|
|
out := make(map[string]bool, len(resolved))
|
|
for _, r := range resolved {
|
|
out[r.FilterID()] = true
|
|
}
|
|
return out
|
|
}
|
|
|
|
// TestOpenAIStreamGateRequiredCapabilityAdmission verifies that a required
|
|
// output-filter capability excludes candidates that do not advertise it, and
|
|
// that a request whose only candidates all lack the capability is rejected
|
|
// before dispatch (S02/S08 pre-admission 400).
|
|
func TestOpenAIStreamGateRequiredCapabilityAdmission(t *testing.T) {
|
|
gateCfg := config.StreamEvidenceGateConf{
|
|
Enabled: true,
|
|
Filters: []config.StreamGateFilterPolicyConf{
|
|
{Filter: config.StreamGateFilterProviderError, Priority: 20},
|
|
},
|
|
}
|
|
fctx := openAIOutputFilterContext{endpoint: openAIRebuildEndpointChat, requestRef: "openai.snap.1"}
|
|
|
|
capable, err := streamgate.NewAttemptTarget("client-model", "ornith:35b", "prov-a", "normalized", []string{"output.provider_error"})
|
|
if err != nil {
|
|
t.Fatalf("NewAttemptTarget(capable): %v", err)
|
|
}
|
|
incapable, err := streamgate.NewAttemptTarget("client-model", "ornith:35b", "prov-b", "normalized", nil)
|
|
if err != nil {
|
|
t.Fatalf("NewAttemptTarget(incapable): %v", err)
|
|
}
|
|
|
|
eligible, err := openAIStreamGateAdmitCandidates(gateCfg, fctx, []streamgate.AttemptTarget{capable, incapable})
|
|
if err != nil {
|
|
t.Fatalf("admit(mixed): %v", err)
|
|
}
|
|
if len(eligible) != 1 || eligible[0].Provider() != "prov-a" {
|
|
t.Fatalf("mixed admission = %+v, want only prov-a", eligible)
|
|
}
|
|
|
|
if _, err := openAIStreamGateAdmitCandidates(gateCfg, fctx, []streamgate.AttemptTarget{incapable}); !errors.Is(err, errStreamGateRequiredCapabilityUnsupported) {
|
|
t.Fatalf("all-incapable admission err = %v, want errStreamGateRequiredCapabilityUnsupported", err)
|
|
}
|
|
|
|
// No configured filters => no required capability => every candidate admitted.
|
|
empty := config.StreamEvidenceGateConf{Enabled: true}
|
|
admitted, err := openAIStreamGateAdmitCandidates(empty, fctx, []streamgate.AttemptTarget{incapable})
|
|
if err != nil {
|
|
t.Fatalf("admit(no filters): %v", err)
|
|
}
|
|
if len(admitted) != 1 {
|
|
t.Fatalf("no-filter admission = %d, want 1 (unchanged)", len(admitted))
|
|
}
|
|
}
|
|
|
|
// TestOpenAIStreamGatePolicySelectorPrecedence verifies that a provider selector
|
|
// disabling a filter is re-resolved per attempt target: the same request snapshot
|
|
// keeps the filter active for one provider and drops it for the disabled provider
|
|
// (S08 provider switch).
|
|
func TestOpenAIStreamGatePolicySelectorPrecedence(t *testing.T) {
|
|
gateCfg := config.StreamEvidenceGateConf{
|
|
Enabled: true,
|
|
Filters: []config.StreamGateFilterPolicyConf{
|
|
{
|
|
Filter: config.StreamGateFilterProviderError,
|
|
Priority: 20,
|
|
Selectors: []config.StreamGateFilterSelectorConf{
|
|
{Type: config.StreamGateFilterSelectorProvider, Key: "prov-disabled", Enabled: boolPtr(false)},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
fctx := openAIOutputFilterContext{endpoint: openAIRebuildEndpointChat, requestRef: "openai.snap.1"}
|
|
reqSnap := beginOutputFilterRequest(t, gateCfg, fctx)
|
|
|
|
enabledTarget := attemptTarget(t, "prov-enabled")
|
|
disabledTarget := attemptTarget(t, "prov-disabled")
|
|
|
|
enabledResolved, err := reqSnap.ResolveAttempt(enabledTarget)
|
|
if err != nil {
|
|
t.Fatalf("ResolveAttempt(enabled): %v", err)
|
|
}
|
|
if !resolvedFilterIDs(enabledResolved)[openAIProviderErrorFilterID] {
|
|
t.Errorf("provider_error not active for prov-enabled")
|
|
}
|
|
|
|
disabledResolved, err := reqSnap.ResolveAttempt(disabledTarget)
|
|
if err != nil {
|
|
t.Fatalf("ResolveAttempt(disabled): %v", err)
|
|
}
|
|
if resolvedFilterIDs(disabledResolved)[openAIProviderErrorFilterID] {
|
|
t.Errorf("provider_error must be inactive for prov-disabled selector")
|
|
}
|
|
}
|
|
|
|
// TestOpenAIStreamGateCallerNeutralResolution verifies resolution depends only on
|
|
// protocol/model/provider/path facts, never on a caller product name (S13).
|
|
// The three fixture labels identify the originating client only to the test;
|
|
// the byte-identical protocol payload and every request/filter context passed to
|
|
// production policy resolution deliberately carry no caller-name field.
|
|
func TestOpenAIStreamGateCallerNeutralResolution(t *testing.T) {
|
|
gateCfg := outputFilterGateCfg(config.StreamGateFilterEnforcementBlocking)
|
|
const protocolPayload = `{"model":"qwen","messages":[{"role":"user","content":"hi"}],"metadata":{"scheme":{"type":"object"}},"stream":true}`
|
|
fixtures := []struct {
|
|
name string
|
|
payload []byte
|
|
}{
|
|
{name: "raw HTTP", payload: []byte(protocolPayload)},
|
|
{name: "OpenAI SDK", payload: []byte(protocolPayload)},
|
|
{name: "Pi", payload: []byte(protocolPayload)},
|
|
}
|
|
var baseline string
|
|
for _, fixture := range fixtures {
|
|
t.Run(fixture.name, func(t *testing.T) {
|
|
if strings.Contains(string(fixture.payload), "caller") || strings.Contains(string(fixture.payload), "agent") {
|
|
t.Fatalf("fixture unexpectedly contains a caller-name field: %s", fixture.payload)
|
|
}
|
|
var req chatCompletionRequest
|
|
if err := json.Unmarshal(fixture.payload, &req); err != nil {
|
|
t.Fatalf("decode protocol payload: %v", err)
|
|
}
|
|
fctx := openAIOutputFilterContext{
|
|
environment: config.StreamGateEnvironmentDev,
|
|
endpoint: openAIRebuildEndpointChat,
|
|
modelGroup: req.Model,
|
|
hasScheme: chatRequestHasSchemeMetadata(req.Metadata),
|
|
requestRef: "openai.snap.caller-neutral",
|
|
}
|
|
reqSnap := beginOutputFilterRequest(t, gateCfg, fctx)
|
|
target, err := streamgate.NewAttemptTarget(req.Model, "qwen:latest", "prov-a", string(edgeservice.ProviderPoolPathNormalized), []string{
|
|
"output.repeat_guard", "output.schema_gate", "output.provider_error",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("NewAttemptTarget: %v", err)
|
|
}
|
|
resolved, err := reqSnap.ResolveAttempt(target)
|
|
if err != nil {
|
|
t.Fatalf("ResolveAttempt: %v", err)
|
|
}
|
|
admitted, err := openAIStreamGateAdmitCandidates(gateCfg, fctx, []streamgate.AttemptTarget{target})
|
|
if err != nil || len(admitted) != 1 {
|
|
t.Fatalf("admission decision=(%d,%v), want one admitted target", len(admitted), err)
|
|
}
|
|
parts := []string{fmt.Sprintf("path=%s;admitted=%d", target.ExecutionPath(), len(admitted))}
|
|
for _, filter := range resolved {
|
|
hold := filter.HoldRequirement()
|
|
parts = append(parts, fmt.Sprintf("%s:%s:%d:%s:%d", filter.FilterID(), filter.Enforcement(), filter.Priority(), hold.Mode(), hold.EvidenceRunes()))
|
|
}
|
|
signature := strings.Join(parts, "|")
|
|
if baseline == "" {
|
|
baseline = signature
|
|
} else if signature != baseline {
|
|
t.Fatalf("caller-neutral path/threshold/decision changed:\n got %s\nwant %s", signature, baseline)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestOpenAIStreamGateConfigReloadIsolation verifies each request resolves against
|
|
// the generation snapshot it began with: an enabled-filter generation keeps the
|
|
// filter, a disabled-filter generation drops it, and a request context cannot
|
|
// begin against a mismatched generation snapshot (S08 config reload isolation).
|
|
func TestOpenAIStreamGateConfigReloadIsolation(t *testing.T) {
|
|
enabledFilter, err := newOpenAIOutputFilter(openAIOutputFilterProviderError, 500, 20, "openai.snap.1")
|
|
if err != nil {
|
|
t.Fatalf("newOpenAIOutputFilter: %v", err)
|
|
}
|
|
enabledReg, err := streamgate.NewFilterRegistration(enabledFilter, "output.provider_error", true, streamgate.FilterEnforcementBlocking, streamGateFilterTimeout, 20)
|
|
if err != nil {
|
|
t.Fatalf("NewFilterRegistration: %v", err)
|
|
}
|
|
snapGen1, err := streamgate.NewFilterRegistrySnapshot("edge.gen.1", []streamgate.FilterRegistration{enabledReg}, nil)
|
|
if err != nil {
|
|
t.Fatalf("snapshot gen1: %v", err)
|
|
}
|
|
snapGen2, err := streamgate.NewFilterRegistrySnapshot("edge.gen.2", nil, nil)
|
|
if err != nil {
|
|
t.Fatalf("snapshot gen2: %v", err)
|
|
}
|
|
|
|
reqGen1 := beginGenerationRequest(t, snapGen1, "edge.gen.1")
|
|
reqGen2 := beginGenerationRequest(t, snapGen2, "edge.gen.2")
|
|
target := attemptTarget(t, "prov-a")
|
|
|
|
gen1Resolved, err := reqGen1.ResolveAttempt(target)
|
|
if err != nil {
|
|
t.Fatalf("ResolveAttempt(gen1): %v", err)
|
|
}
|
|
if !resolvedFilterIDs(gen1Resolved)[openAIProviderErrorFilterID] {
|
|
t.Errorf("gen1 request must keep its enabled provider_error filter")
|
|
}
|
|
gen2Resolved, err := reqGen2.ResolveAttempt(target)
|
|
if err != nil {
|
|
t.Fatalf("ResolveAttempt(gen2): %v", err)
|
|
}
|
|
if len(gen2Resolved) != 0 {
|
|
t.Errorf("gen2 request resolved %d filters, want 0 (its own generation)", len(gen2Resolved))
|
|
}
|
|
|
|
// A request context cannot begin against a mismatched generation snapshot.
|
|
mismatchCtx, err := streamgate.NewRequestFilterContext(
|
|
"edge.gen.2", "attempt.x", streamGateEnvironment, openAIRebuildEndpointChat,
|
|
openAIRebuildFamily, "", streamgate.CommitStateTransportUncommitted, false, false, "",
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("NewRequestFilterContext(mismatch): %v", err)
|
|
}
|
|
if _, err := snapGen1.BeginRequest(mismatchCtx); err == nil {
|
|
t.Errorf("BeginRequest with a mismatched generation succeeded, want generation isolation error")
|
|
}
|
|
}
|
|
|
|
func TestOpenAIStreamGatePolicyTargetMatrix(t *testing.T) {
|
|
gateCfg := config.StreamEvidenceGateConf{
|
|
Environment: config.StreamGateEnvironmentDevCorp,
|
|
Filters: []config.StreamGateFilterPolicyConf{{
|
|
Filter: config.StreamGateFilterProviderError,
|
|
Enabled: boolPtr(false),
|
|
Selectors: []config.StreamGateFilterSelectorConf{
|
|
{Type: config.StreamGateFilterSelectorEnvironment, Key: config.StreamGateEnvironmentDevCorp, Enabled: boolPtr(true)},
|
|
{Type: config.StreamGateFilterSelectorModelGroup, Key: "gemma", Enabled: boolPtr(false)},
|
|
{Type: config.StreamGateFilterSelectorModel, Key: "ornith:35b", Enabled: boolPtr(true)},
|
|
{Type: config.StreamGateFilterSelectorProvider, Key: "prov-off", Enabled: boolPtr(false)},
|
|
},
|
|
}},
|
|
}
|
|
fctx := openAIOutputFilterContext{
|
|
environment: config.StreamGateEnvironmentDevCorp,
|
|
endpoint: openAIRebuildEndpointChat,
|
|
modelGroup: "qwen",
|
|
requestRef: "openai.snap.matrix",
|
|
}
|
|
reqSnap := beginOutputFilterRequest(t, gateCfg, fctx)
|
|
tests := []struct {
|
|
name, group, model, provider string
|
|
wantActive bool
|
|
}{
|
|
{name: "environment enables base-disabled", group: "qwen", model: "generic", provider: "prov-a", wantActive: true},
|
|
{name: "model-group disables environment", group: "gemma", model: "generic", provider: "prov-a", wantActive: false},
|
|
{name: "model overrides model-group", group: "gemma", model: "ornith:35b", provider: "prov-a", wantActive: true},
|
|
{name: "provider overrides model", group: "gemma", model: "ornith:35b", provider: "prov-off", wantActive: false},
|
|
}
|
|
for _, tc := range tests {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
target, err := streamgate.NewAttemptTarget(tc.group, tc.model, tc.provider, string(edgeservice.ProviderPoolPathNormalized), nil)
|
|
if err != nil {
|
|
t.Fatalf("NewAttemptTarget: %v", err)
|
|
}
|
|
resolved, err := reqSnap.ResolveAttempt(target)
|
|
if err != nil {
|
|
t.Fatalf("ResolveAttempt: %v", err)
|
|
}
|
|
active := resolvedFilterIDs(resolved)[openAIProviderErrorFilterID]
|
|
if active != tc.wantActive {
|
|
t.Fatalf("active=%t, want %t", active, tc.wantActive)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestOpenAIStreamGateObserveOnlyDoesNotGateAdmission(t *testing.T) {
|
|
gateCfg := config.StreamEvidenceGateConf{
|
|
Environment: config.StreamGateEnvironmentDev,
|
|
Filters: []config.StreamGateFilterPolicyConf{{
|
|
Filter: config.StreamGateFilterProviderError,
|
|
Enforcement: config.StreamGateFilterEnforcementObserveOnly,
|
|
Selectors: []config.StreamGateFilterSelectorConf{{
|
|
Type: config.StreamGateFilterSelectorProvider, Key: "prov-block", Enforcement: config.StreamGateFilterEnforcementBlocking,
|
|
}},
|
|
}},
|
|
}
|
|
fctx := openAIOutputFilterContext{environment: config.StreamGateEnvironmentDev, endpoint: openAIRebuildEndpointChat, modelGroup: "qwen", requestRef: "openai.snap.observe"}
|
|
observeTarget, _ := streamgate.NewAttemptTarget("qwen", "qwen:latest", "prov-observe", "normalized", nil)
|
|
required, err := openAIStreamGateRequiredCapabilities(gateCfg, fctx, observeTarget)
|
|
if err != nil {
|
|
t.Fatalf("RequiredCapabilities(observe): %v", err)
|
|
}
|
|
if len(required) != 0 {
|
|
t.Fatalf("observe-only required capabilities=%v, want none", required)
|
|
}
|
|
if admitted, err := openAIStreamGateAdmitCandidates(gateCfg, fctx, []streamgate.AttemptTarget{observeTarget}); err != nil || len(admitted) != 1 {
|
|
t.Fatalf("observe-only admission=(%d,%v), want admitted", len(admitted), err)
|
|
}
|
|
blockingTarget, _ := streamgate.NewAttemptTarget("qwen", "qwen:latest", "prov-block", "normalized", nil)
|
|
if _, err := openAIStreamGateAdmitCandidates(gateCfg, fctx, []streamgate.AttemptTarget{blockingTarget}); !errors.Is(err, errStreamGateRequiredCapabilityUnsupported) {
|
|
t.Fatalf("blocking selector err=%v, want capability rejection", err)
|
|
}
|
|
}
|
|
|
|
func beginOutputFilterRequest(t *testing.T, gateCfg config.StreamEvidenceGateConf, fctx openAIOutputFilterContext) streamgate.RequestFilterSnapshot {
|
|
t.Helper()
|
|
regs, policies, err := openAIOutputFilterRegistrations(gateCfg, fctx)
|
|
if err != nil {
|
|
t.Fatalf("openAIOutputFilterRegistrations: %v", err)
|
|
}
|
|
snap, err := streamgate.NewFilterRegistrySnapshot(streamGateConfigGeneration, regs, policies)
|
|
if err != nil {
|
|
t.Fatalf("NewFilterRegistrySnapshot: %v", err)
|
|
}
|
|
reqCtx, err := openAIOutputFilterRequestContext(fctx)
|
|
if err != nil {
|
|
t.Fatalf("openAIOutputFilterRequestContext: %v", err)
|
|
}
|
|
reqSnap, err := snap.BeginRequest(reqCtx)
|
|
if err != nil {
|
|
t.Fatalf("BeginRequest: %v", err)
|
|
}
|
|
return reqSnap
|
|
}
|
|
|
|
func beginGenerationRequest(t *testing.T, snap streamgate.FilterRegistrySnapshot, generation string) streamgate.RequestFilterSnapshot {
|
|
t.Helper()
|
|
reqCtx, err := streamgate.NewRequestFilterContext(
|
|
generation, "attempt.1", streamGateEnvironment, openAIRebuildEndpointChat,
|
|
openAIRebuildFamily, "", streamgate.CommitStateTransportUncommitted, false, false, "",
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("NewRequestFilterContext: %v", err)
|
|
}
|
|
reqSnap, err := snap.BeginRequest(reqCtx)
|
|
if err != nil {
|
|
t.Fatalf("BeginRequest: %v", err)
|
|
}
|
|
return reqSnap
|
|
}
|
|
|
|
func attemptTarget(t *testing.T, provider string) streamgate.AttemptTarget {
|
|
t.Helper()
|
|
target, err := streamgate.NewAttemptTarget("client-model", "ornith:35b", provider, "normalized",
|
|
[]string{"output.repeat_guard", "output.schema_gate", "output.provider_error"})
|
|
if err != nil {
|
|
t.Fatalf("NewAttemptTarget(%s): %v", provider, err)
|
|
}
|
|
return target
|
|
}
|