347 lines
14 KiB
Go
347 lines
14 KiB
Go
package openai
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"testing"
|
|
|
|
edgeservice "iop/apps/edge/internal/service"
|
|
"iop/packages/go/config"
|
|
"iop/packages/go/streamgate"
|
|
)
|
|
|
|
func resolvedFilterIDs(resolved []streamgate.ResolvedFilter) map[string]bool {
|
|
out := make(map[string]bool, len(resolved))
|
|
for _, r := range resolved {
|
|
out[r.FilterID()] = true
|
|
}
|
|
return out
|
|
}
|
|
|
|
// TestOpenAIStreamGateRequiredCapabilityAdmission verifies that a required
|
|
// output-filter capability excludes candidates that do not advertise it, and
|
|
// that a request whose only candidates all lack the capability is rejected
|
|
// before dispatch (S02/S08 pre-admission 400).
|
|
func TestOpenAIStreamGateRequiredCapabilityAdmission(t *testing.T) {
|
|
gateCfg := config.StreamEvidenceGateConf{
|
|
Enabled: true,
|
|
Filters: []config.StreamGateFilterPolicyConf{
|
|
{Filter: config.StreamGateFilterProviderError, Priority: 20},
|
|
},
|
|
}
|
|
fctx := openAIOutputFilterContext{endpoint: openAIRebuildEndpointChat, requestRef: "openai.snap.1"}
|
|
|
|
capable, err := streamgate.NewAttemptTarget("client-model", "ornith:35b", "prov-a", "normalized", []string{"output.provider_error"})
|
|
if err != nil {
|
|
t.Fatalf("NewAttemptTarget(capable): %v", err)
|
|
}
|
|
incapable, err := streamgate.NewAttemptTarget("client-model", "ornith:35b", "prov-b", "normalized", nil)
|
|
if err != nil {
|
|
t.Fatalf("NewAttemptTarget(incapable): %v", err)
|
|
}
|
|
|
|
eligible, err := openAIStreamGateAdmitCandidates(gateCfg, fctx, []streamgate.AttemptTarget{capable, incapable})
|
|
if err != nil {
|
|
t.Fatalf("admit(mixed): %v", err)
|
|
}
|
|
if len(eligible) != 1 || eligible[0].Provider() != "prov-a" {
|
|
t.Fatalf("mixed admission = %+v, want only prov-a", eligible)
|
|
}
|
|
|
|
if _, err := openAIStreamGateAdmitCandidates(gateCfg, fctx, []streamgate.AttemptTarget{incapable}); !errors.Is(err, errStreamGateRequiredCapabilityUnsupported) {
|
|
t.Fatalf("all-incapable admission err = %v, want errStreamGateRequiredCapabilityUnsupported", err)
|
|
}
|
|
|
|
// No configured filters => no required capability => every candidate admitted.
|
|
empty := config.StreamEvidenceGateConf{Enabled: true}
|
|
admitted, err := openAIStreamGateAdmitCandidates(empty, fctx, []streamgate.AttemptTarget{incapable})
|
|
if err != nil {
|
|
t.Fatalf("admit(no filters): %v", err)
|
|
}
|
|
if len(admitted) != 1 {
|
|
t.Fatalf("no-filter admission = %d, want 1 (unchanged)", len(admitted))
|
|
}
|
|
}
|
|
|
|
// TestOpenAIStreamGatePolicySelectorPrecedence verifies that a provider selector
|
|
// disabling a filter is re-resolved per attempt target: the same request snapshot
|
|
// keeps the filter active for one provider and drops it for the disabled provider
|
|
// (S08 provider switch).
|
|
func TestOpenAIStreamGatePolicySelectorPrecedence(t *testing.T) {
|
|
gateCfg := config.StreamEvidenceGateConf{
|
|
Enabled: true,
|
|
Filters: []config.StreamGateFilterPolicyConf{
|
|
{
|
|
Filter: config.StreamGateFilterProviderError,
|
|
Priority: 20,
|
|
Selectors: []config.StreamGateFilterSelectorConf{
|
|
{Type: config.StreamGateFilterSelectorProvider, Key: "prov-disabled", Enabled: boolPtr(false)},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
fctx := openAIOutputFilterContext{endpoint: openAIRebuildEndpointChat, requestRef: "openai.snap.1"}
|
|
reqSnap := beginOutputFilterRequest(t, gateCfg, fctx)
|
|
|
|
enabledTarget := attemptTarget(t, "prov-enabled")
|
|
disabledTarget := attemptTarget(t, "prov-disabled")
|
|
|
|
enabledResolved, err := reqSnap.ResolveAttempt(enabledTarget)
|
|
if err != nil {
|
|
t.Fatalf("ResolveAttempt(enabled): %v", err)
|
|
}
|
|
if !resolvedFilterIDs(enabledResolved)[openAIProviderErrorFilterID] {
|
|
t.Errorf("provider_error not active for prov-enabled")
|
|
}
|
|
|
|
disabledResolved, err := reqSnap.ResolveAttempt(disabledTarget)
|
|
if err != nil {
|
|
t.Fatalf("ResolveAttempt(disabled): %v", err)
|
|
}
|
|
if resolvedFilterIDs(disabledResolved)[openAIProviderErrorFilterID] {
|
|
t.Errorf("provider_error must be inactive for prov-disabled selector")
|
|
}
|
|
}
|
|
|
|
// TestOpenAIStreamGateCallerNeutralResolution verifies resolution depends only on
|
|
// protocol/model/provider/path facts, never on a caller product name (S13).
|
|
// The three fixture labels identify the originating client only to the test;
|
|
// the byte-identical protocol payload and every request/filter context passed to
|
|
// production policy resolution deliberately carry no caller-name field.
|
|
func TestOpenAIStreamGateCallerNeutralResolution(t *testing.T) {
|
|
gateCfg := outputFilterGateCfg(config.StreamGateFilterEnforcementBlocking)
|
|
const protocolPayload = `{"model":"qwen","messages":[{"role":"user","content":"hi"}],"metadata":{"scheme":{"type":"object"}},"stream":true}`
|
|
fixtures := []struct {
|
|
name string
|
|
payload []byte
|
|
}{
|
|
{name: "raw HTTP", payload: []byte(protocolPayload)},
|
|
{name: "OpenAI SDK", payload: []byte(protocolPayload)},
|
|
{name: "Pi", payload: []byte(protocolPayload)},
|
|
}
|
|
var baseline string
|
|
for _, fixture := range fixtures {
|
|
t.Run(fixture.name, func(t *testing.T) {
|
|
if strings.Contains(string(fixture.payload), "caller") || strings.Contains(string(fixture.payload), "agent") {
|
|
t.Fatalf("fixture unexpectedly contains a caller-name field: %s", fixture.payload)
|
|
}
|
|
var req chatCompletionRequest
|
|
if err := json.Unmarshal(fixture.payload, &req); err != nil {
|
|
t.Fatalf("decode protocol payload: %v", err)
|
|
}
|
|
fctx := openAIOutputFilterContext{
|
|
environment: config.StreamGateEnvironmentDev,
|
|
endpoint: openAIRebuildEndpointChat,
|
|
modelGroup: req.Model,
|
|
hasScheme: chatRequestHasSchemeMetadata(req.Metadata),
|
|
requestRef: "openai.snap.caller-neutral",
|
|
}
|
|
reqSnap := beginOutputFilterRequest(t, gateCfg, fctx)
|
|
target, err := streamgate.NewAttemptTarget(req.Model, "qwen:latest", "prov-a", string(edgeservice.ProviderPoolPathNormalized), []string{
|
|
"output.repeat_guard", "output.schema_gate", "output.provider_error",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("NewAttemptTarget: %v", err)
|
|
}
|
|
resolved, err := reqSnap.ResolveAttempt(target)
|
|
if err != nil {
|
|
t.Fatalf("ResolveAttempt: %v", err)
|
|
}
|
|
admitted, err := openAIStreamGateAdmitCandidates(gateCfg, fctx, []streamgate.AttemptTarget{target})
|
|
if err != nil || len(admitted) != 1 {
|
|
t.Fatalf("admission decision=(%d,%v), want one admitted target", len(admitted), err)
|
|
}
|
|
parts := []string{fmt.Sprintf("path=%s;admitted=%d", target.ExecutionPath(), len(admitted))}
|
|
for _, filter := range resolved {
|
|
hold := filter.HoldRequirement()
|
|
parts = append(parts, fmt.Sprintf("%s:%s:%d:%s:%d", filter.FilterID(), filter.Enforcement(), filter.Priority(), hold.Mode(), hold.EvidenceRunes()))
|
|
}
|
|
signature := strings.Join(parts, "|")
|
|
if baseline == "" {
|
|
baseline = signature
|
|
} else if signature != baseline {
|
|
t.Fatalf("caller-neutral path/threshold/decision changed:\n got %s\nwant %s", signature, baseline)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestOpenAIStreamGateConfigReloadIsolation verifies each request resolves against
|
|
// the generation snapshot it began with: an enabled-filter generation keeps the
|
|
// filter, a disabled-filter generation drops it, and a request context cannot
|
|
// begin against a mismatched generation snapshot (S08 config reload isolation).
|
|
func TestOpenAIStreamGateConfigReloadIsolation(t *testing.T) {
|
|
enabledFilter, err := newOpenAIOutputFilter(openAIOutputFilterProviderError, 500, 20, "openai.snap.1")
|
|
if err != nil {
|
|
t.Fatalf("newOpenAIOutputFilter: %v", err)
|
|
}
|
|
enabledReg, err := streamgate.NewFilterRegistration(enabledFilter, "output.provider_error", true, streamgate.FilterEnforcementBlocking, streamGateFilterTimeout, 20)
|
|
if err != nil {
|
|
t.Fatalf("NewFilterRegistration: %v", err)
|
|
}
|
|
snapGen1, err := streamgate.NewFilterRegistrySnapshot("edge.gen.1", []streamgate.FilterRegistration{enabledReg}, nil)
|
|
if err != nil {
|
|
t.Fatalf("snapshot gen1: %v", err)
|
|
}
|
|
snapGen2, err := streamgate.NewFilterRegistrySnapshot("edge.gen.2", nil, nil)
|
|
if err != nil {
|
|
t.Fatalf("snapshot gen2: %v", err)
|
|
}
|
|
|
|
reqGen1 := beginGenerationRequest(t, snapGen1, "edge.gen.1")
|
|
reqGen2 := beginGenerationRequest(t, snapGen2, "edge.gen.2")
|
|
target := attemptTarget(t, "prov-a")
|
|
|
|
gen1Resolved, err := reqGen1.ResolveAttempt(target)
|
|
if err != nil {
|
|
t.Fatalf("ResolveAttempt(gen1): %v", err)
|
|
}
|
|
if !resolvedFilterIDs(gen1Resolved)[openAIProviderErrorFilterID] {
|
|
t.Errorf("gen1 request must keep its enabled provider_error filter")
|
|
}
|
|
gen2Resolved, err := reqGen2.ResolveAttempt(target)
|
|
if err != nil {
|
|
t.Fatalf("ResolveAttempt(gen2): %v", err)
|
|
}
|
|
if len(gen2Resolved) != 0 {
|
|
t.Errorf("gen2 request resolved %d filters, want 0 (its own generation)", len(gen2Resolved))
|
|
}
|
|
|
|
// A request context cannot begin against a mismatched generation snapshot.
|
|
mismatchCtx, err := streamgate.NewRequestFilterContext(
|
|
"edge.gen.2", "attempt.x", streamGateEnvironment, openAIRebuildEndpointChat,
|
|
openAIRebuildFamily, "", streamgate.CommitStateTransportUncommitted, false, false, "",
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("NewRequestFilterContext(mismatch): %v", err)
|
|
}
|
|
if _, err := snapGen1.BeginRequest(mismatchCtx); err == nil {
|
|
t.Errorf("BeginRequest with a mismatched generation succeeded, want generation isolation error")
|
|
}
|
|
}
|
|
|
|
func TestOpenAIStreamGatePolicyTargetMatrix(t *testing.T) {
|
|
gateCfg := config.StreamEvidenceGateConf{
|
|
Environment: config.StreamGateEnvironmentDevCorp,
|
|
Filters: []config.StreamGateFilterPolicyConf{{
|
|
Filter: config.StreamGateFilterProviderError,
|
|
Enabled: boolPtr(false),
|
|
Selectors: []config.StreamGateFilterSelectorConf{
|
|
{Type: config.StreamGateFilterSelectorEnvironment, Key: config.StreamGateEnvironmentDevCorp, Enabled: boolPtr(true)},
|
|
{Type: config.StreamGateFilterSelectorModelGroup, Key: "gemma", Enabled: boolPtr(false)},
|
|
{Type: config.StreamGateFilterSelectorModel, Key: "ornith:35b", Enabled: boolPtr(true)},
|
|
{Type: config.StreamGateFilterSelectorProvider, Key: "prov-off", Enabled: boolPtr(false)},
|
|
},
|
|
}},
|
|
}
|
|
fctx := openAIOutputFilterContext{
|
|
environment: config.StreamGateEnvironmentDevCorp,
|
|
endpoint: openAIRebuildEndpointChat,
|
|
modelGroup: "qwen",
|
|
requestRef: "openai.snap.matrix",
|
|
}
|
|
reqSnap := beginOutputFilterRequest(t, gateCfg, fctx)
|
|
tests := []struct {
|
|
name, group, model, provider string
|
|
wantActive bool
|
|
}{
|
|
{name: "environment enables base-disabled", group: "qwen", model: "generic", provider: "prov-a", wantActive: true},
|
|
{name: "model-group disables environment", group: "gemma", model: "generic", provider: "prov-a", wantActive: false},
|
|
{name: "model overrides model-group", group: "gemma", model: "ornith:35b", provider: "prov-a", wantActive: true},
|
|
{name: "provider overrides model", group: "gemma", model: "ornith:35b", provider: "prov-off", wantActive: false},
|
|
}
|
|
for _, tc := range tests {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
target, err := streamgate.NewAttemptTarget(tc.group, tc.model, tc.provider, string(edgeservice.ProviderPoolPathNormalized), nil)
|
|
if err != nil {
|
|
t.Fatalf("NewAttemptTarget: %v", err)
|
|
}
|
|
resolved, err := reqSnap.ResolveAttempt(target)
|
|
if err != nil {
|
|
t.Fatalf("ResolveAttempt: %v", err)
|
|
}
|
|
active := resolvedFilterIDs(resolved)[openAIProviderErrorFilterID]
|
|
if active != tc.wantActive {
|
|
t.Fatalf("active=%t, want %t", active, tc.wantActive)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestOpenAIStreamGateObserveOnlyDoesNotGateAdmission(t *testing.T) {
|
|
gateCfg := config.StreamEvidenceGateConf{
|
|
Environment: config.StreamGateEnvironmentDev,
|
|
Filters: []config.StreamGateFilterPolicyConf{{
|
|
Filter: config.StreamGateFilterProviderError,
|
|
Enforcement: config.StreamGateFilterEnforcementObserveOnly,
|
|
Selectors: []config.StreamGateFilterSelectorConf{{
|
|
Type: config.StreamGateFilterSelectorProvider, Key: "prov-block", Enforcement: config.StreamGateFilterEnforcementBlocking,
|
|
}},
|
|
}},
|
|
}
|
|
fctx := openAIOutputFilterContext{environment: config.StreamGateEnvironmentDev, endpoint: openAIRebuildEndpointChat, modelGroup: "qwen", requestRef: "openai.snap.observe"}
|
|
observeTarget, _ := streamgate.NewAttemptTarget("qwen", "qwen:latest", "prov-observe", "normalized", nil)
|
|
required, err := openAIStreamGateRequiredCapabilities(gateCfg, fctx, observeTarget)
|
|
if err != nil {
|
|
t.Fatalf("RequiredCapabilities(observe): %v", err)
|
|
}
|
|
if len(required) != 0 {
|
|
t.Fatalf("observe-only required capabilities=%v, want none", required)
|
|
}
|
|
if admitted, err := openAIStreamGateAdmitCandidates(gateCfg, fctx, []streamgate.AttemptTarget{observeTarget}); err != nil || len(admitted) != 1 {
|
|
t.Fatalf("observe-only admission=(%d,%v), want admitted", len(admitted), err)
|
|
}
|
|
blockingTarget, _ := streamgate.NewAttemptTarget("qwen", "qwen:latest", "prov-block", "normalized", nil)
|
|
if _, err := openAIStreamGateAdmitCandidates(gateCfg, fctx, []streamgate.AttemptTarget{blockingTarget}); !errors.Is(err, errStreamGateRequiredCapabilityUnsupported) {
|
|
t.Fatalf("blocking selector err=%v, want capability rejection", err)
|
|
}
|
|
}
|
|
|
|
func beginOutputFilterRequest(t *testing.T, gateCfg config.StreamEvidenceGateConf, fctx openAIOutputFilterContext) streamgate.RequestFilterSnapshot {
|
|
t.Helper()
|
|
regs, policies, err := openAIOutputFilterRegistrations(gateCfg, fctx)
|
|
if err != nil {
|
|
t.Fatalf("openAIOutputFilterRegistrations: %v", err)
|
|
}
|
|
snap, err := streamgate.NewFilterRegistrySnapshot(streamGateConfigGeneration, regs, policies)
|
|
if err != nil {
|
|
t.Fatalf("NewFilterRegistrySnapshot: %v", err)
|
|
}
|
|
reqCtx, err := openAIOutputFilterRequestContext(fctx)
|
|
if err != nil {
|
|
t.Fatalf("openAIOutputFilterRequestContext: %v", err)
|
|
}
|
|
reqSnap, err := snap.BeginRequest(reqCtx)
|
|
if err != nil {
|
|
t.Fatalf("BeginRequest: %v", err)
|
|
}
|
|
return reqSnap
|
|
}
|
|
|
|
func beginGenerationRequest(t *testing.T, snap streamgate.FilterRegistrySnapshot, generation string) streamgate.RequestFilterSnapshot {
|
|
t.Helper()
|
|
reqCtx, err := streamgate.NewRequestFilterContext(
|
|
generation, "attempt.1", streamGateEnvironment, openAIRebuildEndpointChat,
|
|
openAIRebuildFamily, "", streamgate.CommitStateTransportUncommitted, false, false, "",
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("NewRequestFilterContext: %v", err)
|
|
}
|
|
reqSnap, err := snap.BeginRequest(reqCtx)
|
|
if err != nil {
|
|
t.Fatalf("BeginRequest: %v", err)
|
|
}
|
|
return reqSnap
|
|
}
|
|
|
|
func attemptTarget(t *testing.T, provider string) streamgate.AttemptTarget {
|
|
t.Helper()
|
|
target, err := streamgate.NewAttemptTarget("client-model", "ornith:35b", provider, "normalized",
|
|
[]string{"output.repeat_guard", "output.schema_gate", "output.provider_error"})
|
|
if err != nil {
|
|
t.Fatalf("NewAttemptTarget(%s): %v", provider, err)
|
|
}
|
|
return target
|
|
}
|