toki
dc9a9a8c59
승인된 execution preset을 Edge 조정 경계와 Node workspace/tool 실행 경계로 연결해 단일 요청 수명주기와 관측 계약을 일관되게 처리한다.
418 lines
17 KiB
Go
418 lines
17 KiB
Go
package openai
|
|
|
|
import (
|
|
"errors"
|
|
"testing"
|
|
|
|
"iop/apps/edge/internal/authprojection"
|
|
edgeservice "iop/apps/edge/internal/service"
|
|
"iop/packages/go/config"
|
|
)
|
|
|
|
func newTestView(principalRef string, routes []authprojection.Route) authprojection.AuthenticatedView {
|
|
return authprojection.AuthenticatedView{
|
|
Principal: authprojection.Principal{
|
|
PrincipalRef: principalRef,
|
|
},
|
|
Routes: routes,
|
|
}
|
|
}
|
|
|
|
func managedBinding(modelGroupKey, providerID, principalRef, routeID string, managed bool) routeDispatch {
|
|
return routeDispatch{
|
|
Managed: managed,
|
|
ModelGroupKey: modelGroupKey,
|
|
ProviderID: providerID,
|
|
PrincipalRef: principalRef,
|
|
RouteID: routeID,
|
|
}
|
|
}
|
|
|
|
// validSingleRequestPreset returns an approved fixed single-request preset whose
|
|
// selector, allowed modes, and light route exactly match the frozen
|
|
// plan→work→review policy stages: high reasoning on plan/review, none on work,
|
|
// and the selector fused to the plan stage. Each call builds fresh option maps so
|
|
// subtests may mutate one aspect in isolation.
|
|
func validSingleRequestPreset() config.ExecutionPreset {
|
|
return config.ExecutionPreset{
|
|
ID: "preset-single-request",
|
|
Selector: config.ExecutionModelBinding{
|
|
Model: "plan-model",
|
|
Options: map[string]any{"reasoning_effort": "high"},
|
|
},
|
|
AllowedModes: []string{config.ModeLight},
|
|
Routes: map[string]config.ExecutionRoute{
|
|
config.ModeLight: {
|
|
Stages: []config.ExecutionRouteStage{
|
|
{Role: "plan", Model: "plan-model", Options: map[string]any{"reasoning_effort": "high"}},
|
|
{Role: "work", Model: "work-model"},
|
|
{Role: "review", Model: "review-model", Options: map[string]any{"reasoning_effort": "high"}},
|
|
},
|
|
},
|
|
},
|
|
SingleRequest: &config.ExecutionSingleRequestPolicy{
|
|
WorkspaceRef: "ws-opaque-ref",
|
|
Limits: config.ExecutionSingleRequestLimits{
|
|
WallClockMS: 30 * 60 * 1000,
|
|
StageTimeoutMS: 10 * 60 * 1000,
|
|
MaxToolIterations: 64,
|
|
MaxOutputBytes: 16 * 1024 * 1024,
|
|
},
|
|
Stages: config.ExecutionSingleRequestStages{
|
|
Plan: config.ExecutionSingleRequestStageConfig{Model: "plan-model", Options: map[string]any{"reasoning_effort": "high"}},
|
|
Work: config.ExecutionSingleRequestStageConfig{Model: "work-model"},
|
|
Review: config.ExecutionSingleRequestStageConfig{Model: "review-model", Options: map[string]any{"reasoning_effort": "high"}},
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
// validSingleRequestBindings returns managed, same-principal canonical bindings
|
|
// for the plan/work/review models referenced by validSingleRequestPreset.
|
|
func validSingleRequestBindings() map[string]routeDispatch {
|
|
return map[string]routeDispatch{
|
|
"plan-model": managedBinding("plan-model", "prov-1", "principal-1", "route-plan", true),
|
|
"work-model": managedBinding("work-model", "prov-1", "principal-1", "route-work", true),
|
|
"review-model": managedBinding("review-model", "prov-1", "principal-1", "route-review", true),
|
|
}
|
|
}
|
|
|
|
func TestSingleRequestPresetBindingManaged(t *testing.T) {
|
|
preset := validSingleRequestPreset()
|
|
bindings := validSingleRequestBindings()
|
|
view := newTestView("principal-1", nil)
|
|
|
|
binding, err := compileSingleRequestBinding("virtual-public-model", preset, bindings, view)
|
|
if err != nil {
|
|
t.Fatalf("managed compilation failed: %v", err)
|
|
}
|
|
if binding == nil {
|
|
t.Fatal("expected non-nil binding")
|
|
}
|
|
if binding.PublicModel != "virtual-public-model" {
|
|
t.Errorf("PublicModel=%q, want virtual-public-model", binding.PublicModel)
|
|
}
|
|
if binding.WorkspaceRef != "ws-opaque-ref" {
|
|
t.Errorf("WorkspaceRef=%q, want ws-opaque-ref", binding.WorkspaceRef)
|
|
}
|
|
if binding.Plan.Model != "plan-model" {
|
|
t.Errorf("Plan.Model=%q, want plan-model", binding.Plan.Model)
|
|
}
|
|
if binding.Work.Model != "work-model" {
|
|
t.Errorf("Work.Model=%q, want work-model", binding.Work.Model)
|
|
}
|
|
if binding.Review.Model != "review-model" {
|
|
t.Errorf("Review.Model=%q, want review-model", binding.Review.Model)
|
|
}
|
|
if binding.Limits.WallClockMS != 30*60*1000 {
|
|
t.Errorf("WallClockMS=%d, want 1800000", binding.Limits.WallClockMS)
|
|
}
|
|
|
|
// Approved fixed options survive admission: high reasoning on plan/review, and
|
|
// the work stage carries no reasoning option.
|
|
if binding.Plan.Options["reasoning_effort"] != "high" {
|
|
t.Errorf("Plan.Options[reasoning_effort]=%v, want high", binding.Plan.Options["reasoning_effort"])
|
|
}
|
|
if binding.Review.Options["reasoning_effort"] != "high" {
|
|
t.Errorf("Review.Options[reasoning_effort]=%v, want high", binding.Review.Options["reasoning_effort"])
|
|
}
|
|
if _, present := binding.Work.Options["reasoning_effort"]; present {
|
|
t.Errorf("Work.Options unexpectedly declares reasoning_effort: %v", binding.Work.Options)
|
|
}
|
|
}
|
|
|
|
func TestSingleRequestPresetBindingUnmanaged(t *testing.T) {
|
|
preset := validSingleRequestPreset()
|
|
|
|
_, err := compileSingleRequestBindingForUnmanaged("virtual-model", preset)
|
|
if !errors.Is(err, errSingleRequestBindingUnauthorized) {
|
|
t.Fatalf("expected errSingleRequestBindingUnauthorized, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestSingleRequestPresetBindingRejectsInvalidDefenseInDepth(t *testing.T) {
|
|
view := newTestView("principal-1", nil)
|
|
|
|
// Binding-resolution defenses: the fixed shape is valid, so compilation reaches
|
|
// the per-stage authorization checks against the managed bindings.
|
|
t.Run("missing binding", func(t *testing.T) {
|
|
bindings := validSingleRequestBindings()
|
|
delete(bindings, "plan-model")
|
|
_, err := compileSingleRequestBinding("virtual-model", validSingleRequestPreset(), bindings, view)
|
|
if !errors.Is(err, errSingleRequestBindingMissingStage) {
|
|
t.Fatalf("expected missing stage error, got %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("unmanaged binding", func(t *testing.T) {
|
|
bindings := validSingleRequestBindings()
|
|
bindings["plan-model"] = managedBinding("plan-model", "prov-1", "principal-1", "route-plan", false)
|
|
_, err := compileSingleRequestBinding("virtual-model", validSingleRequestPreset(), bindings, view)
|
|
if !errors.Is(err, errSingleRequestBindingUnauthorized) {
|
|
t.Fatalf("expected unauthorized error, got %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("wrong principal", func(t *testing.T) {
|
|
bindings := validSingleRequestBindings()
|
|
bindings["plan-model"] = managedBinding("plan-model", "prov-1", "principal-2", "route-plan", true)
|
|
_, err := compileSingleRequestBinding("virtual-model", validSingleRequestPreset(), bindings, view)
|
|
if !errors.Is(err, errSingleRequestBindingUnauthorized) {
|
|
t.Fatalf("expected unauthorized error, got %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("model group mismatch", func(t *testing.T) {
|
|
bindings := validSingleRequestBindings()
|
|
bindings["plan-model"] = managedBinding("wrong-group", "prov-1", "principal-1", "route-plan", true)
|
|
_, err := compileSingleRequestBinding("virtual-model", validSingleRequestPreset(), bindings, view)
|
|
if !errors.Is(err, errSingleRequestBindingInconsistent) {
|
|
t.Fatalf("expected inconsistent error, got %v", err)
|
|
}
|
|
})
|
|
|
|
// Fixed-shape defenses: these fail before any binding is resolved, so the
|
|
// bindings map is valid to prove the rejection comes from the frozen shape.
|
|
t.Run("allowed modes not light", func(t *testing.T) {
|
|
preset := validSingleRequestPreset()
|
|
preset.AllowedModes = []string{config.ModeDirect}
|
|
_, err := compileSingleRequestBinding("virtual-model", preset, validSingleRequestBindings(), view)
|
|
if !errors.Is(err, errSingleRequestBindingInconsistent) {
|
|
t.Fatalf("expected inconsistent error, got %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("selector model mismatch", func(t *testing.T) {
|
|
preset := validSingleRequestPreset()
|
|
preset.Selector.Model = "other-model"
|
|
_, err := compileSingleRequestBinding("virtual-model", preset, validSingleRequestBindings(), view)
|
|
if !errors.Is(err, errSingleRequestBindingInconsistent) {
|
|
t.Fatalf("expected inconsistent error, got %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("selector options mismatch", func(t *testing.T) {
|
|
preset := validSingleRequestPreset()
|
|
preset.Selector.Options = map[string]any{"reasoning_effort": "low"}
|
|
_, err := compileSingleRequestBinding("virtual-model", preset, validSingleRequestBindings(), view)
|
|
if !errors.Is(err, errSingleRequestBindingInconsistent) {
|
|
t.Fatalf("expected inconsistent error, got %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("duplicate role", func(t *testing.T) {
|
|
duplicateSequences := [][]config.ExecutionRouteStage{
|
|
{
|
|
{Role: "plan", Model: "plan-model", Options: map[string]any{"reasoning_effort": "high"}},
|
|
{Role: "plan", Model: "plan-model", Options: map[string]any{"reasoning_effort": "high"}},
|
|
{Role: "review", Model: "review-model", Options: map[string]any{"reasoning_effort": "high"}},
|
|
},
|
|
{
|
|
{Role: "work", Model: "work-model"},
|
|
{Role: "work", Model: "work-model"},
|
|
{Role: "review", Model: "review-model", Options: map[string]any{"reasoning_effort": "high"}},
|
|
},
|
|
{
|
|
{Role: "plan", Model: "plan-model", Options: map[string]any{"reasoning_effort": "high"}},
|
|
{Role: "review", Model: "review-model", Options: map[string]any{"reasoning_effort": "high"}},
|
|
{Role: "review", Model: "review-model", Options: map[string]any{"reasoning_effort": "high"}},
|
|
},
|
|
}
|
|
for _, seq := range duplicateSequences {
|
|
preset := validSingleRequestPreset()
|
|
route := preset.Routes[config.ModeLight]
|
|
route.Stages = seq
|
|
preset.Routes[config.ModeLight] = route
|
|
_, err := compileSingleRequestBinding("virtual-model", preset, validSingleRequestBindings(), view)
|
|
if !errors.Is(err, errSingleRequestBindingDuplicate) {
|
|
t.Fatalf("expected duplicate error for sequence %+v, got %v", seq, err)
|
|
}
|
|
}
|
|
})
|
|
|
|
t.Run("extra route key", func(t *testing.T) {
|
|
preset := validSingleRequestPreset()
|
|
preset.Routes[config.ModeDirect] = config.ExecutionRoute{
|
|
Stages: []config.ExecutionRouteStage{
|
|
{Role: "plan", Model: "plan-model"},
|
|
},
|
|
}
|
|
_, err := compileSingleRequestBinding("virtual-model", preset, validSingleRequestBindings(), view)
|
|
if !errors.Is(err, errSingleRequestBindingInconsistent) {
|
|
t.Fatalf("expected inconsistent error for extra route key, got %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("route policy model mismatch", func(t *testing.T) {
|
|
preset := validSingleRequestPreset()
|
|
route := preset.Routes[config.ModeLight]
|
|
route.Stages[1].Model = "other-work-model"
|
|
preset.Routes[config.ModeLight] = route
|
|
_, err := compileSingleRequestBinding("virtual-model", preset, validSingleRequestBindings(), view)
|
|
if !errors.Is(err, errSingleRequestBindingDynamic) {
|
|
t.Fatalf("expected dynamic error, got %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("plan option mismatch", func(t *testing.T) {
|
|
preset := validSingleRequestPreset()
|
|
route := preset.Routes[config.ModeLight]
|
|
route.Stages[0].Options = map[string]any{"reasoning_effort": "low"}
|
|
preset.Routes[config.ModeLight] = route
|
|
_, err := compileSingleRequestBinding("virtual-model", preset, validSingleRequestBindings(), view)
|
|
if !errors.Is(err, errSingleRequestBindingInconsistent) {
|
|
t.Fatalf("expected inconsistent error, got %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("review option mismatch", func(t *testing.T) {
|
|
preset := validSingleRequestPreset()
|
|
route := preset.Routes[config.ModeLight]
|
|
route.Stages[2].Options = map[string]any{"reasoning_effort": "low"}
|
|
preset.Routes[config.ModeLight] = route
|
|
_, err := compileSingleRequestBinding("virtual-model", preset, validSingleRequestBindings(), view)
|
|
if !errors.Is(err, errSingleRequestBindingInconsistent) {
|
|
t.Fatalf("expected inconsistent error, got %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("work reasoning option", func(t *testing.T) {
|
|
preset := validSingleRequestPreset()
|
|
preset.SingleRequest.Stages.Work.Options = map[string]any{"reasoning_effort": "high"}
|
|
_, err := compileSingleRequestBinding("virtual-model", preset, validSingleRequestBindings(), view)
|
|
if !errors.Is(err, errSingleRequestBindingInconsistent) {
|
|
t.Fatalf("expected inconsistent error, got %v", err)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestSingleRequestPresetBindingNoPresetPolicy(t *testing.T) {
|
|
preset := config.ExecutionPreset{
|
|
ID: "preset-no-single-request",
|
|
Selector: config.ExecutionModelBinding{Model: "selector-model"},
|
|
AllowedModes: []string{config.ModeLight},
|
|
Routes: map[string]config.ExecutionRoute{
|
|
config.ModeLight: {
|
|
Stages: []config.ExecutionRouteStage{
|
|
{Role: "local", Model: "local-model"},
|
|
{Role: "review", Model: "review-model"},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
bindings := map[string]routeDispatch{
|
|
"selector-model": managedBinding("selector-model", "prov-1", "principal-1", "route-s", true),
|
|
}
|
|
view := newTestView("principal-1", nil)
|
|
|
|
binding, err := compileSingleRequestBinding("virtual-model", preset, bindings, view)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if binding != nil {
|
|
t.Fatalf("expected nil binding for preset without SingleRequest policy, got %+v", binding)
|
|
}
|
|
}
|
|
|
|
func TestSingleRequestPresetBindingRefreshIsolation(t *testing.T) {
|
|
preset := validSingleRequestPreset()
|
|
bindings := validSingleRequestBindings()
|
|
view := newTestView("principal-1", nil)
|
|
|
|
binding, err := compileSingleRequestBinding("virtual-public-model", preset, bindings, view)
|
|
if err != nil {
|
|
t.Fatalf("compilation failed: %v", err)
|
|
}
|
|
|
|
// The admitted binding must start with non-empty approved options.
|
|
if binding.Plan.Options["reasoning_effort"] != "high" || binding.Review.Options["reasoning_effort"] != "high" {
|
|
t.Fatalf("expected admitted high options, got plan=%v review=%v", binding.Plan.Options, binding.Review.Options)
|
|
}
|
|
|
|
// Simulate a config refresh: mutate the approved stage option values after
|
|
// compilation. The already admitted binding must not reflect the mutation.
|
|
preset.SingleRequest.Stages.Plan.Options["reasoning_effort"] = "low"
|
|
preset.SingleRequest.Stages.Review.Options["reasoning_effort"] = "low"
|
|
if binding.Plan.Options["reasoning_effort"] != "high" {
|
|
t.Errorf("binding Plan.Options reflected refresh mutation: %v", binding.Plan.Options["reasoning_effort"])
|
|
}
|
|
if binding.Review.Options["reasoning_effort"] != "high" {
|
|
t.Errorf("binding Review.Options reflected refresh mutation: %v", binding.Review.Options["reasoning_effort"])
|
|
}
|
|
|
|
// Simulate a catalog refresh: add a new entry to the bindings map. The
|
|
// compiled binding must still reference the original models.
|
|
bindings["extra-model"] = managedBinding("extra-model", "prov-2", "principal-1", "route-extra", true)
|
|
if binding.Plan.Model != "plan-model" || binding.Work.Model != "work-model" || binding.Review.Model != "review-model" {
|
|
t.Errorf("binding models changed after refresh: plan=%q work=%q review=%q",
|
|
binding.Plan.Model, binding.Work.Model, binding.Review.Model)
|
|
}
|
|
}
|
|
|
|
func TestSingleRequestPresetBindingPublicModelEcho(t *testing.T) {
|
|
preset := validSingleRequestPreset()
|
|
bindings := validSingleRequestBindings()
|
|
view := newTestView("principal-1", nil)
|
|
|
|
// Public model echo: the binding's PublicModel equals the requested virtual
|
|
// model ID, not the selector's route ID or canonical model group.
|
|
expectedPublicModels := []string{"virtual-gpt-combo", "my-cool-preset", "preset-alpha"}
|
|
for _, publicModel := range expectedPublicModels {
|
|
binding, err := compileSingleRequestBinding(publicModel, preset, bindings, view)
|
|
if err != nil {
|
|
t.Fatalf("compilation failed for %q: %v", publicModel, err)
|
|
}
|
|
if binding.PublicModel != publicModel {
|
|
t.Errorf("PublicModel=%q, want %q", binding.PublicModel, publicModel)
|
|
}
|
|
// The canonical stage models must never equal the public model.
|
|
if binding.Plan.Model == publicModel || binding.Work.Model == publicModel || binding.Review.Model == publicModel {
|
|
t.Errorf("stage model unexpectedly equals public model %q", publicModel)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestSingleRequestPresetBindingDefensiveCopies(t *testing.T) {
|
|
preset := validSingleRequestPreset()
|
|
bindings := validSingleRequestBindings()
|
|
view := newTestView("principal-1", nil)
|
|
|
|
binding, err := compileSingleRequestBinding("virtual-model", preset, bindings, view)
|
|
if err != nil {
|
|
t.Fatalf("compilation failed: %v", err)
|
|
}
|
|
|
|
// The admitted binding carries the approved non-empty plan/review options.
|
|
if binding.Plan.Options["reasoning_effort"] != "high" {
|
|
t.Fatalf("Plan.Options[reasoning_effort]=%v, want high", binding.Plan.Options["reasoning_effort"])
|
|
}
|
|
|
|
// Clone the binding and verify deep-copy isolation of both structural values
|
|
// and the stage option maps in both mutation directions.
|
|
clone := binding.Clone()
|
|
if clone == nil {
|
|
t.Fatal("Clone returned nil")
|
|
}
|
|
if clone.PublicModel != binding.PublicModel {
|
|
t.Errorf("clone PublicModel mismatch")
|
|
}
|
|
if clone.Plan.Model != binding.Plan.Model || clone.Work.Model != binding.Work.Model || clone.Review.Model != binding.Review.Model {
|
|
t.Errorf("clone stage model mismatch: %+v", clone)
|
|
}
|
|
|
|
// Mutating the clone's options must not affect the original.
|
|
clone.Plan.Options["reasoning_effort"] = "low"
|
|
if binding.Plan.Options["reasoning_effort"] != "high" {
|
|
t.Errorf("original Plan.Options mutated through clone: %v", binding.Plan.Options["reasoning_effort"])
|
|
}
|
|
|
|
// Mutating the original's options must not affect the clone.
|
|
binding.Review.Options["reasoning_effort"] = "medium"
|
|
if clone.Review.Options["reasoning_effort"] != "high" {
|
|
t.Errorf("clone Review.Options mutated through original: %v", clone.Review.Options["reasoning_effort"])
|
|
}
|
|
}
|
|
|
|
// ensure edgeservice import is used
|
|
var _ = edgeservice.SingleRequestBinding{}
|