iop/packages/go/agentpolicy/evaluator_test.go
toki 3c48879c44 feat(agent-runtime): 독립 Agent CLI 런타임 기반을 구현한다
독립 호스트에서 안전한 작업 실행과 복구를 제공하기 위해 런타임 설정, 정책, 상태 저장소, 워크스페이스 격리 및 AgentTask 오케스트레이션을 확장한다.
2026-07-29 13:12:21 +09:00

1200 lines
34 KiB
Go

package agentpolicy
import (
"bytes"
"context"
"encoding/json"
"errors"
"strings"
"testing"
"time"
"iop/packages/go/agentconfig"
)
// --- fixture helpers ---
func snapshotWithRules(rules []agentconfig.SelectionRule, defaultProfile string) agentconfig.RuntimeSnapshot {
global := `version: "1"
catalog:
version: "1"
providers:
- id: codex
command: codex
version_probe:
args: [--version]
authentication:
args: [login, status]
capabilities: [run]
models:
- id: gpt-4
provider: codex
target: gpt-4-native
- id: gpt-35
provider: codex
target: gpt-35-native
profiles:
- id: primary
provider: codex
model: gpt-4
capabilities: [run]
- id: backup
provider: codex
model: gpt-35
capabilities: [run]
- id: default-profile
provider: codex
model: gpt-4
capabilities: [run]
selection:
timezone: UTC
default:
profile: ` + defaultProfile + `
rules:
`
for _, rule := range rules {
global += yamlMarshalRule(rule)
}
local := `version: "1"
device:
state_root: "/tmp/state"
overlay_root: "/tmp/overlays"
log_root: "/tmp/logs"
override:
defaults:
default_profile: ` + defaultProfile + `
`
snapshot, err := agentconfig.LoadRuntimeConfigBytes([]byte(global), []byte(local))
if err != nil {
panic("fixture load error: " + err.Error())
}
return snapshot
}
func yamlMarshalRule(rule agentconfig.SelectionRule) string {
var sb strings.Builder
sb.WriteString(" - id: " + rule.ID + "\n")
hasMatch := rule.Match.TimeWindows != nil || rule.Match.QuotaStates != nil ||
rule.Match.MinRemainingToken != nil || rule.Match.Agents != nil ||
rule.Match.Stages != nil || rule.Match.Lanes != nil ||
rule.Match.MinGrade != 0 || rule.Match.MaxGrade != 0 ||
rule.Match.Capabilities != nil || rule.Match.FailureCodes != nil
if hasMatch {
sb.WriteString(" match:\n")
if rule.Match.TimeWindows != nil {
sb.WriteString(" time_windows:\n")
for _, w := range rule.Match.TimeWindows {
sb.WriteString(" - start: \"" + w.Start + "\"\n")
sb.WriteString(" end: \"" + w.End + "\"\n")
if w.Days != nil {
sb.WriteString(" days: [")
for i, d := range w.Days {
if i > 0 {
sb.WriteString(", ")
}
sb.WriteString(d)
}
sb.WriteString("]\n")
}
}
}
if rule.Match.QuotaStates != nil {
sb.WriteString(" quota_states: [")
for i, s := range rule.Match.QuotaStates {
if i > 0 {
sb.WriteString(", ")
}
sb.WriteString(s)
}
sb.WriteString("]\n")
}
if rule.Match.MinRemainingToken != nil {
sb.WriteString(" min_remaining_tokens: " + intToStr(int(*rule.Match.MinRemainingToken)) + "\n")
}
if rule.Match.Agents != nil {
sb.WriteString(" agents: [")
for i, s := range rule.Match.Agents {
if i > 0 {
sb.WriteString(", ")
}
sb.WriteString(s)
}
sb.WriteString("]\n")
}
if rule.Match.Stages != nil {
sb.WriteString(" stages: [")
for i, s := range rule.Match.Stages {
if i > 0 {
sb.WriteString(", ")
}
sb.WriteString(s)
}
sb.WriteString("]\n")
}
if rule.Match.Lanes != nil {
sb.WriteString(" lanes: [")
for i, s := range rule.Match.Lanes {
if i > 0 {
sb.WriteString(", ")
}
sb.WriteString(s)
}
sb.WriteString("]\n")
}
if rule.Match.MinGrade != 0 {
sb.WriteString(" min_grade: " + intToStr(rule.Match.MinGrade) + "\n")
}
if rule.Match.MaxGrade != 0 {
sb.WriteString(" max_grade: " + intToStr(rule.Match.MaxGrade) + "\n")
}
if rule.Match.Capabilities != nil {
sb.WriteString(" capabilities: [")
for i, s := range rule.Match.Capabilities {
if i > 0 {
sb.WriteString(", ")
}
sb.WriteString(s)
}
sb.WriteString("]\n")
}
if rule.Match.FailureCodes != nil {
sb.WriteString(" failure_codes: [")
for i, s := range rule.Match.FailureCodes {
if i > 0 {
sb.WriteString(", ")
}
sb.WriteString(s)
}
sb.WriteString("]\n")
}
}
sb.WriteString(" target:\n profile: " + rule.Target.Profile + "\n")
return sb.String()
}
func intToStr(i int) string {
if i == 0 {
return "0"
}
neg := i < 0
if neg {
i = -i
}
var digits []byte
for i > 0 {
digits = append([]byte{byte('0' + i%10)}, digits...)
i /= 10
}
if neg {
digits = append([]byte{'-'}, digits...)
}
return string(digits)
}
func ruleWithID(id, profile string, match agentconfig.SelectionMatch) agentconfig.SelectionRule {
return agentconfig.SelectionRule{ID: id, Match: match, Target: agentconfig.TargetRef{Profile: profile}}
}
// --- tests ---
func TestEvaluatorFirstMatchWins(t *testing.T) {
rules := []agentconfig.SelectionRule{
ruleWithID("rule-a", "primary", agentconfig.SelectionMatch{
Stages: []string{"worker"},
}),
ruleWithID("rule-b", "backup", agentconfig.SelectionMatch{
Stages: []string{"worker"},
}),
}
snapshot := snapshotWithRules(rules, "default-profile")
evaluator := NewEvaluator()
decision, err := evaluator.SelectPolicy(context.Background(), snapshot, SelectionContext{
Now: time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC),
Stage: "worker",
})
if err != nil {
t.Fatalf("SelectPolicy: %v", err)
}
if decision.ProfileID != "primary" {
t.Fatalf("profile = %q, want %q (first match)", decision.ProfileID, "primary")
}
if decision.SelectedRuleID != "rule-a" {
t.Fatalf("selected rule = %q, want %q", decision.SelectedRuleID, "rule-a")
}
if len(decision.Candidates) != 3 {
t.Fatalf("candidates = %v, want rule-a, rule-b, default", decision.Candidates)
}
if !decision.Candidates[0].Used ||
decision.Candidates[1].Evaluated ||
decision.Candidates[2].Evaluated {
t.Fatalf("ordered candidate evidence = %#v", decision.Candidates)
}
}
func TestEvaluatorOverlappingRulesFirstMatchWins(t *testing.T) {
rules := []agentconfig.SelectionRule{
ruleWithID("early", "primary", agentconfig.SelectionMatch{
MinGrade: 1,
}),
ruleWithID("late", "backup", agentconfig.SelectionMatch{
MinGrade: 5,
}),
}
snapshot := snapshotWithRules(rules, "default-profile")
evaluator := NewEvaluator()
// Grade 7 matches both rules; the first one should win.
decision, err := evaluator.SelectPolicy(context.Background(), snapshot, SelectionContext{
Now: time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC),
Grade: 7,
})
if err != nil {
t.Fatalf("SelectPolicy: %v", err)
}
if decision.SelectedRuleID != "early" {
t.Fatalf("selected rule = %q, want %q", decision.SelectedRuleID, "early")
}
if decision.ProfileID != "primary" {
t.Fatalf("profile = %q, want %q", decision.ProfileID, "primary")
}
if len(decision.Candidates) != 3 ||
decision.Candidates[1].Reason != CandidateReasonNotEvaluatedAfterFirstHit ||
decision.Candidates[2].Reason != CandidateReasonNotEvaluatedAfterFirstHit {
t.Fatalf("ordered candidate evidence = %#v", decision.Candidates)
}
}
func TestEvaluatorRejectsNonMatchingFirstRule(t *testing.T) {
rules := []agentconfig.SelectionRule{
ruleWithID("rule-a", "primary", agentconfig.SelectionMatch{
Stages: []string{"selfcheck"},
}),
ruleWithID("rule-b", "backup", agentconfig.SelectionMatch{
Stages: []string{"worker"},
}),
}
snapshot := snapshotWithRules(rules, "default-profile")
evaluator := NewEvaluator()
decision, err := evaluator.SelectPolicy(context.Background(), snapshot, SelectionContext{
Now: time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC),
Stage: "worker",
})
if err != nil {
t.Fatalf("SelectPolicy: %v", err)
}
if decision.SelectedRuleID != "rule-b" {
t.Fatalf("selected rule = %q, want %q", decision.SelectedRuleID, "rule-b")
}
if len(decision.Candidates) != 3 ||
!decision.Candidates[0].Evaluated ||
decision.Candidates[0].Eligible ||
decision.Candidates[0].RuleID != "rule-a" ||
!decision.Candidates[1].Used {
t.Fatalf("ordered candidate evidence = %#v", decision.Candidates)
}
}
func TestEvaluatorBoundaryTimeWindow(t *testing.T) {
rules := []agentconfig.SelectionRule{
ruleWithID("windowed", "primary", agentconfig.SelectionMatch{
TimeWindows: []agentconfig.SelectionTimeWindow{
{Start: "09:00", End: "17:00"},
},
}),
}
// No default target: when the time window doesn't match, ErrNoMatch is
// returned instead of silently falling back.
snapshot := snapshotWithRules(rules, "")
evaluator := NewEvaluator()
tests := []struct {
name string
now time.Time
wantMatch bool
}{
{"before window", time.Date(2026, 7, 28, 8, 59, 0, 0, time.UTC), false},
{"at start boundary", time.Date(2026, 7, 28, 9, 0, 0, 0, time.UTC), true},
{"middle of window", time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC), true},
{"at end boundary", time.Date(2026, 7, 28, 17, 0, 0, 0, time.UTC), true},
{"after window", time.Date(2026, 7, 28, 17, 1, 0, 0, time.UTC), false},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
decision, err := evaluator.SelectPolicy(context.Background(), snapshot, SelectionContext{
Now: test.now,
})
if test.wantMatch {
if err != nil {
t.Fatalf("SelectPolicy: %v", err)
}
if decision.SelectedRuleID != "windowed" {
t.Fatalf("selected rule = %q, want %q", decision.SelectedRuleID, "windowed")
}
} else {
if !errors.Is(err, ErrNoMatch) {
t.Fatalf("SelectPolicy error = %v, want ErrNoMatch", err)
}
}
})
}
}
func TestEvaluatorGradeStageMatch(t *testing.T) {
rules := []agentconfig.SelectionRule{
ruleWithID("graded", "primary", agentconfig.SelectionMatch{
MinGrade: 3,
MaxGrade: 7,
Stages: []string{"worker"},
}),
}
snapshot := snapshotWithRules(rules, "")
evaluator := NewEvaluator()
tests := []struct {
name string
grade int
stage string
want bool
}{
{"grade below min", 2, "worker", false},
{"grade at min", 3, "worker", true},
{"grade in range", 5, "worker", true},
{"grade at max", 7, "worker", true},
{"grade above max", 8, "worker", false},
{"wrong stage", 5, "selfcheck", false},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
decision, err := evaluator.SelectPolicy(context.Background(), snapshot, SelectionContext{
Now: time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC),
Grade: test.grade,
Stage: test.stage,
})
if test.want {
if err != nil {
t.Fatalf("SelectPolicy: %v", err)
}
if decision.SelectedRuleID != "graded" {
t.Fatalf("selected rule = %q, want %q", decision.SelectedRuleID, "graded")
}
} else {
if !errors.Is(err, ErrNoMatch) {
t.Fatalf("SelectPolicy error = %v, want ErrNoMatch", err)
}
}
})
}
}
func TestEvaluatorNoMatchBlocker(t *testing.T) {
rules := []agentconfig.SelectionRule{
ruleWithID("unmatched", "primary", agentconfig.SelectionMatch{
Stages: []string{"worker"},
}),
}
snapshot := snapshotWithRules(rules, "")
evaluator := NewEvaluator()
_, err := evaluator.SelectPolicy(context.Background(), snapshot, SelectionContext{
Now: time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC),
Stage: "selfcheck",
})
if !errors.Is(err, ErrNoMatch) {
t.Fatalf("SelectPolicy error = %v, want ErrNoMatch", err)
}
}
func TestEvaluatorDefaultTargetWhenNoRuleMatches(t *testing.T) {
rules := []agentconfig.SelectionRule{
ruleWithID("unmatched", "primary", agentconfig.SelectionMatch{
Stages: []string{"worker"},
}),
}
snapshot := snapshotWithRules(rules, "default-profile")
evaluator := NewEvaluator()
decision, err := evaluator.SelectPolicy(context.Background(), snapshot, SelectionContext{
Now: time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC),
Stage: "selfcheck",
})
if err != nil {
t.Fatalf("SelectPolicy: %v", err)
}
if decision.ProfileID != "default-profile" {
t.Fatalf("profile = %q, want %q", decision.ProfileID, "default-profile")
}
if decision.SelectedRuleID != "" {
t.Fatalf("selected rule = %q, want empty (default)", decision.SelectedRuleID)
}
if len(decision.Candidates) != 2 ||
decision.Candidates[0].RuleID != "unmatched" ||
decision.Candidates[0].Eligible ||
!decision.Candidates[1].Used ||
decision.Candidates[1].Source != CandidateSourceDefault {
t.Fatalf("ordered candidate evidence = %#v", decision.Candidates)
}
}
func TestEvaluatorCapabilityMatch(t *testing.T) {
rules := []agentconfig.SelectionRule{
ruleWithID("capable", "primary", agentconfig.SelectionMatch{
Capabilities: []string{"run", "cancel"},
}),
}
snapshot := snapshotWithRules(rules, "")
evaluator := NewEvaluator()
_, err := evaluator.SelectPolicy(context.Background(), snapshot, SelectionContext{
Now: time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC),
Capabilities: []string{"run"},
})
if !errors.Is(err, ErrNoMatch) {
t.Fatalf("SelectPolicy error = %v, want ErrNoMatch (missing cancel)", err)
}
decision, err := evaluator.SelectPolicy(context.Background(), snapshot, SelectionContext{
Now: time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC),
Capabilities: []string{"run", "cancel"},
})
if err != nil {
t.Fatalf("SelectPolicy: %v", err)
}
if decision.SelectedRuleID != "capable" {
t.Fatalf("selected rule = %q, want %q", decision.SelectedRuleID, "capable")
}
}
func TestEvaluatorTimezoneApplied(t *testing.T) {
// Override the timezone to Asia/Seoul (UTC+9).
global := `version: "1"
catalog:
version: "1"
providers:
- id: codex
command: codex
version_probe:
args: [--version]
authentication:
args: [login, status]
capabilities: [run]
models:
- id: gpt-4
provider: codex
target: gpt-4-native
profiles:
- id: primary
provider: codex
model: gpt-4
capabilities: [run]
- id: default-profile
provider: codex
model: gpt-4
capabilities: [run]
selection:
timezone: Asia/Seoul
rules:
- id: tz-window
match:
time_windows:
- start: "09:00"
end: "17:00"
days: [monday]
target:
profile: primary
`
local := `version: "1"
device:
state_root: "/tmp/state"
overlay_root: "/tmp/overlays"
log_root: "/tmp/logs"
`
snapshot, err := agentconfig.LoadRuntimeConfigBytes([]byte(global), []byte(local))
if err != nil {
t.Fatalf("LoadRuntimeConfigBytes: %v", err)
}
evaluator := NewEvaluator()
// 2026-07-28 is a Tuesday in UTC. In Seoul (UTC+9), it's still Tuesday.
// So the Monday window should NOT match.
_, err = evaluator.SelectPolicy(context.Background(), snapshot, SelectionContext{
Now: time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC),
})
if !errors.Is(err, ErrNoMatch) {
t.Fatalf("SelectPolicy error = %v, want ErrNoMatch (Tuesday, not Monday)", err)
}
// 2026-07-27 is a Monday. 05:00 UTC = 14:00 Seoul, within 09:00-17:00.
decision, err := evaluator.SelectPolicy(context.Background(), snapshot, SelectionContext{
Now: time.Date(2026, 7, 27, 5, 0, 0, 0, time.UTC),
})
if err != nil {
t.Fatalf("SelectPolicy: %v", err)
}
if decision.SelectedRuleID != "tz-window" {
t.Fatalf("selected rule = %q, want %q", decision.SelectedRuleID, "tz-window")
}
}
func TestEncodeDecodeDecisionRoundTrip(t *testing.T) {
_, original, data := durableDecisionFixture(t)
decoded, err := DecodeDecision(data, decisionExpectation(original))
if err != nil {
t.Fatalf("DecodeDecision: %v", err)
}
encodedAgain, err := EncodeDecision(decoded)
if err != nil {
t.Fatalf("EncodeDecision(decoded): %v", err)
}
if !bytes.Equal(encodedAgain, data) {
t.Fatalf("round trip changed canonical bytes\nfirst: %s\nsecond: %s", data, encodedAgain)
}
}
func TestDecodeDecisionRejectsCorruptJSON(t *testing.T) {
_, decision, _ := durableDecisionFixture(t)
_, err := DecodeDecision([]byte("{not valid json"), decisionExpectation(decision))
if !errors.Is(err, ErrCorruptDecision) {
t.Fatalf("DecodeDecision error = %v, want ErrCorruptDecision", err)
}
}
func TestDecodeDecisionRejectsWrongVersion(t *testing.T) {
_, decision, data := durableDecisionFixture(t)
data = bytes.Replace(data, []byte(`"version":"1"`), []byte(`"version":"9"`), 1)
_, err := DecodeDecision(data, decisionExpectation(decision))
if !errors.Is(err, ErrCorruptDecision) {
t.Fatalf("DecodeDecision error = %v, want ErrCorruptDecision", err)
}
}
func TestDecodeDecisionRejectsRevisionMismatch(t *testing.T) {
_, decision, data := durableDecisionFixture(t)
_, err := DecodeDecision(data, DecisionExpectation{
ConfigRevision: "sha256:foreign",
SelectionRevision: decision.SelectionRevision,
})
if !errors.Is(err, ErrRevisionMismatch) {
t.Fatalf("DecodeDecision error = %v, want ErrRevisionMismatch", err)
}
_, err = DecodeDecision(data, DecisionExpectation{
ConfigRevision: decision.ConfigRevision,
SelectionRevision: "sha256:foreign",
})
if !errors.Is(err, ErrRevisionMismatch) {
t.Fatalf("DecodeDecision selection error = %v, want ErrRevisionMismatch", err)
}
}
func TestDecodeDecisionRequiresCompleteExpectation(t *testing.T) {
_, _, data := durableDecisionFixture(t)
for _, expected := range []DecisionExpectation{
{},
{ConfigRevision: "sha256:config"},
{SelectionRevision: "sha256:selection"},
} {
if _, err := DecodeDecision(data, expected); !errors.Is(err, ErrRevisionMismatch) {
t.Fatalf("DecodeDecision(%#v) error = %v, want ErrRevisionMismatch", expected, err)
}
}
}
func TestDecodeDecisionRejectsMutationMatrix(t *testing.T) {
_, decision, data := durableDecisionFixture(t)
expected := decisionExpectation(decision)
tests := []struct {
name string
mutate func(*decisionEnvelope, map[string]any)
reseal bool
}{
{
name: "envelope config revision",
mutate: func(envelope *decisionEnvelope, _ map[string]any) {
envelope.ConfigRevision = "sha256:tampered"
},
},
{
name: "payload config revision",
mutate: func(_ *decisionEnvelope, payload map[string]any) {
payload["config_revision"] = "sha256:tampered"
},
},
{
name: "payload selection revision",
mutate: func(_ *decisionEnvelope, payload map[string]any) {
payload["selection_revision"] = "sha256:tampered"
},
},
{
name: "provider identity",
mutate: func(_ *decisionEnvelope, payload map[string]any) {
payload["provider_id"] = "tampered"
},
},
{
name: "model identity",
mutate: func(_ *decisionEnvelope, payload map[string]any) {
payload["model_id"] = "tampered"
},
},
{
name: "profile identity",
mutate: func(_ *decisionEnvelope, payload map[string]any) {
payload["profile_id"] = "tampered"
},
},
{
name: "profile revision",
mutate: func(_ *decisionEnvelope, payload map[string]any) {
payload["profile_revision"] = "sha256:tampered"
},
},
{
name: "candidate history",
mutate: func(_ *decisionEnvelope, payload map[string]any) {
candidates := payload["candidates"].([]any)
candidates[0].(map[string]any)["reason"] = "tampered"
},
},
{
name: "used route history",
mutate: func(_ *decisionEnvelope, payload map[string]any) {
history := payload["history"].(map[string]any)
used := history["used_routes"].([]any)
used[0].(map[string]any)["provider_id"] = "tampered"
},
},
{
name: "decision history identity",
mutate: func(_ *decisionEnvelope, payload map[string]any) {
history := payload["history"].(map[string]any)
history["decision_id"] = "sha256:tampered"
},
},
{
name: "integrity value",
mutate: func(envelope *decisionEnvelope, _ map[string]any) {
envelope.Integrity = "sha256:tampered"
},
},
{
name: "unknown payload field with matching seal",
mutate: func(_ *decisionEnvelope, payload map[string]any) {
payload["unknown"] = true
},
reseal: true,
},
{
name: "envelope payload disagreement with matching seal",
mutate: func(envelope *decisionEnvelope, _ map[string]any) {
envelope.ConfigRevision = "sha256:tampered"
},
reseal: true,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
tampered := mutateDecision(t, data, test.mutate, test.reseal)
if _, err := DecodeDecision(tampered, expected); !errors.Is(err, ErrCorruptDecision) {
t.Fatalf("DecodeDecision error = %v, want ErrCorruptDecision", err)
}
})
}
unknownEnvelope := bytes.Replace(
data,
[]byte(`{"version"`),
[]byte(`{"unknown":true,"version"`),
1,
)
if _, err := DecodeDecision(unknownEnvelope, expected); !errors.Is(err, ErrCorruptDecision) {
t.Fatalf("unknown envelope error = %v, want ErrCorruptDecision", err)
}
}
func TestEvaluatorUsesProjectSelectionOverride(t *testing.T) {
snapshot := projectPolicySnapshot(t)
project, ok := snapshot.Project("project-a")
if !ok {
t.Fatal("project-a missing from fixture")
}
decision, err := NewEvaluator().SelectPolicy(
context.Background(),
snapshot,
SelectionContext{
ProjectID: "project-a",
Now: time.Date(2026, 7, 29, 12, 0, 0, 0, time.UTC),
Stage: "worker",
},
)
if err != nil {
t.Fatalf("SelectPolicy: %v", err)
}
if decision.ProfileID != "backup" || decision.SelectedRuleID != "project-rule" {
t.Fatalf("project decision = %#v", decision)
}
if decision.ConfigRevision != snapshot.Revision() ||
decision.SelectionRevision != project.Selection.Revision ||
decision.ProfileRevision == "" {
t.Fatalf("project revisions = %#v", decision)
}
data, err := EncodeDecision(decision)
if err != nil {
t.Fatalf("EncodeDecision: %v", err)
}
resumed, err := NewEvaluator().ResumePolicy(
context.Background(),
snapshot,
SelectionContext{ProjectID: "project-a"},
data,
)
if err != nil {
t.Fatalf("ResumePolicy: %v", err)
}
if resumed.History.DecisionID != decision.History.DecisionID {
t.Fatal("project resume changed the pinned decision")
}
}
func TestEvaluatorRejectsUnknownProject(t *testing.T) {
_, err := NewEvaluator().SelectPolicy(
context.Background(),
projectPolicySnapshot(t),
SelectionContext{ProjectID: "missing-project", Now: time.Now().UTC()},
)
if !errors.Is(err, ErrUnknownProject) {
t.Fatalf("SelectPolicy error = %v, want ErrUnknownProject", err)
}
}
func TestEvaluatorRejectsMissingCurrentTime(t *testing.T) {
_, err := NewEvaluator().SelectPolicy(
context.Background(),
snapshotWithRules(nil, "primary"),
SelectionContext{},
)
if !errors.Is(err, ErrInvalidSelectionContext) {
t.Fatalf("SelectPolicy error = %v, want ErrInvalidSelectionContext", err)
}
}
func TestEvaluatorRejectsUnresolvedProfile(t *testing.T) {
global := `version: "1"
selection:
default:
profile: missing-profile
`
local := `version: "1"
device:
state_root: "/tmp/state"
overlay_root: "/tmp/overlays"
log_root: "/tmp/logs"
`
snapshot, err := agentconfig.LoadRuntimeConfigBytes([]byte(global), []byte(local))
if err != nil {
t.Fatalf("LoadRuntimeConfigBytes: %v", err)
}
_, err = NewEvaluator().SelectPolicy(
context.Background(),
snapshot,
SelectionContext{Now: time.Date(2026, 7, 29, 12, 0, 0, 0, time.UTC)},
)
if !errors.Is(err, ErrUnknownProfile) {
t.Fatalf("SelectPolicy error = %v, want ErrUnknownProfile", err)
}
}
func TestEvaluatorRejectsDeclaredTargetIdentityMismatch(t *testing.T) {
config := snapshotWithRules(nil, "primary").Config()
_, err := resolveTarget(config, agentconfig.TargetRef{
Provider: "other",
Model: "gpt-4",
Profile: "primary",
})
if !errors.Is(err, ErrTargetIdentityMismatch) {
t.Fatalf("resolveTarget error = %v, want ErrTargetIdentityMismatch", err)
}
}
func TestEvaluatorQuotaAndMinimumTokenOrdering(t *testing.T) {
minimum := int64(100)
rules := []agentconfig.SelectionRule{
ruleWithID("quota-with-budget", "primary", agentconfig.SelectionMatch{
QuotaStates: []string{"available"},
MinRemainingToken: &minimum,
}),
ruleWithID("quota-fallback", "backup", agentconfig.SelectionMatch{
QuotaStates: []string{"available"},
}),
}
snapshot := snapshotWithRules(rules, "default-profile")
for _, test := range []struct {
name string
remaining int64
wantRule string
}{
{name: "at minimum first match", remaining: 100, wantRule: "quota-with-budget"},
{name: "below minimum fallback", remaining: 99, wantRule: "quota-fallback"},
} {
t.Run(test.name, func(t *testing.T) {
decision, err := NewEvaluator().SelectPolicy(
context.Background(),
snapshot,
SelectionContext{
Now: time.Date(2026, 7, 29, 12, 0, 0, 0, time.UTC),
QuotaState: "available",
RemainingTokens: test.remaining,
},
)
if err != nil {
t.Fatalf("SelectPolicy: %v", err)
}
if decision.SelectedRuleID != test.wantRule {
t.Fatalf("selected rule = %q, want %q", decision.SelectedRuleID, test.wantRule)
}
})
}
}
func TestEvaluatorOrderedCandidateEvidence(t *testing.T) {
rules := []agentconfig.SelectionRule{
ruleWithID("rejected", "primary", agentconfig.SelectionMatch{
Stages: []string{"selfcheck"},
}),
ruleWithID("selected", "backup", agentconfig.SelectionMatch{
Stages: []string{"worker"},
}),
ruleWithID("later", "primary", agentconfig.SelectionMatch{
Stages: []string{"worker"},
}),
}
decision, err := NewEvaluator().SelectPolicy(
context.Background(),
snapshotWithRules(rules, "default-profile"),
SelectionContext{
Now: time.Date(2026, 7, 29, 12, 0, 0, 0, time.UTC),
Stage: "worker",
},
)
if err != nil {
t.Fatalf("SelectPolicy: %v", err)
}
if len(decision.Candidates) != 4 {
t.Fatalf("candidate count = %d, want 4", len(decision.Candidates))
}
rejected, selected, later, fallback := decision.Candidates[0],
decision.Candidates[1], decision.Candidates[2], decision.Candidates[3]
if !rejected.Evaluated || rejected.Eligible || rejected.Used ||
rejected.Reason != "stage not matched" {
t.Fatalf("rejected candidate = %#v", rejected)
}
if !selected.Evaluated || !selected.Eligible || !selected.Used ||
selected.Reason != CandidateReasonMatched {
t.Fatalf("selected candidate = %#v", selected)
}
for _, candidate := range []CandidateEvaluation{later, fallback} {
if candidate.Evaluated || candidate.Eligible || candidate.Used ||
candidate.Reason != CandidateReasonNotEvaluatedAfterFirstHit {
t.Fatalf("unevaluated candidate = %#v", candidate)
}
}
if decision.SelectedReason != CandidateReasonMatched ||
len(decision.History.UsedRoutes) != 1 ||
decision.History.UsedRoutes[0].ProfileID != decision.ProfileID {
t.Fatalf("decision history = %#v", decision)
}
}
func TestResumePolicyReturnsPinnedDecisionWithoutReselection(t *testing.T) {
snapshot, decision, data := durableDecisionFixture(t)
resumed, err := NewEvaluator().ResumePolicy(
context.Background(),
snapshot,
SelectionContext{
Now: time.Date(2030, 1, 1, 0, 0, 0, 0, time.UTC),
Stage: "selfcheck",
},
data,
)
if err != nil {
t.Fatalf("ResumePolicy: %v", err)
}
resumedData, err := EncodeDecision(resumed)
if err != nil {
t.Fatalf("EncodeDecision(resumed): %v", err)
}
if !bytes.Equal(resumedData, data) ||
resumed.History.DecisionID != decision.History.DecisionID {
t.Fatal("resume changed the pinned route decision")
}
changedSnapshot := snapshotWithRules([]agentconfig.SelectionRule{
ruleWithID("replacement", "backup", agentconfig.SelectionMatch{
Stages: []string{"selfcheck"},
}),
}, "backup")
_, err = NewEvaluator().ResumePolicy(
context.Background(),
changedSnapshot,
SelectionContext{Stage: "selfcheck"},
data,
)
if !errors.Is(err, ErrRevisionMismatch) {
t.Fatalf("ResumePolicy changed snapshot error = %v, want ErrRevisionMismatch", err)
}
}
func TestResumePolicyRejectsCorruptOrForeignTarget(t *testing.T) {
snapshot, decision, data := durableDecisionFixture(t)
corrupt := bytes.Replace(data, []byte(decision.ProviderID), []byte("tampered"), 1)
if _, err := NewEvaluator().ResumePolicy(
context.Background(),
snapshot,
SelectionContext{},
corrupt,
); !errors.Is(err, ErrCorruptDecision) {
t.Fatalf("corrupt ResumePolicy error = %v, want ErrCorruptDecision", err)
}
foreign := decision
foreign.ProviderID = "foreign"
for index := range foreign.Candidates {
if foreign.Candidates[index].Used {
foreign.Candidates[index].ProviderID = "foreign"
}
}
foreign.History.UsedRoutes[len(foreign.History.UsedRoutes)-1].ProviderID = "foreign"
foreignData, err := EncodeDecision(foreign)
if err != nil {
t.Fatalf("EncodeDecision(foreign): %v", err)
}
if _, err := NewEvaluator().ResumePolicy(
context.Background(),
snapshot,
SelectionContext{},
foreignData,
); !errors.Is(err, ErrCorruptDecision) {
t.Fatalf("foreign ResumePolicy error = %v, want ErrCorruptDecision", err)
}
}
func TestResumePolicyRejectsResealedKnownTargetMutation(t *testing.T) {
snapshot, decision, _ := durableDecisionFixture(t)
replacement, err := resolveTarget(
snapshot.Config(),
agentconfig.TargetRef{Profile: "backup"},
)
if err != nil {
t.Fatalf("resolveTarget(backup): %v", err)
}
mutated := decision
mutated.Candidates = append([]CandidateEvaluation(nil), decision.Candidates...)
mutated.History.UsedRoutes = append([]UsedRoute(nil), decision.History.UsedRoutes...)
mutated.ProviderID = replacement.providerID
mutated.ModelID = replacement.modelID
mutated.ProfileID = replacement.profileID
mutated.ProfileRevision = replacement.profileRevision
for index := range mutated.Candidates {
if mutated.Candidates[index].Used {
mutated.Candidates[index].ProviderID = replacement.providerID
mutated.Candidates[index].ModelID = replacement.modelID
mutated.Candidates[index].ProfileID = replacement.profileID
mutated.Candidates[index].ProfileRevision = replacement.profileRevision
}
}
used := &mutated.History.UsedRoutes[len(mutated.History.UsedRoutes)-1]
used.ProviderID = replacement.providerID
used.ModelID = replacement.modelID
used.ProfileID = replacement.profileID
used.ProfileRevision = replacement.profileRevision
data, err := EncodeDecision(mutated)
if err != nil {
t.Fatalf("EncodeDecision(mutated): %v", err)
}
if _, err := NewEvaluator().ResumePolicy(
context.Background(),
snapshot,
SelectionContext{},
data,
); !errors.Is(err, ErrCorruptDecision) {
t.Fatalf("ResumePolicy error = %v, want ErrCorruptDecision", err)
}
}
func durableDecisionFixture(
t *testing.T,
) (agentconfig.RuntimeSnapshot, RouteDecision, []byte) {
t.Helper()
rules := []agentconfig.SelectionRule{
ruleWithID("rejected", "backup", agentconfig.SelectionMatch{
Stages: []string{"selfcheck"},
}),
ruleWithID("selected", "primary", agentconfig.SelectionMatch{
Stages: []string{"worker"},
}),
ruleWithID("later", "backup", agentconfig.SelectionMatch{
Stages: []string{"worker"},
}),
}
snapshot := snapshotWithRules(rules, "default-profile")
decision, err := NewEvaluator().SelectPolicy(
context.Background(),
snapshot,
SelectionContext{
Now: time.Date(2026, 7, 29, 12, 0, 0, 0, time.UTC),
Stage: "worker",
},
)
if err != nil {
t.Fatalf("SelectPolicy: %v", err)
}
data, err := EncodeDecision(decision)
if err != nil {
t.Fatalf("EncodeDecision: %v", err)
}
return snapshot, decision, data
}
func decisionExpectation(decision RouteDecision) DecisionExpectation {
return DecisionExpectation{
ConfigRevision: decision.ConfigRevision,
SelectionRevision: decision.SelectionRevision,
}
}
func mutateDecision(
t *testing.T,
data []byte,
mutate func(*decisionEnvelope, map[string]any),
reseal bool,
) []byte {
t.Helper()
var envelope decisionEnvelope
if err := json.Unmarshal(data, &envelope); err != nil {
t.Fatalf("json.Unmarshal(envelope): %v", err)
}
var payload map[string]any
if err := json.Unmarshal(envelope.Decision, &payload); err != nil {
t.Fatalf("json.Unmarshal(decision): %v", err)
}
mutate(&envelope, payload)
decisionData, err := json.Marshal(payload)
if err != nil {
t.Fatalf("json.Marshal(decision): %v", err)
}
envelope.Decision = decisionData
if reseal {
envelope.Integrity = decisionIntegrity(
envelope.Version,
envelope.ConfigRevision,
envelope.SelectionRevision,
envelope.Decision,
)
}
out, err := json.Marshal(envelope)
if err != nil {
t.Fatalf("json.Marshal(envelope): %v", err)
}
return out
}
func projectPolicySnapshot(t *testing.T) agentconfig.RuntimeSnapshot {
t.Helper()
global := `version: "1"
catalog:
version: "1"
providers:
- id: codex
command: codex
version_probe:
args: [--version]
authentication:
args: [login, status]
capabilities: [run]
models:
- id: gpt-4
provider: codex
target: gpt-4-native
- id: gpt-35
provider: codex
target: gpt-35-native
profiles:
- id: primary
provider: codex
model: gpt-4
capabilities: [run]
- id: backup
provider: codex
model: gpt-35
capabilities: [run]
selection:
timezone: UTC
default:
profile: primary
rules:
- id: global-rule
match:
stages: [worker]
target:
profile: primary
`
local := `version: "1"
device:
state_root: "/tmp/state"
overlay_root: "/tmp/overlays"
log_root: "/tmp/logs"
projects:
project-a:
workspace: "/tmp/project-a"
override:
selection:
rules:
- id: project-rule
match:
stages: [worker]
target:
profile: backup
`
snapshot, err := agentconfig.LoadRuntimeConfigBytes([]byte(global), []byte(local))
if err != nil {
t.Fatalf("LoadRuntimeConfigBytes: %v", err)
}
return snapshot
}
// --- integration: evaluator satisfies agenttask.PolicySelector ---
func TestEvaluatorSatisfiesPolicySelectorShape(t *testing.T) {
// This test verifies that *Evaluator has the SelectPolicy method
// signature expected by agenttask.PolicySelector. We check the method
// exists and returns the right types by calling it through a local
// interface that mirrors agenttask.PolicySelector.
type policySelector interface {
SelectPolicy(
context.Context,
agentconfig.RuntimeSnapshot,
SelectionContext,
) (RouteDecision, error)
}
var selector policySelector = NewEvaluator()
rules := []agentconfig.SelectionRule{
ruleWithID("direct", "primary", agentconfig.SelectionMatch{
Stages: []string{"worker"},
}),
}
snapshot := snapshotWithRules(rules, "default-profile")
decision, err := selector.SelectPolicy(context.Background(), snapshot, SelectionContext{
Now: time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC),
Stage: "worker",
})
if err != nil {
t.Fatalf("SelectPolicy: %v", err)
}
if decision.ProfileID != "primary" {
t.Fatalf("profile = %q, want %q", decision.ProfileID, "primary")
}
}