toki
3c48879c44
독립 호스트에서 안전한 작업 실행과 복구를 제공하기 위해 런타임 설정, 정책, 상태 저장소, 워크스페이스 격리 및 AgentTask 오케스트레이션을 확장한다.
559 lines
17 KiB
Go
559 lines
17 KiB
Go
package agentpolicy
|
|
|
|
import (
|
|
"encoding/json"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"iop/packages/go/agentprovider/cli/status"
|
|
"iop/packages/go/agentruntime"
|
|
)
|
|
|
|
var policyTestNow = time.Date(2026, 7, 29, 12, 0, 0, 0, time.UTC)
|
|
|
|
func TestNormalizeAttemptObservationExcludesFailureDiagnostics(t *testing.T) {
|
|
observation := NormalizeAttemptObservation(
|
|
policyQuotaSnapshot("quota-safe", QuotaStateAvailable, policyTestNow),
|
|
&agentruntime.Failure{
|
|
Code: agentruntime.FailureCodeUnavailable,
|
|
Message: "token=provider-secret",
|
|
Retryable: true,
|
|
Metadata: map[string]string{
|
|
"authorization": "Bearer provider-secret",
|
|
},
|
|
},
|
|
policyTestNow,
|
|
time.Minute,
|
|
)
|
|
encoded, err := json.Marshal(observation)
|
|
if err != nil {
|
|
t.Fatalf("Marshal observation: %v", err)
|
|
}
|
|
if strings.Contains(string(encoded), "provider-secret") ||
|
|
strings.Contains(string(encoded), "authorization") {
|
|
t.Fatalf("observation retained a provider diagnostic: %s", encoded)
|
|
}
|
|
if observation.Failure != (FailureObservation{
|
|
Code: agentruntime.FailureCodeUnavailable, Retryable: true,
|
|
}) {
|
|
t.Fatalf("failure observation = %#v", observation.Failure)
|
|
}
|
|
}
|
|
func TestNormalizeQuotaObservationMarksUnsupportedSchemaAndMalformedIdentityCorrupt(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
mutate func(*status.QuotaSnapshot)
|
|
}{
|
|
{
|
|
name: "unsupported schema",
|
|
mutate: func(snapshot *status.QuotaSnapshot) {
|
|
snapshot.SchemaVersion = "2.0"
|
|
},
|
|
},
|
|
{
|
|
name: "whitespace snapshot identity",
|
|
mutate: func(snapshot *status.QuotaSnapshot) {
|
|
snapshot.SnapshotID = " quota-safe "
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
snapshot := policyQuotaSnapshot("quota-safe", QuotaStateAvailable, policyTestNow)
|
|
test.mutate(&snapshot)
|
|
observation := NormalizeQuotaObservation(snapshot, policyTestNow, time.Minute)
|
|
if observation.Validity != ObservationCorrupt {
|
|
t.Fatalf("observation = %#v", observation)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestNormalizeQuotaObservationSanitizesCorruptEvidence(t *testing.T) {
|
|
snapshot := policyQuotaSnapshot("ignored", QuotaStateAvailable, policyTestNow)
|
|
snapshot.SnapshotID = "token=provider-secret\n"
|
|
snapshot.ReasonCodes = []string{"authorization=Bearer provider-secret"}
|
|
|
|
observation := NormalizeQuotaObservation(snapshot, policyTestNow, time.Minute)
|
|
if !validQuotaObservation(observation) ||
|
|
observation.Validity != ObservationCorrupt ||
|
|
observation.SnapshotID != "" ||
|
|
len(observation.Reasons) != 0 {
|
|
t.Fatalf("observation = %#v, want canonical corrupt evidence", observation)
|
|
}
|
|
|
|
untrusted := policyObservation(
|
|
"ignored",
|
|
QuotaStateAvailable,
|
|
agentruntime.FailureCodeUnavailable,
|
|
true,
|
|
)
|
|
untrusted.Quota.Reasons = []string{"token=provider-secret"}
|
|
sanitized := SanitizeAttemptObservation(untrusted, policyTestNow)
|
|
encoded, err := json.Marshal(sanitized)
|
|
if err != nil {
|
|
t.Fatalf("Marshal sanitized observation: %v", err)
|
|
}
|
|
if sanitized.Quota.Validity != ObservationCorrupt ||
|
|
strings.Contains(string(encoded), "provider-secret") {
|
|
t.Fatalf("sanitized observation = %s", encoded)
|
|
}
|
|
if !ValidateAttemptObservation(sanitized) {
|
|
t.Fatalf("sanitized observation is not durable: %#v", sanitized)
|
|
}
|
|
|
|
unknownSnapshot := policyQuotaSnapshot("ignored", QuotaStateUnknown, policyTestNow)
|
|
immutable := NormalizeQuotaObservation(unknownSnapshot, policyTestNow, time.Minute)
|
|
unknownSnapshot.ReasonCodes[0] = "token=mutated-secret"
|
|
if len(immutable.Reasons) != 1 || immutable.Reasons[0] != "cap_evidence_unknown" {
|
|
t.Fatalf("normalized observation shares source reasons: %#v", immutable)
|
|
}
|
|
copied := SanitizeQuotaObservation(immutable)
|
|
immutable.Reasons[0] = "token=mutated-secret"
|
|
if copied.Reasons[0] != "cap_evidence_unknown" {
|
|
t.Fatalf("sanitized observation shares caller reasons: %#v", copied)
|
|
}
|
|
}
|
|
|
|
func TestSanitizeQuotaObservationRejectsProjectionTampering(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
observation QuotaObservation
|
|
mutate func(*QuotaObservation)
|
|
}{
|
|
{
|
|
name: "state",
|
|
observation: policyQuotaObservation("ignored", QuotaStateAvailable),
|
|
mutate: func(observation *QuotaObservation) {
|
|
observation.State = QuotaStateExhausted
|
|
},
|
|
},
|
|
{
|
|
name: "snapshot ID",
|
|
observation: policyQuotaObservation("ignored", QuotaStateAvailable),
|
|
mutate: func(observation *QuotaObservation) {
|
|
observation.SnapshotID = "quota-" + strings.Repeat("0", 64)
|
|
},
|
|
},
|
|
{
|
|
name: "adapter identity",
|
|
observation: policyQuotaObservation("ignored", QuotaStateAvailable),
|
|
mutate: func(observation *QuotaObservation) {
|
|
observation.Adapter = "other-provider"
|
|
},
|
|
},
|
|
{
|
|
name: "target identity",
|
|
observation: policyQuotaObservation("ignored", QuotaStateAvailable),
|
|
mutate: func(observation *QuotaObservation) {
|
|
observation.Target = "other-profile"
|
|
},
|
|
},
|
|
{
|
|
name: "checked time",
|
|
observation: policyQuotaObservation("ignored", QuotaStateAvailable),
|
|
mutate: func(observation *QuotaObservation) {
|
|
observation.CheckedAt = observation.CheckedAt.Add(-time.Second)
|
|
},
|
|
},
|
|
{
|
|
name: "validity",
|
|
observation: policyQuotaObservation("ignored", QuotaStateAvailable),
|
|
mutate: func(observation *QuotaObservation) {
|
|
observation.Validity = ObservationStale
|
|
},
|
|
},
|
|
{
|
|
name: "ordered reason",
|
|
observation: policyQuotaObservation("ignored", QuotaStateUnknown),
|
|
mutate: func(observation *QuotaObservation) {
|
|
observation.Reasons[0] = "checker_error"
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
observation := test.observation
|
|
observation.Reasons = append([]string(nil), test.observation.Reasons...)
|
|
test.mutate(&observation)
|
|
if got := SanitizeQuotaObservation(observation); !reflect.DeepEqual(got, CorruptQuotaObservation()) {
|
|
t.Fatalf("SanitizeQuotaObservation() = %#v, want canonical corrupt evidence", got)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestQuotaObservationJSONRoundTripRejectsTampering(t *testing.T) {
|
|
valid := policyQuotaObservation("ignored", QuotaStateAvailable)
|
|
stale := NormalizeQuotaObservation(
|
|
policyQuotaSnapshot("ignored", QuotaStateAvailable, policyTestNow.Add(-2*time.Minute)),
|
|
policyTestNow,
|
|
time.Minute,
|
|
)
|
|
unknown := policyQuotaObservation("ignored", QuotaStateUnknown)
|
|
notApplicable := policyQuotaObservation("ignored", QuotaStateNotApplicable)
|
|
for _, observation := range []QuotaObservation{valid, stale, unknown, notApplicable, CorruptQuotaObservation()} {
|
|
encoded, err := json.Marshal(observation)
|
|
if err != nil {
|
|
t.Fatalf("Marshal(%#v): %v", observation, err)
|
|
}
|
|
var decoded QuotaObservation
|
|
if err := json.Unmarshal(encoded, &decoded); err != nil {
|
|
t.Fatalf("Unmarshal(%s): %v", encoded, err)
|
|
}
|
|
if !reflect.DeepEqual(decoded, observation) {
|
|
t.Fatalf("round trip = %#v, want %#v", decoded, observation)
|
|
}
|
|
if len(decoded.Reasons) > 0 {
|
|
decoded.Reasons[0] = "checker_error"
|
|
if reflect.DeepEqual(decoded.Reasons, observation.Reasons) {
|
|
t.Fatalf("round trip shares reason storage: %#v", decoded)
|
|
}
|
|
}
|
|
}
|
|
|
|
tests := []struct {
|
|
name string
|
|
observation QuotaObservation
|
|
mutate func(map[string]any)
|
|
}{
|
|
{
|
|
name: "state",
|
|
observation: valid,
|
|
mutate: func(encoded map[string]any) {
|
|
encoded["state"] = string(QuotaStateExhausted)
|
|
},
|
|
},
|
|
{
|
|
name: "target",
|
|
observation: valid,
|
|
mutate: func(encoded map[string]any) {
|
|
encoded["target"] = "other-profile"
|
|
},
|
|
},
|
|
{
|
|
name: "snapshot id",
|
|
observation: valid,
|
|
mutate: func(encoded map[string]any) {
|
|
encoded["snapshot_id"] = "quota-" + strings.Repeat("0", 64)
|
|
},
|
|
},
|
|
{
|
|
name: "adapter",
|
|
observation: valid,
|
|
mutate: func(encoded map[string]any) {
|
|
encoded["adapter"] = "other-provider"
|
|
},
|
|
},
|
|
{
|
|
name: "checked time",
|
|
observation: valid,
|
|
mutate: func(encoded map[string]any) {
|
|
encoded["checked_at"] = policyTestNow.Add(-time.Second).Format(time.RFC3339Nano)
|
|
},
|
|
},
|
|
{
|
|
name: "validity",
|
|
observation: valid,
|
|
mutate: func(encoded map[string]any) {
|
|
encoded["validity"] = string(ObservationStale)
|
|
},
|
|
},
|
|
{
|
|
name: "reason",
|
|
observation: unknown,
|
|
mutate: func(encoded map[string]any) {
|
|
encoded["reasons"] = []any{"checker_error"}
|
|
},
|
|
},
|
|
{
|
|
name: "integrity",
|
|
observation: valid,
|
|
mutate: func(encoded map[string]any) {
|
|
encoded["projection_integrity"] = "sha256:tampered"
|
|
},
|
|
},
|
|
{
|
|
name: "unknown field",
|
|
observation: valid,
|
|
mutate: func(encoded map[string]any) {
|
|
encoded["unexpected"] = true
|
|
},
|
|
},
|
|
}
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
payload, err := json.Marshal(test.observation)
|
|
if err != nil {
|
|
t.Fatalf("Marshal: %v", err)
|
|
}
|
|
var encoded map[string]any
|
|
if err := json.Unmarshal(payload, &encoded); err != nil {
|
|
t.Fatalf("Unmarshal object: %v", err)
|
|
}
|
|
test.mutate(encoded)
|
|
payload, err = json.Marshal(encoded)
|
|
if err != nil {
|
|
t.Fatalf("Marshal tampered object: %v", err)
|
|
}
|
|
var decoded QuotaObservation
|
|
if err := json.Unmarshal(payload, &decoded); err == nil {
|
|
t.Fatalf("Unmarshal accepted tampered projection: %s", payload)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestDecideContinuationRetryFailoverAndBlockers(t *testing.T) {
|
|
current := policyTarget("provider", "model", "primary", "profile-r1")
|
|
used := policyTarget("provider", "model", "used", "profile-r2")
|
|
exhausted := policyTarget("provider", "model", "exhausted", "profile-r3")
|
|
alternate := policyTarget("provider", "model", "alternate", "profile-r4")
|
|
|
|
tests := []struct {
|
|
name string
|
|
request ContinuationRequest
|
|
action ContinuationAction
|
|
target TargetIdentity
|
|
blocker ContinuationBlockerCode
|
|
}{
|
|
{
|
|
name: "known retryable failure retries same available target",
|
|
request: ContinuationRequest{
|
|
Policy: FailurePolicy{
|
|
RetryableCodes: []agentruntime.FailureCode{agentruntime.FailureCodeUnavailable},
|
|
},
|
|
Current: current,
|
|
Budget: FailureBudget{Used: 1, Limit: 3},
|
|
Observation: policyObservation("quota-current", QuotaStateAvailable, agentruntime.FailureCodeUnavailable, true),
|
|
},
|
|
action: ContinuationRetry,
|
|
target: current,
|
|
},
|
|
{
|
|
name: "quota exhaustion fails over past used and exhausted candidates",
|
|
request: ContinuationRequest{
|
|
Policy: FailurePolicy{
|
|
FailoverCodes: []agentruntime.FailureCode{agentruntime.FailureCodeQuotaExhausted},
|
|
},
|
|
Current: current,
|
|
Used: []TargetIdentity{used},
|
|
Budget: FailureBudget{Used: 1, Limit: 3},
|
|
Observation: policyObservation(
|
|
"quota-current", QuotaStateExhausted, agentruntime.FailureCodeQuotaExhausted, false,
|
|
),
|
|
Candidates: []ContinuationCandidate{
|
|
{Target: used, Eligible: true, Quota: policyQuotaObservation("quota-used", QuotaStateAvailable)},
|
|
{Target: exhausted, Eligible: true, Quota: policyQuotaObservation("quota-exhausted", QuotaStateExhausted)},
|
|
{Target: alternate, Eligible: true, Quota: policyQuotaObservation("quota-alternate", QuotaStateAvailable)},
|
|
},
|
|
},
|
|
action: ContinuationFailover,
|
|
target: alternate,
|
|
},
|
|
{
|
|
name: "unknown quota blocks without choosing a later candidate",
|
|
request: ContinuationRequest{
|
|
Policy: FailurePolicy{
|
|
FailoverCodes: []agentruntime.FailureCode{agentruntime.FailureCodeQuotaExhausted},
|
|
},
|
|
Current: current,
|
|
Budget: FailureBudget{Used: 1, Limit: 3},
|
|
Observation: policyObservation(
|
|
"quota-current", QuotaStateUnknown, agentruntime.FailureCodeQuotaExhausted, false,
|
|
),
|
|
Candidates: []ContinuationCandidate{
|
|
{Target: alternate, Eligible: true, Quota: policyQuotaObservation("quota-alternate", QuotaStateAvailable)},
|
|
},
|
|
},
|
|
action: ContinuationBlock,
|
|
blocker: ContinuationBlockerUnknownQuota,
|
|
},
|
|
{
|
|
name: "stale observation blocks",
|
|
request: ContinuationRequest{
|
|
Policy: FailurePolicy{
|
|
RetryableCodes: []agentruntime.FailureCode{agentruntime.FailureCodeUnavailable},
|
|
},
|
|
Current: current,
|
|
Budget: FailureBudget{Used: 1, Limit: 3},
|
|
Observation: NormalizeAttemptObservation(
|
|
policyQuotaSnapshot("quota-stale", QuotaStateAvailable, policyTestNow.Add(-2*time.Minute)),
|
|
&agentruntime.Failure{Code: agentruntime.FailureCodeUnavailable, Retryable: true},
|
|
policyTestNow,
|
|
time.Minute,
|
|
),
|
|
},
|
|
action: ContinuationBlock,
|
|
blocker: ContinuationBlockerStaleObservation,
|
|
},
|
|
{
|
|
name: "unknown failure blocks",
|
|
request: ContinuationRequest{
|
|
Policy: FailurePolicy{RetryableCodes: []agentruntime.FailureCode{agentruntime.FailureCodeUnavailable}},
|
|
Current: current,
|
|
Budget: FailureBudget{Used: 1, Limit: 3},
|
|
Observation: policyObservation("quota-current", QuotaStateAvailable, agentruntime.FailureCodeUnknown, false),
|
|
},
|
|
action: ContinuationBlock,
|
|
blocker: ContinuationBlockerUnknownFailure,
|
|
},
|
|
{
|
|
name: "failure budget blocks before retry",
|
|
request: ContinuationRequest{
|
|
Policy: FailurePolicy{RetryableCodes: []agentruntime.FailureCode{agentruntime.FailureCodeUnavailable}},
|
|
Current: current,
|
|
Budget: FailureBudget{Used: 3, Limit: 3},
|
|
Observation: policyObservation("quota-current", QuotaStateAvailable, agentruntime.FailureCodeUnavailable, true),
|
|
},
|
|
action: ContinuationBlock,
|
|
blocker: ContinuationBlockerBudgetExhausted,
|
|
},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
decision, err := DecideContinuation(test.request)
|
|
if err != nil {
|
|
t.Fatalf("DecideContinuation: %v", err)
|
|
}
|
|
if decision.Action != test.action || decision.Target != test.target || decision.Blocker != test.blocker {
|
|
t.Fatalf("decision = %#v", decision)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestDecideContinuationCorruptCandidateBlocksInsteadOfSkipping(t *testing.T) {
|
|
current := policyTarget("provider", "model", "primary", "profile-r1")
|
|
alternate := policyTarget("provider", "model", "alternate", "profile-r2")
|
|
decision, err := DecideContinuation(ContinuationRequest{
|
|
Policy: FailurePolicy{
|
|
FailoverCodes: []agentruntime.FailureCode{agentruntime.FailureCodeQuotaExhausted},
|
|
},
|
|
Current: current,
|
|
Budget: FailureBudget{Used: 1, Limit: 2},
|
|
Observation: policyObservation(
|
|
"quota-current", QuotaStateExhausted, agentruntime.FailureCodeQuotaExhausted, false,
|
|
),
|
|
Candidates: []ContinuationCandidate{{
|
|
Target: alternate, Eligible: true, Quota: CorruptQuotaObservation(),
|
|
}},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("DecideContinuation: %v", err)
|
|
}
|
|
if decision.Action != ContinuationBlock || decision.Blocker != ContinuationBlockerCorruptObservation {
|
|
t.Fatalf("decision = %#v", decision)
|
|
}
|
|
}
|
|
|
|
func TestDecideContinuationNotApplicable(t *testing.T) {
|
|
current := policyTarget("provider", "model", "primary", "profile-r1")
|
|
alternate := policyTarget("provider", "model", "alternate", "profile-r2")
|
|
|
|
retry, err := DecideContinuation(ContinuationRequest{
|
|
Policy: FailurePolicy{
|
|
RetryableCodes: []agentruntime.FailureCode{agentruntime.FailureCodeUnavailable},
|
|
},
|
|
Current: current,
|
|
Budget: FailureBudget{Used: 1, Limit: 3},
|
|
Observation: policyObservation(
|
|
"ignored",
|
|
QuotaStateNotApplicable,
|
|
agentruntime.FailureCodeUnavailable,
|
|
true,
|
|
),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("retry DecideContinuation: %v", err)
|
|
}
|
|
if retry.Action != ContinuationRetry || retry.Target != current {
|
|
t.Fatalf("retry decision = %#v", retry)
|
|
}
|
|
|
|
failover, err := DecideContinuation(ContinuationRequest{
|
|
Policy: FailurePolicy{
|
|
FailoverCodes: []agentruntime.FailureCode{agentruntime.FailureCodeQuotaExhausted},
|
|
},
|
|
Current: current,
|
|
Budget: FailureBudget{Used: 1, Limit: 3},
|
|
Observation: policyObservation(
|
|
"ignored",
|
|
QuotaStateExhausted,
|
|
agentruntime.FailureCodeQuotaExhausted,
|
|
false,
|
|
),
|
|
Candidates: []ContinuationCandidate{{
|
|
Target: alternate,
|
|
Eligible: true,
|
|
Quota: policyQuotaObservation("ignored", QuotaStateNotApplicable),
|
|
}},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("failover DecideContinuation: %v", err)
|
|
}
|
|
if failover.Action != ContinuationFailover || failover.Target != alternate {
|
|
t.Fatalf("failover decision = %#v", failover)
|
|
}
|
|
}
|
|
|
|
func policyTarget(provider, model, profile, revision string) TargetIdentity {
|
|
return TargetIdentity{
|
|
ProviderID: provider, ModelID: model, ProfileID: profile, ProfileRevision: revision,
|
|
}
|
|
}
|
|
|
|
func policyObservation(
|
|
snapshotID string,
|
|
quota QuotaState,
|
|
code agentruntime.FailureCode,
|
|
retryable bool,
|
|
) AttemptObservation {
|
|
return NormalizeAttemptObservation(
|
|
policyQuotaSnapshot(snapshotID, quota, policyTestNow),
|
|
&agentruntime.Failure{Code: code, Retryable: retryable},
|
|
policyTestNow,
|
|
time.Minute,
|
|
)
|
|
}
|
|
|
|
func policyQuotaObservation(snapshotID string, state QuotaState) QuotaObservation {
|
|
return NormalizeQuotaObservation(
|
|
policyQuotaSnapshot(snapshotID, state, policyTestNow),
|
|
policyTestNow,
|
|
time.Minute,
|
|
)
|
|
}
|
|
|
|
func policyQuotaSnapshot(_ string, state QuotaState, checkedAt time.Time) status.QuotaSnapshot {
|
|
var requiredCaps []string
|
|
var usage *status.UsageStatus
|
|
switch state {
|
|
case QuotaStateAvailable:
|
|
requiredCaps = []string{"overall"}
|
|
usage = &status.UsageStatus{DailyLimit: "50%"}
|
|
case QuotaStateExhausted:
|
|
requiredCaps = []string{"overall"}
|
|
usage = &status.UsageStatus{DailyLimit: "0%"}
|
|
case QuotaStateUnknown:
|
|
requiredCaps = []string{"overall"}
|
|
usage = &status.UsageStatus{DailyLimit: "not-a-percent"}
|
|
case QuotaStateNotApplicable:
|
|
requiredCaps = nil
|
|
default:
|
|
panic("unsupported quota state")
|
|
}
|
|
return status.NormalizeQuotaSnapshot(
|
|
"provider",
|
|
"profile",
|
|
requiredCaps,
|
|
checkedAt,
|
|
usage,
|
|
nil,
|
|
)
|
|
}
|