toki
3c48879c44
독립 호스트에서 안전한 작업 실행과 복구를 제공하기 위해 런타임 설정, 정책, 상태 저장소, 워크스페이스 격리 및 AgentTask 오케스트레이션을 확장한다.
592 lines
18 KiB
Go
592 lines
18 KiB
Go
package agenttask
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"iop/packages/go/agentpolicy"
|
|
"iop/packages/go/agentprovider/cli/status"
|
|
"iop/packages/go/agentruntime"
|
|
)
|
|
|
|
var continuationTestNow = time.Date(2026, 7, 28, 0, 0, 0, 0, time.UTC)
|
|
|
|
func TestFailureContinuationUsesCommonPolicyAndSkipsUsedCandidate(t *testing.T) {
|
|
current := continuationTarget("profile", "profile-r1")
|
|
firstAlternate := continuationTarget("backup-one", "profile-r2")
|
|
secondAlternate := continuationTarget("backup-two", "profile-r3")
|
|
policy := FailureContinuationPolicy{
|
|
Policy: agentpolicy.FailurePolicy{
|
|
FailoverCodes: []agentruntime.FailureCode{agentruntime.FailureCodeQuotaExhausted},
|
|
},
|
|
Candidates: []FailureContinuationCandidate{
|
|
continuationCandidate(firstAlternate, agentpolicy.QuotaStateAvailable),
|
|
continuationCandidate(secondAlternate, agentpolicy.QuotaStateNotApplicable),
|
|
},
|
|
}
|
|
selector := &continuationTestSelector{initial: current, policy: policy}
|
|
invoker := &continuationTestInvoker{
|
|
observations: []agentpolicy.AttemptObservation{
|
|
continuationObservationForTarget(
|
|
current,
|
|
agentpolicy.QuotaStateExhausted,
|
|
agentruntime.FailureCodeQuotaExhausted,
|
|
false,
|
|
),
|
|
continuationObservationForTarget(
|
|
firstAlternate,
|
|
agentpolicy.QuotaStateExhausted,
|
|
agentruntime.FailureCodeQuotaExhausted,
|
|
false,
|
|
),
|
|
},
|
|
failures: 2,
|
|
}
|
|
snapshot := testSnapshot("project", "workspace", testUnit("work", WriteSetUnknown))
|
|
harness := newHarness(t, map[ProjectID]ProjectWorkflowSnapshot{"project": snapshot}, 1)
|
|
harness.manager.selector = selector
|
|
harness.manager.invoker = invoker
|
|
harness.start("project", "workspace", nil)
|
|
|
|
if err := harness.manager.Reconcile(context.Background()); err != nil {
|
|
t.Fatalf("Reconcile: %v", err)
|
|
}
|
|
if len(invoker.requests) != 3 {
|
|
t.Fatalf("invocations = %d, want 3", len(invoker.requests))
|
|
}
|
|
wantTargets := []ExecutionTarget{current, firstAlternate, secondAlternate}
|
|
for index, want := range wantTargets {
|
|
if got := invoker.requests[index].Target; !sameExecutionTarget(got, want) {
|
|
t.Fatalf("target %d = %#v, want %#v", index, got, want)
|
|
}
|
|
}
|
|
if len(selector.requests) != 2 {
|
|
t.Fatalf("policy-source requests = %d, want 2", len(selector.requests))
|
|
}
|
|
|
|
state := harness.store.snapshot()
|
|
work := state.Projects["project"].Works["work"]
|
|
if work.State != WorkStateCompleted || work.Attempt != 3 ||
|
|
work.ContinuationTarget != nil || len(work.AttemptObservations) != 2 {
|
|
t.Fatalf("work = %#v", work)
|
|
}
|
|
if !sameExecutionTarget(work.AttemptObservations[0].Target, current) ||
|
|
!sameExecutionTarget(work.AttemptObservations[1].Target, firstAlternate) {
|
|
t.Fatalf("attempt target history = %#v", work.AttemptObservations)
|
|
}
|
|
}
|
|
|
|
func TestFailureContinuationMalformedObservationBecomesTypedBlocker(t *testing.T) {
|
|
current := continuationTarget("profile", "profile-r1")
|
|
observation := continuationObservationForTarget(
|
|
current,
|
|
agentpolicy.QuotaStateAvailable,
|
|
agentruntime.FailureCodeUnavailable,
|
|
true,
|
|
)
|
|
observation.Quota.SnapshotID = "token=provider-secret\n"
|
|
observation.Quota.Reasons = []string{"authorization=Bearer provider-secret"}
|
|
|
|
snapshot := testSnapshot("project", "workspace", testUnit("work", WriteSetUnknown))
|
|
harness := newHarness(t, map[ProjectID]ProjectWorkflowSnapshot{"project": snapshot}, 1)
|
|
selector := &continuationTestSelector{
|
|
initial: current,
|
|
policy: FailureContinuationPolicy{Policy: agentpolicy.FailurePolicy{
|
|
RetryableCodes: []agentruntime.FailureCode{agentruntime.FailureCodeUnavailable},
|
|
}},
|
|
}
|
|
harness.manager.selector = selector
|
|
harness.manager.invoker = &continuationTestInvoker{
|
|
observations: []agentpolicy.AttemptObservation{observation},
|
|
failures: 1,
|
|
}
|
|
harness.start("project", "workspace", nil)
|
|
|
|
if err := harness.manager.Reconcile(context.Background()); err != nil {
|
|
t.Fatalf("Reconcile: %v", err)
|
|
}
|
|
state := harness.store.snapshot()
|
|
work := state.Projects["project"].Works["work"]
|
|
if work.State != WorkStateBlocked || work.Blocker == nil ||
|
|
work.Blocker.Code != BlockerFailureObservationCorrupt {
|
|
t.Fatalf("work = %#v", work)
|
|
}
|
|
if len(work.AttemptObservations) != 1 {
|
|
t.Fatalf("attempt observations = %#v", work.AttemptObservations)
|
|
}
|
|
got := work.AttemptObservations[0].Observation
|
|
if got.Quota.Validity != agentpolicy.ObservationCorrupt ||
|
|
got.Quota.SnapshotID != "" || len(got.Quota.Reasons) != 0 {
|
|
t.Fatalf("durable observation = %#v", got)
|
|
}
|
|
encoded, err := json.Marshal(state)
|
|
if err != nil {
|
|
t.Fatalf("Marshal state: %v", err)
|
|
}
|
|
if strings.Contains(string(encoded), "provider-secret") ||
|
|
strings.Contains(string(encoded), "authorization") {
|
|
t.Fatalf("durable state retained unsafe evidence: %s", encoded)
|
|
}
|
|
}
|
|
|
|
func TestFailureContinuationProjectionTamperBecomesTypedBlocker(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
state agentpolicy.QuotaState
|
|
mutate func(*agentpolicy.QuotaObservation)
|
|
}{
|
|
{
|
|
name: "state",
|
|
state: agentpolicy.QuotaStateAvailable,
|
|
mutate: func(observation *agentpolicy.QuotaObservation) {
|
|
observation.State = agentpolicy.QuotaStateExhausted
|
|
},
|
|
},
|
|
{
|
|
name: "target",
|
|
state: agentpolicy.QuotaStateAvailable,
|
|
mutate: func(observation *agentpolicy.QuotaObservation) {
|
|
observation.Target = "other-profile"
|
|
},
|
|
},
|
|
{
|
|
name: "snapshot id",
|
|
state: agentpolicy.QuotaStateAvailable,
|
|
mutate: func(observation *agentpolicy.QuotaObservation) {
|
|
observation.SnapshotID = "quota-" + strings.Repeat("0", 64)
|
|
},
|
|
},
|
|
{
|
|
name: "adapter",
|
|
state: agentpolicy.QuotaStateAvailable,
|
|
mutate: func(observation *agentpolicy.QuotaObservation) {
|
|
observation.Adapter = "other-provider"
|
|
},
|
|
},
|
|
{
|
|
name: "checked time",
|
|
state: agentpolicy.QuotaStateAvailable,
|
|
mutate: func(observation *agentpolicy.QuotaObservation) {
|
|
observation.CheckedAt = observation.CheckedAt.Add(-time.Second)
|
|
},
|
|
},
|
|
{
|
|
name: "validity",
|
|
state: agentpolicy.QuotaStateAvailable,
|
|
mutate: func(observation *agentpolicy.QuotaObservation) {
|
|
observation.Validity = agentpolicy.ObservationStale
|
|
},
|
|
},
|
|
{
|
|
name: "reason",
|
|
state: agentpolicy.QuotaStateUnknown,
|
|
mutate: func(observation *agentpolicy.QuotaObservation) {
|
|
observation.Reasons[0] = "checker_error"
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
current := continuationTarget("profile", "profile-r1")
|
|
observation := continuationObservationForTarget(
|
|
current,
|
|
test.state,
|
|
agentruntime.FailureCodeUnavailable,
|
|
true,
|
|
)
|
|
test.mutate(&observation.Quota)
|
|
|
|
snapshot := testSnapshot("project", "workspace", testUnit("work", WriteSetUnknown))
|
|
harness := newHarness(t, map[ProjectID]ProjectWorkflowSnapshot{"project": snapshot}, 1)
|
|
harness.manager.selector = &continuationTestSelector{
|
|
initial: current,
|
|
policy: FailureContinuationPolicy{Policy: agentpolicy.FailurePolicy{
|
|
RetryableCodes: []agentruntime.FailureCode{agentruntime.FailureCodeUnavailable},
|
|
}},
|
|
}
|
|
invoker := &continuationTestInvoker{
|
|
observations: []agentpolicy.AttemptObservation{observation},
|
|
failures: 1,
|
|
}
|
|
harness.manager.invoker = invoker
|
|
harness.start("project", "workspace", nil)
|
|
|
|
if err := harness.manager.Reconcile(context.Background()); err != nil {
|
|
t.Fatalf("Reconcile: %v", err)
|
|
}
|
|
if len(invoker.requests) != 1 {
|
|
t.Fatalf("invocations = %d, want exactly one", len(invoker.requests))
|
|
}
|
|
work := harness.store.snapshot().Projects["project"].Works["work"]
|
|
if work.State != WorkStateBlocked || work.Blocker == nil ||
|
|
work.Blocker.Code != BlockerFailureObservationCorrupt {
|
|
t.Fatalf("work = %#v", work)
|
|
}
|
|
if len(work.AttemptObservations) != 1 ||
|
|
!reflect.DeepEqual(
|
|
work.AttemptObservations[0].Observation.Quota,
|
|
agentpolicy.CorruptQuotaObservation(),
|
|
) {
|
|
t.Fatalf("durable observations = %#v", work.AttemptObservations)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestFailureContinuationUnknownAndBudgetBlockWork(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
observation agentpolicy.AttemptObservation
|
|
policy agentpolicy.FailurePolicy
|
|
maxFailures uint32
|
|
wantBlocker BlockerCode
|
|
}{
|
|
{
|
|
name: "unknown policy failure becomes typed blocker",
|
|
observation: continuationObservationForTarget(
|
|
continuationTarget("profile", "profile-r1"),
|
|
agentpolicy.QuotaStateAvailable,
|
|
agentruntime.FailureCodeUnknown,
|
|
false,
|
|
),
|
|
policy: agentpolicy.FailurePolicy{
|
|
RetryableCodes: []agentruntime.FailureCode{agentruntime.FailureCodeUnavailable},
|
|
},
|
|
wantBlocker: BlockerFailureUnknown,
|
|
},
|
|
{
|
|
name: "manager failure budget blocks a declared retry",
|
|
observation: continuationObservationForTarget(
|
|
continuationTarget("profile", "profile-r1"),
|
|
agentpolicy.QuotaStateNotApplicable,
|
|
agentruntime.FailureCodeUnavailable,
|
|
true,
|
|
),
|
|
policy: agentpolicy.FailurePolicy{
|
|
RetryableCodes: []agentruntime.FailureCode{agentruntime.FailureCodeUnavailable},
|
|
},
|
|
maxFailures: 1,
|
|
wantBlocker: BlockerFailureBudgetExhausted,
|
|
},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
current := continuationTarget("profile", "profile-r1")
|
|
snapshot := testSnapshot("project", "workspace", testUnit("work", WriteSetUnknown))
|
|
harness := newHarness(t, map[ProjectID]ProjectWorkflowSnapshot{"project": snapshot}, 1)
|
|
if test.maxFailures != 0 {
|
|
harness.manager.config.MaxFailureAttempts = test.maxFailures
|
|
}
|
|
selector := &continuationTestSelector{
|
|
initial: current,
|
|
policy: FailureContinuationPolicy{Policy: test.policy},
|
|
}
|
|
invoker := &continuationTestInvoker{
|
|
observations: []agentpolicy.AttemptObservation{test.observation},
|
|
failures: 1,
|
|
}
|
|
harness.manager.selector = selector
|
|
harness.manager.invoker = invoker
|
|
harness.start("project", "workspace", nil)
|
|
|
|
if err := harness.manager.Reconcile(context.Background()); err != nil {
|
|
t.Fatalf("Reconcile: %v", err)
|
|
}
|
|
if len(invoker.requests) != 1 {
|
|
t.Fatalf("invocations = %d, want 1", len(invoker.requests))
|
|
}
|
|
work := harness.store.snapshot().Projects["project"].Works["work"]
|
|
if work.State != WorkStateBlocked || work.Blocker == nil ||
|
|
work.Blocker.Code != test.wantBlocker {
|
|
t.Fatalf("work = %#v", work)
|
|
}
|
|
if len(work.AttemptObservations) != 1 {
|
|
t.Fatalf("attempt observations = %#v", work.AttemptObservations)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestFailureWithoutObservationRedactsProviderError(t *testing.T) {
|
|
current := continuationTarget("profile", "profile-r1")
|
|
snapshot := testSnapshot("project", "workspace", testUnit("work", WriteSetUnknown))
|
|
harness := newHarness(t, map[ProjectID]ProjectWorkflowSnapshot{"project": snapshot}, 1)
|
|
invoker := &unobservedFailureInvoker{
|
|
owner: &continuationTestInvoker{
|
|
observations: []agentpolicy.AttemptObservation{
|
|
continuationObservationForTarget(
|
|
current,
|
|
agentpolicy.QuotaStateAvailable,
|
|
agentruntime.FailureCodeUnavailable,
|
|
true,
|
|
),
|
|
},
|
|
failures: 1,
|
|
},
|
|
}
|
|
harness.manager.selector = &continuationTestSelector{initial: current}
|
|
harness.manager.invoker = invoker
|
|
harness.start("project", "workspace", nil)
|
|
|
|
if err := harness.manager.Reconcile(context.Background()); err != nil {
|
|
t.Fatalf("Reconcile: %v", err)
|
|
}
|
|
state := harness.store.snapshot()
|
|
work := state.Projects["project"].Works["work"]
|
|
if work.State != WorkStateBlocked || work.Blocker == nil || work.Blocker.Code != BlockerInvocationFailed {
|
|
t.Fatalf("work = %#v", work)
|
|
}
|
|
encoded, err := json.Marshal(state)
|
|
if err != nil {
|
|
t.Fatalf("Marshal state: %v", err)
|
|
}
|
|
if strings.Contains(string(encoded), "credential=provider-secret") {
|
|
t.Fatalf("durable state retained provider error: %s", encoded)
|
|
}
|
|
}
|
|
|
|
type continuationTestSelector struct {
|
|
initial ExecutionTarget
|
|
policy FailureContinuationPolicy
|
|
policies []FailureContinuationPolicy
|
|
requests []FailureContinuationPolicyRequest
|
|
}
|
|
|
|
func (s *continuationTestSelector) Select(context.Context, SelectionRequest) (ExecutionTarget, error) {
|
|
return s.initial, nil
|
|
}
|
|
|
|
func (s *continuationTestSelector) ContinuationPolicy(
|
|
_ context.Context,
|
|
request FailureContinuationPolicyRequest,
|
|
) (FailureContinuationPolicy, error) {
|
|
s.requests = append(s.requests, request)
|
|
if len(s.policies) == 0 {
|
|
return s.policy, nil
|
|
}
|
|
policy := s.policies[0]
|
|
s.policies = s.policies[1:]
|
|
return policy, nil
|
|
}
|
|
|
|
type unobservedFailureInvoker struct {
|
|
owner *continuationTestInvoker
|
|
}
|
|
|
|
func (i *unobservedFailureInvoker) Prepare(
|
|
ctx context.Context,
|
|
request DispatchRequest,
|
|
) (ProviderLaunch, error) {
|
|
launch, err := i.owner.Prepare(ctx, request)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return unobservedFailureLaunch{ProviderLaunch: launch}, nil
|
|
}
|
|
|
|
type unobservedFailureLaunch struct {
|
|
ProviderLaunch
|
|
}
|
|
|
|
func (l unobservedFailureLaunch) BindStarted(
|
|
started StartedConfinement,
|
|
) (ProviderInvocation, error) {
|
|
invocation, err := l.ProviderLaunch.BindStarted(started)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return unobservedFailureInvocation{ProviderInvocation: invocation}, nil
|
|
}
|
|
|
|
type unobservedFailureInvocation struct {
|
|
ProviderInvocation
|
|
}
|
|
|
|
type continuationTestInvoker struct {
|
|
observations []agentpolicy.AttemptObservation
|
|
failures int
|
|
requests []DispatchRequest
|
|
}
|
|
|
|
func (i *continuationTestInvoker) Prepare(
|
|
_ context.Context,
|
|
request DispatchRequest,
|
|
) (ProviderLaunch, error) {
|
|
i.requests = append(i.requests, request)
|
|
return &continuationTestLaunch{
|
|
owner: i,
|
|
request: request,
|
|
index: len(i.requests),
|
|
}, nil
|
|
}
|
|
|
|
type continuationTestLaunch struct {
|
|
owner *continuationTestInvoker
|
|
request DispatchRequest
|
|
index int
|
|
}
|
|
|
|
func (continuationTestLaunch) Command() ConfinementCommand {
|
|
return ConfinementCommand{Name: "true"}
|
|
}
|
|
|
|
func (l *continuationTestLaunch) BindStarted(
|
|
started StartedConfinement,
|
|
) (ProviderInvocation, error) {
|
|
if started == nil || started.Child() == nil {
|
|
return nil, errors.New("missing confined started handle")
|
|
}
|
|
locator := LocatorRecord{
|
|
Kind: LocatorProcess,
|
|
Opaque: fmt.Sprintf("continuation-process-%d", l.index),
|
|
Revision: "process-r1",
|
|
ProjectID: l.request.Project.ProjectID,
|
|
WorkspaceID: l.request.Project.WorkspaceID,
|
|
WorkUnitID: l.request.Work.Unit.ID,
|
|
AttemptID: l.request.Work.AttemptID,
|
|
}
|
|
return &continuationTestInvocation{
|
|
request: l.request,
|
|
locators: []LocatorRecord{locator},
|
|
observation: l.owner.observation(l.index),
|
|
fail: l.index <= l.owner.failures,
|
|
started: started,
|
|
}, nil
|
|
}
|
|
|
|
func (i *continuationTestInvoker) observation(index int) agentpolicy.AttemptObservation {
|
|
if index <= 0 || index > len(i.observations) {
|
|
return agentpolicy.AttemptObservation{}
|
|
}
|
|
return i.observations[index-1].Clone()
|
|
}
|
|
|
|
type continuationTestInvocation struct {
|
|
request DispatchRequest
|
|
locators []LocatorRecord
|
|
observation agentpolicy.AttemptObservation
|
|
fail bool
|
|
started StartedConfinement
|
|
}
|
|
|
|
func (i *continuationTestInvocation) Locators() []LocatorRecord {
|
|
return append([]LocatorRecord(nil), i.locators...)
|
|
}
|
|
|
|
func (i *continuationTestInvocation) Wait(context.Context) (Submission, error) {
|
|
if i.started != nil {
|
|
if stdin := i.started.Stdin(); stdin != nil {
|
|
_ = stdin.Close()
|
|
}
|
|
if child := i.started.Child(); child != nil {
|
|
_ = child.Wait()
|
|
}
|
|
if stdout := i.started.Stdout(); stdout != nil {
|
|
_ = stdout.Close()
|
|
}
|
|
if stderr := i.started.Stderr(); stderr != nil {
|
|
_ = stderr.Close()
|
|
}
|
|
}
|
|
if i.fail {
|
|
return Submission{}, errors.New("credential=provider-secret")
|
|
}
|
|
return Submission{
|
|
ProjectID: i.request.Project.ProjectID,
|
|
WorkUnitID: i.request.Work.Unit.ID,
|
|
AttemptID: i.request.Work.AttemptID,
|
|
ArtifactID: ArtifactID("artifact-" + string(i.request.Work.AttemptID)),
|
|
Ready: true,
|
|
Locators: append([]LocatorRecord(nil), i.locators...),
|
|
}, nil
|
|
}
|
|
|
|
func (i *continuationTestInvocation) Cancel(context.Context) error {
|
|
if i.started != nil {
|
|
_ = i.started.Abort()
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (i *continuationTestInvocation) FailureObservation() agentpolicy.AttemptObservation {
|
|
return i.observation.Clone()
|
|
}
|
|
|
|
func continuationTarget(profile, revision string) ExecutionTarget {
|
|
return ExecutionTarget{
|
|
ProviderID: "provider",
|
|
ModelID: "model",
|
|
ProfileID: profile,
|
|
ProfileRevision: revision,
|
|
ConfigRevision: "config-r1",
|
|
Capacity: 1,
|
|
}
|
|
}
|
|
|
|
func continuationObservationForTarget(
|
|
target ExecutionTarget,
|
|
state agentpolicy.QuotaState,
|
|
code agentruntime.FailureCode,
|
|
retryable bool,
|
|
) agentpolicy.AttemptObservation {
|
|
return agentpolicy.NormalizeAttemptObservation(
|
|
continuationQuotaSnapshot(target, state),
|
|
&agentruntime.Failure{Code: code, Retryable: retryable},
|
|
continuationTestNow,
|
|
0,
|
|
)
|
|
}
|
|
|
|
func continuationQuotaSnapshot(
|
|
target ExecutionTarget,
|
|
state agentpolicy.QuotaState,
|
|
) status.QuotaSnapshot {
|
|
var requiredCaps []string
|
|
var usage *status.UsageStatus
|
|
switch state {
|
|
case agentpolicy.QuotaStateAvailable:
|
|
requiredCaps = []string{"overall"}
|
|
usage = &status.UsageStatus{DailyLimit: "50%"}
|
|
case agentpolicy.QuotaStateExhausted:
|
|
requiredCaps = []string{"overall"}
|
|
usage = &status.UsageStatus{DailyLimit: "0%"}
|
|
case agentpolicy.QuotaStateUnknown:
|
|
requiredCaps = []string{"overall"}
|
|
usage = &status.UsageStatus{DailyLimit: "not-a-percent"}
|
|
case agentpolicy.QuotaStateNotApplicable:
|
|
requiredCaps = nil
|
|
default:
|
|
panic("unsupported quota state")
|
|
}
|
|
return status.NormalizeQuotaSnapshot(
|
|
target.ProviderID,
|
|
target.ProfileID,
|
|
requiredCaps,
|
|
continuationTestNow,
|
|
usage,
|
|
nil,
|
|
)
|
|
}
|
|
|
|
func continuationCandidate(
|
|
target ExecutionTarget,
|
|
state agentpolicy.QuotaState,
|
|
) FailureContinuationCandidate {
|
|
return FailureContinuationCandidate{
|
|
Target: target,
|
|
Eligible: true,
|
|
Quota: agentpolicy.NormalizeQuotaObservation(
|
|
continuationQuotaSnapshot(target, state),
|
|
continuationTestNow,
|
|
time.Minute,
|
|
),
|
|
}
|
|
}
|