package agenttask import ( "context" "encoding/json" "errors" "fmt" "reflect" "strings" "testing" "time" "iop/packages/go/agentpolicy" "iop/packages/go/agentprovider/cli/status" "iop/packages/go/agentruntime" ) var continuationTestNow = time.Date(2026, 7, 28, 0, 0, 0, 0, time.UTC) func TestFailureContinuationUsesCommonPolicyAndSkipsUsedCandidate(t *testing.T) { current := continuationTarget("profile", "profile-r1") firstAlternate := continuationTarget("backup-one", "profile-r2") secondAlternate := continuationTarget("backup-two", "profile-r3") policy := FailureContinuationPolicy{ Policy: agentpolicy.FailurePolicy{ FailoverCodes: []agentruntime.FailureCode{agentruntime.FailureCodeQuotaExhausted}, }, Candidates: []FailureContinuationCandidate{ continuationCandidate(firstAlternate, agentpolicy.QuotaStateAvailable), continuationCandidate(secondAlternate, agentpolicy.QuotaStateNotApplicable), }, } selector := &continuationTestSelector{initial: current, policy: policy} invoker := &continuationTestInvoker{ observations: []agentpolicy.AttemptObservation{ continuationObservationForTarget( current, agentpolicy.QuotaStateExhausted, agentruntime.FailureCodeQuotaExhausted, false, ), continuationObservationForTarget( firstAlternate, agentpolicy.QuotaStateExhausted, agentruntime.FailureCodeQuotaExhausted, false, ), }, failures: 2, } snapshot := testSnapshot("project", "workspace", testUnit("work", WriteSetUnknown)) harness := newHarness(t, map[ProjectID]ProjectWorkflowSnapshot{"project": snapshot}, 1) harness.manager.selector = selector harness.manager.invoker = invoker harness.start("project", "workspace", nil) if err := harness.manager.Reconcile(context.Background()); err != nil { t.Fatalf("Reconcile: %v", err) } if len(invoker.requests) != 3 { t.Fatalf("invocations = %d, want 3", len(invoker.requests)) } wantTargets := []ExecutionTarget{current, firstAlternate, secondAlternate} for index, want := range wantTargets { if got := invoker.requests[index].Target; !sameExecutionTarget(got, want) { t.Fatalf("target %d = %#v, want %#v", index, got, want) } } if len(selector.requests) != 2 { t.Fatalf("policy-source requests = %d, want 2", len(selector.requests)) } for index, want := range invoker.observations { got := selector.requests[index] if !reflect.DeepEqual(got.Observation, agentpolicy.SanitizeAttemptObservation( want, continuationTestNow, )) { t.Fatalf( "policy-source observation %d = %#v, want sanitized %#v", index, got.Observation, want, ) } if !sameExecutionTarget(got.CurrentTarget, wantTargets[index]) { t.Fatalf( "policy-source current target %d = %#v, want %#v", index, got.CurrentTarget, wantTargets[index], ) } } state := harness.store.snapshot() work := state.Projects["project"].Works["work"] if work.State != WorkStateCompleted || work.Attempt != 3 || work.ContinuationTarget != nil || len(work.AttemptObservations) != 2 { t.Fatalf("work = %#v", work) } if !sameExecutionTarget(work.AttemptObservations[0].Target, current) || !sameExecutionTarget(work.AttemptObservations[1].Target, firstAlternate) { t.Fatalf("attempt target history = %#v", work.AttemptObservations) } } func TestFailureContinuationMalformedObservationBecomesTypedBlocker(t *testing.T) { current := continuationTarget("profile", "profile-r1") observation := continuationObservationForTarget( current, agentpolicy.QuotaStateAvailable, agentruntime.FailureCodeUnavailable, true, ) observation.Quota.SnapshotID = "token=provider-secret\n" observation.Quota.Reasons = []string{"authorization=Bearer provider-secret"} snapshot := testSnapshot("project", "workspace", testUnit("work", WriteSetUnknown)) harness := newHarness(t, map[ProjectID]ProjectWorkflowSnapshot{"project": snapshot}, 1) selector := &continuationTestSelector{ initial: current, policy: FailureContinuationPolicy{Policy: agentpolicy.FailurePolicy{ RetryableCodes: []agentruntime.FailureCode{agentruntime.FailureCodeUnavailable}, }}, } harness.manager.selector = selector harness.manager.invoker = &continuationTestInvoker{ observations: []agentpolicy.AttemptObservation{observation}, failures: 1, } harness.start("project", "workspace", nil) if err := harness.manager.Reconcile(context.Background()); err != nil { t.Fatalf("Reconcile: %v", err) } state := harness.store.snapshot() work := state.Projects["project"].Works["work"] if work.State != WorkStateBlocked || work.Blocker == nil || work.Blocker.Code != BlockerFailureObservationCorrupt { t.Fatalf("work = %#v", work) } if len(work.AttemptObservations) != 1 { t.Fatalf("attempt observations = %#v", work.AttemptObservations) } got := work.AttemptObservations[0].Observation if got.Quota.Validity != agentpolicy.ObservationCorrupt || got.Quota.SnapshotID != "" || len(got.Quota.Reasons) != 0 { t.Fatalf("durable observation = %#v", got) } encoded, err := json.Marshal(state) if err != nil { t.Fatalf("Marshal state: %v", err) } if strings.Contains(string(encoded), "provider-secret") || strings.Contains(string(encoded), "authorization") { t.Fatalf("durable state retained unsafe evidence: %s", encoded) } } func TestFailureContinuationProjectionTamperBecomesTypedBlocker(t *testing.T) { tests := []struct { name string state agentpolicy.QuotaState mutate func(*agentpolicy.QuotaObservation) }{ { name: "state", state: agentpolicy.QuotaStateAvailable, mutate: func(observation *agentpolicy.QuotaObservation) { observation.State = agentpolicy.QuotaStateExhausted }, }, { name: "target", state: agentpolicy.QuotaStateAvailable, mutate: func(observation *agentpolicy.QuotaObservation) { observation.Target = "other-profile" }, }, { name: "snapshot id", state: agentpolicy.QuotaStateAvailable, mutate: func(observation *agentpolicy.QuotaObservation) { observation.SnapshotID = "quota-" + strings.Repeat("0", 64) }, }, { name: "adapter", state: agentpolicy.QuotaStateAvailable, mutate: func(observation *agentpolicy.QuotaObservation) { observation.Adapter = "other-provider" }, }, { name: "checked time", state: agentpolicy.QuotaStateAvailable, mutate: func(observation *agentpolicy.QuotaObservation) { observation.CheckedAt = observation.CheckedAt.Add(-time.Second) }, }, { name: "validity", state: agentpolicy.QuotaStateAvailable, mutate: func(observation *agentpolicy.QuotaObservation) { observation.Validity = agentpolicy.ObservationStale }, }, { name: "reason", state: agentpolicy.QuotaStateUnknown, mutate: func(observation *agentpolicy.QuotaObservation) { observation.Reasons[0] = "checker_error" }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { current := continuationTarget("profile", "profile-r1") observation := continuationObservationForTarget( current, test.state, agentruntime.FailureCodeUnavailable, true, ) test.mutate(&observation.Quota) snapshot := testSnapshot("project", "workspace", testUnit("work", WriteSetUnknown)) harness := newHarness(t, map[ProjectID]ProjectWorkflowSnapshot{"project": snapshot}, 1) harness.manager.selector = &continuationTestSelector{ initial: current, policy: FailureContinuationPolicy{Policy: agentpolicy.FailurePolicy{ RetryableCodes: []agentruntime.FailureCode{agentruntime.FailureCodeUnavailable}, }}, } invoker := &continuationTestInvoker{ observations: []agentpolicy.AttemptObservation{observation}, failures: 1, } harness.manager.invoker = invoker harness.start("project", "workspace", nil) if err := harness.manager.Reconcile(context.Background()); err != nil { t.Fatalf("Reconcile: %v", err) } if len(invoker.requests) != 1 { t.Fatalf("invocations = %d, want exactly one", len(invoker.requests)) } work := harness.store.snapshot().Projects["project"].Works["work"] if work.State != WorkStateBlocked || work.Blocker == nil || work.Blocker.Code != BlockerFailureObservationCorrupt { t.Fatalf("work = %#v", work) } if len(work.AttemptObservations) != 1 || !reflect.DeepEqual( work.AttemptObservations[0].Observation.Quota, agentpolicy.CorruptQuotaObservation(), ) { t.Fatalf("durable observations = %#v", work.AttemptObservations) } }) } } func TestFailureContinuationUnknownAndBudgetBlockWork(t *testing.T) { tests := []struct { name string observation agentpolicy.AttemptObservation policy agentpolicy.FailurePolicy maxFailures uint32 wantBlocker BlockerCode }{ { name: "unknown policy failure becomes typed blocker", observation: continuationObservationForTarget( continuationTarget("profile", "profile-r1"), agentpolicy.QuotaStateAvailable, agentruntime.FailureCodeUnknown, false, ), policy: agentpolicy.FailurePolicy{ RetryableCodes: []agentruntime.FailureCode{agentruntime.FailureCodeUnavailable}, }, wantBlocker: BlockerFailureUnknown, }, { name: "manager failure budget blocks a declared retry", observation: continuationObservationForTarget( continuationTarget("profile", "profile-r1"), agentpolicy.QuotaStateNotApplicable, agentruntime.FailureCodeUnavailable, true, ), policy: agentpolicy.FailurePolicy{ RetryableCodes: []agentruntime.FailureCode{agentruntime.FailureCodeUnavailable}, }, maxFailures: 1, wantBlocker: BlockerFailureBudgetExhausted, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { current := continuationTarget("profile", "profile-r1") snapshot := testSnapshot("project", "workspace", testUnit("work", WriteSetUnknown)) harness := newHarness(t, map[ProjectID]ProjectWorkflowSnapshot{"project": snapshot}, 1) if test.maxFailures != 0 { harness.manager.config.MaxFailureAttempts = test.maxFailures } selector := &continuationTestSelector{ initial: current, policy: FailureContinuationPolicy{Policy: test.policy}, } invoker := &continuationTestInvoker{ observations: []agentpolicy.AttemptObservation{test.observation}, failures: 1, } harness.manager.selector = selector harness.manager.invoker = invoker harness.start("project", "workspace", nil) if err := harness.manager.Reconcile(context.Background()); err != nil { t.Fatalf("Reconcile: %v", err) } if len(invoker.requests) != 1 { t.Fatalf("invocations = %d, want 1", len(invoker.requests)) } work := harness.store.snapshot().Projects["project"].Works["work"] if work.State != WorkStateBlocked || work.Blocker == nil || work.Blocker.Code != test.wantBlocker { t.Fatalf("work = %#v", work) } if len(work.AttemptObservations) != 1 { t.Fatalf("attempt observations = %#v", work.AttemptObservations) } }) } } func TestFailureWithoutObservationRedactsProviderError(t *testing.T) { current := continuationTarget("profile", "profile-r1") snapshot := testSnapshot("project", "workspace", testUnit("work", WriteSetUnknown)) harness := newHarness(t, map[ProjectID]ProjectWorkflowSnapshot{"project": snapshot}, 1) invoker := &unobservedFailureInvoker{ owner: &continuationTestInvoker{ observations: []agentpolicy.AttemptObservation{ continuationObservationForTarget( current, agentpolicy.QuotaStateAvailable, agentruntime.FailureCodeUnavailable, true, ), }, failures: 1, }, } harness.manager.selector = &continuationTestSelector{initial: current} harness.manager.invoker = invoker harness.start("project", "workspace", nil) if err := harness.manager.Reconcile(context.Background()); err != nil { t.Fatalf("Reconcile: %v", err) } state := harness.store.snapshot() work := state.Projects["project"].Works["work"] if work.State != WorkStateBlocked || work.Blocker == nil || work.Blocker.Code != BlockerInvocationFailed { t.Fatalf("work = %#v", work) } encoded, err := json.Marshal(state) if err != nil { t.Fatalf("Marshal state: %v", err) } if strings.Contains(string(encoded), "credential=provider-secret") { t.Fatalf("durable state retained provider error: %s", encoded) } } type continuationTestSelector struct { initial ExecutionTarget policy FailureContinuationPolicy policies []FailureContinuationPolicy requests []FailureContinuationPolicyRequest } func (s *continuationTestSelector) Select(context.Context, SelectionRequest) (ExecutionTarget, error) { return s.initial, nil } func (s *continuationTestSelector) ContinuationPolicy( _ context.Context, request FailureContinuationPolicyRequest, ) (FailureContinuationPolicy, error) { s.requests = append(s.requests, request) if len(s.policies) == 0 { return s.policy, nil } policy := s.policies[0] s.policies = s.policies[1:] return policy, nil } type unobservedFailureInvoker struct { owner *continuationTestInvoker } func (i *unobservedFailureInvoker) Prepare( ctx context.Context, request DispatchRequest, ) (ProviderLaunch, error) { launch, err := i.owner.Prepare(ctx, request) if err != nil { return nil, err } return unobservedFailureLaunch{ProviderLaunch: launch}, nil } type unobservedFailureLaunch struct { ProviderLaunch } func (l unobservedFailureLaunch) BindStarted( started StartedConfinement, ) (ProviderInvocation, error) { invocation, err := l.ProviderLaunch.BindStarted(started) if err != nil { return nil, err } return unobservedFailureInvocation{ProviderInvocation: invocation}, nil } type unobservedFailureInvocation struct { ProviderInvocation } type continuationTestInvoker struct { observations []agentpolicy.AttemptObservation failures int requests []DispatchRequest } func (i *continuationTestInvoker) Prepare( _ context.Context, request DispatchRequest, ) (ProviderLaunch, error) { i.requests = append(i.requests, request) return &continuationTestLaunch{ owner: i, request: request, index: len(i.requests), }, nil } type continuationTestLaunch struct { owner *continuationTestInvoker request DispatchRequest index int } func (continuationTestLaunch) Command() ConfinementCommand { return ConfinementCommand{Name: "true"} } func (l *continuationTestLaunch) BindStarted( started StartedConfinement, ) (ProviderInvocation, error) { if started == nil || started.Child() == nil { return nil, errors.New("missing confined started handle") } locator := LocatorRecord{ Kind: LocatorProcess, Opaque: fmt.Sprintf("continuation-process-%d", l.index), Revision: "process-r1", ProjectID: l.request.Project.ProjectID, WorkspaceID: l.request.Project.WorkspaceID, WorkUnitID: l.request.Work.Unit.ID, AttemptID: l.request.Work.AttemptID, } return &continuationTestInvocation{ request: l.request, locators: []LocatorRecord{locator}, observation: l.owner.observation(l.index), fail: l.index <= l.owner.failures, started: started, }, nil } func (i *continuationTestInvoker) observation(index int) agentpolicy.AttemptObservation { if index <= 0 || index > len(i.observations) { return agentpolicy.AttemptObservation{} } return i.observations[index-1].Clone() } type continuationTestInvocation struct { request DispatchRequest locators []LocatorRecord observation agentpolicy.AttemptObservation fail bool started StartedConfinement } func (i *continuationTestInvocation) Locators() []LocatorRecord { return append([]LocatorRecord(nil), i.locators...) } func (i *continuationTestInvocation) Wait(context.Context) (Submission, error) { if i.started != nil { if stdin := i.started.Stdin(); stdin != nil { _ = stdin.Close() } if child := i.started.Child(); child != nil { _ = child.Wait() } if stdout := i.started.Stdout(); stdout != nil { _ = stdout.Close() } if stderr := i.started.Stderr(); stderr != nil { _ = stderr.Close() } } if i.fail { return Submission{}, errors.New("credential=provider-secret") } return Submission{ ProjectID: i.request.Project.ProjectID, WorkUnitID: i.request.Work.Unit.ID, AttemptID: i.request.Work.AttemptID, ArtifactID: ArtifactID("artifact-" + string(i.request.Work.AttemptID)), Ready: true, Locators: append([]LocatorRecord(nil), i.locators...), }, nil } func (i *continuationTestInvocation) Cancel(context.Context) error { if i.started != nil { _ = i.started.Abort() } return nil } func (i *continuationTestInvocation) FailureObservation() agentpolicy.AttemptObservation { return i.observation.Clone() } func continuationTarget(profile, revision string) ExecutionTarget { return ExecutionTarget{ ProviderID: "provider", ModelID: "model", ProfileID: profile, ProfileRevision: revision, ConfigRevision: "config-r1", Capacity: 1, } } func continuationObservationForTarget( target ExecutionTarget, state agentpolicy.QuotaState, code agentruntime.FailureCode, retryable bool, ) agentpolicy.AttemptObservation { return agentpolicy.NormalizeAttemptObservation( continuationQuotaSnapshot(target, state), &agentruntime.Failure{Code: code, Retryable: retryable}, continuationTestNow, 0, ) } func continuationQuotaSnapshot( target ExecutionTarget, state agentpolicy.QuotaState, ) status.QuotaSnapshot { var requiredCaps []string var usage *status.UsageStatus switch state { case agentpolicy.QuotaStateAvailable: requiredCaps = []string{"overall"} usage = &status.UsageStatus{DailyLimit: "50%"} case agentpolicy.QuotaStateExhausted: requiredCaps = []string{"overall"} usage = &status.UsageStatus{DailyLimit: "0%"} case agentpolicy.QuotaStateUnknown: requiredCaps = []string{"overall"} usage = &status.UsageStatus{DailyLimit: "not-a-percent"} case agentpolicy.QuotaStateNotApplicable: requiredCaps = nil default: panic("unsupported quota state") } return status.NormalizeQuotaSnapshot( target.ProviderID, target.ProfileID, requiredCaps, continuationTestNow, usage, nil, ) } func continuationCandidate( target ExecutionTarget, state agentpolicy.QuotaState, ) FailureContinuationCandidate { return FailureContinuationCandidate{ Target: target, Eligible: true, Quota: agentpolicy.NormalizeQuotaObservation( continuationQuotaSnapshot(target, state), continuationTestNow, time.Minute, ), } }