독립 호스트에서 안전한 작업 실행과 복구를 제공하기 위해 런타임 설정, 정책, 상태 저장소, 워크스페이스 격리 및 AgentTask 오케스트레이션을 확장한다.
222 lines
7.5 KiB
Go
222 lines
7.5 KiB
Go
package agentpolicy
|
|
|
|
import (
|
|
"fmt"
|
|
|
|
"iop/packages/go/agentruntime"
|
|
)
|
|
|
|
// TargetIdentity is the complete immutable target identity used by a
|
|
// continuation decision. Capacity and host-only execution settings remain at
|
|
// the manager port boundary.
|
|
type TargetIdentity struct {
|
|
ProviderID string
|
|
ModelID string
|
|
ProfileID string
|
|
ProfileRevision string
|
|
}
|
|
|
|
// FailurePolicy is the declared recovery policy for one immutable selection
|
|
// revision. Retry and failover permissions are intentionally separate.
|
|
type FailurePolicy struct {
|
|
RetryableCodes []agentruntime.FailureCode
|
|
FailoverCodes []agentruntime.FailureCode
|
|
}
|
|
|
|
// FailureBudget is the dispatch-stage failure count including the failure
|
|
// currently being evaluated.
|
|
type FailureBudget struct {
|
|
Used uint32
|
|
Limit uint32
|
|
}
|
|
|
|
// ContinuationCandidate is one ordered policy candidate with fresh quota
|
|
// evidence. Candidates are never selected unless the policy marked them
|
|
// eligible and they have not already been used by this work unit.
|
|
type ContinuationCandidate struct {
|
|
Target TargetIdentity
|
|
Eligible bool
|
|
Quota QuotaObservation
|
|
}
|
|
|
|
// ContinuationRequest contains only immutable input captured for one failed
|
|
// attempt. The policy never mutates these values or performs a new selection.
|
|
type ContinuationRequest struct {
|
|
Policy FailurePolicy
|
|
Current TargetIdentity
|
|
Candidates []ContinuationCandidate
|
|
Used []TargetIdentity
|
|
Budget FailureBudget
|
|
Observation AttemptObservation
|
|
}
|
|
|
|
type ContinuationAction string
|
|
|
|
const (
|
|
ContinuationRetry ContinuationAction = "retry"
|
|
ContinuationFailover ContinuationAction = "failover"
|
|
ContinuationBlock ContinuationAction = "block"
|
|
)
|
|
|
|
type ContinuationBlockerCode string
|
|
|
|
const (
|
|
ContinuationBlockerUnknownQuota ContinuationBlockerCode = "unknown_quota_observation"
|
|
ContinuationBlockerStaleObservation ContinuationBlockerCode = "stale_quota_observation"
|
|
ContinuationBlockerCorruptObservation ContinuationBlockerCode = "corrupt_quota_observation"
|
|
ContinuationBlockerUnknownFailure ContinuationBlockerCode = "unknown_failure"
|
|
ContinuationBlockerBudgetExhausted ContinuationBlockerCode = "failure_budget_exhausted"
|
|
ContinuationBlockerPolicyDenied ContinuationBlockerCode = "failure_not_declared_by_policy"
|
|
ContinuationBlockerNoAlternate ContinuationBlockerCode = "no_eligible_failover_target"
|
|
)
|
|
|
|
// ContinuationDecision is a closed result: retry and failover include the
|
|
// exact next target, while block includes a typed reason and no target.
|
|
type ContinuationDecision struct {
|
|
Action ContinuationAction
|
|
Target TargetIdentity
|
|
Blocker ContinuationBlockerCode
|
|
}
|
|
|
|
// DecideContinuation applies the declared recovery policy without any silent
|
|
// re-selection. A valid known failure can retry only the current target, and
|
|
// failover can use only the first eligible unused candidate in policy order.
|
|
func DecideContinuation(request ContinuationRequest) (ContinuationDecision, error) {
|
|
if err := validateContinuationRequest(request); err != nil {
|
|
return ContinuationDecision{}, err
|
|
}
|
|
if request.Observation.Quota.Validity == ObservationCorrupt {
|
|
return blocked(ContinuationBlockerCorruptObservation), nil
|
|
}
|
|
if request.Observation.Quota.Validity == ObservationStale {
|
|
return blocked(ContinuationBlockerStaleObservation), nil
|
|
}
|
|
if request.Observation.Quota.State == QuotaStateUnknown {
|
|
return blocked(ContinuationBlockerUnknownQuota), nil
|
|
}
|
|
if request.Observation.Failure.Code == agentruntime.FailureCodeUnknown {
|
|
return blocked(ContinuationBlockerUnknownFailure), nil
|
|
}
|
|
if request.Budget.Used >= request.Budget.Limit {
|
|
return blocked(ContinuationBlockerBudgetExhausted), nil
|
|
}
|
|
|
|
failure := request.Observation.Failure
|
|
if failure.Retryable && containsFailureCode(request.Policy.RetryableCodes, failure.Code) &&
|
|
quotaAllowsContinuation(request.Observation.Quota.State) {
|
|
return ContinuationDecision{Action: ContinuationRetry, Target: request.Current}, nil
|
|
}
|
|
if !containsFailureCode(request.Policy.FailoverCodes, failure.Code) {
|
|
return blocked(ContinuationBlockerPolicyDenied), nil
|
|
}
|
|
|
|
used := make(map[TargetIdentity]struct{}, len(request.Used)+1)
|
|
used[request.Current] = struct{}{}
|
|
for _, target := range request.Used {
|
|
used[target] = struct{}{}
|
|
}
|
|
for _, candidate := range request.Candidates {
|
|
if !candidate.Eligible {
|
|
continue
|
|
}
|
|
if _, alreadyUsed := used[candidate.Target]; alreadyUsed {
|
|
continue
|
|
}
|
|
switch candidate.Quota.Validity {
|
|
case ObservationCorrupt:
|
|
return blocked(ContinuationBlockerCorruptObservation), nil
|
|
case ObservationStale:
|
|
return blocked(ContinuationBlockerStaleObservation), nil
|
|
}
|
|
switch candidate.Quota.State {
|
|
case QuotaStateAvailable, QuotaStateNotApplicable:
|
|
return ContinuationDecision{Action: ContinuationFailover, Target: candidate.Target}, nil
|
|
case QuotaStateUnknown:
|
|
return blocked(ContinuationBlockerUnknownQuota), nil
|
|
case QuotaStateExhausted:
|
|
continue
|
|
}
|
|
}
|
|
return blocked(ContinuationBlockerNoAlternate), nil
|
|
}
|
|
|
|
func blocked(code ContinuationBlockerCode) ContinuationDecision {
|
|
return ContinuationDecision{Action: ContinuationBlock, Blocker: code}
|
|
}
|
|
|
|
func validateContinuationRequest(request ContinuationRequest) error {
|
|
if err := validateTargetIdentity(request.Current); err != nil {
|
|
return fmt.Errorf("agentpolicy: invalid current target: %w", err)
|
|
}
|
|
if request.Budget.Limit == 0 || request.Budget.Used > request.Budget.Limit {
|
|
return fmt.Errorf("agentpolicy: invalid failure budget")
|
|
}
|
|
if !ValidateAttemptObservation(request.Observation) {
|
|
return fmt.Errorf("agentpolicy: invalid attempt observation")
|
|
}
|
|
if err := validatePolicyCodes(request.Policy.RetryableCodes); err != nil {
|
|
return err
|
|
}
|
|
if err := validatePolicyCodes(request.Policy.FailoverCodes); err != nil {
|
|
return err
|
|
}
|
|
for index, target := range request.Used {
|
|
if err := validateTargetIdentity(target); err != nil {
|
|
return fmt.Errorf("agentpolicy: invalid used target %d: %w", index, err)
|
|
}
|
|
}
|
|
for index, candidate := range request.Candidates {
|
|
if err := validateTargetIdentity(candidate.Target); err != nil {
|
|
return fmt.Errorf("agentpolicy: invalid candidate %d: %w", index, err)
|
|
}
|
|
if !validQuotaObservation(candidate.Quota) {
|
|
return fmt.Errorf("agentpolicy: invalid candidate %d quota observation", index)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validQuotaObservation(observation QuotaObservation) bool {
|
|
return validateQuotaObservation(observation)
|
|
}
|
|
|
|
func validateTargetIdentity(target TargetIdentity) error {
|
|
for name, value := range map[string]string{
|
|
"provider": target.ProviderID,
|
|
"model": target.ModelID,
|
|
"profile": target.ProfileID,
|
|
"profile revision": target.ProfileRevision,
|
|
} {
|
|
if !safeObservationIdentity(value) {
|
|
return fmt.Errorf("%s identity is missing or malformed", name)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validatePolicyCodes(codes []agentruntime.FailureCode) error {
|
|
seen := make(map[agentruntime.FailureCode]struct{}, len(codes))
|
|
for _, code := range codes {
|
|
if code == agentruntime.FailureCodeUnknown || !knownFailureCode(code) {
|
|
return fmt.Errorf("agentpolicy: policy contains unknown failure code %q", code)
|
|
}
|
|
if _, duplicate := seen[code]; duplicate {
|
|
return fmt.Errorf("agentpolicy: policy repeats failure code %q", code)
|
|
}
|
|
seen[code] = struct{}{}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func containsFailureCode(codes []agentruntime.FailureCode, want agentruntime.FailureCode) bool {
|
|
for _, code := range codes {
|
|
if code == want {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func quotaAllowsContinuation(state QuotaState) bool {
|
|
return state == QuotaStateAvailable || state == QuotaStateNotApplicable
|
|
}
|