iop/apps/agent/internal/projectlog/record.go

666 lines
21 KiB
Go

// Package projectlog provides immutable, versioned presentation and recovery
// project-log record schemas and validation matrices for the standalone host.
package projectlog
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"iop/packages/go/agentpolicy"
"iop/packages/go/agenttask"
)
const (
RecordSchemaVersion uint32 = 1
ArchiveManifestSchemaVersion uint32 = 1
)
const (
MaxIDLength = 256
MaxMessageLength = 4096
MaxMetadataKeys = 32
MaxMetadataValLen = 1024
MaxLocatorsCount = 16
)
var (
ErrInvalidSchemaVersion = errors.New("projectlog: invalid schema version")
ErrInvalidIdentity = errors.New("projectlog: invalid identity")
ErrUnboundedField = errors.New("projectlog: field size exceeds allowed limit")
ErrSensitiveContent = errors.New("projectlog: sensitive content detected")
ErrInvalidTimestamp = errors.New("projectlog: invalid timestamp")
)
type RouteStatus struct {
ProviderID string `json:"provider_id"`
ProfileID string `json:"profile_id"`
ModelID string `json:"model_id"`
RuleID string `json:"rule_id,omitempty"`
ProfileRevision string `json:"profile_revision"`
ConfigRevision string `json:"config_revision"`
SelectionRevision string `json:"selection_revision,omitempty"`
ReasonCode string `json:"reason_code,omitempty"`
}
type ProjectLogRecord struct {
SchemaVersion uint32 `json:"schema_version"`
RecordID string `json:"record_id"`
Sequence uint64 `json:"sequence"`
LoopOrdinal uint64 `json:"loop_ordinal"`
DispatchOrdinal agenttask.DispatchOrdinal `json:"dispatch_ordinal"`
ProjectID agenttask.ProjectID `json:"project_id"`
WorkspaceID agenttask.WorkspaceID `json:"workspace_id"`
WorkUnitID agenttask.WorkUnitID `json:"work_unit_id,omitempty"`
AttemptID agenttask.AttemptID `json:"attempt_id,omitempty"`
CommandID agenttask.CommandID `json:"command_id,omitempty"`
EventType agenttask.EventType `json:"event_type"`
State agenttask.WorkState `json:"state,omitempty"`
StateRevision agenttask.StateRevision `json:"state_revision,omitempty"`
RouteStatus *RouteStatus `json:"route_status,omitempty"`
QuotaObservation *agentpolicy.QuotaObservation `json:"quota_observation,omitempty"`
Locators []agenttask.LocatorRecord `json:"locators,omitempty"`
ChangeSetID agenttask.ChangeSetID `json:"change_set_id,omitempty"`
ChangeSetRevision string `json:"change_set_revision,omitempty"`
IntegrationAttempt agenttask.IntegrationAttempt `json:"integration_attempt,omitempty"`
IntegrationOutcome agenttask.IntegrationOutcome `json:"integration_outcome,omitempty"`
BlockerCode agenttask.BlockerCode `json:"blocker_code,omitempty"`
Result string `json:"result,omitempty"`
Message string `json:"message,omitempty"`
Metadata map[string]string `json:"metadata,omitempty"`
Timestamp time.Time `json:"timestamp"`
Terminal bool `json:"terminal"`
}
type ArchiveManifest struct {
SchemaVersion uint32 `json:"schema_version"`
ProjectID agenttask.ProjectID `json:"project_id"`
WorkspaceID agenttask.WorkspaceID `json:"workspace_id"`
ArchiveOrdinal uint64 `json:"archive_ordinal"`
RecordCount int `json:"record_count"`
FirstSequence uint64 `json:"first_sequence"`
LastSequence uint64 `json:"last_sequence"`
Checksum string `json:"checksum"`
ArchivedAt time.Time `json:"archived_at"`
Terminal bool `json:"terminal"`
WorkUnitIDs []agenttask.WorkUnitID `json:"work_unit_ids,omitempty"`
}
func (r ProjectLogRecord) Validate() error {
if r.SchemaVersion != RecordSchemaVersion {
return fmt.Errorf("%w: record schema version %d expected %d", ErrInvalidSchemaVersion, r.SchemaVersion, RecordSchemaVersion)
}
if err := validateIdentity("RecordID", r.RecordID, true); err != nil {
return err
}
if err := validateIdentity("ProjectID", string(r.ProjectID), true); err != nil {
return err
}
if err := validateIdentity("WorkspaceID", string(r.WorkspaceID), true); err != nil {
return err
}
if r.WorkUnitID != "" {
if err := validateIdentity("WorkUnitID", string(r.WorkUnitID), false); err != nil {
return err
}
}
if r.AttemptID != "" {
if err := validateIdentity("AttemptID", string(r.AttemptID), false); err != nil {
return err
}
}
if r.CommandID != "" {
if err := validateIdentity("CommandID", string(r.CommandID), false); err != nil {
return err
}
}
if r.Sequence == 0 {
return fmt.Errorf("%w: Sequence is required", ErrInvalidIdentity)
}
if !knownEventType(r.EventType) {
return fmt.Errorf("%w: unsupported EventType %q", ErrInvalidIdentity, r.EventType)
}
if r.State == "" {
if r.StateRevision != "" {
return fmt.Errorf("%w: StateRevision requires State", ErrInvalidIdentity)
}
if r.Terminal {
return fmt.Errorf("%w: terminal record requires State", ErrInvalidIdentity)
}
} else {
if !knownWorkState(r.State) {
return fmt.Errorf("%w: unsupported State %q", ErrInvalidIdentity, r.State)
}
if r.StateRevision != "" {
if err := validateIdentity("StateRevision", string(r.StateRevision), true); err != nil {
return err
}
}
if r.Terminal != r.State.Terminal() {
return fmt.Errorf("%w: Terminal disagrees with State %q", ErrInvalidIdentity, r.State)
}
}
if r.BlockerCode != "" && !knownBlockerCode(r.BlockerCode) {
return fmt.Errorf("%w: unsupported BlockerCode %q", ErrInvalidIdentity, r.BlockerCode)
}
if r.Timestamp.IsZero() {
return ErrInvalidTimestamp
}
if len(r.Message) > MaxMessageLength {
return fmt.Errorf("%w: message length %d exceeds max %d", ErrUnboundedField, len(r.Message), MaxMessageLength)
}
if containsSensitiveText(r.Message) {
return fmt.Errorf("%w: sensitive content in message", ErrSensitiveContent)
}
if len(r.Result) > MaxIDLength {
return fmt.Errorf("%w: result length %d exceeds max %d", ErrUnboundedField, len(r.Result), MaxIDLength)
}
if containsSensitiveText(r.Result) {
return fmt.Errorf("%w: sensitive content in result", ErrSensitiveContent)
}
if err := validateRouteStatus(r.RouteStatus); err != nil {
return err
}
if err := validateQuotaObservation(r.QuotaObservation); err != nil {
return err
}
if len(r.Locators) > MaxLocatorsCount {
return fmt.Errorf("%w: locators count %d exceeds max %d", ErrUnboundedField, len(r.Locators), MaxLocatorsCount)
}
for _, locator := range r.Locators {
if err := validateLocator(r, locator); err != nil {
return err
}
}
if err := validateChangeSetIntegration(r); err != nil {
return err
}
if len(r.Metadata) > MaxMetadataKeys {
return fmt.Errorf("%w: metadata key count %d exceeds max %d", ErrUnboundedField, len(r.Metadata), MaxMetadataKeys)
}
for k, v := range r.Metadata {
if len(k) > MaxMetadataValLen || len(v) > MaxMetadataValLen {
return fmt.Errorf("%w: metadata key or value length exceeds limit", ErrUnboundedField)
}
if containsSensitiveKey(k) {
return fmt.Errorf("%w: sensitive key %q in metadata", ErrSensitiveContent, k)
}
if containsSensitiveText(v) {
return fmt.Errorf("%w: sensitive value in metadata key %q", ErrSensitiveContent, k)
}
}
return nil
}
func (m ArchiveManifest) Validate() error {
if m.SchemaVersion != ArchiveManifestSchemaVersion {
return fmt.Errorf("%w: manifest schema version %d expected %d", ErrInvalidSchemaVersion, m.SchemaVersion, ArchiveManifestSchemaVersion)
}
if err := validateIdentity("ProjectID", string(m.ProjectID), true); err != nil {
return err
}
if err := validateIdentity("WorkspaceID", string(m.WorkspaceID), true); err != nil {
return err
}
if m.ArchiveOrdinal == 0 {
return fmt.Errorf("%w: ArchiveOrdinal is required", ErrInvalidIdentity)
}
if m.RecordCount <= 0 {
return fmt.Errorf("%w: RecordCount must be positive", ErrInvalidIdentity)
}
if m.FirstSequence == 0 || m.LastSequence < m.FirstSequence {
return fmt.Errorf("%w: archive sequence endpoints are invalid", ErrInvalidIdentity)
}
if m.RecordCount == 1 && m.FirstSequence != m.LastSequence {
return fmt.Errorf("%w: single-record archive endpoints disagree", ErrInvalidIdentity)
}
if m.RecordCount > 1 && m.FirstSequence >= m.LastSequence {
return fmt.Errorf("%w: multi-record archive endpoints are not increasing", ErrInvalidIdentity)
}
if m.ArchivedAt.IsZero() {
return ErrInvalidTimestamp
}
if !validArchiveChecksum(m.Checksum) {
return fmt.Errorf("%w: checksum must be exact lowercase sha256", ErrInvalidIdentity)
}
if !m.Terminal {
return fmt.Errorf("%w: archive must be terminal", ErrInvalidIdentity)
}
seenWorkUnits := make(map[agenttask.WorkUnitID]struct{}, len(m.WorkUnitIDs))
for _, w := range m.WorkUnitIDs {
if err := validateIdentity("WorkUnitID", string(w), true); err != nil {
return err
}
if _, duplicate := seenWorkUnits[w]; duplicate {
return fmt.Errorf("%w: duplicate WorkUnitID %q", ErrInvalidIdentity, w)
}
seenWorkUnits[w] = struct{}{}
}
return nil
}
func NewArchiveManifest(
projectID agenttask.ProjectID,
workspaceID agenttask.WorkspaceID,
archiveOrdinal uint64,
records []ProjectLogRecord,
archivedAt time.Time,
terminal bool,
) (ArchiveManifest, error) {
if archiveOrdinal == 0 {
return ArchiveManifest{}, fmt.Errorf("%w: ArchiveOrdinal is required", ErrInvalidIdentity)
}
if len(records) == 0 {
return ArchiveManifest{}, fmt.Errorf("%w: archive requires records", ErrInvalidIdentity)
}
if archivedAt.IsZero() {
return ArchiveManifest{}, ErrInvalidTimestamp
}
manifest := ArchiveManifest{
SchemaVersion: ArchiveManifestSchemaVersion,
ProjectID: projectID,
WorkspaceID: workspaceID,
ArchiveOrdinal: archiveOrdinal,
RecordCount: len(records),
ArchivedAt: archivedAt,
}
h := sha256.New()
workUnitsMap := make(map[agenttask.WorkUnitID]bool)
workUnits := make([]agenttask.WorkUnitID, 0)
for index, rec := range records {
if err := validateArchiveRecord(manifest, records, index); err != nil {
return ArchiveManifest{}, err
}
payload, err := json.Marshal(rec)
if err != nil {
return ArchiveManifest{}, fmt.Errorf("projectlog: encode record for checksum: %w", err)
}
h.Write(payload)
h.Write([]byte("\n"))
if rec.WorkUnitID != "" && !workUnitsMap[rec.WorkUnitID] {
workUnitsMap[rec.WorkUnitID] = true
workUnits = append(workUnits, rec.WorkUnitID)
}
}
manifest.FirstSequence = records[0].Sequence
manifest.LastSequence = records[len(records)-1].Sequence
manifest.Terminal = records[len(records)-1].Terminal
if terminal != manifest.Terminal {
return ArchiveManifest{}, fmt.Errorf("%w: caller terminal flag disagrees with final record", ErrInvalidIdentity)
}
manifest.Checksum = "sha256:" + hex.EncodeToString(h.Sum(nil))
manifest.WorkUnitIDs = workUnits
if err := manifest.Validate(); err != nil {
return ArchiveManifest{}, err
}
return manifest, nil
}
func validateRouteStatus(route *RouteStatus) error {
if route == nil {
return nil
}
for _, identity := range []struct {
field string
value string
}{
{"RouteStatus.ProviderID", route.ProviderID},
{"RouteStatus.ProfileID", route.ProfileID},
{"RouteStatus.ModelID", route.ModelID},
{"RouteStatus.ProfileRevision", route.ProfileRevision},
{"RouteStatus.ConfigRevision", route.ConfigRevision},
} {
if err := validateIdentity(identity.field, identity.value, true); err != nil {
return err
}
}
if err := validateIdentity("RouteStatus.RuleID", route.RuleID, false); err != nil {
return err
}
if err := validateIdentity("RouteStatus.SelectionRevision", route.SelectionRevision, false); err != nil {
return err
}
if len(route.ReasonCode) > MaxIDLength {
return fmt.Errorf("%w: invalid route reason code", ErrInvalidIdentity)
}
for _, char := range route.ReasonCode {
if (char < 'a' || char > 'z') && (char < '0' || char > '9') && char != '_' {
return fmt.Errorf("%w: route reason code must be lowercase snake case", ErrInvalidIdentity)
}
}
return nil
}
func validateChangeSetIntegration(record ProjectLogRecord) error {
if record.ChangeSetID == "" {
if record.ChangeSetRevision != "" ||
record.IntegrationAttempt != 0 ||
record.IntegrationOutcome != "" {
return fmt.Errorf("%w: integration evidence requires ChangeSetID", ErrInvalidIdentity)
}
return nil
}
if err := validateIdentity("ChangeSetID", string(record.ChangeSetID), true); err != nil {
return err
}
if err := validateIdentity("ChangeSetRevision", record.ChangeSetRevision, true); err != nil {
return err
}
if record.IntegrationOutcome == "" {
return nil
}
if record.IntegrationAttempt == 0 {
return fmt.Errorf("%w: IntegrationOutcome requires IntegrationAttempt", ErrInvalidIdentity)
}
switch record.IntegrationOutcome {
case agenttask.IntegrationOutcomeIntegrated,
agenttask.IntegrationOutcomeTerminalDeferred:
return nil
default:
return fmt.Errorf(
"%w: unsupported IntegrationOutcome %q",
ErrInvalidIdentity,
record.IntegrationOutcome,
)
}
}
func validateQuotaObservation(obs *agentpolicy.QuotaObservation) error {
if obs == nil {
return nil
}
if _, err := json.Marshal(obs); err != nil {
return fmt.Errorf("%w: invalid safe quota observation", ErrInvalidIdentity)
}
if obs.Validity == agentpolicy.ObservationCorrupt {
return nil
}
for _, identity := range []struct {
field string
value string
}{
{"QuotaObservation.SnapshotID", obs.SnapshotID},
{"QuotaObservation.Adapter", obs.Adapter},
{"QuotaObservation.Target", obs.Target},
} {
if err := validateIdentity(identity.field, identity.value, true); err != nil {
return err
}
}
return nil
}
func validateLocator(record ProjectLogRecord, locator agenttask.LocatorRecord) error {
if !knownLocatorKind(locator.Kind) {
return fmt.Errorf("%w: unsupported locator kind %q", ErrInvalidIdentity, locator.Kind)
}
if err := validateIdentity("Locator.Opaque", locator.Opaque, true); err != nil {
return err
}
if err := validateIdentity("Locator.Revision", locator.Revision, true); err != nil {
return err
}
for _, identity := range []struct {
field string
value string
}{
{"Locator.ProjectID", string(locator.ProjectID)},
{"Locator.WorkspaceID", string(locator.WorkspaceID)},
{"Locator.WorkUnitID", string(locator.WorkUnitID)},
{"Locator.AttemptID", string(locator.AttemptID)},
} {
if err := validateIdentity(identity.field, identity.value, true); err != nil {
return err
}
}
if locator.ProjectID != record.ProjectID ||
locator.WorkspaceID != record.WorkspaceID ||
locator.WorkUnitID != record.WorkUnitID ||
locator.AttemptID != record.AttemptID {
return fmt.Errorf("%w: locator identity disagrees with enclosing record", ErrInvalidIdentity)
}
return nil
}
func validateArchiveRecord(manifest ArchiveManifest, records []ProjectLogRecord, index int) error {
record := records[index]
if err := record.Validate(); err != nil {
return fmt.Errorf("projectlog: invalid record in archive: %w", err)
}
if record.ProjectID != manifest.ProjectID || record.WorkspaceID != manifest.WorkspaceID {
return fmt.Errorf("%w: archive record identity disagrees with manifest", ErrInvalidIdentity)
}
if index > 0 && record.Sequence <= records[index-1].Sequence {
return fmt.Errorf("%w: archive record sequences must be strictly increasing", ErrInvalidIdentity)
}
last := index == len(records)-1
if !last && record.Terminal {
return fmt.Errorf("%w: archive contains an early terminal record", ErrInvalidIdentity)
}
if last && !record.Terminal {
return fmt.Errorf("%w: archive final record must be terminal", ErrInvalidIdentity)
}
return nil
}
func validArchiveChecksum(checksum string) bool {
const prefix = "sha256:"
if !strings.HasPrefix(checksum, prefix) || len(checksum) != len(prefix)+sha256.Size*2 {
return false
}
digest := strings.TrimPrefix(checksum, prefix)
if digest != strings.ToLower(digest) {
return false
}
_, err := hex.DecodeString(digest)
return err == nil
}
func knownEventType(eventType agenttask.EventType) bool {
switch eventType {
case agenttask.EventObserved,
agenttask.EventManualStart,
agenttask.EventAutoResume,
agenttask.EventStopped,
agenttask.EventDependencyReady,
agenttask.EventDispatchStarted,
agenttask.EventSubmissionAccepted,
agenttask.EventReviewResult,
agenttask.EventFollowup,
agenttask.EventIntegrationResult,
agenttask.EventBlocked,
agenttask.EventCompleted:
return true
default:
return false
}
}
func knownWorkState(state agenttask.WorkState) bool {
switch state {
case agenttask.WorkStateObserved,
agenttask.WorkStateReady,
agenttask.WorkStatePreparing,
agenttask.WorkStateDispatching,
agenttask.WorkStateSubmitted,
agenttask.WorkStateReviewing,
agenttask.WorkStatePendingIntegration,
agenttask.WorkStateIntegrating,
agenttask.WorkStateCompleted,
agenttask.WorkStateTerminalDeferred,
agenttask.WorkStateBlocked,
agenttask.WorkStateStopped:
return true
default:
return false
}
}
func knownLocatorKind(kind agenttask.LocatorKind) bool {
switch kind {
case agenttask.LocatorProcess,
agenttask.LocatorSession,
agenttask.LocatorOverlay,
agenttask.LocatorChangeSet,
agenttask.LocatorCompletion:
return true
default:
return false
}
}
func knownBlockerCode(code agenttask.BlockerCode) bool {
switch code {
case agenttask.BlockerInvalidIdentity,
agenttask.BlockerWorkflowUnavailable,
agenttask.BlockerWorkflowRevisionDrift,
agenttask.BlockerDependencyMissing,
agenttask.BlockerDependencyAmbiguous,
agenttask.BlockerDependencyBlocked,
agenttask.BlockerSelectionFailed,
agenttask.BlockerIsolationFailed,
agenttask.BlockerAdmissionFailed,
agenttask.BlockerProviderCapacity,
agenttask.BlockerInvocationFailed,
agenttask.BlockerSubmissionIncomplete,
agenttask.BlockerArtifactMismatch,
agenttask.BlockerEvidenceUnavailable,
agenttask.BlockerEvidenceRepairDenied,
agenttask.BlockerEvidenceRepairFailed,
agenttask.BlockerReviewFailed,
agenttask.BlockerReviewReworkExhausted,
agenttask.BlockerUserReview,
agenttask.BlockerIntegrationFailed,
agenttask.BlockerCorruptCheckpoint,
agenttask.BlockerStaleCheckpoint,
agenttask.BlockerAmbiguousCheckpoint,
agenttask.BlockerFailurePolicyUnavailable,
agenttask.BlockerFailureObservationUnknown,
agenttask.BlockerFailureObservationStale,
agenttask.BlockerFailureObservationCorrupt,
agenttask.BlockerFailureUnknown,
agenttask.BlockerFailurePolicyDenied,
agenttask.BlockerNoEligibleFailover,
agenttask.BlockerPartialCompletion,
agenttask.BlockerFailureBudgetExhausted,
agenttask.BlockerDuplicateDeviceLease,
agenttask.BlockerDuplicateProjectLease,
agenttask.BlockerDuplicateWorkspaceCall:
return true
default:
return false
}
}
func validateIdentity(field string, value string, required bool) error {
if required && strings.TrimSpace(value) == "" {
return fmt.Errorf("%w: %s is required", ErrInvalidIdentity, field)
}
if value != strings.TrimSpace(value) {
return fmt.Errorf("%w: %s has leading or trailing whitespace", ErrInvalidIdentity, field)
}
if len(value) > MaxIDLength {
return fmt.Errorf("%w: %s length %d exceeds max %d", ErrUnboundedField, field, len(value), MaxIDLength)
}
if strings.ContainsAny(value, "\x00\r\n") {
return fmt.Errorf("%w: %s contains invalid characters", ErrInvalidIdentity, field)
}
if containsSensitiveText(value) {
return fmt.Errorf("%w: sensitive content in %s", ErrSensitiveContent, field)
}
return nil
}
func containsSensitiveText(s string) bool {
if s == "" {
return false
}
lower := strings.ToLower(s)
sensitiveTokens := []string{
"bearer ",
"sk-",
"ghp_",
"eyj",
"aws_secret_access_key",
"begin private key",
"password=",
"secret=",
"api_key=",
"apikey=",
"auth_token",
"private_key",
}
for _, token := range sensitiveTokens {
if strings.Contains(lower, token) {
return true
}
}
rawEnvPatterns := []string{
"path=",
"home=",
"shell=",
"user=",
"ld_library_path=",
"http_proxy=",
"https_proxy=",
}
for _, env := range rawEnvPatterns {
if strings.Contains(lower, env) {
return true
}
}
return false
}
func containsSensitiveKey(key string) bool {
if key == "" {
return false
}
lower := strings.ToLower(key)
sensitiveKeys := []string{
"secret",
"password",
"token",
"credential",
"api_key",
"apikey",
"private_key",
"auth",
"authorization",
"env",
"environment",
"raw_output",
"provider_output",
"stdout",
"stderr",
}
for _, k := range sensitiveKeys {
if strings.Contains(lower, k) {
return true
}
}
return false
}