// Package projectlog provides immutable, versioned presentation and recovery // project-log record schemas and validation matrices for the standalone host. // // This file implements the CAS-backed journal and terminal archive // reconciliation over a device-local root. It owns the persistence invariant: // append by CAS with bounded retry, retention only after terminal evidence, // restart-safe materialization of redacted JSONL/timeline files, and // idempotent reconciliation that fails closed on conflicting content. package projectlog import ( "bytes" "context" "crypto/sha256" "encoding/binary" "encoding/hex" "encoding/json" "errors" "fmt" "math" "os" "path/filepath" "strconv" "strings" "syscall" "time" "iop/packages/go/agentstate" "iop/packages/go/agenttask" ) const journalSchemaVersion uint32 = 1 const eventReplayIndexSchemaVersion uint32 = 1 const maxCASRetries = 16 const ( journalIntegrationPrefix = "projectlog:" eventReplayIntegrationPrefix = "projectlog-event-replay:" ) var ( ErrNoTerminalRecord = errors.New("projectlog: no terminal record to archive") ErrArchiveIncomplete = errors.New("projectlog: archive incomplete") ErrArchiveConflict = errors.New("projectlog: archive content conflict") ErrRecordReplayConflict = errors.New("projectlog: record replay content conflict") ErrCASExhausted = errors.New("projectlog: CAS retry exhausted") ) type seenRecord struct { Sequence uint64 `json:"sequence"` RecordFingerprint string `json:"record_fingerprint"` EventFingerprint string `json:"event_fingerprint,omitempty"` Fingerprint string `json:"fingerprint,omitempty"` } // journal is the durable CAS payload for one project or work-unit scope. type journal struct { SchemaVersion uint32 `json:"schema_version"` ProjectID agenttask.ProjectID `json:"project_id"` WorkspaceID agenttask.WorkspaceID `json:"workspace_id"` WorkUnitID agenttask.WorkUnitID `json:"work_unit_id,omitempty"` NextSequence uint64 `json:"next_sequence"` ArchiveOrdinal uint64 `json:"archive_ordinal"` SeenRecords map[string]seenRecord `json:"seen_records"` Records []ProjectLogRecord `json:"records"` } type eventReplayEntry struct { RecordID string `json:"record_id"` WorkUnitID agenttask.WorkUnitID `json:"work_unit_id,omitempty"` Sequence uint64 `json:"sequence"` EventFingerprint string `json:"event_fingerprint"` } // eventReplayIndex is the retained project/workspace-wide ownership record for // manager EventID projections. Task-scoped journals retain their independent // sequence and archive lifecycle. type eventReplayIndex struct { SchemaVersion uint32 `json:"schema_version"` ProjectID agenttask.ProjectID `json:"project_id"` WorkspaceID agenttask.WorkspaceID `json:"workspace_id"` Entries map[string]eventReplayEntry `json:"entries"` } // archiveIntent is the durable crash-recovery marker written before any // artifact. It records the manifest checksum so Reconcile can verify or // recompute the manifest without trusting partial state. type archiveIntent struct { SchemaVersion uint32 `json:"schema_version"` ProjectID agenttask.ProjectID `json:"project_id"` WorkspaceID agenttask.WorkspaceID `json:"workspace_id"` WorkUnitID agenttask.WorkUnitID `json:"work_unit_id,omitempty"` ArchiveOrdinal uint64 `json:"archive_ordinal"` FirstSequence uint64 `json:"first_sequence"` LastSequence uint64 `json:"last_sequence"` RecordCount int `json:"record_count"` Checksum string `json:"checksum"` ArchivedAt time.Time `json:"archived_at"` Terminal bool `json:"terminal"` } // Store is the CAS-backed project log journal with device-local archive roots. // It wraps an agentstate.Store for the durable CAS journal and a device-local // root directory for materialized archive artifacts. type Store struct { state *agentstate.Store root string projectID agenttask.ProjectID workspaceID agenttask.WorkspaceID workUnitID agenttask.WorkUnitID key string replayKey string failureHook func(phase string) error } // NewStore creates a CAS-backed project log store over a device-local root. // The state store provides the crash-safe CAS journal; root holds archive // artifacts (intent, JSONL, timeline, manifest). func NewStore( state *agentstate.Store, root string, projectID agenttask.ProjectID, workspaceID agenttask.WorkspaceID, ) (*Store, error) { if state == nil { return nil, fmt.Errorf("projectlog: state store is required") } if root == "" { return nil, fmt.Errorf("projectlog: root is required") } if projectID == "" || workspaceID == "" { return nil, fmt.Errorf("projectlog: project and workspace IDs are required") } return &Store{ state: state, root: filepath.Clean(root), projectID: projectID, workspaceID: workspaceID, key: integrationRecordKey(projectID, workspaceID, ""), replayKey: eventReplayIndexKey(projectID, workspaceID), }, nil } // ForWorkUnit returns a task-scoped journal below the same project/workspace. // Each scoped journal owns its own sequence and archive ordinal. func (s *Store) ForWorkUnit(workUnitID agenttask.WorkUnitID) (*Store, error) { if s == nil { return nil, fmt.Errorf("projectlog: store is required") } if err := validateIdentity("WorkUnitID", string(workUnitID), true); err != nil { return nil, err } return &Store{ state: s.state, root: s.root, projectID: s.projectID, workspaceID: s.workspaceID, workUnitID: workUnitID, key: integrationRecordKey(s.projectID, s.workspaceID, workUnitID), replayKey: s.replayKey, failureHook: s.failureHook, }, nil } // WithFailureHook installs a phase-gated failure hook for testing crash // windows. The hook is invoked at named phases during Archive; if it returns // a non-nil error the operation aborts at that phase. Reconcile is never // gated by the hook. func (s *Store) WithFailureHook(hook func(phase string) error) *Store { s.failureHook = hook return s } func (s *Store) failAt(phase string) error { if s.failureHook != nil { return s.failureHook(phase) } return nil } func (s *Store) loadJournal(ctx context.Context) (journal, string, bool, error) { payload, revision, found, err := s.state.LoadIntegrationRecord(ctx, s.key) if err != nil { return journal{}, "", false, err } if !found { return newJournal(s.projectID, s.workspaceID, s.workUnitID), revision, false, nil } var j journal if err := json.Unmarshal(payload, &j); err != nil { return journal{}, "", false, fmt.Errorf("projectlog: decode journal: %w", err) } normalizeJournal(&j) if err := s.validateJournal(j); err != nil { return journal{}, "", false, err } return j, revision, true, nil } func (s *Store) validateJournal(j journal) error { if j.SchemaVersion != journalSchemaVersion { return fmt.Errorf( "%w: journal schema version %d expected %d", ErrInvalidSchemaVersion, j.SchemaVersion, journalSchemaVersion, ) } if j.ProjectID != s.projectID || j.WorkspaceID != s.workspaceID { return fmt.Errorf("%w: journal identity disagrees with store", ErrInvalidIdentity) } if j.WorkUnitID != s.workUnitID { return fmt.Errorf("%w: journal work-unit scope disagrees with store", ErrInvalidIdentity) } if j.NextSequence == 0 || j.NextSequence == math.MaxUint64 { return fmt.Errorf("%w: journal next sequence %d cannot be assigned safely", ErrInvalidIdentity, j.NextSequence) } for index, record := range j.Records { if err := record.Validate(); err != nil { return fmt.Errorf("projectlog: invalid journal record %d: %w", index, err) } if record.ProjectID != s.projectID || record.WorkspaceID != s.workspaceID { return fmt.Errorf("%w: journal record %d identity disagrees with store", ErrInvalidIdentity, index) } if s.workUnitID != "" && record.WorkUnitID != s.workUnitID { return fmt.Errorf("%w: journal record %d work unit disagrees with scope", ErrInvalidIdentity, index) } if index > 0 && record.Sequence != j.Records[index-1].Sequence+1 { return fmt.Errorf( "%w: journal record sequence %d does not follow %d", ErrInvalidIdentity, record.Sequence, j.Records[index-1].Sequence, ) } seen, ok := j.SeenRecords[record.RecordID] if !ok { return fmt.Errorf("%w: record %q is absent from replay index", ErrInvalidIdentity, record.RecordID) } fingerprint, err := recordFingerprint(record) if err != nil { return err } if seen.Sequence != record.Sequence || seen.RecordFingerprint != fingerprint { return fmt.Errorf("%w: replay index disagrees with record %q", ErrInvalidIdentity, record.RecordID) } } if len(j.Records) > 0 { lastSequence := j.Records[len(j.Records)-1].Sequence if lastSequence == math.MaxUint64 || j.NextSequence != lastSequence+1 { return fmt.Errorf( "%w: journal next sequence %d does not follow retained sequence %d", ErrInvalidIdentity, j.NextSequence, lastSequence, ) } } for recordID, seen := range j.SeenRecords { if err := validateIdentity("SeenRecords.RecordID", recordID, true); err != nil { return err } if seen.Sequence == 0 || seen.Sequence >= j.NextSequence { return fmt.Errorf("%w: replay sequence for %q is outside journal range", ErrInvalidIdentity, recordID) } if !validRecordFingerprint(seen.RecordFingerprint) { return fmt.Errorf("%w: replay fingerprint for %q is invalid", ErrInvalidIdentity, recordID) } if seen.EventFingerprint != "" && !validRecordFingerprint(seen.EventFingerprint) { return fmt.Errorf("%w: event replay fingerprint for %q is invalid", ErrInvalidIdentity, recordID) } } return nil } // AppendRecord validates the record, assigns a monotonic sequence, and // CAS-appends it to the project journal with bounded retry on revision // conflict. It returns the assigned sequence number. func (s *Store) AppendRecord( ctx context.Context, record ProjectLogRecord, ) (uint64, error) { return s.appendRecord(ctx, record, "") } func (s *Store) appendRecord( ctx context.Context, record ProjectLogRecord, eventFingerprint string, ) (uint64, error) { if record.Sequence != 0 { return 0, fmt.Errorf("%w: append sequence must be unassigned", ErrInvalidIdentity) } if record.ProjectID != s.projectID || record.WorkspaceID != s.workspaceID { return 0, fmt.Errorf("%w: record identity disagrees with store", ErrInvalidIdentity) } if s.workUnitID != "" && record.WorkUnitID != s.workUnitID { return 0, fmt.Errorf("%w: record work unit disagrees with store scope", ErrInvalidIdentity) } if eventFingerprint != "" && !validRecordFingerprint(eventFingerprint) { return 0, fmt.Errorf("%w: event fingerprint is invalid", ErrInvalidIdentity) } fingerprint, err := recordFingerprint(record) if err != nil { return 0, err } for attempt := 0; attempt < maxCASRetries; attempt++ { if err := ctx.Err(); err != nil { return 0, err } j, revision, _, err := s.loadJournal(ctx) if err != nil { return 0, err } if seen, ok := j.SeenRecords[record.RecordID]; ok { if eventFingerprint != "" && seen.EventFingerprint != "" { if seen.EventFingerprint != eventFingerprint { return 0, fmt.Errorf( "%w: manager event id %q", ErrRecordReplayConflict, record.RecordID, ) } return seen.Sequence, nil } if seen.RecordFingerprint != fingerprint { return 0, fmt.Errorf( "%w: record id %q", ErrRecordReplayConflict, record.RecordID, ) } if eventFingerprint != "" { seen.EventFingerprint = eventFingerprint j.SeenRecords[record.RecordID] = seen if err := s.validateJournal(j); err != nil { return 0, err } next, err := json.Marshal(j) if err != nil { return 0, fmt.Errorf("projectlog: encode journal: %w", err) } if _, err := s.state.CompareAndSwapIntegrationRecord(ctx, s.key, revision, next); err != nil { if errors.Is(err, agenttask.ErrRevisionConflict) { continue } return 0, err } } return seen.Sequence, nil } rec := record seq := j.NextSequence rec.Sequence = seq if err := rec.Validate(); err != nil { return 0, err } j.Records = append(j.Records, rec) j.SeenRecords[rec.RecordID] = seenRecord{ Sequence: seq, RecordFingerprint: fingerprint, EventFingerprint: eventFingerprint, } j.NextSequence = seq + 1 if err := s.validateJournal(j); err != nil { return 0, err } next, err := json.Marshal(j) if err != nil { return 0, fmt.Errorf("projectlog: encode journal: %w", err) } if _, err := s.state.CompareAndSwapIntegrationRecord(ctx, s.key, revision, next); err != nil { if errors.Is(err, agenttask.ErrRevisionConflict) { continue } return 0, err } return seq, nil } return 0, fmt.Errorf("%w: after %d retries", ErrCASExhausted, maxCASRetries) } func (s *Store) scopeForWorkUnit(workUnitID agenttask.WorkUnitID) (*Store, error) { if workUnitID != "" { if s.workUnitID == workUnitID { return s, nil } return s.ForWorkUnit(workUnitID) } if s.workUnitID == "" { return s, nil } return &Store{ state: s.state, root: s.root, projectID: s.projectID, workspaceID: s.workspaceID, key: integrationRecordKey(s.projectID, s.workspaceID, ""), replayKey: s.replayKey, failureHook: s.failureHook, }, nil } func (s *Store) loadEventReplayIndex( ctx context.Context, ) (eventReplayIndex, string, bool, error) { payload, revision, found, err := s.state.LoadIntegrationRecord(ctx, s.replayKey) if err != nil { return eventReplayIndex{}, "", false, err } if !found { return newEventReplayIndex(s.projectID, s.workspaceID), revision, false, nil } var index eventReplayIndex if err := json.Unmarshal(payload, &index); err != nil { return eventReplayIndex{}, "", false, fmt.Errorf( "projectlog: decode event replay index: %w", err, ) } normalizeEventReplayIndex(&index) if err := s.validateEventReplayIndex(index); err != nil { return eventReplayIndex{}, "", false, err } return index, revision, true, nil } func (s *Store) validateEventReplayIndex(index eventReplayIndex) error { if index.SchemaVersion != eventReplayIndexSchemaVersion { return fmt.Errorf( "%w: event replay index schema version %d expected %d", ErrInvalidSchemaVersion, index.SchemaVersion, eventReplayIndexSchemaVersion, ) } if index.ProjectID != s.projectID || index.WorkspaceID != s.workspaceID { return fmt.Errorf("%w: event replay index identity disagrees with store", ErrInvalidIdentity) } for recordID, entry := range index.Entries { if err := validateIdentity("EventReplayIndex.RecordID", recordID, true); err != nil { return err } if entry.RecordID != recordID { return fmt.Errorf("%w: event replay entry key disagrees with record id", ErrInvalidIdentity) } if entry.WorkUnitID != "" { if err := validateIdentity( "EventReplayIndex.WorkUnitID", string(entry.WorkUnitID), false, ); err != nil { return err } } if entry.Sequence == 0 || entry.Sequence == math.MaxUint64 { return fmt.Errorf( "%w: event replay sequence for %q cannot be retained safely", ErrInvalidIdentity, recordID, ) } if !validRecordFingerprint(entry.EventFingerprint) { return fmt.Errorf( "%w: event replay fingerprint for %q is invalid", ErrInvalidIdentity, recordID, ) } } return nil } func (s *Store) loadOrRecoverEventReplayIndex( ctx context.Context, recordID string, ) (eventReplayIndex, string, error) { for attempt := 0; attempt < maxCASRetries; attempt++ { index, revision, _, err := s.loadEventReplayIndex(ctx) if err != nil { return eventReplayIndex{}, "", err } if _, retained := index.Entries[recordID]; retained { return index, revision, nil } recovered, err := s.legacyEventReplayEntries(ctx) if err != nil { return eventReplayIndex{}, "", err } changed := false for recoveredID, entry := range recovered { if retained, ok := index.Entries[recoveredID]; ok { if retained != entry { return eventReplayIndex{}, "", eventReplayConflict(recoveredID) } continue } index.Entries[recoveredID] = entry changed = true } if !changed { return index, revision, nil } if err := s.validateEventReplayIndex(index); err != nil { return eventReplayIndex{}, "", err } payload, err := json.Marshal(index) if err != nil { return eventReplayIndex{}, "", fmt.Errorf( "projectlog: encode recovered event replay index: %w", err, ) } committed, err := s.state.CompareAndSwapIntegrationRecord( ctx, s.replayKey, revision, payload, ) if errors.Is(err, agenttask.ErrRevisionConflict) { continue } if err != nil { return eventReplayIndex{}, "", err } return index, committed, nil } return eventReplayIndex{}, "", fmt.Errorf( "%w: event replay recovery after %d retries", ErrCASExhausted, maxCASRetries, ) } func (s *Store) legacyEventReplayEntries( ctx context.Context, ) (map[string]eventReplayEntry, error) { snapshots, err := s.state.LoadIntegrationRecords(ctx, journalIntegrationPrefix) if err != nil { return nil, err } recovered := make(map[string]eventReplayEntry) for key, snapshot := range snapshots { var candidate journal if err := json.Unmarshal(snapshot.Payload, &candidate); err != nil { return nil, fmt.Errorf("projectlog: decode legacy journal %q: %w", key, err) } if candidate.ProjectID != s.projectID || candidate.WorkspaceID != s.workspaceID { continue } scope, err := s.scopeForWorkUnit(candidate.WorkUnitID) if err != nil { return nil, err } if key != scope.key { return nil, fmt.Errorf( "%w: legacy journal key disagrees with retained scope", ErrInvalidIdentity, ) } normalizeJournal(&candidate) if err := scope.validateJournal(candidate); err != nil { return nil, err } for recordID, seen := range candidate.SeenRecords { if seen.EventFingerprint == "" { continue } entry := eventReplayEntry{ RecordID: recordID, WorkUnitID: candidate.WorkUnitID, Sequence: seen.Sequence, EventFingerprint: seen.EventFingerprint, } if retained, ok := recovered[recordID]; ok && retained != entry { return nil, eventReplayConflict(recordID) } recovered[recordID] = entry } } return recovered, nil } func eventReplayConflict(recordID string) error { return fmt.Errorf( "%w: manager event id %q", ErrRecordReplayConflict, recordID, ) } // AppendEventRecord routes work events to their deterministic task scope and // keeps project-only events in the project journal. func (s *Store) AppendEventRecord( ctx context.Context, record ProjectLogRecord, eventFingerprint string, ) (uint64, error) { if record.Sequence != 0 { return 0, fmt.Errorf("%w: append sequence must be unassigned", ErrInvalidIdentity) } if record.ProjectID != s.projectID || record.WorkspaceID != s.workspaceID { return 0, fmt.Errorf("%w: record identity disagrees with store", ErrInvalidIdentity) } if !validRecordFingerprint(eventFingerprint) { return 0, fmt.Errorf("%w: event fingerprint is invalid", ErrInvalidIdentity) } scope, err := s.scopeForWorkUnit(record.WorkUnitID) if err != nil { return 0, err } recordFingerprint, err := recordFingerprint(record) if err != nil { return 0, err } for attempt := 0; attempt < maxCASRetries; attempt++ { if err := ctx.Err(); err != nil { return 0, err } index, indexRevision, err := s.loadOrRecoverEventReplayIndex(ctx, record.RecordID) if err != nil { return 0, err } if retained, ok := index.Entries[record.RecordID]; ok { if retained.WorkUnitID != record.WorkUnitID || retained.EventFingerprint != eventFingerprint { return 0, eventReplayConflict(record.RecordID) } return retained.Sequence, nil } j, journalRevision, _, err := scope.loadJournal(ctx) if err != nil { return 0, err } sequence := j.NextSequence if seen, ok := j.SeenRecords[record.RecordID]; ok { if seen.EventFingerprint != "" && seen.EventFingerprint != eventFingerprint { return 0, eventReplayConflict(record.RecordID) } if seen.RecordFingerprint != recordFingerprint { return 0, fmt.Errorf( "%w: record id %q", ErrRecordReplayConflict, record.RecordID, ) } sequence = seen.Sequence seen.EventFingerprint = eventFingerprint j.SeenRecords[record.RecordID] = seen } else { rec := record rec.Sequence = sequence if err := rec.Validate(); err != nil { return 0, err } j.Records = append(j.Records, rec) j.SeenRecords[rec.RecordID] = seenRecord{ Sequence: sequence, RecordFingerprint: recordFingerprint, EventFingerprint: eventFingerprint, } j.NextSequence = sequence + 1 } index.Entries[record.RecordID] = eventReplayEntry{ RecordID: record.RecordID, WorkUnitID: record.WorkUnitID, Sequence: sequence, EventFingerprint: eventFingerprint, } if err := s.validateEventReplayIndex(index); err != nil { return 0, err } if err := scope.validateJournal(j); err != nil { return 0, err } indexPayload, err := json.Marshal(index) if err != nil { return 0, fmt.Errorf("projectlog: encode event replay index: %w", err) } journalPayload, err := json.Marshal(j) if err != nil { return 0, fmt.Errorf("projectlog: encode journal: %w", err) } _, err = s.state.CompareAndSwapIntegrationRecords( ctx, []agentstate.IntegrationRecordUpdate{ { Key: s.replayKey, Expected: indexRevision, Payload: indexPayload, }, { Key: scope.key, Expected: journalRevision, Payload: journalPayload, }, }, ) if errors.Is(err, agenttask.ErrRevisionConflict) { continue } if err != nil { return 0, err } return sequence, nil } return 0, fmt.Errorf("%w: after %d retries", ErrCASExhausted, maxCASRetries) } // CheckEventReplay checks the retained stable manager-event index before // volatile manager evidence is resolved. Legacy and generic entries without // an event fingerprint require normal evidence resolution and record replay. func (s *Store) CheckEventReplay( ctx context.Context, workUnitID agenttask.WorkUnitID, recordID string, eventFingerprint string, ) (bool, error) { if err := validateIdentity("RecordID", recordID, true); err != nil { return false, err } if !validRecordFingerprint(eventFingerprint) { return false, fmt.Errorf("%w: event fingerprint is invalid", ErrInvalidIdentity) } if _, err := s.scopeForWorkUnit(workUnitID); err != nil { return false, err } index, _, err := s.loadOrRecoverEventReplayIndex(ctx, recordID) if err != nil { return false, err } retained, ok := index.Entries[recordID] if !ok { return false, nil } if retained.WorkUnitID != workUnitID || retained.EventFingerprint != eventFingerprint { return false, eventReplayConflict(recordID) } return true, nil } // ReplayRecords returns all un-archived records from the journal in sequence // order. Returns nil (not an error) when the journal has not been created. func (s *Store) ReplayRecords(ctx context.Context) ([]ProjectLogRecord, error) { if err := ctx.Err(); err != nil { return nil, err } j, _, found, err := s.loadJournal(ctx) if err != nil { return nil, err } if !found { return nil, nil } return append([]ProjectLogRecord(nil), j.Records...), nil } // Archive materializes a terminal archive from the journal. It writes the // archive intent first (crash-recovery point), then redacted JSONL and // timeline files via temp-file sync/rename, then the manifest, and finally // prunes the journal. On any failure the journal is left intact and Reconcile // converges on restart. Calling Archive on an empty journal is a no-op, // providing exactly-once archive semantics. func (s *Store) Archive(ctx context.Context) error { if err := ctx.Err(); err != nil { return err } if err := s.failAt("archive_before_lock"); err != nil { return err } return s.withArchiveLock(ctx, func() error { return s.archiveLocked(ctx) }) } func (s *Store) archiveLocked(ctx context.Context) error { j, _, found, err := s.loadJournal(ctx) if err != nil { return err } if !found || len(j.Records) == 0 { return nil } ordinal := j.ArchiveOrdinal + 1 artifactsExist, err := s.archiveArtifactsExist(ordinal) if err != nil { return err } if artifactsExist { verifiedIntent, verifiedRecords, err := s.verifyArchiveArtifacts(ordinal) if err != nil { return err } return s.reconcilePrune(ctx, verifiedIntent, verifiedRecords) } if !j.Records[len(j.Records)-1].Terminal { return fmt.Errorf("%w: last record is not terminal", ErrNoTerminalRecord) } records := j.Records now := time.Now().UTC() manifest, err := NewArchiveManifest(s.projectID, s.workspaceID, ordinal, records, now, true) if err != nil { return err } intent := archiveIntent{ SchemaVersion: journalSchemaVersion, ProjectID: s.projectID, WorkspaceID: s.workspaceID, WorkUnitID: s.workUnitID, ArchiveOrdinal: ordinal, FirstSequence: manifest.FirstSequence, LastSequence: manifest.LastSequence, RecordCount: manifest.RecordCount, Checksum: manifest.Checksum, ArchivedAt: now, Terminal: true, } archiveDir := s.archiveDir() if err := os.MkdirAll(archiveDir, 0o700); err != nil { return err } intentPath := s.intentPath(ordinal) jsonlPath := s.jsonlPath(ordinal) timelinePath := s.timelinePath(ordinal) manifestPath := s.manifestPath(ordinal) // 1. Persist archive intent (crash-recovery anchor). intentData, err := json.Marshal(intent) if err != nil { return err } if err := writeAtomically(intentPath, append(intentData, '\n')); err != nil { return err } if err := s.failAt("archive_before_write"); err != nil { return err } // 2. Materialize redacted JSONL and timeline via temp-file sync/rename. jsonlData, err := encodeJSONL(records) if err != nil { return err } if err := writeAtomically(jsonlPath, jsonlData); err != nil { return err } timelineData, err := encodeTimeline(records) if err != nil { return err } if err := writeAtomically(timelinePath, timelineData); err != nil { return err } if err := s.failAt("archive_after_rename"); err != nil { return err } // 3. Write manifest to a temp file, then commit via rename. manifestData, err := json.Marshal(manifest) if err != nil { return err } manifestTemp, err := writeTemp(archiveDir, "manifest", append(manifestData, '\n')) if err != nil { return err } if err := s.failAt("archive_before_manifest"); err != nil { return err } if err := os.Rename(manifestTemp, manifestPath); err != nil { return err } if err := syncDir(archiveDir); err != nil { return err } if err := s.failAt("archive_before_cleanup"); err != nil { return err } // 4. Verify the complete committed artifact set, then prune only the // records proven by that immutable set. verifiedIntent, verifiedRecords, err := s.verifyArchiveArtifacts(ordinal) if err != nil { return err } return s.reconcilePrune(ctx, verifiedIntent, verifiedRecords) } // Reconcile scans the archive directory for intent files and converges any // incomplete archive idempotently. If the manifest is missing it is // recomputed from the JSONL and verified against the intent checksum. // Conflicting content fails closed with ErrArchiveConflict. After a manifest // is verified or reconstructed, the journal is pruned if it still holds the // archived records. func (s *Store) Reconcile(ctx context.Context) error { if err := ctx.Err(); err != nil { return err } return s.withArchiveLock(ctx, func() error { return s.reconcileLocked(ctx) }) } func (s *Store) reconcileLocked(ctx context.Context) error { archiveDir := s.archiveDir() entries, err := os.ReadDir(archiveDir) if err != nil { if os.IsNotExist(err) { return nil } return err } foundIntent := false for _, entry := range entries { if entry.IsDir() { continue } name := entry.Name() if !strings.HasSuffix(name, ".intent") { continue } foundIntent = true ordinal := parseOrdinal(name) if ordinal == 0 { continue } if err := s.reconcileArchive(ctx, ordinal); err != nil { return err } } if !foundIntent { j, _, found, err := s.loadJournal(ctx) if err != nil { return err } if found && len(j.Records) > 0 && j.Records[len(j.Records)-1].Terminal { return s.archiveLocked(ctx) } } return nil } func (s *Store) reconcileArchive(ctx context.Context, ordinal uint64) error { intent, records, err := s.verifyArchiveArtifacts(ordinal) if err != nil { if !errors.Is(err, ErrArchiveIncomplete) { return err } if restoreErr := s.restoreMissingJSONL(ctx, ordinal); restoreErr != nil { return restoreErr } intent, records, err = s.verifyArchiveArtifacts(ordinal) if err != nil { return err } } return s.reconcilePrune(ctx, intent, records) } func (s *Store) restoreMissingJSONL(ctx context.Context, ordinal uint64) error { if _, err := os.Stat(s.jsonlPath(ordinal)); err == nil { return nil } else if !os.IsNotExist(err) { return err } intentData, err := os.ReadFile(s.intentPath(ordinal)) if err != nil { return fmt.Errorf("%w: read intent for recovery: %v", ErrArchiveIncomplete, err) } var intent archiveIntent if err := json.Unmarshal(intentData, &intent); err != nil { return fmt.Errorf("%w: decode intent for recovery: %v", ErrArchiveConflict, err) } if intent.SchemaVersion != journalSchemaVersion || intent.ProjectID != s.projectID || intent.WorkspaceID != s.workspaceID || intent.WorkUnitID != s.workUnitID || intent.ArchiveOrdinal != ordinal || intent.RecordCount <= 0 { return fmt.Errorf("%w: recovery intent identity is invalid", ErrArchiveConflict) } j, _, found, err := s.loadJournal(ctx) if err != nil { return err } if !found || len(j.Records) < intent.RecordCount { return fmt.Errorf("%w: journal cannot reconstruct ordinal %d", ErrArchiveIncomplete, ordinal) } records := append([]ProjectLogRecord(nil), j.Records[:intent.RecordCount]...) manifest, err := NewArchiveManifest( s.projectID, s.workspaceID, ordinal, records, intent.ArchivedAt, true, ) if err != nil { return fmt.Errorf("%w: reconstruct manifest: %v", ErrArchiveConflict, err) } if manifest.FirstSequence != intent.FirstSequence || manifest.LastSequence != intent.LastSequence || manifest.RecordCount != intent.RecordCount || manifest.Checksum != intent.Checksum { return fmt.Errorf("%w: journal does not match recovery intent", ErrArchiveConflict) } jsonlData, err := encodeJSONL(records) if err != nil { return err } return writeAtomically(s.jsonlPath(ordinal), jsonlData) } func (s *Store) archiveArtifactsExist(ordinal uint64) (bool, error) { for _, path := range []string{ s.intentPath(ordinal), s.jsonlPath(ordinal), s.timelinePath(ordinal), s.manifestPath(ordinal), } { if _, err := os.Stat(path); err == nil { return true, nil } else if !os.IsNotExist(err) { return false, err } } return false, nil } func (s *Store) verifyArchiveArtifacts(ordinal uint64) (archiveIntent, []ProjectLogRecord, error) { intentPath := s.intentPath(ordinal) manifestPath := s.manifestPath(ordinal) jsonlPath := s.jsonlPath(ordinal) timelinePath := s.timelinePath(ordinal) intentData, err := os.ReadFile(intentPath) if err != nil { if os.IsNotExist(err) { return archiveIntent{}, nil, fmt.Errorf("%w: missing intent for ordinal %d", ErrArchiveIncomplete, ordinal) } return archiveIntent{}, nil, fmt.Errorf("projectlog: read intent: %w", err) } var intent archiveIntent if err := json.Unmarshal(intentData, &intent); err != nil { return archiveIntent{}, nil, fmt.Errorf("%w: decode intent: %v", ErrArchiveConflict, err) } if intent.SchemaVersion != journalSchemaVersion { return archiveIntent{}, nil, fmt.Errorf("%w: intent schema version %d", ErrArchiveConflict, intent.SchemaVersion) } if intent.ProjectID != s.projectID || intent.WorkspaceID != s.workspaceID || intent.WorkUnitID != s.workUnitID { return archiveIntent{}, nil, fmt.Errorf("%w: intent identity disagrees with store", ErrArchiveConflict) } if intent.ArchiveOrdinal != ordinal { return archiveIntent{}, nil, fmt.Errorf("%w: intent ordinal %d != filename %d", ErrArchiveConflict, intent.ArchiveOrdinal, ordinal) } if intent.RecordCount <= 0 || intent.FirstSequence == 0 || intent.LastSequence < intent.FirstSequence { return archiveIntent{}, nil, fmt.Errorf("%w: invalid intent sequence range or count", ErrArchiveConflict) } if !validArchiveChecksum(intent.Checksum) { return archiveIntent{}, nil, fmt.Errorf("%w: invalid intent checksum format", ErrArchiveConflict) } if intent.ArchivedAt.IsZero() { return archiveIntent{}, nil, fmt.Errorf("%w: zero intent timestamp", ErrArchiveConflict) } if !intent.Terminal { return archiveIntent{}, nil, fmt.Errorf("%w: intent is not terminal", ErrArchiveConflict) } jsonlData, err := os.ReadFile(jsonlPath) if err != nil { if os.IsNotExist(err) { return archiveIntent{}, nil, fmt.Errorf("%w: missing jsonl for ordinal %d", ErrArchiveIncomplete, ordinal) } return archiveIntent{}, nil, err } if computeJSONLChecksum(jsonlData) != intent.Checksum { return archiveIntent{}, nil, fmt.Errorf("%w: jsonl checksum mismatch for ordinal %d", ErrArchiveConflict, ordinal) } records, err := decodeJSONL(jsonlData) if err != nil { return archiveIntent{}, nil, fmt.Errorf("%w: decode jsonl: %v", ErrArchiveConflict, err) } if len(records) != intent.RecordCount { return archiveIntent{}, nil, fmt.Errorf("%w: jsonl record count %d != intent count %d", ErrArchiveConflict, len(records), intent.RecordCount) } if records[0].Sequence != intent.FirstSequence || records[len(records)-1].Sequence != intent.LastSequence { return archiveIntent{}, nil, fmt.Errorf("%w: jsonl sequence range disagrees with intent", ErrArchiveConflict) } expectedManifest, err := NewArchiveManifest(s.projectID, s.workspaceID, ordinal, records, intent.ArchivedAt, true) if err != nil { return archiveIntent{}, nil, fmt.Errorf("%w: derive manifest: %w", ErrArchiveConflict, err) } if expectedManifest.Checksum != intent.Checksum { return archiveIntent{}, nil, fmt.Errorf("%w: derived checksum %s != intent %s", ErrArchiveConflict, expectedManifest.Checksum, intent.Checksum) } if s.workUnitID != "" && (len(expectedManifest.WorkUnitIDs) != 1 || expectedManifest.WorkUnitIDs[0] != s.workUnitID) { return archiveIntent{}, nil, fmt.Errorf( "%w: manifest work unit disagrees with store scope", ErrArchiveConflict, ) } manifestBytes, err := os.ReadFile(manifestPath) if err != nil { if os.IsNotExist(err) { mBytes, marshalErr := json.Marshal(expectedManifest) if marshalErr != nil { return archiveIntent{}, nil, marshalErr } if err := writeAtomically(manifestPath, append(mBytes, '\n')); err != nil { return archiveIntent{}, nil, err } } else { return archiveIntent{}, nil, err } } else { var existingManifest ArchiveManifest if err := json.Unmarshal(manifestBytes, &existingManifest); err != nil { return archiveIntent{}, nil, fmt.Errorf("%w: decode manifest: %w", ErrArchiveConflict, err) } if err := existingManifest.Validate(); err != nil { return archiveIntent{}, nil, fmt.Errorf("%w: invalid manifest: %w", ErrArchiveConflict, err) } if !manifestsEqual(existingManifest, expectedManifest) { return archiveIntent{}, nil, fmt.Errorf("%w: manifest field drift for ordinal %d", ErrArchiveConflict, ordinal) } } expectedTimelineData, err := encodeTimeline(records) if err != nil { return archiveIntent{}, nil, fmt.Errorf("projectlog: encode timeline: %w", err) } timelineData, err := os.ReadFile(timelinePath) if err != nil { if os.IsNotExist(err) { if err := writeAtomically(timelinePath, expectedTimelineData); err != nil { return archiveIntent{}, nil, err } } else { return archiveIntent{}, nil, err } } else { if !bytes.Equal(bytes.TrimSpace(timelineData), bytes.TrimSpace(expectedTimelineData)) { return archiveIntent{}, nil, fmt.Errorf("%w: timeline content mismatch for ordinal %d", ErrArchiveConflict, ordinal) } } return intent, records, nil } func manifestsEqual(m1, m2 ArchiveManifest) bool { if m1.SchemaVersion != m2.SchemaVersion || m1.ProjectID != m2.ProjectID || m1.WorkspaceID != m2.WorkspaceID || m1.ArchiveOrdinal != m2.ArchiveOrdinal || m1.RecordCount != m2.RecordCount || m1.FirstSequence != m2.FirstSequence || m1.LastSequence != m2.LastSequence || m1.Checksum != m2.Checksum || !m1.ArchivedAt.Equal(m2.ArchivedAt) || m1.Terminal != m2.Terminal || len(m1.WorkUnitIDs) != len(m2.WorkUnitIDs) { return false } for i := range m1.WorkUnitIDs { if m1.WorkUnitIDs[i] != m2.WorkUnitIDs[i] { return false } } return true } func recordsMatchPrefix(jRecords, archived []ProjectLogRecord) bool { if len(jRecords) < len(archived) { return false } for i := range archived { if !recordsEqual(jRecords[i], archived[i]) { return false } } return true } func recordsEqual(a, b ProjectLogRecord) bool { if a.Sequence != b.Sequence || a.RecordID != b.RecordID || a.EventType != b.EventType || a.Terminal != b.Terminal || a.State != b.State { return false } aBytes, errA := json.Marshal(a) bBytes, errB := json.Marshal(b) if errA != nil || errB != nil { return false } return bytes.Equal(aBytes, bBytes) } func (s *Store) reconcilePrune( ctx context.Context, intent archiveIntent, archivedRecords []ProjectLogRecord, ) error { for attempt := 0; attempt < maxCASRetries; attempt++ { if err := ctx.Err(); err != nil { return err } j, revision, found, err := s.loadJournal(ctx) if err != nil { return err } if !found { return nil } if len(j.Records) == 0 { if j.ArchiveOrdinal < intent.ArchiveOrdinal { j.ArchiveOrdinal = intent.ArchiveOrdinal next, err := json.Marshal(j) if err != nil { return err } if _, err := s.state.CompareAndSwapIntegrationRecord(ctx, s.key, revision, next); err != nil { if errors.Is(err, agenttask.ErrRevisionConflict) { continue } return err } } return nil } if j.Records[0].Sequence > intent.LastSequence { if j.ArchiveOrdinal < intent.ArchiveOrdinal { j.ArchiveOrdinal = intent.ArchiveOrdinal next, err := json.Marshal(j) if err != nil { return err } if _, err := s.state.CompareAndSwapIntegrationRecord(ctx, s.key, revision, next); err != nil { if errors.Is(err, agenttask.ErrRevisionConflict) { continue } return err } } return nil } if !recordsMatchPrefix(j.Records, archivedRecords) { return fmt.Errorf("%w: journal prefix does not match archived records", ErrArchiveConflict) } j.Records = append([]ProjectLogRecord(nil), j.Records[len(archivedRecords):]...) if j.ArchiveOrdinal < intent.ArchiveOrdinal { j.ArchiveOrdinal = intent.ArchiveOrdinal } next, err := json.Marshal(j) if err != nil { return err } if _, err := s.state.CompareAndSwapIntegrationRecord(ctx, s.key, revision, next); err != nil { if errors.Is(err, agenttask.ErrRevisionConflict) { continue } return err } return nil } return fmt.Errorf("%w: reconcile prune after %d retries", ErrCASExhausted, maxCASRetries) } func integrationRecordKey( projectID agenttask.ProjectID, workspaceID agenttask.WorkspaceID, workUnitID agenttask.WorkUnitID, ) string { h := sha256.New() writeLengthPrefixed(h, string(projectID)) writeLengthPrefixed(h, string(workspaceID)) if workUnitID != "" { writeLengthPrefixed(h, string(workUnitID)) } return journalIntegrationPrefix + hex.EncodeToString(h.Sum(nil)) } func eventReplayIndexKey( projectID agenttask.ProjectID, workspaceID agenttask.WorkspaceID, ) string { h := sha256.New() writeLengthPrefixed(h, "event-replay-index") writeLengthPrefixed(h, string(projectID)) writeLengthPrefixed(h, string(workspaceID)) return eventReplayIntegrationPrefix + hex.EncodeToString(h.Sum(nil)) } func newJournal( projectID agenttask.ProjectID, workspaceID agenttask.WorkspaceID, workUnitID agenttask.WorkUnitID, ) journal { return journal{ SchemaVersion: journalSchemaVersion, ProjectID: projectID, WorkspaceID: workspaceID, WorkUnitID: workUnitID, NextSequence: 1, ArchiveOrdinal: 0, SeenRecords: make(map[string]seenRecord), Records: nil, } } func normalizeJournal(j *journal) { if j.SeenRecords == nil { j.SeenRecords = make(map[string]seenRecord, len(j.Records)) } for recordID, seen := range j.SeenRecords { if seen.RecordFingerprint == "" { seen.RecordFingerprint = seen.Fingerprint } seen.Fingerprint = "" j.SeenRecords[recordID] = seen } for _, record := range j.Records { if _, exists := j.SeenRecords[record.RecordID]; exists { continue } fingerprint, err := recordFingerprint(record) if err != nil { continue } j.SeenRecords[record.RecordID] = seenRecord{ Sequence: record.Sequence, RecordFingerprint: fingerprint, } } } func newEventReplayIndex( projectID agenttask.ProjectID, workspaceID agenttask.WorkspaceID, ) eventReplayIndex { return eventReplayIndex{ SchemaVersion: eventReplayIndexSchemaVersion, ProjectID: projectID, WorkspaceID: workspaceID, Entries: make(map[string]eventReplayEntry), } } func normalizeEventReplayIndex(index *eventReplayIndex) { if index.Entries == nil { index.Entries = make(map[string]eventReplayEntry) } } func (s *Store) archiveDir() string { h := sha256.New() writeLengthPrefixed(h, string(s.projectID)) writeLengthPrefixed(h, string(s.workspaceID)) if s.workUnitID != "" { writeLengthPrefixed(h, string(s.workUnitID)) } return filepath.Join(s.root, hex.EncodeToString(h.Sum(nil)), "archive") } func writeLengthPrefixed(h interface{ Write([]byte) (int, error) }, value string) { var length [4]byte binary.BigEndian.PutUint32(length[:], uint32(len(value))) _, _ = h.Write(length[:]) _, _ = h.Write([]byte(value)) } func (s *Store) withArchiveLock(ctx context.Context, action func() error) error { archiveDir := s.archiveDir() if err := os.MkdirAll(archiveDir, 0o700); err != nil { return err } lock, err := os.OpenFile(filepath.Join(archiveDir, ".archive.lock"), os.O_CREATE|os.O_RDWR, 0o600) if err != nil { return err } defer lock.Close() if err := ctx.Err(); err != nil { return err } if err := syscall.Flock(int(lock.Fd()), syscall.LOCK_EX); err != nil { return err } defer syscall.Flock(int(lock.Fd()), syscall.LOCK_UN) //nolint:errcheck if err := ctx.Err(); err != nil { return err } return action() } func (s *Store) intentPath(ordinal uint64) string { return filepath.Join(s.archiveDir(), fmt.Sprintf("%06d.intent", ordinal)) } func (s *Store) jsonlPath(ordinal uint64) string { return filepath.Join(s.archiveDir(), fmt.Sprintf("%06d.jsonl", ordinal)) } func (s *Store) timelinePath(ordinal uint64) string { return filepath.Join(s.archiveDir(), fmt.Sprintf("%06d.timeline.jsonl", ordinal)) } func (s *Store) manifestPath(ordinal uint64) string { return filepath.Join(s.archiveDir(), fmt.Sprintf("%06d.manifest.json", ordinal)) } func parseOrdinal(name string) uint64 { base := strings.TrimSuffix(name, ".intent") ordinal, err := strconv.ParseUint(base, 10, 64) if err != nil { return 0 } return ordinal } func writeTemp(dir, base string, data []byte) (string, error) { temp, err := os.CreateTemp(dir, "."+base+".tmp-*") if err != nil { return "", err } tempPath := temp.Name() removeTemp := true defer func() { if removeTemp { _ = os.Remove(tempPath) } }() if err := temp.Chmod(0o600); err != nil { _ = temp.Close() return "", err } if _, err := temp.Write(data); err != nil { _ = temp.Close() return "", err } if err := temp.Sync(); err != nil { _ = temp.Close() return "", err } if err := temp.Close(); err != nil { return "", err } removeTemp = false return tempPath, nil } func writeAtomically(path string, data []byte) error { tempPath, err := writeTemp(filepath.Dir(path), filepath.Base(path), data) if err != nil { return err } if err := os.Rename(tempPath, path); err != nil { _ = os.Remove(tempPath) return err } return syncDir(filepath.Dir(path)) } func syncDir(path string) error { dirHandle, err := os.Open(path) if err != nil { return err } defer dirHandle.Close() return dirHandle.Sync() } func encodeJSONL(records []ProjectLogRecord) ([]byte, error) { var buf bytes.Buffer for _, rec := range records { data, err := json.Marshal(rec) if err != nil { return nil, err } buf.Write(data) buf.WriteByte('\n') } return buf.Bytes(), nil } func encodeTimeline(records []ProjectLogRecord) ([]byte, error) { return encodeWorkLogEntries(ProjectWorkLog(records).Entries) } func encodeWorkLogEntries(entries []WorkLogEntry) ([]byte, error) { var buf bytes.Buffer for _, entry := range entries { data, err := json.Marshal(entry) if err != nil { return nil, err } buf.Write(data) buf.WriteByte('\n') } return buf.Bytes(), nil } func recordFingerprint(record ProjectLogRecord) (string, error) { record.Sequence = 0 payload, err := json.Marshal(record) if err != nil { return "", fmt.Errorf("projectlog: encode record fingerprint: %w", err) } sum := sha256.Sum256(payload) return "sha256:" + hex.EncodeToString(sum[:]), nil } func validRecordFingerprint(fingerprint string) bool { return validArchiveChecksum(fingerprint) } func decodeJSONL(data []byte) ([]ProjectLogRecord, error) { var records []ProjectLogRecord for len(data) > 0 { idx := bytes.IndexByte(data, '\n') var line []byte if idx == -1 { line = data data = nil } else { line = data[:idx] data = data[idx+1:] } line = bytes.TrimSpace(line) if len(line) == 0 { continue } var rec ProjectLogRecord if err := json.Unmarshal(line, &rec); err != nil { return nil, err } records = append(records, rec) } return records, nil } func computeJSONLChecksum(data []byte) string { sum := sha256.Sum256(data) return "sha256:" + hex.EncodeToString(sum[:]) }