1564 lines
46 KiB
Go
1564 lines
46 KiB
Go
// Package projectlog provides immutable, versioned presentation and recovery
|
|
// project-log record schemas and validation matrices for the standalone host.
|
|
//
|
|
// This file implements the CAS-backed journal and terminal archive
|
|
// reconciliation over a device-local root. It owns the persistence invariant:
|
|
// append by CAS with bounded retry, retention only after terminal evidence,
|
|
// restart-safe materialization of redacted JSONL/timeline files, and
|
|
// idempotent reconciliation that fails closed on conflicting content.
|
|
package projectlog
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/binary"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"math"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"syscall"
|
|
"time"
|
|
|
|
"iop/packages/go/agentstate"
|
|
"iop/packages/go/agenttask"
|
|
)
|
|
|
|
const journalSchemaVersion uint32 = 1
|
|
const eventReplayIndexSchemaVersion uint32 = 1
|
|
const maxCASRetries = 16
|
|
|
|
const (
|
|
journalIntegrationPrefix = "projectlog:"
|
|
eventReplayIntegrationPrefix = "projectlog-event-replay:"
|
|
)
|
|
|
|
var (
|
|
ErrNoTerminalRecord = errors.New("projectlog: no terminal record to archive")
|
|
ErrArchiveIncomplete = errors.New("projectlog: archive incomplete")
|
|
ErrArchiveConflict = errors.New("projectlog: archive content conflict")
|
|
ErrRecordReplayConflict = errors.New("projectlog: record replay content conflict")
|
|
ErrCASExhausted = errors.New("projectlog: CAS retry exhausted")
|
|
)
|
|
|
|
type seenRecord struct {
|
|
Sequence uint64 `json:"sequence"`
|
|
RecordFingerprint string `json:"record_fingerprint"`
|
|
EventFingerprint string `json:"event_fingerprint,omitempty"`
|
|
Fingerprint string `json:"fingerprint,omitempty"`
|
|
}
|
|
|
|
// journal is the durable CAS payload for one project or work-unit scope.
|
|
type journal struct {
|
|
SchemaVersion uint32 `json:"schema_version"`
|
|
ProjectID agenttask.ProjectID `json:"project_id"`
|
|
WorkspaceID agenttask.WorkspaceID `json:"workspace_id"`
|
|
WorkUnitID agenttask.WorkUnitID `json:"work_unit_id,omitempty"`
|
|
NextSequence uint64 `json:"next_sequence"`
|
|
ArchiveOrdinal uint64 `json:"archive_ordinal"`
|
|
SeenRecords map[string]seenRecord `json:"seen_records"`
|
|
Records []ProjectLogRecord `json:"records"`
|
|
}
|
|
|
|
type eventReplayEntry struct {
|
|
RecordID string `json:"record_id"`
|
|
WorkUnitID agenttask.WorkUnitID `json:"work_unit_id,omitempty"`
|
|
Sequence uint64 `json:"sequence"`
|
|
EventFingerprint string `json:"event_fingerprint"`
|
|
}
|
|
|
|
// eventReplayIndex is the retained project/workspace-wide ownership record for
|
|
// manager EventID projections. Task-scoped journals retain their independent
|
|
// sequence and archive lifecycle.
|
|
type eventReplayIndex struct {
|
|
SchemaVersion uint32 `json:"schema_version"`
|
|
ProjectID agenttask.ProjectID `json:"project_id"`
|
|
WorkspaceID agenttask.WorkspaceID `json:"workspace_id"`
|
|
Entries map[string]eventReplayEntry `json:"entries"`
|
|
}
|
|
|
|
// archiveIntent is the durable crash-recovery marker written before any
|
|
// artifact. It records the manifest checksum so Reconcile can verify or
|
|
// recompute the manifest without trusting partial state.
|
|
type archiveIntent struct {
|
|
SchemaVersion uint32 `json:"schema_version"`
|
|
ProjectID agenttask.ProjectID `json:"project_id"`
|
|
WorkspaceID agenttask.WorkspaceID `json:"workspace_id"`
|
|
WorkUnitID agenttask.WorkUnitID `json:"work_unit_id,omitempty"`
|
|
ArchiveOrdinal uint64 `json:"archive_ordinal"`
|
|
FirstSequence uint64 `json:"first_sequence"`
|
|
LastSequence uint64 `json:"last_sequence"`
|
|
RecordCount int `json:"record_count"`
|
|
Checksum string `json:"checksum"`
|
|
ArchivedAt time.Time `json:"archived_at"`
|
|
Terminal bool `json:"terminal"`
|
|
}
|
|
|
|
// Store is the CAS-backed project log journal with device-local archive roots.
|
|
// It wraps an agentstate.Store for the durable CAS journal and a device-local
|
|
// root directory for materialized archive artifacts.
|
|
type Store struct {
|
|
state *agentstate.Store
|
|
root string
|
|
projectID agenttask.ProjectID
|
|
workspaceID agenttask.WorkspaceID
|
|
workUnitID agenttask.WorkUnitID
|
|
key string
|
|
replayKey string
|
|
failureHook func(phase string) error
|
|
}
|
|
|
|
// NewStore creates a CAS-backed project log store over a device-local root.
|
|
// The state store provides the crash-safe CAS journal; root holds archive
|
|
// artifacts (intent, JSONL, timeline, manifest).
|
|
func NewStore(
|
|
state *agentstate.Store,
|
|
root string,
|
|
projectID agenttask.ProjectID,
|
|
workspaceID agenttask.WorkspaceID,
|
|
) (*Store, error) {
|
|
if state == nil {
|
|
return nil, fmt.Errorf("projectlog: state store is required")
|
|
}
|
|
if root == "" {
|
|
return nil, fmt.Errorf("projectlog: root is required")
|
|
}
|
|
if projectID == "" || workspaceID == "" {
|
|
return nil, fmt.Errorf("projectlog: project and workspace IDs are required")
|
|
}
|
|
return &Store{
|
|
state: state,
|
|
root: filepath.Clean(root),
|
|
projectID: projectID,
|
|
workspaceID: workspaceID,
|
|
key: integrationRecordKey(projectID, workspaceID, ""),
|
|
replayKey: eventReplayIndexKey(projectID, workspaceID),
|
|
}, nil
|
|
}
|
|
|
|
// ForWorkUnit returns a task-scoped journal below the same project/workspace.
|
|
// Each scoped journal owns its own sequence and archive ordinal.
|
|
func (s *Store) ForWorkUnit(workUnitID agenttask.WorkUnitID) (*Store, error) {
|
|
if s == nil {
|
|
return nil, fmt.Errorf("projectlog: store is required")
|
|
}
|
|
if err := validateIdentity("WorkUnitID", string(workUnitID), true); err != nil {
|
|
return nil, err
|
|
}
|
|
return &Store{
|
|
state: s.state,
|
|
root: s.root,
|
|
projectID: s.projectID,
|
|
workspaceID: s.workspaceID,
|
|
workUnitID: workUnitID,
|
|
key: integrationRecordKey(s.projectID, s.workspaceID, workUnitID),
|
|
replayKey: s.replayKey,
|
|
failureHook: s.failureHook,
|
|
}, nil
|
|
}
|
|
|
|
// WithFailureHook installs a phase-gated failure hook for testing crash
|
|
// windows. The hook is invoked at named phases during Archive; if it returns
|
|
// a non-nil error the operation aborts at that phase. Reconcile is never
|
|
// gated by the hook.
|
|
func (s *Store) WithFailureHook(hook func(phase string) error) *Store {
|
|
s.failureHook = hook
|
|
return s
|
|
}
|
|
|
|
func (s *Store) failAt(phase string) error {
|
|
if s.failureHook != nil {
|
|
return s.failureHook(phase)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *Store) loadJournal(ctx context.Context) (journal, string, bool, error) {
|
|
payload, revision, found, err := s.state.LoadIntegrationRecord(ctx, s.key)
|
|
if err != nil {
|
|
return journal{}, "", false, err
|
|
}
|
|
if !found {
|
|
return newJournal(s.projectID, s.workspaceID, s.workUnitID), revision, false, nil
|
|
}
|
|
var j journal
|
|
if err := json.Unmarshal(payload, &j); err != nil {
|
|
return journal{}, "", false, fmt.Errorf("projectlog: decode journal: %w", err)
|
|
}
|
|
normalizeJournal(&j)
|
|
if err := s.validateJournal(j); err != nil {
|
|
return journal{}, "", false, err
|
|
}
|
|
return j, revision, true, nil
|
|
}
|
|
|
|
func (s *Store) validateJournal(j journal) error {
|
|
if j.SchemaVersion != journalSchemaVersion {
|
|
return fmt.Errorf(
|
|
"%w: journal schema version %d expected %d",
|
|
ErrInvalidSchemaVersion,
|
|
j.SchemaVersion,
|
|
journalSchemaVersion,
|
|
)
|
|
}
|
|
if j.ProjectID != s.projectID || j.WorkspaceID != s.workspaceID {
|
|
return fmt.Errorf("%w: journal identity disagrees with store", ErrInvalidIdentity)
|
|
}
|
|
if j.WorkUnitID != s.workUnitID {
|
|
return fmt.Errorf("%w: journal work-unit scope disagrees with store", ErrInvalidIdentity)
|
|
}
|
|
if j.NextSequence == 0 || j.NextSequence == math.MaxUint64 {
|
|
return fmt.Errorf("%w: journal next sequence %d cannot be assigned safely", ErrInvalidIdentity, j.NextSequence)
|
|
}
|
|
for index, record := range j.Records {
|
|
if err := record.Validate(); err != nil {
|
|
return fmt.Errorf("projectlog: invalid journal record %d: %w", index, err)
|
|
}
|
|
if record.ProjectID != s.projectID || record.WorkspaceID != s.workspaceID {
|
|
return fmt.Errorf("%w: journal record %d identity disagrees with store", ErrInvalidIdentity, index)
|
|
}
|
|
if s.workUnitID != "" && record.WorkUnitID != s.workUnitID {
|
|
return fmt.Errorf("%w: journal record %d work unit disagrees with scope", ErrInvalidIdentity, index)
|
|
}
|
|
if index > 0 && record.Sequence != j.Records[index-1].Sequence+1 {
|
|
return fmt.Errorf(
|
|
"%w: journal record sequence %d does not follow %d",
|
|
ErrInvalidIdentity,
|
|
record.Sequence,
|
|
j.Records[index-1].Sequence,
|
|
)
|
|
}
|
|
seen, ok := j.SeenRecords[record.RecordID]
|
|
if !ok {
|
|
return fmt.Errorf("%w: record %q is absent from replay index", ErrInvalidIdentity, record.RecordID)
|
|
}
|
|
fingerprint, err := recordFingerprint(record)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if seen.Sequence != record.Sequence || seen.RecordFingerprint != fingerprint {
|
|
return fmt.Errorf("%w: replay index disagrees with record %q", ErrInvalidIdentity, record.RecordID)
|
|
}
|
|
}
|
|
if len(j.Records) > 0 {
|
|
lastSequence := j.Records[len(j.Records)-1].Sequence
|
|
if lastSequence == math.MaxUint64 || j.NextSequence != lastSequence+1 {
|
|
return fmt.Errorf(
|
|
"%w: journal next sequence %d does not follow retained sequence %d",
|
|
ErrInvalidIdentity,
|
|
j.NextSequence,
|
|
lastSequence,
|
|
)
|
|
}
|
|
}
|
|
for recordID, seen := range j.SeenRecords {
|
|
if err := validateIdentity("SeenRecords.RecordID", recordID, true); err != nil {
|
|
return err
|
|
}
|
|
if seen.Sequence == 0 || seen.Sequence >= j.NextSequence {
|
|
return fmt.Errorf("%w: replay sequence for %q is outside journal range", ErrInvalidIdentity, recordID)
|
|
}
|
|
if !validRecordFingerprint(seen.RecordFingerprint) {
|
|
return fmt.Errorf("%w: replay fingerprint for %q is invalid", ErrInvalidIdentity, recordID)
|
|
}
|
|
if seen.EventFingerprint != "" && !validRecordFingerprint(seen.EventFingerprint) {
|
|
return fmt.Errorf("%w: event replay fingerprint for %q is invalid", ErrInvalidIdentity, recordID)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// AppendRecord validates the record, assigns a monotonic sequence, and
|
|
// CAS-appends it to the project journal with bounded retry on revision
|
|
// conflict. It returns the assigned sequence number.
|
|
func (s *Store) AppendRecord(
|
|
ctx context.Context,
|
|
record ProjectLogRecord,
|
|
) (uint64, error) {
|
|
return s.appendRecord(ctx, record, "")
|
|
}
|
|
|
|
func (s *Store) appendRecord(
|
|
ctx context.Context,
|
|
record ProjectLogRecord,
|
|
eventFingerprint string,
|
|
) (uint64, error) {
|
|
if record.Sequence != 0 {
|
|
return 0, fmt.Errorf("%w: append sequence must be unassigned", ErrInvalidIdentity)
|
|
}
|
|
if record.ProjectID != s.projectID || record.WorkspaceID != s.workspaceID {
|
|
return 0, fmt.Errorf("%w: record identity disagrees with store", ErrInvalidIdentity)
|
|
}
|
|
if s.workUnitID != "" && record.WorkUnitID != s.workUnitID {
|
|
return 0, fmt.Errorf("%w: record work unit disagrees with store scope", ErrInvalidIdentity)
|
|
}
|
|
if eventFingerprint != "" && !validRecordFingerprint(eventFingerprint) {
|
|
return 0, fmt.Errorf("%w: event fingerprint is invalid", ErrInvalidIdentity)
|
|
}
|
|
fingerprint, err := recordFingerprint(record)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
for attempt := 0; attempt < maxCASRetries; attempt++ {
|
|
if err := ctx.Err(); err != nil {
|
|
return 0, err
|
|
}
|
|
j, revision, _, err := s.loadJournal(ctx)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
if seen, ok := j.SeenRecords[record.RecordID]; ok {
|
|
if eventFingerprint != "" && seen.EventFingerprint != "" {
|
|
if seen.EventFingerprint != eventFingerprint {
|
|
return 0, fmt.Errorf(
|
|
"%w: manager event id %q",
|
|
ErrRecordReplayConflict,
|
|
record.RecordID,
|
|
)
|
|
}
|
|
return seen.Sequence, nil
|
|
}
|
|
if seen.RecordFingerprint != fingerprint {
|
|
return 0, fmt.Errorf(
|
|
"%w: record id %q",
|
|
ErrRecordReplayConflict,
|
|
record.RecordID,
|
|
)
|
|
}
|
|
if eventFingerprint != "" {
|
|
seen.EventFingerprint = eventFingerprint
|
|
j.SeenRecords[record.RecordID] = seen
|
|
if err := s.validateJournal(j); err != nil {
|
|
return 0, err
|
|
}
|
|
next, err := json.Marshal(j)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("projectlog: encode journal: %w", err)
|
|
}
|
|
if _, err := s.state.CompareAndSwapIntegrationRecord(ctx, s.key, revision, next); err != nil {
|
|
if errors.Is(err, agenttask.ErrRevisionConflict) {
|
|
continue
|
|
}
|
|
return 0, err
|
|
}
|
|
}
|
|
return seen.Sequence, nil
|
|
}
|
|
rec := record
|
|
seq := j.NextSequence
|
|
rec.Sequence = seq
|
|
if err := rec.Validate(); err != nil {
|
|
return 0, err
|
|
}
|
|
j.Records = append(j.Records, rec)
|
|
j.SeenRecords[rec.RecordID] = seenRecord{
|
|
Sequence: seq,
|
|
RecordFingerprint: fingerprint,
|
|
EventFingerprint: eventFingerprint,
|
|
}
|
|
j.NextSequence = seq + 1
|
|
if err := s.validateJournal(j); err != nil {
|
|
return 0, err
|
|
}
|
|
next, err := json.Marshal(j)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("projectlog: encode journal: %w", err)
|
|
}
|
|
if _, err := s.state.CompareAndSwapIntegrationRecord(ctx, s.key, revision, next); err != nil {
|
|
if errors.Is(err, agenttask.ErrRevisionConflict) {
|
|
continue
|
|
}
|
|
return 0, err
|
|
}
|
|
return seq, nil
|
|
}
|
|
return 0, fmt.Errorf("%w: after %d retries", ErrCASExhausted, maxCASRetries)
|
|
}
|
|
|
|
func (s *Store) scopeForWorkUnit(workUnitID agenttask.WorkUnitID) (*Store, error) {
|
|
if workUnitID != "" {
|
|
if s.workUnitID == workUnitID {
|
|
return s, nil
|
|
}
|
|
return s.ForWorkUnit(workUnitID)
|
|
}
|
|
if s.workUnitID == "" {
|
|
return s, nil
|
|
}
|
|
return &Store{
|
|
state: s.state,
|
|
root: s.root,
|
|
projectID: s.projectID,
|
|
workspaceID: s.workspaceID,
|
|
key: integrationRecordKey(s.projectID, s.workspaceID, ""),
|
|
replayKey: s.replayKey,
|
|
failureHook: s.failureHook,
|
|
}, nil
|
|
}
|
|
|
|
func (s *Store) loadEventReplayIndex(
|
|
ctx context.Context,
|
|
) (eventReplayIndex, string, bool, error) {
|
|
payload, revision, found, err := s.state.LoadIntegrationRecord(ctx, s.replayKey)
|
|
if err != nil {
|
|
return eventReplayIndex{}, "", false, err
|
|
}
|
|
if !found {
|
|
return newEventReplayIndex(s.projectID, s.workspaceID), revision, false, nil
|
|
}
|
|
var index eventReplayIndex
|
|
if err := json.Unmarshal(payload, &index); err != nil {
|
|
return eventReplayIndex{}, "", false, fmt.Errorf(
|
|
"projectlog: decode event replay index: %w",
|
|
err,
|
|
)
|
|
}
|
|
normalizeEventReplayIndex(&index)
|
|
if err := s.validateEventReplayIndex(index); err != nil {
|
|
return eventReplayIndex{}, "", false, err
|
|
}
|
|
return index, revision, true, nil
|
|
}
|
|
|
|
func (s *Store) validateEventReplayIndex(index eventReplayIndex) error {
|
|
if index.SchemaVersion != eventReplayIndexSchemaVersion {
|
|
return fmt.Errorf(
|
|
"%w: event replay index schema version %d expected %d",
|
|
ErrInvalidSchemaVersion,
|
|
index.SchemaVersion,
|
|
eventReplayIndexSchemaVersion,
|
|
)
|
|
}
|
|
if index.ProjectID != s.projectID || index.WorkspaceID != s.workspaceID {
|
|
return fmt.Errorf("%w: event replay index identity disagrees with store", ErrInvalidIdentity)
|
|
}
|
|
for recordID, entry := range index.Entries {
|
|
if err := validateIdentity("EventReplayIndex.RecordID", recordID, true); err != nil {
|
|
return err
|
|
}
|
|
if entry.RecordID != recordID {
|
|
return fmt.Errorf("%w: event replay entry key disagrees with record id", ErrInvalidIdentity)
|
|
}
|
|
if entry.WorkUnitID != "" {
|
|
if err := validateIdentity(
|
|
"EventReplayIndex.WorkUnitID",
|
|
string(entry.WorkUnitID),
|
|
false,
|
|
); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if entry.Sequence == 0 || entry.Sequence == math.MaxUint64 {
|
|
return fmt.Errorf(
|
|
"%w: event replay sequence for %q cannot be retained safely",
|
|
ErrInvalidIdentity,
|
|
recordID,
|
|
)
|
|
}
|
|
if !validRecordFingerprint(entry.EventFingerprint) {
|
|
return fmt.Errorf(
|
|
"%w: event replay fingerprint for %q is invalid",
|
|
ErrInvalidIdentity,
|
|
recordID,
|
|
)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *Store) loadOrRecoverEventReplayIndex(
|
|
ctx context.Context,
|
|
recordID string,
|
|
) (eventReplayIndex, string, error) {
|
|
for attempt := 0; attempt < maxCASRetries; attempt++ {
|
|
index, revision, _, err := s.loadEventReplayIndex(ctx)
|
|
if err != nil {
|
|
return eventReplayIndex{}, "", err
|
|
}
|
|
if _, retained := index.Entries[recordID]; retained {
|
|
return index, revision, nil
|
|
}
|
|
recovered, err := s.legacyEventReplayEntries(ctx)
|
|
if err != nil {
|
|
return eventReplayIndex{}, "", err
|
|
}
|
|
changed := false
|
|
for recoveredID, entry := range recovered {
|
|
if retained, ok := index.Entries[recoveredID]; ok {
|
|
if retained != entry {
|
|
return eventReplayIndex{}, "", eventReplayConflict(recoveredID)
|
|
}
|
|
continue
|
|
}
|
|
index.Entries[recoveredID] = entry
|
|
changed = true
|
|
}
|
|
if !changed {
|
|
return index, revision, nil
|
|
}
|
|
if err := s.validateEventReplayIndex(index); err != nil {
|
|
return eventReplayIndex{}, "", err
|
|
}
|
|
payload, err := json.Marshal(index)
|
|
if err != nil {
|
|
return eventReplayIndex{}, "", fmt.Errorf(
|
|
"projectlog: encode recovered event replay index: %w",
|
|
err,
|
|
)
|
|
}
|
|
committed, err := s.state.CompareAndSwapIntegrationRecord(
|
|
ctx,
|
|
s.replayKey,
|
|
revision,
|
|
payload,
|
|
)
|
|
if errors.Is(err, agenttask.ErrRevisionConflict) {
|
|
continue
|
|
}
|
|
if err != nil {
|
|
return eventReplayIndex{}, "", err
|
|
}
|
|
return index, committed, nil
|
|
}
|
|
return eventReplayIndex{}, "", fmt.Errorf(
|
|
"%w: event replay recovery after %d retries",
|
|
ErrCASExhausted,
|
|
maxCASRetries,
|
|
)
|
|
}
|
|
|
|
func (s *Store) legacyEventReplayEntries(
|
|
ctx context.Context,
|
|
) (map[string]eventReplayEntry, error) {
|
|
snapshots, err := s.state.LoadIntegrationRecords(ctx, journalIntegrationPrefix)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
recovered := make(map[string]eventReplayEntry)
|
|
for key, snapshot := range snapshots {
|
|
var candidate journal
|
|
if err := json.Unmarshal(snapshot.Payload, &candidate); err != nil {
|
|
return nil, fmt.Errorf("projectlog: decode legacy journal %q: %w", key, err)
|
|
}
|
|
if candidate.ProjectID != s.projectID || candidate.WorkspaceID != s.workspaceID {
|
|
continue
|
|
}
|
|
scope, err := s.scopeForWorkUnit(candidate.WorkUnitID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if key != scope.key {
|
|
return nil, fmt.Errorf(
|
|
"%w: legacy journal key disagrees with retained scope",
|
|
ErrInvalidIdentity,
|
|
)
|
|
}
|
|
normalizeJournal(&candidate)
|
|
if err := scope.validateJournal(candidate); err != nil {
|
|
return nil, err
|
|
}
|
|
for recordID, seen := range candidate.SeenRecords {
|
|
if seen.EventFingerprint == "" {
|
|
continue
|
|
}
|
|
entry := eventReplayEntry{
|
|
RecordID: recordID,
|
|
WorkUnitID: candidate.WorkUnitID,
|
|
Sequence: seen.Sequence,
|
|
EventFingerprint: seen.EventFingerprint,
|
|
}
|
|
if retained, ok := recovered[recordID]; ok && retained != entry {
|
|
return nil, eventReplayConflict(recordID)
|
|
}
|
|
recovered[recordID] = entry
|
|
}
|
|
}
|
|
return recovered, nil
|
|
}
|
|
|
|
func eventReplayConflict(recordID string) error {
|
|
return fmt.Errorf(
|
|
"%w: manager event id %q",
|
|
ErrRecordReplayConflict,
|
|
recordID,
|
|
)
|
|
}
|
|
|
|
// AppendEventRecord routes work events to their deterministic task scope and
|
|
// keeps project-only events in the project journal.
|
|
func (s *Store) AppendEventRecord(
|
|
ctx context.Context,
|
|
record ProjectLogRecord,
|
|
eventFingerprint string,
|
|
) (uint64, error) {
|
|
if record.Sequence != 0 {
|
|
return 0, fmt.Errorf("%w: append sequence must be unassigned", ErrInvalidIdentity)
|
|
}
|
|
if record.ProjectID != s.projectID || record.WorkspaceID != s.workspaceID {
|
|
return 0, fmt.Errorf("%w: record identity disagrees with store", ErrInvalidIdentity)
|
|
}
|
|
if !validRecordFingerprint(eventFingerprint) {
|
|
return 0, fmt.Errorf("%w: event fingerprint is invalid", ErrInvalidIdentity)
|
|
}
|
|
scope, err := s.scopeForWorkUnit(record.WorkUnitID)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
recordFingerprint, err := recordFingerprint(record)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
|
|
for attempt := 0; attempt < maxCASRetries; attempt++ {
|
|
if err := ctx.Err(); err != nil {
|
|
return 0, err
|
|
}
|
|
index, indexRevision, err := s.loadOrRecoverEventReplayIndex(ctx, record.RecordID)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
if retained, ok := index.Entries[record.RecordID]; ok {
|
|
if retained.WorkUnitID != record.WorkUnitID ||
|
|
retained.EventFingerprint != eventFingerprint {
|
|
return 0, eventReplayConflict(record.RecordID)
|
|
}
|
|
return retained.Sequence, nil
|
|
}
|
|
|
|
j, journalRevision, _, err := scope.loadJournal(ctx)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
sequence := j.NextSequence
|
|
if seen, ok := j.SeenRecords[record.RecordID]; ok {
|
|
if seen.EventFingerprint != "" &&
|
|
seen.EventFingerprint != eventFingerprint {
|
|
return 0, eventReplayConflict(record.RecordID)
|
|
}
|
|
if seen.RecordFingerprint != recordFingerprint {
|
|
return 0, fmt.Errorf(
|
|
"%w: record id %q",
|
|
ErrRecordReplayConflict,
|
|
record.RecordID,
|
|
)
|
|
}
|
|
sequence = seen.Sequence
|
|
seen.EventFingerprint = eventFingerprint
|
|
j.SeenRecords[record.RecordID] = seen
|
|
} else {
|
|
rec := record
|
|
rec.Sequence = sequence
|
|
if err := rec.Validate(); err != nil {
|
|
return 0, err
|
|
}
|
|
j.Records = append(j.Records, rec)
|
|
j.SeenRecords[rec.RecordID] = seenRecord{
|
|
Sequence: sequence,
|
|
RecordFingerprint: recordFingerprint,
|
|
EventFingerprint: eventFingerprint,
|
|
}
|
|
j.NextSequence = sequence + 1
|
|
}
|
|
index.Entries[record.RecordID] = eventReplayEntry{
|
|
RecordID: record.RecordID,
|
|
WorkUnitID: record.WorkUnitID,
|
|
Sequence: sequence,
|
|
EventFingerprint: eventFingerprint,
|
|
}
|
|
if err := s.validateEventReplayIndex(index); err != nil {
|
|
return 0, err
|
|
}
|
|
if err := scope.validateJournal(j); err != nil {
|
|
return 0, err
|
|
}
|
|
indexPayload, err := json.Marshal(index)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("projectlog: encode event replay index: %w", err)
|
|
}
|
|
journalPayload, err := json.Marshal(j)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("projectlog: encode journal: %w", err)
|
|
}
|
|
_, err = s.state.CompareAndSwapIntegrationRecords(
|
|
ctx,
|
|
[]agentstate.IntegrationRecordUpdate{
|
|
{
|
|
Key: s.replayKey,
|
|
Expected: indexRevision,
|
|
Payload: indexPayload,
|
|
},
|
|
{
|
|
Key: scope.key,
|
|
Expected: journalRevision,
|
|
Payload: journalPayload,
|
|
},
|
|
},
|
|
)
|
|
if errors.Is(err, agenttask.ErrRevisionConflict) {
|
|
continue
|
|
}
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
return sequence, nil
|
|
}
|
|
return 0, fmt.Errorf("%w: after %d retries", ErrCASExhausted, maxCASRetries)
|
|
}
|
|
|
|
// CheckEventReplay checks the retained stable manager-event index before
|
|
// volatile manager evidence is resolved. Legacy and generic entries without
|
|
// an event fingerprint require normal evidence resolution and record replay.
|
|
func (s *Store) CheckEventReplay(
|
|
ctx context.Context,
|
|
workUnitID agenttask.WorkUnitID,
|
|
recordID string,
|
|
eventFingerprint string,
|
|
) (bool, error) {
|
|
if err := validateIdentity("RecordID", recordID, true); err != nil {
|
|
return false, err
|
|
}
|
|
if !validRecordFingerprint(eventFingerprint) {
|
|
return false, fmt.Errorf("%w: event fingerprint is invalid", ErrInvalidIdentity)
|
|
}
|
|
if _, err := s.scopeForWorkUnit(workUnitID); err != nil {
|
|
return false, err
|
|
}
|
|
index, _, err := s.loadOrRecoverEventReplayIndex(ctx, recordID)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
retained, ok := index.Entries[recordID]
|
|
if !ok {
|
|
return false, nil
|
|
}
|
|
if retained.WorkUnitID != workUnitID ||
|
|
retained.EventFingerprint != eventFingerprint {
|
|
return false, eventReplayConflict(recordID)
|
|
}
|
|
return true, nil
|
|
}
|
|
|
|
// ReplayRecords returns all un-archived records from the journal in sequence
|
|
// order. Returns nil (not an error) when the journal has not been created.
|
|
func (s *Store) ReplayRecords(ctx context.Context) ([]ProjectLogRecord, error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
j, _, found, err := s.loadJournal(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if !found {
|
|
return nil, nil
|
|
}
|
|
return append([]ProjectLogRecord(nil), j.Records...), nil
|
|
}
|
|
|
|
// Archive materializes a terminal archive from the journal. It writes the
|
|
// archive intent first (crash-recovery point), then redacted JSONL and
|
|
// timeline files via temp-file sync/rename, then the manifest, and finally
|
|
// prunes the journal. On any failure the journal is left intact and Reconcile
|
|
// converges on restart. Calling Archive on an empty journal is a no-op,
|
|
// providing exactly-once archive semantics.
|
|
func (s *Store) Archive(ctx context.Context) error {
|
|
if err := ctx.Err(); err != nil {
|
|
return err
|
|
}
|
|
if err := s.failAt("archive_before_lock"); err != nil {
|
|
return err
|
|
}
|
|
return s.withArchiveLock(ctx, func() error {
|
|
return s.archiveLocked(ctx)
|
|
})
|
|
}
|
|
|
|
func (s *Store) archiveLocked(ctx context.Context) error {
|
|
j, _, found, err := s.loadJournal(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !found || len(j.Records) == 0 {
|
|
return nil
|
|
}
|
|
ordinal := j.ArchiveOrdinal + 1
|
|
artifactsExist, err := s.archiveArtifactsExist(ordinal)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if artifactsExist {
|
|
verifiedIntent, verifiedRecords, err := s.verifyArchiveArtifacts(ordinal)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return s.reconcilePrune(ctx, verifiedIntent, verifiedRecords)
|
|
}
|
|
if !j.Records[len(j.Records)-1].Terminal {
|
|
return fmt.Errorf("%w: last record is not terminal", ErrNoTerminalRecord)
|
|
}
|
|
records := j.Records
|
|
now := time.Now().UTC()
|
|
|
|
manifest, err := NewArchiveManifest(s.projectID, s.workspaceID, ordinal, records, now, true)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
intent := archiveIntent{
|
|
SchemaVersion: journalSchemaVersion,
|
|
ProjectID: s.projectID,
|
|
WorkspaceID: s.workspaceID,
|
|
WorkUnitID: s.workUnitID,
|
|
ArchiveOrdinal: ordinal,
|
|
FirstSequence: manifest.FirstSequence,
|
|
LastSequence: manifest.LastSequence,
|
|
RecordCount: manifest.RecordCount,
|
|
Checksum: manifest.Checksum,
|
|
ArchivedAt: now,
|
|
Terminal: true,
|
|
}
|
|
|
|
archiveDir := s.archiveDir()
|
|
if err := os.MkdirAll(archiveDir, 0o700); err != nil {
|
|
return err
|
|
}
|
|
intentPath := s.intentPath(ordinal)
|
|
jsonlPath := s.jsonlPath(ordinal)
|
|
timelinePath := s.timelinePath(ordinal)
|
|
manifestPath := s.manifestPath(ordinal)
|
|
|
|
// 1. Persist archive intent (crash-recovery anchor).
|
|
intentData, err := json.Marshal(intent)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := writeAtomically(intentPath, append(intentData, '\n')); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := s.failAt("archive_before_write"); err != nil {
|
|
return err
|
|
}
|
|
|
|
// 2. Materialize redacted JSONL and timeline via temp-file sync/rename.
|
|
jsonlData, err := encodeJSONL(records)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := writeAtomically(jsonlPath, jsonlData); err != nil {
|
|
return err
|
|
}
|
|
timelineData, err := encodeTimeline(records)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := writeAtomically(timelinePath, timelineData); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := s.failAt("archive_after_rename"); err != nil {
|
|
return err
|
|
}
|
|
|
|
// 3. Write manifest to a temp file, then commit via rename.
|
|
manifestData, err := json.Marshal(manifest)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
manifestTemp, err := writeTemp(archiveDir, "manifest", append(manifestData, '\n'))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := s.failAt("archive_before_manifest"); err != nil {
|
|
return err
|
|
}
|
|
if err := os.Rename(manifestTemp, manifestPath); err != nil {
|
|
return err
|
|
}
|
|
if err := syncDir(archiveDir); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := s.failAt("archive_before_cleanup"); err != nil {
|
|
return err
|
|
}
|
|
|
|
// 4. Verify the complete committed artifact set, then prune only the
|
|
// records proven by that immutable set.
|
|
verifiedIntent, verifiedRecords, err := s.verifyArchiveArtifacts(ordinal)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return s.reconcilePrune(ctx, verifiedIntent, verifiedRecords)
|
|
}
|
|
|
|
// Reconcile scans the archive directory for intent files and converges any
|
|
// incomplete archive idempotently. If the manifest is missing it is
|
|
// recomputed from the JSONL and verified against the intent checksum.
|
|
// Conflicting content fails closed with ErrArchiveConflict. After a manifest
|
|
// is verified or reconstructed, the journal is pruned if it still holds the
|
|
// archived records.
|
|
func (s *Store) Reconcile(ctx context.Context) error {
|
|
if err := ctx.Err(); err != nil {
|
|
return err
|
|
}
|
|
return s.withArchiveLock(ctx, func() error {
|
|
return s.reconcileLocked(ctx)
|
|
})
|
|
}
|
|
|
|
func (s *Store) reconcileLocked(ctx context.Context) error {
|
|
archiveDir := s.archiveDir()
|
|
entries, err := os.ReadDir(archiveDir)
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
foundIntent := false
|
|
for _, entry := range entries {
|
|
if entry.IsDir() {
|
|
continue
|
|
}
|
|
name := entry.Name()
|
|
if !strings.HasSuffix(name, ".intent") {
|
|
continue
|
|
}
|
|
foundIntent = true
|
|
ordinal := parseOrdinal(name)
|
|
if ordinal == 0 {
|
|
continue
|
|
}
|
|
if err := s.reconcileArchive(ctx, ordinal); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if !foundIntent {
|
|
j, _, found, err := s.loadJournal(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if found && len(j.Records) > 0 && j.Records[len(j.Records)-1].Terminal {
|
|
return s.archiveLocked(ctx)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *Store) reconcileArchive(ctx context.Context, ordinal uint64) error {
|
|
intent, records, err := s.verifyArchiveArtifacts(ordinal)
|
|
if err != nil {
|
|
if !errors.Is(err, ErrArchiveIncomplete) {
|
|
return err
|
|
}
|
|
if restoreErr := s.restoreMissingJSONL(ctx, ordinal); restoreErr != nil {
|
|
return restoreErr
|
|
}
|
|
intent, records, err = s.verifyArchiveArtifacts(ordinal)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return s.reconcilePrune(ctx, intent, records)
|
|
}
|
|
|
|
func (s *Store) restoreMissingJSONL(ctx context.Context, ordinal uint64) error {
|
|
if _, err := os.Stat(s.jsonlPath(ordinal)); err == nil {
|
|
return nil
|
|
} else if !os.IsNotExist(err) {
|
|
return err
|
|
}
|
|
intentData, err := os.ReadFile(s.intentPath(ordinal))
|
|
if err != nil {
|
|
return fmt.Errorf("%w: read intent for recovery: %v", ErrArchiveIncomplete, err)
|
|
}
|
|
var intent archiveIntent
|
|
if err := json.Unmarshal(intentData, &intent); err != nil {
|
|
return fmt.Errorf("%w: decode intent for recovery: %v", ErrArchiveConflict, err)
|
|
}
|
|
if intent.SchemaVersion != journalSchemaVersion ||
|
|
intent.ProjectID != s.projectID ||
|
|
intent.WorkspaceID != s.workspaceID ||
|
|
intent.WorkUnitID != s.workUnitID ||
|
|
intent.ArchiveOrdinal != ordinal ||
|
|
intent.RecordCount <= 0 {
|
|
return fmt.Errorf("%w: recovery intent identity is invalid", ErrArchiveConflict)
|
|
}
|
|
j, _, found, err := s.loadJournal(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !found || len(j.Records) < intent.RecordCount {
|
|
return fmt.Errorf("%w: journal cannot reconstruct ordinal %d", ErrArchiveIncomplete, ordinal)
|
|
}
|
|
records := append([]ProjectLogRecord(nil), j.Records[:intent.RecordCount]...)
|
|
manifest, err := NewArchiveManifest(
|
|
s.projectID,
|
|
s.workspaceID,
|
|
ordinal,
|
|
records,
|
|
intent.ArchivedAt,
|
|
true,
|
|
)
|
|
if err != nil {
|
|
return fmt.Errorf("%w: reconstruct manifest: %v", ErrArchiveConflict, err)
|
|
}
|
|
if manifest.FirstSequence != intent.FirstSequence ||
|
|
manifest.LastSequence != intent.LastSequence ||
|
|
manifest.RecordCount != intent.RecordCount ||
|
|
manifest.Checksum != intent.Checksum {
|
|
return fmt.Errorf("%w: journal does not match recovery intent", ErrArchiveConflict)
|
|
}
|
|
jsonlData, err := encodeJSONL(records)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return writeAtomically(s.jsonlPath(ordinal), jsonlData)
|
|
}
|
|
|
|
func (s *Store) archiveArtifactsExist(ordinal uint64) (bool, error) {
|
|
for _, path := range []string{
|
|
s.intentPath(ordinal),
|
|
s.jsonlPath(ordinal),
|
|
s.timelinePath(ordinal),
|
|
s.manifestPath(ordinal),
|
|
} {
|
|
if _, err := os.Stat(path); err == nil {
|
|
return true, nil
|
|
} else if !os.IsNotExist(err) {
|
|
return false, err
|
|
}
|
|
}
|
|
return false, nil
|
|
}
|
|
|
|
func (s *Store) verifyArchiveArtifacts(ordinal uint64) (archiveIntent, []ProjectLogRecord, error) {
|
|
intentPath := s.intentPath(ordinal)
|
|
manifestPath := s.manifestPath(ordinal)
|
|
jsonlPath := s.jsonlPath(ordinal)
|
|
timelinePath := s.timelinePath(ordinal)
|
|
|
|
intentData, err := os.ReadFile(intentPath)
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
return archiveIntent{}, nil, fmt.Errorf("%w: missing intent for ordinal %d", ErrArchiveIncomplete, ordinal)
|
|
}
|
|
return archiveIntent{}, nil, fmt.Errorf("projectlog: read intent: %w", err)
|
|
}
|
|
var intent archiveIntent
|
|
if err := json.Unmarshal(intentData, &intent); err != nil {
|
|
return archiveIntent{}, nil, fmt.Errorf("%w: decode intent: %v", ErrArchiveConflict, err)
|
|
}
|
|
|
|
if intent.SchemaVersion != journalSchemaVersion {
|
|
return archiveIntent{}, nil, fmt.Errorf("%w: intent schema version %d", ErrArchiveConflict, intent.SchemaVersion)
|
|
}
|
|
if intent.ProjectID != s.projectID ||
|
|
intent.WorkspaceID != s.workspaceID ||
|
|
intent.WorkUnitID != s.workUnitID {
|
|
return archiveIntent{}, nil, fmt.Errorf("%w: intent identity disagrees with store", ErrArchiveConflict)
|
|
}
|
|
if intent.ArchiveOrdinal != ordinal {
|
|
return archiveIntent{}, nil, fmt.Errorf("%w: intent ordinal %d != filename %d", ErrArchiveConflict, intent.ArchiveOrdinal, ordinal)
|
|
}
|
|
if intent.RecordCount <= 0 || intent.FirstSequence == 0 || intent.LastSequence < intent.FirstSequence {
|
|
return archiveIntent{}, nil, fmt.Errorf("%w: invalid intent sequence range or count", ErrArchiveConflict)
|
|
}
|
|
if !validArchiveChecksum(intent.Checksum) {
|
|
return archiveIntent{}, nil, fmt.Errorf("%w: invalid intent checksum format", ErrArchiveConflict)
|
|
}
|
|
if intent.ArchivedAt.IsZero() {
|
|
return archiveIntent{}, nil, fmt.Errorf("%w: zero intent timestamp", ErrArchiveConflict)
|
|
}
|
|
if !intent.Terminal {
|
|
return archiveIntent{}, nil, fmt.Errorf("%w: intent is not terminal", ErrArchiveConflict)
|
|
}
|
|
|
|
jsonlData, err := os.ReadFile(jsonlPath)
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
return archiveIntent{}, nil, fmt.Errorf("%w: missing jsonl for ordinal %d", ErrArchiveIncomplete, ordinal)
|
|
}
|
|
return archiveIntent{}, nil, err
|
|
}
|
|
|
|
if computeJSONLChecksum(jsonlData) != intent.Checksum {
|
|
return archiveIntent{}, nil, fmt.Errorf("%w: jsonl checksum mismatch for ordinal %d", ErrArchiveConflict, ordinal)
|
|
}
|
|
|
|
records, err := decodeJSONL(jsonlData)
|
|
if err != nil {
|
|
return archiveIntent{}, nil, fmt.Errorf("%w: decode jsonl: %v", ErrArchiveConflict, err)
|
|
}
|
|
|
|
if len(records) != intent.RecordCount {
|
|
return archiveIntent{}, nil, fmt.Errorf("%w: jsonl record count %d != intent count %d", ErrArchiveConflict, len(records), intent.RecordCount)
|
|
}
|
|
if records[0].Sequence != intent.FirstSequence || records[len(records)-1].Sequence != intent.LastSequence {
|
|
return archiveIntent{}, nil, fmt.Errorf("%w: jsonl sequence range disagrees with intent", ErrArchiveConflict)
|
|
}
|
|
|
|
expectedManifest, err := NewArchiveManifest(s.projectID, s.workspaceID, ordinal, records, intent.ArchivedAt, true)
|
|
if err != nil {
|
|
return archiveIntent{}, nil, fmt.Errorf("%w: derive manifest: %w", ErrArchiveConflict, err)
|
|
}
|
|
if expectedManifest.Checksum != intent.Checksum {
|
|
return archiveIntent{}, nil, fmt.Errorf("%w: derived checksum %s != intent %s", ErrArchiveConflict, expectedManifest.Checksum, intent.Checksum)
|
|
}
|
|
if s.workUnitID != "" &&
|
|
(len(expectedManifest.WorkUnitIDs) != 1 || expectedManifest.WorkUnitIDs[0] != s.workUnitID) {
|
|
return archiveIntent{}, nil, fmt.Errorf(
|
|
"%w: manifest work unit disagrees with store scope",
|
|
ErrArchiveConflict,
|
|
)
|
|
}
|
|
|
|
manifestBytes, err := os.ReadFile(manifestPath)
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
mBytes, marshalErr := json.Marshal(expectedManifest)
|
|
if marshalErr != nil {
|
|
return archiveIntent{}, nil, marshalErr
|
|
}
|
|
if err := writeAtomically(manifestPath, append(mBytes, '\n')); err != nil {
|
|
return archiveIntent{}, nil, err
|
|
}
|
|
} else {
|
|
return archiveIntent{}, nil, err
|
|
}
|
|
} else {
|
|
var existingManifest ArchiveManifest
|
|
if err := json.Unmarshal(manifestBytes, &existingManifest); err != nil {
|
|
return archiveIntent{}, nil, fmt.Errorf("%w: decode manifest: %w", ErrArchiveConflict, err)
|
|
}
|
|
if err := existingManifest.Validate(); err != nil {
|
|
return archiveIntent{}, nil, fmt.Errorf("%w: invalid manifest: %w", ErrArchiveConflict, err)
|
|
}
|
|
if !manifestsEqual(existingManifest, expectedManifest) {
|
|
return archiveIntent{}, nil, fmt.Errorf("%w: manifest field drift for ordinal %d", ErrArchiveConflict, ordinal)
|
|
}
|
|
}
|
|
|
|
expectedTimelineData, err := encodeTimeline(records)
|
|
if err != nil {
|
|
return archiveIntent{}, nil, fmt.Errorf("projectlog: encode timeline: %w", err)
|
|
}
|
|
|
|
timelineData, err := os.ReadFile(timelinePath)
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
if err := writeAtomically(timelinePath, expectedTimelineData); err != nil {
|
|
return archiveIntent{}, nil, err
|
|
}
|
|
} else {
|
|
return archiveIntent{}, nil, err
|
|
}
|
|
} else {
|
|
if !bytes.Equal(bytes.TrimSpace(timelineData), bytes.TrimSpace(expectedTimelineData)) {
|
|
return archiveIntent{}, nil, fmt.Errorf("%w: timeline content mismatch for ordinal %d", ErrArchiveConflict, ordinal)
|
|
}
|
|
}
|
|
|
|
return intent, records, nil
|
|
}
|
|
|
|
func manifestsEqual(m1, m2 ArchiveManifest) bool {
|
|
if m1.SchemaVersion != m2.SchemaVersion ||
|
|
m1.ProjectID != m2.ProjectID ||
|
|
m1.WorkspaceID != m2.WorkspaceID ||
|
|
m1.ArchiveOrdinal != m2.ArchiveOrdinal ||
|
|
m1.RecordCount != m2.RecordCount ||
|
|
m1.FirstSequence != m2.FirstSequence ||
|
|
m1.LastSequence != m2.LastSequence ||
|
|
m1.Checksum != m2.Checksum ||
|
|
!m1.ArchivedAt.Equal(m2.ArchivedAt) ||
|
|
m1.Terminal != m2.Terminal ||
|
|
len(m1.WorkUnitIDs) != len(m2.WorkUnitIDs) {
|
|
return false
|
|
}
|
|
for i := range m1.WorkUnitIDs {
|
|
if m1.WorkUnitIDs[i] != m2.WorkUnitIDs[i] {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func recordsMatchPrefix(jRecords, archived []ProjectLogRecord) bool {
|
|
if len(jRecords) < len(archived) {
|
|
return false
|
|
}
|
|
for i := range archived {
|
|
if !recordsEqual(jRecords[i], archived[i]) {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func recordsEqual(a, b ProjectLogRecord) bool {
|
|
if a.Sequence != b.Sequence || a.RecordID != b.RecordID || a.EventType != b.EventType || a.Terminal != b.Terminal || a.State != b.State {
|
|
return false
|
|
}
|
|
aBytes, errA := json.Marshal(a)
|
|
bBytes, errB := json.Marshal(b)
|
|
if errA != nil || errB != nil {
|
|
return false
|
|
}
|
|
return bytes.Equal(aBytes, bBytes)
|
|
}
|
|
|
|
func (s *Store) reconcilePrune(
|
|
ctx context.Context,
|
|
intent archiveIntent,
|
|
archivedRecords []ProjectLogRecord,
|
|
) error {
|
|
for attempt := 0; attempt < maxCASRetries; attempt++ {
|
|
if err := ctx.Err(); err != nil {
|
|
return err
|
|
}
|
|
j, revision, found, err := s.loadJournal(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !found {
|
|
return nil
|
|
}
|
|
|
|
if len(j.Records) == 0 {
|
|
if j.ArchiveOrdinal < intent.ArchiveOrdinal {
|
|
j.ArchiveOrdinal = intent.ArchiveOrdinal
|
|
next, err := json.Marshal(j)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if _, err := s.state.CompareAndSwapIntegrationRecord(ctx, s.key, revision, next); err != nil {
|
|
if errors.Is(err, agenttask.ErrRevisionConflict) {
|
|
continue
|
|
}
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
if j.Records[0].Sequence > intent.LastSequence {
|
|
if j.ArchiveOrdinal < intent.ArchiveOrdinal {
|
|
j.ArchiveOrdinal = intent.ArchiveOrdinal
|
|
next, err := json.Marshal(j)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if _, err := s.state.CompareAndSwapIntegrationRecord(ctx, s.key, revision, next); err != nil {
|
|
if errors.Is(err, agenttask.ErrRevisionConflict) {
|
|
continue
|
|
}
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
if !recordsMatchPrefix(j.Records, archivedRecords) {
|
|
return fmt.Errorf("%w: journal prefix does not match archived records", ErrArchiveConflict)
|
|
}
|
|
|
|
j.Records = append([]ProjectLogRecord(nil), j.Records[len(archivedRecords):]...)
|
|
if j.ArchiveOrdinal < intent.ArchiveOrdinal {
|
|
j.ArchiveOrdinal = intent.ArchiveOrdinal
|
|
}
|
|
next, err := json.Marshal(j)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if _, err := s.state.CompareAndSwapIntegrationRecord(ctx, s.key, revision, next); err != nil {
|
|
if errors.Is(err, agenttask.ErrRevisionConflict) {
|
|
continue
|
|
}
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
return fmt.Errorf("%w: reconcile prune after %d retries", ErrCASExhausted, maxCASRetries)
|
|
}
|
|
|
|
func integrationRecordKey(
|
|
projectID agenttask.ProjectID,
|
|
workspaceID agenttask.WorkspaceID,
|
|
workUnitID agenttask.WorkUnitID,
|
|
) string {
|
|
h := sha256.New()
|
|
writeLengthPrefixed(h, string(projectID))
|
|
writeLengthPrefixed(h, string(workspaceID))
|
|
if workUnitID != "" {
|
|
writeLengthPrefixed(h, string(workUnitID))
|
|
}
|
|
return journalIntegrationPrefix + hex.EncodeToString(h.Sum(nil))
|
|
}
|
|
|
|
func eventReplayIndexKey(
|
|
projectID agenttask.ProjectID,
|
|
workspaceID agenttask.WorkspaceID,
|
|
) string {
|
|
h := sha256.New()
|
|
writeLengthPrefixed(h, "event-replay-index")
|
|
writeLengthPrefixed(h, string(projectID))
|
|
writeLengthPrefixed(h, string(workspaceID))
|
|
return eventReplayIntegrationPrefix + hex.EncodeToString(h.Sum(nil))
|
|
}
|
|
|
|
func newJournal(
|
|
projectID agenttask.ProjectID,
|
|
workspaceID agenttask.WorkspaceID,
|
|
workUnitID agenttask.WorkUnitID,
|
|
) journal {
|
|
return journal{
|
|
SchemaVersion: journalSchemaVersion,
|
|
ProjectID: projectID,
|
|
WorkspaceID: workspaceID,
|
|
WorkUnitID: workUnitID,
|
|
NextSequence: 1,
|
|
ArchiveOrdinal: 0,
|
|
SeenRecords: make(map[string]seenRecord),
|
|
Records: nil,
|
|
}
|
|
}
|
|
|
|
func normalizeJournal(j *journal) {
|
|
if j.SeenRecords == nil {
|
|
j.SeenRecords = make(map[string]seenRecord, len(j.Records))
|
|
}
|
|
for recordID, seen := range j.SeenRecords {
|
|
if seen.RecordFingerprint == "" {
|
|
seen.RecordFingerprint = seen.Fingerprint
|
|
}
|
|
seen.Fingerprint = ""
|
|
j.SeenRecords[recordID] = seen
|
|
}
|
|
for _, record := range j.Records {
|
|
if _, exists := j.SeenRecords[record.RecordID]; exists {
|
|
continue
|
|
}
|
|
fingerprint, err := recordFingerprint(record)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
j.SeenRecords[record.RecordID] = seenRecord{
|
|
Sequence: record.Sequence,
|
|
RecordFingerprint: fingerprint,
|
|
}
|
|
}
|
|
}
|
|
|
|
func newEventReplayIndex(
|
|
projectID agenttask.ProjectID,
|
|
workspaceID agenttask.WorkspaceID,
|
|
) eventReplayIndex {
|
|
return eventReplayIndex{
|
|
SchemaVersion: eventReplayIndexSchemaVersion,
|
|
ProjectID: projectID,
|
|
WorkspaceID: workspaceID,
|
|
Entries: make(map[string]eventReplayEntry),
|
|
}
|
|
}
|
|
|
|
func normalizeEventReplayIndex(index *eventReplayIndex) {
|
|
if index.Entries == nil {
|
|
index.Entries = make(map[string]eventReplayEntry)
|
|
}
|
|
}
|
|
|
|
func (s *Store) archiveDir() string {
|
|
h := sha256.New()
|
|
writeLengthPrefixed(h, string(s.projectID))
|
|
writeLengthPrefixed(h, string(s.workspaceID))
|
|
if s.workUnitID != "" {
|
|
writeLengthPrefixed(h, string(s.workUnitID))
|
|
}
|
|
return filepath.Join(s.root, hex.EncodeToString(h.Sum(nil)), "archive")
|
|
}
|
|
|
|
func writeLengthPrefixed(h interface{ Write([]byte) (int, error) }, value string) {
|
|
var length [4]byte
|
|
binary.BigEndian.PutUint32(length[:], uint32(len(value)))
|
|
_, _ = h.Write(length[:])
|
|
_, _ = h.Write([]byte(value))
|
|
}
|
|
|
|
func (s *Store) withArchiveLock(ctx context.Context, action func() error) error {
|
|
archiveDir := s.archiveDir()
|
|
if err := os.MkdirAll(archiveDir, 0o700); err != nil {
|
|
return err
|
|
}
|
|
lock, err := os.OpenFile(filepath.Join(archiveDir, ".archive.lock"), os.O_CREATE|os.O_RDWR, 0o600)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer lock.Close()
|
|
if err := ctx.Err(); err != nil {
|
|
return err
|
|
}
|
|
if err := syscall.Flock(int(lock.Fd()), syscall.LOCK_EX); err != nil {
|
|
return err
|
|
}
|
|
defer syscall.Flock(int(lock.Fd()), syscall.LOCK_UN) //nolint:errcheck
|
|
if err := ctx.Err(); err != nil {
|
|
return err
|
|
}
|
|
return action()
|
|
}
|
|
|
|
func (s *Store) intentPath(ordinal uint64) string {
|
|
return filepath.Join(s.archiveDir(), fmt.Sprintf("%06d.intent", ordinal))
|
|
}
|
|
|
|
func (s *Store) jsonlPath(ordinal uint64) string {
|
|
return filepath.Join(s.archiveDir(), fmt.Sprintf("%06d.jsonl", ordinal))
|
|
}
|
|
|
|
func (s *Store) timelinePath(ordinal uint64) string {
|
|
return filepath.Join(s.archiveDir(), fmt.Sprintf("%06d.timeline.jsonl", ordinal))
|
|
}
|
|
|
|
func (s *Store) manifestPath(ordinal uint64) string {
|
|
return filepath.Join(s.archiveDir(), fmt.Sprintf("%06d.manifest.json", ordinal))
|
|
}
|
|
|
|
func parseOrdinal(name string) uint64 {
|
|
base := strings.TrimSuffix(name, ".intent")
|
|
ordinal, err := strconv.ParseUint(base, 10, 64)
|
|
if err != nil {
|
|
return 0
|
|
}
|
|
return ordinal
|
|
}
|
|
|
|
func writeTemp(dir, base string, data []byte) (string, error) {
|
|
temp, err := os.CreateTemp(dir, "."+base+".tmp-*")
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
tempPath := temp.Name()
|
|
removeTemp := true
|
|
defer func() {
|
|
if removeTemp {
|
|
_ = os.Remove(tempPath)
|
|
}
|
|
}()
|
|
if err := temp.Chmod(0o600); err != nil {
|
|
_ = temp.Close()
|
|
return "", err
|
|
}
|
|
if _, err := temp.Write(data); err != nil {
|
|
_ = temp.Close()
|
|
return "", err
|
|
}
|
|
if err := temp.Sync(); err != nil {
|
|
_ = temp.Close()
|
|
return "", err
|
|
}
|
|
if err := temp.Close(); err != nil {
|
|
return "", err
|
|
}
|
|
removeTemp = false
|
|
return tempPath, nil
|
|
}
|
|
|
|
func writeAtomically(path string, data []byte) error {
|
|
tempPath, err := writeTemp(filepath.Dir(path), filepath.Base(path), data)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := os.Rename(tempPath, path); err != nil {
|
|
_ = os.Remove(tempPath)
|
|
return err
|
|
}
|
|
return syncDir(filepath.Dir(path))
|
|
}
|
|
|
|
func syncDir(path string) error {
|
|
dirHandle, err := os.Open(path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer dirHandle.Close()
|
|
return dirHandle.Sync()
|
|
}
|
|
|
|
func encodeJSONL(records []ProjectLogRecord) ([]byte, error) {
|
|
var buf bytes.Buffer
|
|
for _, rec := range records {
|
|
data, err := json.Marshal(rec)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
buf.Write(data)
|
|
buf.WriteByte('\n')
|
|
}
|
|
return buf.Bytes(), nil
|
|
}
|
|
|
|
func encodeTimeline(records []ProjectLogRecord) ([]byte, error) {
|
|
return encodeWorkLogEntries(ProjectWorkLog(records).Entries)
|
|
}
|
|
|
|
func encodeWorkLogEntries(entries []WorkLogEntry) ([]byte, error) {
|
|
var buf bytes.Buffer
|
|
for _, entry := range entries {
|
|
data, err := json.Marshal(entry)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
buf.Write(data)
|
|
buf.WriteByte('\n')
|
|
}
|
|
return buf.Bytes(), nil
|
|
}
|
|
|
|
func recordFingerprint(record ProjectLogRecord) (string, error) {
|
|
record.Sequence = 0
|
|
payload, err := json.Marshal(record)
|
|
if err != nil {
|
|
return "", fmt.Errorf("projectlog: encode record fingerprint: %w", err)
|
|
}
|
|
sum := sha256.Sum256(payload)
|
|
return "sha256:" + hex.EncodeToString(sum[:]), nil
|
|
}
|
|
|
|
func validRecordFingerprint(fingerprint string) bool {
|
|
return validArchiveChecksum(fingerprint)
|
|
}
|
|
|
|
func decodeJSONL(data []byte) ([]ProjectLogRecord, error) {
|
|
var records []ProjectLogRecord
|
|
for len(data) > 0 {
|
|
idx := bytes.IndexByte(data, '\n')
|
|
var line []byte
|
|
if idx == -1 {
|
|
line = data
|
|
data = nil
|
|
} else {
|
|
line = data[:idx]
|
|
data = data[idx+1:]
|
|
}
|
|
line = bytes.TrimSpace(line)
|
|
if len(line) == 0 {
|
|
continue
|
|
}
|
|
var rec ProjectLogRecord
|
|
if err := json.Unmarshal(line, &rec); err != nil {
|
|
return nil, err
|
|
}
|
|
records = append(records, rec)
|
|
}
|
|
return records, nil
|
|
}
|
|
|
|
func computeJSONLChecksum(data []byte) string {
|
|
sum := sha256.Sum256(data)
|
|
return "sha256:" + hex.EncodeToString(sum[:])
|
|
}
|