iop/apps/agent/internal/projectlog/store.go

1564 lines
46 KiB
Go

// Package projectlog provides immutable, versioned presentation and recovery
// project-log record schemas and validation matrices for the standalone host.
//
// This file implements the CAS-backed journal and terminal archive
// reconciliation over a device-local root. It owns the persistence invariant:
// append by CAS with bounded retry, retention only after terminal evidence,
// restart-safe materialization of redacted JSONL/timeline files, and
// idempotent reconciliation that fails closed on conflicting content.
package projectlog
import (
"bytes"
"context"
"crypto/sha256"
"encoding/binary"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"math"
"os"
"path/filepath"
"strconv"
"strings"
"syscall"
"time"
"iop/packages/go/agentstate"
"iop/packages/go/agenttask"
)
const journalSchemaVersion uint32 = 1
const eventReplayIndexSchemaVersion uint32 = 1
const maxCASRetries = 16
const (
journalIntegrationPrefix = "projectlog:"
eventReplayIntegrationPrefix = "projectlog-event-replay:"
)
var (
ErrNoTerminalRecord = errors.New("projectlog: no terminal record to archive")
ErrArchiveIncomplete = errors.New("projectlog: archive incomplete")
ErrArchiveConflict = errors.New("projectlog: archive content conflict")
ErrRecordReplayConflict = errors.New("projectlog: record replay content conflict")
ErrCASExhausted = errors.New("projectlog: CAS retry exhausted")
)
type seenRecord struct {
Sequence uint64 `json:"sequence"`
RecordFingerprint string `json:"record_fingerprint"`
EventFingerprint string `json:"event_fingerprint,omitempty"`
Fingerprint string `json:"fingerprint,omitempty"`
}
// journal is the durable CAS payload for one project or work-unit scope.
type journal struct {
SchemaVersion uint32 `json:"schema_version"`
ProjectID agenttask.ProjectID `json:"project_id"`
WorkspaceID agenttask.WorkspaceID `json:"workspace_id"`
WorkUnitID agenttask.WorkUnitID `json:"work_unit_id,omitempty"`
NextSequence uint64 `json:"next_sequence"`
ArchiveOrdinal uint64 `json:"archive_ordinal"`
SeenRecords map[string]seenRecord `json:"seen_records"`
Records []ProjectLogRecord `json:"records"`
}
type eventReplayEntry struct {
RecordID string `json:"record_id"`
WorkUnitID agenttask.WorkUnitID `json:"work_unit_id,omitempty"`
Sequence uint64 `json:"sequence"`
EventFingerprint string `json:"event_fingerprint"`
}
// eventReplayIndex is the retained project/workspace-wide ownership record for
// manager EventID projections. Task-scoped journals retain their independent
// sequence and archive lifecycle.
type eventReplayIndex struct {
SchemaVersion uint32 `json:"schema_version"`
ProjectID agenttask.ProjectID `json:"project_id"`
WorkspaceID agenttask.WorkspaceID `json:"workspace_id"`
Entries map[string]eventReplayEntry `json:"entries"`
}
// archiveIntent is the durable crash-recovery marker written before any
// artifact. It records the manifest checksum so Reconcile can verify or
// recompute the manifest without trusting partial state.
type archiveIntent struct {
SchemaVersion uint32 `json:"schema_version"`
ProjectID agenttask.ProjectID `json:"project_id"`
WorkspaceID agenttask.WorkspaceID `json:"workspace_id"`
WorkUnitID agenttask.WorkUnitID `json:"work_unit_id,omitempty"`
ArchiveOrdinal uint64 `json:"archive_ordinal"`
FirstSequence uint64 `json:"first_sequence"`
LastSequence uint64 `json:"last_sequence"`
RecordCount int `json:"record_count"`
Checksum string `json:"checksum"`
ArchivedAt time.Time `json:"archived_at"`
Terminal bool `json:"terminal"`
}
// Store is the CAS-backed project log journal with device-local archive roots.
// It wraps an agentstate.Store for the durable CAS journal and a device-local
// root directory for materialized archive artifacts.
type Store struct {
state *agentstate.Store
root string
projectID agenttask.ProjectID
workspaceID agenttask.WorkspaceID
workUnitID agenttask.WorkUnitID
key string
replayKey string
failureHook func(phase string) error
}
// NewStore creates a CAS-backed project log store over a device-local root.
// The state store provides the crash-safe CAS journal; root holds archive
// artifacts (intent, JSONL, timeline, manifest).
func NewStore(
state *agentstate.Store,
root string,
projectID agenttask.ProjectID,
workspaceID agenttask.WorkspaceID,
) (*Store, error) {
if state == nil {
return nil, fmt.Errorf("projectlog: state store is required")
}
if root == "" {
return nil, fmt.Errorf("projectlog: root is required")
}
if projectID == "" || workspaceID == "" {
return nil, fmt.Errorf("projectlog: project and workspace IDs are required")
}
return &Store{
state: state,
root: filepath.Clean(root),
projectID: projectID,
workspaceID: workspaceID,
key: integrationRecordKey(projectID, workspaceID, ""),
replayKey: eventReplayIndexKey(projectID, workspaceID),
}, nil
}
// ForWorkUnit returns a task-scoped journal below the same project/workspace.
// Each scoped journal owns its own sequence and archive ordinal.
func (s *Store) ForWorkUnit(workUnitID agenttask.WorkUnitID) (*Store, error) {
if s == nil {
return nil, fmt.Errorf("projectlog: store is required")
}
if err := validateIdentity("WorkUnitID", string(workUnitID), true); err != nil {
return nil, err
}
return &Store{
state: s.state,
root: s.root,
projectID: s.projectID,
workspaceID: s.workspaceID,
workUnitID: workUnitID,
key: integrationRecordKey(s.projectID, s.workspaceID, workUnitID),
replayKey: s.replayKey,
failureHook: s.failureHook,
}, nil
}
// WithFailureHook installs a phase-gated failure hook for testing crash
// windows. The hook is invoked at named phases during Archive; if it returns
// a non-nil error the operation aborts at that phase. Reconcile is never
// gated by the hook.
func (s *Store) WithFailureHook(hook func(phase string) error) *Store {
s.failureHook = hook
return s
}
func (s *Store) failAt(phase string) error {
if s.failureHook != nil {
return s.failureHook(phase)
}
return nil
}
func (s *Store) loadJournal(ctx context.Context) (journal, string, bool, error) {
payload, revision, found, err := s.state.LoadIntegrationRecord(ctx, s.key)
if err != nil {
return journal{}, "", false, err
}
if !found {
return newJournal(s.projectID, s.workspaceID, s.workUnitID), revision, false, nil
}
var j journal
if err := json.Unmarshal(payload, &j); err != nil {
return journal{}, "", false, fmt.Errorf("projectlog: decode journal: %w", err)
}
normalizeJournal(&j)
if err := s.validateJournal(j); err != nil {
return journal{}, "", false, err
}
return j, revision, true, nil
}
func (s *Store) validateJournal(j journal) error {
if j.SchemaVersion != journalSchemaVersion {
return fmt.Errorf(
"%w: journal schema version %d expected %d",
ErrInvalidSchemaVersion,
j.SchemaVersion,
journalSchemaVersion,
)
}
if j.ProjectID != s.projectID || j.WorkspaceID != s.workspaceID {
return fmt.Errorf("%w: journal identity disagrees with store", ErrInvalidIdentity)
}
if j.WorkUnitID != s.workUnitID {
return fmt.Errorf("%w: journal work-unit scope disagrees with store", ErrInvalidIdentity)
}
if j.NextSequence == 0 || j.NextSequence == math.MaxUint64 {
return fmt.Errorf("%w: journal next sequence %d cannot be assigned safely", ErrInvalidIdentity, j.NextSequence)
}
for index, record := range j.Records {
if err := record.Validate(); err != nil {
return fmt.Errorf("projectlog: invalid journal record %d: %w", index, err)
}
if record.ProjectID != s.projectID || record.WorkspaceID != s.workspaceID {
return fmt.Errorf("%w: journal record %d identity disagrees with store", ErrInvalidIdentity, index)
}
if s.workUnitID != "" && record.WorkUnitID != s.workUnitID {
return fmt.Errorf("%w: journal record %d work unit disagrees with scope", ErrInvalidIdentity, index)
}
if index > 0 && record.Sequence != j.Records[index-1].Sequence+1 {
return fmt.Errorf(
"%w: journal record sequence %d does not follow %d",
ErrInvalidIdentity,
record.Sequence,
j.Records[index-1].Sequence,
)
}
seen, ok := j.SeenRecords[record.RecordID]
if !ok {
return fmt.Errorf("%w: record %q is absent from replay index", ErrInvalidIdentity, record.RecordID)
}
fingerprint, err := recordFingerprint(record)
if err != nil {
return err
}
if seen.Sequence != record.Sequence || seen.RecordFingerprint != fingerprint {
return fmt.Errorf("%w: replay index disagrees with record %q", ErrInvalidIdentity, record.RecordID)
}
}
if len(j.Records) > 0 {
lastSequence := j.Records[len(j.Records)-1].Sequence
if lastSequence == math.MaxUint64 || j.NextSequence != lastSequence+1 {
return fmt.Errorf(
"%w: journal next sequence %d does not follow retained sequence %d",
ErrInvalidIdentity,
j.NextSequence,
lastSequence,
)
}
}
for recordID, seen := range j.SeenRecords {
if err := validateIdentity("SeenRecords.RecordID", recordID, true); err != nil {
return err
}
if seen.Sequence == 0 || seen.Sequence >= j.NextSequence {
return fmt.Errorf("%w: replay sequence for %q is outside journal range", ErrInvalidIdentity, recordID)
}
if !validRecordFingerprint(seen.RecordFingerprint) {
return fmt.Errorf("%w: replay fingerprint for %q is invalid", ErrInvalidIdentity, recordID)
}
if seen.EventFingerprint != "" && !validRecordFingerprint(seen.EventFingerprint) {
return fmt.Errorf("%w: event replay fingerprint for %q is invalid", ErrInvalidIdentity, recordID)
}
}
return nil
}
// AppendRecord validates the record, assigns a monotonic sequence, and
// CAS-appends it to the project journal with bounded retry on revision
// conflict. It returns the assigned sequence number.
func (s *Store) AppendRecord(
ctx context.Context,
record ProjectLogRecord,
) (uint64, error) {
return s.appendRecord(ctx, record, "")
}
func (s *Store) appendRecord(
ctx context.Context,
record ProjectLogRecord,
eventFingerprint string,
) (uint64, error) {
if record.Sequence != 0 {
return 0, fmt.Errorf("%w: append sequence must be unassigned", ErrInvalidIdentity)
}
if record.ProjectID != s.projectID || record.WorkspaceID != s.workspaceID {
return 0, fmt.Errorf("%w: record identity disagrees with store", ErrInvalidIdentity)
}
if s.workUnitID != "" && record.WorkUnitID != s.workUnitID {
return 0, fmt.Errorf("%w: record work unit disagrees with store scope", ErrInvalidIdentity)
}
if eventFingerprint != "" && !validRecordFingerprint(eventFingerprint) {
return 0, fmt.Errorf("%w: event fingerprint is invalid", ErrInvalidIdentity)
}
fingerprint, err := recordFingerprint(record)
if err != nil {
return 0, err
}
for attempt := 0; attempt < maxCASRetries; attempt++ {
if err := ctx.Err(); err != nil {
return 0, err
}
j, revision, _, err := s.loadJournal(ctx)
if err != nil {
return 0, err
}
if seen, ok := j.SeenRecords[record.RecordID]; ok {
if eventFingerprint != "" && seen.EventFingerprint != "" {
if seen.EventFingerprint != eventFingerprint {
return 0, fmt.Errorf(
"%w: manager event id %q",
ErrRecordReplayConflict,
record.RecordID,
)
}
return seen.Sequence, nil
}
if seen.RecordFingerprint != fingerprint {
return 0, fmt.Errorf(
"%w: record id %q",
ErrRecordReplayConflict,
record.RecordID,
)
}
if eventFingerprint != "" {
seen.EventFingerprint = eventFingerprint
j.SeenRecords[record.RecordID] = seen
if err := s.validateJournal(j); err != nil {
return 0, err
}
next, err := json.Marshal(j)
if err != nil {
return 0, fmt.Errorf("projectlog: encode journal: %w", err)
}
if _, err := s.state.CompareAndSwapIntegrationRecord(ctx, s.key, revision, next); err != nil {
if errors.Is(err, agenttask.ErrRevisionConflict) {
continue
}
return 0, err
}
}
return seen.Sequence, nil
}
rec := record
seq := j.NextSequence
rec.Sequence = seq
if err := rec.Validate(); err != nil {
return 0, err
}
j.Records = append(j.Records, rec)
j.SeenRecords[rec.RecordID] = seenRecord{
Sequence: seq,
RecordFingerprint: fingerprint,
EventFingerprint: eventFingerprint,
}
j.NextSequence = seq + 1
if err := s.validateJournal(j); err != nil {
return 0, err
}
next, err := json.Marshal(j)
if err != nil {
return 0, fmt.Errorf("projectlog: encode journal: %w", err)
}
if _, err := s.state.CompareAndSwapIntegrationRecord(ctx, s.key, revision, next); err != nil {
if errors.Is(err, agenttask.ErrRevisionConflict) {
continue
}
return 0, err
}
return seq, nil
}
return 0, fmt.Errorf("%w: after %d retries", ErrCASExhausted, maxCASRetries)
}
func (s *Store) scopeForWorkUnit(workUnitID agenttask.WorkUnitID) (*Store, error) {
if workUnitID != "" {
if s.workUnitID == workUnitID {
return s, nil
}
return s.ForWorkUnit(workUnitID)
}
if s.workUnitID == "" {
return s, nil
}
return &Store{
state: s.state,
root: s.root,
projectID: s.projectID,
workspaceID: s.workspaceID,
key: integrationRecordKey(s.projectID, s.workspaceID, ""),
replayKey: s.replayKey,
failureHook: s.failureHook,
}, nil
}
func (s *Store) loadEventReplayIndex(
ctx context.Context,
) (eventReplayIndex, string, bool, error) {
payload, revision, found, err := s.state.LoadIntegrationRecord(ctx, s.replayKey)
if err != nil {
return eventReplayIndex{}, "", false, err
}
if !found {
return newEventReplayIndex(s.projectID, s.workspaceID), revision, false, nil
}
var index eventReplayIndex
if err := json.Unmarshal(payload, &index); err != nil {
return eventReplayIndex{}, "", false, fmt.Errorf(
"projectlog: decode event replay index: %w",
err,
)
}
normalizeEventReplayIndex(&index)
if err := s.validateEventReplayIndex(index); err != nil {
return eventReplayIndex{}, "", false, err
}
return index, revision, true, nil
}
func (s *Store) validateEventReplayIndex(index eventReplayIndex) error {
if index.SchemaVersion != eventReplayIndexSchemaVersion {
return fmt.Errorf(
"%w: event replay index schema version %d expected %d",
ErrInvalidSchemaVersion,
index.SchemaVersion,
eventReplayIndexSchemaVersion,
)
}
if index.ProjectID != s.projectID || index.WorkspaceID != s.workspaceID {
return fmt.Errorf("%w: event replay index identity disagrees with store", ErrInvalidIdentity)
}
for recordID, entry := range index.Entries {
if err := validateIdentity("EventReplayIndex.RecordID", recordID, true); err != nil {
return err
}
if entry.RecordID != recordID {
return fmt.Errorf("%w: event replay entry key disagrees with record id", ErrInvalidIdentity)
}
if entry.WorkUnitID != "" {
if err := validateIdentity(
"EventReplayIndex.WorkUnitID",
string(entry.WorkUnitID),
false,
); err != nil {
return err
}
}
if entry.Sequence == 0 || entry.Sequence == math.MaxUint64 {
return fmt.Errorf(
"%w: event replay sequence for %q cannot be retained safely",
ErrInvalidIdentity,
recordID,
)
}
if !validRecordFingerprint(entry.EventFingerprint) {
return fmt.Errorf(
"%w: event replay fingerprint for %q is invalid",
ErrInvalidIdentity,
recordID,
)
}
}
return nil
}
func (s *Store) loadOrRecoverEventReplayIndex(
ctx context.Context,
recordID string,
) (eventReplayIndex, string, error) {
for attempt := 0; attempt < maxCASRetries; attempt++ {
index, revision, _, err := s.loadEventReplayIndex(ctx)
if err != nil {
return eventReplayIndex{}, "", err
}
if _, retained := index.Entries[recordID]; retained {
return index, revision, nil
}
recovered, err := s.legacyEventReplayEntries(ctx)
if err != nil {
return eventReplayIndex{}, "", err
}
changed := false
for recoveredID, entry := range recovered {
if retained, ok := index.Entries[recoveredID]; ok {
if retained != entry {
return eventReplayIndex{}, "", eventReplayConflict(recoveredID)
}
continue
}
index.Entries[recoveredID] = entry
changed = true
}
if !changed {
return index, revision, nil
}
if err := s.validateEventReplayIndex(index); err != nil {
return eventReplayIndex{}, "", err
}
payload, err := json.Marshal(index)
if err != nil {
return eventReplayIndex{}, "", fmt.Errorf(
"projectlog: encode recovered event replay index: %w",
err,
)
}
committed, err := s.state.CompareAndSwapIntegrationRecord(
ctx,
s.replayKey,
revision,
payload,
)
if errors.Is(err, agenttask.ErrRevisionConflict) {
continue
}
if err != nil {
return eventReplayIndex{}, "", err
}
return index, committed, nil
}
return eventReplayIndex{}, "", fmt.Errorf(
"%w: event replay recovery after %d retries",
ErrCASExhausted,
maxCASRetries,
)
}
func (s *Store) legacyEventReplayEntries(
ctx context.Context,
) (map[string]eventReplayEntry, error) {
snapshots, err := s.state.LoadIntegrationRecords(ctx, journalIntegrationPrefix)
if err != nil {
return nil, err
}
recovered := make(map[string]eventReplayEntry)
for key, snapshot := range snapshots {
var candidate journal
if err := json.Unmarshal(snapshot.Payload, &candidate); err != nil {
return nil, fmt.Errorf("projectlog: decode legacy journal %q: %w", key, err)
}
if candidate.ProjectID != s.projectID || candidate.WorkspaceID != s.workspaceID {
continue
}
scope, err := s.scopeForWorkUnit(candidate.WorkUnitID)
if err != nil {
return nil, err
}
if key != scope.key {
return nil, fmt.Errorf(
"%w: legacy journal key disagrees with retained scope",
ErrInvalidIdentity,
)
}
normalizeJournal(&candidate)
if err := scope.validateJournal(candidate); err != nil {
return nil, err
}
for recordID, seen := range candidate.SeenRecords {
if seen.EventFingerprint == "" {
continue
}
entry := eventReplayEntry{
RecordID: recordID,
WorkUnitID: candidate.WorkUnitID,
Sequence: seen.Sequence,
EventFingerprint: seen.EventFingerprint,
}
if retained, ok := recovered[recordID]; ok && retained != entry {
return nil, eventReplayConflict(recordID)
}
recovered[recordID] = entry
}
}
return recovered, nil
}
func eventReplayConflict(recordID string) error {
return fmt.Errorf(
"%w: manager event id %q",
ErrRecordReplayConflict,
recordID,
)
}
// AppendEventRecord routes work events to their deterministic task scope and
// keeps project-only events in the project journal.
func (s *Store) AppendEventRecord(
ctx context.Context,
record ProjectLogRecord,
eventFingerprint string,
) (uint64, error) {
if record.Sequence != 0 {
return 0, fmt.Errorf("%w: append sequence must be unassigned", ErrInvalidIdentity)
}
if record.ProjectID != s.projectID || record.WorkspaceID != s.workspaceID {
return 0, fmt.Errorf("%w: record identity disagrees with store", ErrInvalidIdentity)
}
if !validRecordFingerprint(eventFingerprint) {
return 0, fmt.Errorf("%w: event fingerprint is invalid", ErrInvalidIdentity)
}
scope, err := s.scopeForWorkUnit(record.WorkUnitID)
if err != nil {
return 0, err
}
recordFingerprint, err := recordFingerprint(record)
if err != nil {
return 0, err
}
for attempt := 0; attempt < maxCASRetries; attempt++ {
if err := ctx.Err(); err != nil {
return 0, err
}
index, indexRevision, err := s.loadOrRecoverEventReplayIndex(ctx, record.RecordID)
if err != nil {
return 0, err
}
if retained, ok := index.Entries[record.RecordID]; ok {
if retained.WorkUnitID != record.WorkUnitID ||
retained.EventFingerprint != eventFingerprint {
return 0, eventReplayConflict(record.RecordID)
}
return retained.Sequence, nil
}
j, journalRevision, _, err := scope.loadJournal(ctx)
if err != nil {
return 0, err
}
sequence := j.NextSequence
if seen, ok := j.SeenRecords[record.RecordID]; ok {
if seen.EventFingerprint != "" &&
seen.EventFingerprint != eventFingerprint {
return 0, eventReplayConflict(record.RecordID)
}
if seen.RecordFingerprint != recordFingerprint {
return 0, fmt.Errorf(
"%w: record id %q",
ErrRecordReplayConflict,
record.RecordID,
)
}
sequence = seen.Sequence
seen.EventFingerprint = eventFingerprint
j.SeenRecords[record.RecordID] = seen
} else {
rec := record
rec.Sequence = sequence
if err := rec.Validate(); err != nil {
return 0, err
}
j.Records = append(j.Records, rec)
j.SeenRecords[rec.RecordID] = seenRecord{
Sequence: sequence,
RecordFingerprint: recordFingerprint,
EventFingerprint: eventFingerprint,
}
j.NextSequence = sequence + 1
}
index.Entries[record.RecordID] = eventReplayEntry{
RecordID: record.RecordID,
WorkUnitID: record.WorkUnitID,
Sequence: sequence,
EventFingerprint: eventFingerprint,
}
if err := s.validateEventReplayIndex(index); err != nil {
return 0, err
}
if err := scope.validateJournal(j); err != nil {
return 0, err
}
indexPayload, err := json.Marshal(index)
if err != nil {
return 0, fmt.Errorf("projectlog: encode event replay index: %w", err)
}
journalPayload, err := json.Marshal(j)
if err != nil {
return 0, fmt.Errorf("projectlog: encode journal: %w", err)
}
_, err = s.state.CompareAndSwapIntegrationRecords(
ctx,
[]agentstate.IntegrationRecordUpdate{
{
Key: s.replayKey,
Expected: indexRevision,
Payload: indexPayload,
},
{
Key: scope.key,
Expected: journalRevision,
Payload: journalPayload,
},
},
)
if errors.Is(err, agenttask.ErrRevisionConflict) {
continue
}
if err != nil {
return 0, err
}
return sequence, nil
}
return 0, fmt.Errorf("%w: after %d retries", ErrCASExhausted, maxCASRetries)
}
// CheckEventReplay checks the retained stable manager-event index before
// volatile manager evidence is resolved. Legacy and generic entries without
// an event fingerprint require normal evidence resolution and record replay.
func (s *Store) CheckEventReplay(
ctx context.Context,
workUnitID agenttask.WorkUnitID,
recordID string,
eventFingerprint string,
) (bool, error) {
if err := validateIdentity("RecordID", recordID, true); err != nil {
return false, err
}
if !validRecordFingerprint(eventFingerprint) {
return false, fmt.Errorf("%w: event fingerprint is invalid", ErrInvalidIdentity)
}
if _, err := s.scopeForWorkUnit(workUnitID); err != nil {
return false, err
}
index, _, err := s.loadOrRecoverEventReplayIndex(ctx, recordID)
if err != nil {
return false, err
}
retained, ok := index.Entries[recordID]
if !ok {
return false, nil
}
if retained.WorkUnitID != workUnitID ||
retained.EventFingerprint != eventFingerprint {
return false, eventReplayConflict(recordID)
}
return true, nil
}
// ReplayRecords returns all un-archived records from the journal in sequence
// order. Returns nil (not an error) when the journal has not been created.
func (s *Store) ReplayRecords(ctx context.Context) ([]ProjectLogRecord, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
j, _, found, err := s.loadJournal(ctx)
if err != nil {
return nil, err
}
if !found {
return nil, nil
}
return append([]ProjectLogRecord(nil), j.Records...), nil
}
// Archive materializes a terminal archive from the journal. It writes the
// archive intent first (crash-recovery point), then redacted JSONL and
// timeline files via temp-file sync/rename, then the manifest, and finally
// prunes the journal. On any failure the journal is left intact and Reconcile
// converges on restart. Calling Archive on an empty journal is a no-op,
// providing exactly-once archive semantics.
func (s *Store) Archive(ctx context.Context) error {
if err := ctx.Err(); err != nil {
return err
}
if err := s.failAt("archive_before_lock"); err != nil {
return err
}
return s.withArchiveLock(ctx, func() error {
return s.archiveLocked(ctx)
})
}
func (s *Store) archiveLocked(ctx context.Context) error {
j, _, found, err := s.loadJournal(ctx)
if err != nil {
return err
}
if !found || len(j.Records) == 0 {
return nil
}
ordinal := j.ArchiveOrdinal + 1
artifactsExist, err := s.archiveArtifactsExist(ordinal)
if err != nil {
return err
}
if artifactsExist {
verifiedIntent, verifiedRecords, err := s.verifyArchiveArtifacts(ordinal)
if err != nil {
return err
}
return s.reconcilePrune(ctx, verifiedIntent, verifiedRecords)
}
if !j.Records[len(j.Records)-1].Terminal {
return fmt.Errorf("%w: last record is not terminal", ErrNoTerminalRecord)
}
records := j.Records
now := time.Now().UTC()
manifest, err := NewArchiveManifest(s.projectID, s.workspaceID, ordinal, records, now, true)
if err != nil {
return err
}
intent := archiveIntent{
SchemaVersion: journalSchemaVersion,
ProjectID: s.projectID,
WorkspaceID: s.workspaceID,
WorkUnitID: s.workUnitID,
ArchiveOrdinal: ordinal,
FirstSequence: manifest.FirstSequence,
LastSequence: manifest.LastSequence,
RecordCount: manifest.RecordCount,
Checksum: manifest.Checksum,
ArchivedAt: now,
Terminal: true,
}
archiveDir := s.archiveDir()
if err := os.MkdirAll(archiveDir, 0o700); err != nil {
return err
}
intentPath := s.intentPath(ordinal)
jsonlPath := s.jsonlPath(ordinal)
timelinePath := s.timelinePath(ordinal)
manifestPath := s.manifestPath(ordinal)
// 1. Persist archive intent (crash-recovery anchor).
intentData, err := json.Marshal(intent)
if err != nil {
return err
}
if err := writeAtomically(intentPath, append(intentData, '\n')); err != nil {
return err
}
if err := s.failAt("archive_before_write"); err != nil {
return err
}
// 2. Materialize redacted JSONL and timeline via temp-file sync/rename.
jsonlData, err := encodeJSONL(records)
if err != nil {
return err
}
if err := writeAtomically(jsonlPath, jsonlData); err != nil {
return err
}
timelineData, err := encodeTimeline(records)
if err != nil {
return err
}
if err := writeAtomically(timelinePath, timelineData); err != nil {
return err
}
if err := s.failAt("archive_after_rename"); err != nil {
return err
}
// 3. Write manifest to a temp file, then commit via rename.
manifestData, err := json.Marshal(manifest)
if err != nil {
return err
}
manifestTemp, err := writeTemp(archiveDir, "manifest", append(manifestData, '\n'))
if err != nil {
return err
}
if err := s.failAt("archive_before_manifest"); err != nil {
return err
}
if err := os.Rename(manifestTemp, manifestPath); err != nil {
return err
}
if err := syncDir(archiveDir); err != nil {
return err
}
if err := s.failAt("archive_before_cleanup"); err != nil {
return err
}
// 4. Verify the complete committed artifact set, then prune only the
// records proven by that immutable set.
verifiedIntent, verifiedRecords, err := s.verifyArchiveArtifacts(ordinal)
if err != nil {
return err
}
return s.reconcilePrune(ctx, verifiedIntent, verifiedRecords)
}
// Reconcile scans the archive directory for intent files and converges any
// incomplete archive idempotently. If the manifest is missing it is
// recomputed from the JSONL and verified against the intent checksum.
// Conflicting content fails closed with ErrArchiveConflict. After a manifest
// is verified or reconstructed, the journal is pruned if it still holds the
// archived records.
func (s *Store) Reconcile(ctx context.Context) error {
if err := ctx.Err(); err != nil {
return err
}
return s.withArchiveLock(ctx, func() error {
return s.reconcileLocked(ctx)
})
}
func (s *Store) reconcileLocked(ctx context.Context) error {
archiveDir := s.archiveDir()
entries, err := os.ReadDir(archiveDir)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
foundIntent := false
for _, entry := range entries {
if entry.IsDir() {
continue
}
name := entry.Name()
if !strings.HasSuffix(name, ".intent") {
continue
}
foundIntent = true
ordinal := parseOrdinal(name)
if ordinal == 0 {
continue
}
if err := s.reconcileArchive(ctx, ordinal); err != nil {
return err
}
}
if !foundIntent {
j, _, found, err := s.loadJournal(ctx)
if err != nil {
return err
}
if found && len(j.Records) > 0 && j.Records[len(j.Records)-1].Terminal {
return s.archiveLocked(ctx)
}
}
return nil
}
func (s *Store) reconcileArchive(ctx context.Context, ordinal uint64) error {
intent, records, err := s.verifyArchiveArtifacts(ordinal)
if err != nil {
if !errors.Is(err, ErrArchiveIncomplete) {
return err
}
if restoreErr := s.restoreMissingJSONL(ctx, ordinal); restoreErr != nil {
return restoreErr
}
intent, records, err = s.verifyArchiveArtifacts(ordinal)
if err != nil {
return err
}
}
return s.reconcilePrune(ctx, intent, records)
}
func (s *Store) restoreMissingJSONL(ctx context.Context, ordinal uint64) error {
if _, err := os.Stat(s.jsonlPath(ordinal)); err == nil {
return nil
} else if !os.IsNotExist(err) {
return err
}
intentData, err := os.ReadFile(s.intentPath(ordinal))
if err != nil {
return fmt.Errorf("%w: read intent for recovery: %v", ErrArchiveIncomplete, err)
}
var intent archiveIntent
if err := json.Unmarshal(intentData, &intent); err != nil {
return fmt.Errorf("%w: decode intent for recovery: %v", ErrArchiveConflict, err)
}
if intent.SchemaVersion != journalSchemaVersion ||
intent.ProjectID != s.projectID ||
intent.WorkspaceID != s.workspaceID ||
intent.WorkUnitID != s.workUnitID ||
intent.ArchiveOrdinal != ordinal ||
intent.RecordCount <= 0 {
return fmt.Errorf("%w: recovery intent identity is invalid", ErrArchiveConflict)
}
j, _, found, err := s.loadJournal(ctx)
if err != nil {
return err
}
if !found || len(j.Records) < intent.RecordCount {
return fmt.Errorf("%w: journal cannot reconstruct ordinal %d", ErrArchiveIncomplete, ordinal)
}
records := append([]ProjectLogRecord(nil), j.Records[:intent.RecordCount]...)
manifest, err := NewArchiveManifest(
s.projectID,
s.workspaceID,
ordinal,
records,
intent.ArchivedAt,
true,
)
if err != nil {
return fmt.Errorf("%w: reconstruct manifest: %v", ErrArchiveConflict, err)
}
if manifest.FirstSequence != intent.FirstSequence ||
manifest.LastSequence != intent.LastSequence ||
manifest.RecordCount != intent.RecordCount ||
manifest.Checksum != intent.Checksum {
return fmt.Errorf("%w: journal does not match recovery intent", ErrArchiveConflict)
}
jsonlData, err := encodeJSONL(records)
if err != nil {
return err
}
return writeAtomically(s.jsonlPath(ordinal), jsonlData)
}
func (s *Store) archiveArtifactsExist(ordinal uint64) (bool, error) {
for _, path := range []string{
s.intentPath(ordinal),
s.jsonlPath(ordinal),
s.timelinePath(ordinal),
s.manifestPath(ordinal),
} {
if _, err := os.Stat(path); err == nil {
return true, nil
} else if !os.IsNotExist(err) {
return false, err
}
}
return false, nil
}
func (s *Store) verifyArchiveArtifacts(ordinal uint64) (archiveIntent, []ProjectLogRecord, error) {
intentPath := s.intentPath(ordinal)
manifestPath := s.manifestPath(ordinal)
jsonlPath := s.jsonlPath(ordinal)
timelinePath := s.timelinePath(ordinal)
intentData, err := os.ReadFile(intentPath)
if err != nil {
if os.IsNotExist(err) {
return archiveIntent{}, nil, fmt.Errorf("%w: missing intent for ordinal %d", ErrArchiveIncomplete, ordinal)
}
return archiveIntent{}, nil, fmt.Errorf("projectlog: read intent: %w", err)
}
var intent archiveIntent
if err := json.Unmarshal(intentData, &intent); err != nil {
return archiveIntent{}, nil, fmt.Errorf("%w: decode intent: %v", ErrArchiveConflict, err)
}
if intent.SchemaVersion != journalSchemaVersion {
return archiveIntent{}, nil, fmt.Errorf("%w: intent schema version %d", ErrArchiveConflict, intent.SchemaVersion)
}
if intent.ProjectID != s.projectID ||
intent.WorkspaceID != s.workspaceID ||
intent.WorkUnitID != s.workUnitID {
return archiveIntent{}, nil, fmt.Errorf("%w: intent identity disagrees with store", ErrArchiveConflict)
}
if intent.ArchiveOrdinal != ordinal {
return archiveIntent{}, nil, fmt.Errorf("%w: intent ordinal %d != filename %d", ErrArchiveConflict, intent.ArchiveOrdinal, ordinal)
}
if intent.RecordCount <= 0 || intent.FirstSequence == 0 || intent.LastSequence < intent.FirstSequence {
return archiveIntent{}, nil, fmt.Errorf("%w: invalid intent sequence range or count", ErrArchiveConflict)
}
if !validArchiveChecksum(intent.Checksum) {
return archiveIntent{}, nil, fmt.Errorf("%w: invalid intent checksum format", ErrArchiveConflict)
}
if intent.ArchivedAt.IsZero() {
return archiveIntent{}, nil, fmt.Errorf("%w: zero intent timestamp", ErrArchiveConflict)
}
if !intent.Terminal {
return archiveIntent{}, nil, fmt.Errorf("%w: intent is not terminal", ErrArchiveConflict)
}
jsonlData, err := os.ReadFile(jsonlPath)
if err != nil {
if os.IsNotExist(err) {
return archiveIntent{}, nil, fmt.Errorf("%w: missing jsonl for ordinal %d", ErrArchiveIncomplete, ordinal)
}
return archiveIntent{}, nil, err
}
if computeJSONLChecksum(jsonlData) != intent.Checksum {
return archiveIntent{}, nil, fmt.Errorf("%w: jsonl checksum mismatch for ordinal %d", ErrArchiveConflict, ordinal)
}
records, err := decodeJSONL(jsonlData)
if err != nil {
return archiveIntent{}, nil, fmt.Errorf("%w: decode jsonl: %v", ErrArchiveConflict, err)
}
if len(records) != intent.RecordCount {
return archiveIntent{}, nil, fmt.Errorf("%w: jsonl record count %d != intent count %d", ErrArchiveConflict, len(records), intent.RecordCount)
}
if records[0].Sequence != intent.FirstSequence || records[len(records)-1].Sequence != intent.LastSequence {
return archiveIntent{}, nil, fmt.Errorf("%w: jsonl sequence range disagrees with intent", ErrArchiveConflict)
}
expectedManifest, err := NewArchiveManifest(s.projectID, s.workspaceID, ordinal, records, intent.ArchivedAt, true)
if err != nil {
return archiveIntent{}, nil, fmt.Errorf("%w: derive manifest: %w", ErrArchiveConflict, err)
}
if expectedManifest.Checksum != intent.Checksum {
return archiveIntent{}, nil, fmt.Errorf("%w: derived checksum %s != intent %s", ErrArchiveConflict, expectedManifest.Checksum, intent.Checksum)
}
if s.workUnitID != "" &&
(len(expectedManifest.WorkUnitIDs) != 1 || expectedManifest.WorkUnitIDs[0] != s.workUnitID) {
return archiveIntent{}, nil, fmt.Errorf(
"%w: manifest work unit disagrees with store scope",
ErrArchiveConflict,
)
}
manifestBytes, err := os.ReadFile(manifestPath)
if err != nil {
if os.IsNotExist(err) {
mBytes, marshalErr := json.Marshal(expectedManifest)
if marshalErr != nil {
return archiveIntent{}, nil, marshalErr
}
if err := writeAtomically(manifestPath, append(mBytes, '\n')); err != nil {
return archiveIntent{}, nil, err
}
} else {
return archiveIntent{}, nil, err
}
} else {
var existingManifest ArchiveManifest
if err := json.Unmarshal(manifestBytes, &existingManifest); err != nil {
return archiveIntent{}, nil, fmt.Errorf("%w: decode manifest: %w", ErrArchiveConflict, err)
}
if err := existingManifest.Validate(); err != nil {
return archiveIntent{}, nil, fmt.Errorf("%w: invalid manifest: %w", ErrArchiveConflict, err)
}
if !manifestsEqual(existingManifest, expectedManifest) {
return archiveIntent{}, nil, fmt.Errorf("%w: manifest field drift for ordinal %d", ErrArchiveConflict, ordinal)
}
}
expectedTimelineData, err := encodeTimeline(records)
if err != nil {
return archiveIntent{}, nil, fmt.Errorf("projectlog: encode timeline: %w", err)
}
timelineData, err := os.ReadFile(timelinePath)
if err != nil {
if os.IsNotExist(err) {
if err := writeAtomically(timelinePath, expectedTimelineData); err != nil {
return archiveIntent{}, nil, err
}
} else {
return archiveIntent{}, nil, err
}
} else {
if !bytes.Equal(bytes.TrimSpace(timelineData), bytes.TrimSpace(expectedTimelineData)) {
return archiveIntent{}, nil, fmt.Errorf("%w: timeline content mismatch for ordinal %d", ErrArchiveConflict, ordinal)
}
}
return intent, records, nil
}
func manifestsEqual(m1, m2 ArchiveManifest) bool {
if m1.SchemaVersion != m2.SchemaVersion ||
m1.ProjectID != m2.ProjectID ||
m1.WorkspaceID != m2.WorkspaceID ||
m1.ArchiveOrdinal != m2.ArchiveOrdinal ||
m1.RecordCount != m2.RecordCount ||
m1.FirstSequence != m2.FirstSequence ||
m1.LastSequence != m2.LastSequence ||
m1.Checksum != m2.Checksum ||
!m1.ArchivedAt.Equal(m2.ArchivedAt) ||
m1.Terminal != m2.Terminal ||
len(m1.WorkUnitIDs) != len(m2.WorkUnitIDs) {
return false
}
for i := range m1.WorkUnitIDs {
if m1.WorkUnitIDs[i] != m2.WorkUnitIDs[i] {
return false
}
}
return true
}
func recordsMatchPrefix(jRecords, archived []ProjectLogRecord) bool {
if len(jRecords) < len(archived) {
return false
}
for i := range archived {
if !recordsEqual(jRecords[i], archived[i]) {
return false
}
}
return true
}
func recordsEqual(a, b ProjectLogRecord) bool {
if a.Sequence != b.Sequence || a.RecordID != b.RecordID || a.EventType != b.EventType || a.Terminal != b.Terminal || a.State != b.State {
return false
}
aBytes, errA := json.Marshal(a)
bBytes, errB := json.Marshal(b)
if errA != nil || errB != nil {
return false
}
return bytes.Equal(aBytes, bBytes)
}
func (s *Store) reconcilePrune(
ctx context.Context,
intent archiveIntent,
archivedRecords []ProjectLogRecord,
) error {
for attempt := 0; attempt < maxCASRetries; attempt++ {
if err := ctx.Err(); err != nil {
return err
}
j, revision, found, err := s.loadJournal(ctx)
if err != nil {
return err
}
if !found {
return nil
}
if len(j.Records) == 0 {
if j.ArchiveOrdinal < intent.ArchiveOrdinal {
j.ArchiveOrdinal = intent.ArchiveOrdinal
next, err := json.Marshal(j)
if err != nil {
return err
}
if _, err := s.state.CompareAndSwapIntegrationRecord(ctx, s.key, revision, next); err != nil {
if errors.Is(err, agenttask.ErrRevisionConflict) {
continue
}
return err
}
}
return nil
}
if j.Records[0].Sequence > intent.LastSequence {
if j.ArchiveOrdinal < intent.ArchiveOrdinal {
j.ArchiveOrdinal = intent.ArchiveOrdinal
next, err := json.Marshal(j)
if err != nil {
return err
}
if _, err := s.state.CompareAndSwapIntegrationRecord(ctx, s.key, revision, next); err != nil {
if errors.Is(err, agenttask.ErrRevisionConflict) {
continue
}
return err
}
}
return nil
}
if !recordsMatchPrefix(j.Records, archivedRecords) {
return fmt.Errorf("%w: journal prefix does not match archived records", ErrArchiveConflict)
}
j.Records = append([]ProjectLogRecord(nil), j.Records[len(archivedRecords):]...)
if j.ArchiveOrdinal < intent.ArchiveOrdinal {
j.ArchiveOrdinal = intent.ArchiveOrdinal
}
next, err := json.Marshal(j)
if err != nil {
return err
}
if _, err := s.state.CompareAndSwapIntegrationRecord(ctx, s.key, revision, next); err != nil {
if errors.Is(err, agenttask.ErrRevisionConflict) {
continue
}
return err
}
return nil
}
return fmt.Errorf("%w: reconcile prune after %d retries", ErrCASExhausted, maxCASRetries)
}
func integrationRecordKey(
projectID agenttask.ProjectID,
workspaceID agenttask.WorkspaceID,
workUnitID agenttask.WorkUnitID,
) string {
h := sha256.New()
writeLengthPrefixed(h, string(projectID))
writeLengthPrefixed(h, string(workspaceID))
if workUnitID != "" {
writeLengthPrefixed(h, string(workUnitID))
}
return journalIntegrationPrefix + hex.EncodeToString(h.Sum(nil))
}
func eventReplayIndexKey(
projectID agenttask.ProjectID,
workspaceID agenttask.WorkspaceID,
) string {
h := sha256.New()
writeLengthPrefixed(h, "event-replay-index")
writeLengthPrefixed(h, string(projectID))
writeLengthPrefixed(h, string(workspaceID))
return eventReplayIntegrationPrefix + hex.EncodeToString(h.Sum(nil))
}
func newJournal(
projectID agenttask.ProjectID,
workspaceID agenttask.WorkspaceID,
workUnitID agenttask.WorkUnitID,
) journal {
return journal{
SchemaVersion: journalSchemaVersion,
ProjectID: projectID,
WorkspaceID: workspaceID,
WorkUnitID: workUnitID,
NextSequence: 1,
ArchiveOrdinal: 0,
SeenRecords: make(map[string]seenRecord),
Records: nil,
}
}
func normalizeJournal(j *journal) {
if j.SeenRecords == nil {
j.SeenRecords = make(map[string]seenRecord, len(j.Records))
}
for recordID, seen := range j.SeenRecords {
if seen.RecordFingerprint == "" {
seen.RecordFingerprint = seen.Fingerprint
}
seen.Fingerprint = ""
j.SeenRecords[recordID] = seen
}
for _, record := range j.Records {
if _, exists := j.SeenRecords[record.RecordID]; exists {
continue
}
fingerprint, err := recordFingerprint(record)
if err != nil {
continue
}
j.SeenRecords[record.RecordID] = seenRecord{
Sequence: record.Sequence,
RecordFingerprint: fingerprint,
}
}
}
func newEventReplayIndex(
projectID agenttask.ProjectID,
workspaceID agenttask.WorkspaceID,
) eventReplayIndex {
return eventReplayIndex{
SchemaVersion: eventReplayIndexSchemaVersion,
ProjectID: projectID,
WorkspaceID: workspaceID,
Entries: make(map[string]eventReplayEntry),
}
}
func normalizeEventReplayIndex(index *eventReplayIndex) {
if index.Entries == nil {
index.Entries = make(map[string]eventReplayEntry)
}
}
func (s *Store) archiveDir() string {
h := sha256.New()
writeLengthPrefixed(h, string(s.projectID))
writeLengthPrefixed(h, string(s.workspaceID))
if s.workUnitID != "" {
writeLengthPrefixed(h, string(s.workUnitID))
}
return filepath.Join(s.root, hex.EncodeToString(h.Sum(nil)), "archive")
}
func writeLengthPrefixed(h interface{ Write([]byte) (int, error) }, value string) {
var length [4]byte
binary.BigEndian.PutUint32(length[:], uint32(len(value)))
_, _ = h.Write(length[:])
_, _ = h.Write([]byte(value))
}
func (s *Store) withArchiveLock(ctx context.Context, action func() error) error {
archiveDir := s.archiveDir()
if err := os.MkdirAll(archiveDir, 0o700); err != nil {
return err
}
lock, err := os.OpenFile(filepath.Join(archiveDir, ".archive.lock"), os.O_CREATE|os.O_RDWR, 0o600)
if err != nil {
return err
}
defer lock.Close()
if err := ctx.Err(); err != nil {
return err
}
if err := syscall.Flock(int(lock.Fd()), syscall.LOCK_EX); err != nil {
return err
}
defer syscall.Flock(int(lock.Fd()), syscall.LOCK_UN) //nolint:errcheck
if err := ctx.Err(); err != nil {
return err
}
return action()
}
func (s *Store) intentPath(ordinal uint64) string {
return filepath.Join(s.archiveDir(), fmt.Sprintf("%06d.intent", ordinal))
}
func (s *Store) jsonlPath(ordinal uint64) string {
return filepath.Join(s.archiveDir(), fmt.Sprintf("%06d.jsonl", ordinal))
}
func (s *Store) timelinePath(ordinal uint64) string {
return filepath.Join(s.archiveDir(), fmt.Sprintf("%06d.timeline.jsonl", ordinal))
}
func (s *Store) manifestPath(ordinal uint64) string {
return filepath.Join(s.archiveDir(), fmt.Sprintf("%06d.manifest.json", ordinal))
}
func parseOrdinal(name string) uint64 {
base := strings.TrimSuffix(name, ".intent")
ordinal, err := strconv.ParseUint(base, 10, 64)
if err != nil {
return 0
}
return ordinal
}
func writeTemp(dir, base string, data []byte) (string, error) {
temp, err := os.CreateTemp(dir, "."+base+".tmp-*")
if err != nil {
return "", err
}
tempPath := temp.Name()
removeTemp := true
defer func() {
if removeTemp {
_ = os.Remove(tempPath)
}
}()
if err := temp.Chmod(0o600); err != nil {
_ = temp.Close()
return "", err
}
if _, err := temp.Write(data); err != nil {
_ = temp.Close()
return "", err
}
if err := temp.Sync(); err != nil {
_ = temp.Close()
return "", err
}
if err := temp.Close(); err != nil {
return "", err
}
removeTemp = false
return tempPath, nil
}
func writeAtomically(path string, data []byte) error {
tempPath, err := writeTemp(filepath.Dir(path), filepath.Base(path), data)
if err != nil {
return err
}
if err := os.Rename(tempPath, path); err != nil {
_ = os.Remove(tempPath)
return err
}
return syncDir(filepath.Dir(path))
}
func syncDir(path string) error {
dirHandle, err := os.Open(path)
if err != nil {
return err
}
defer dirHandle.Close()
return dirHandle.Sync()
}
func encodeJSONL(records []ProjectLogRecord) ([]byte, error) {
var buf bytes.Buffer
for _, rec := range records {
data, err := json.Marshal(rec)
if err != nil {
return nil, err
}
buf.Write(data)
buf.WriteByte('\n')
}
return buf.Bytes(), nil
}
func encodeTimeline(records []ProjectLogRecord) ([]byte, error) {
return encodeWorkLogEntries(ProjectWorkLog(records).Entries)
}
func encodeWorkLogEntries(entries []WorkLogEntry) ([]byte, error) {
var buf bytes.Buffer
for _, entry := range entries {
data, err := json.Marshal(entry)
if err != nil {
return nil, err
}
buf.Write(data)
buf.WriteByte('\n')
}
return buf.Bytes(), nil
}
func recordFingerprint(record ProjectLogRecord) (string, error) {
record.Sequence = 0
payload, err := json.Marshal(record)
if err != nil {
return "", fmt.Errorf("projectlog: encode record fingerprint: %w", err)
}
sum := sha256.Sum256(payload)
return "sha256:" + hex.EncodeToString(sum[:]), nil
}
func validRecordFingerprint(fingerprint string) bool {
return validArchiveChecksum(fingerprint)
}
func decodeJSONL(data []byte) ([]ProjectLogRecord, error) {
var records []ProjectLogRecord
for len(data) > 0 {
idx := bytes.IndexByte(data, '\n')
var line []byte
if idx == -1 {
line = data
data = nil
} else {
line = data[:idx]
data = data[idx+1:]
}
line = bytes.TrimSpace(line)
if len(line) == 0 {
continue
}
var rec ProjectLogRecord
if err := json.Unmarshal(line, &rec); err != nil {
return nil, err
}
records = append(records, rec)
}
return records, nil
}
func computeJSONLChecksum(data []byte) string {
sum := sha256.Sum256(data)
return "sha256:" + hex.EncodeToString(sum[:])
}