2035 lines
92 KiB
Python
2035 lines
92 KiB
Python
"""Durable, append-only benchmark run and attempt state.
|
|
|
|
Attempt-directory creation is the allocation marker. The directory remains
|
|
empty until the workspace layer has prepared it; only then is the identity
|
|
bound running record published. This preserves the workspace API's
|
|
empty-root contract while retaining a crash-recoverable allocation boundary.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import contextlib
|
|
import datetime as _datetime
|
|
import fcntl
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import secrets
|
|
import stat
|
|
import tempfile
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
from typing import Any, Callable, Iterator, Mapping, Protocol
|
|
|
|
from scripts.agent_benchmark.connectivity import (
|
|
CallerCapability,
|
|
ConnectivityIssue,
|
|
ConnectivityResult,
|
|
EffectiveBinding,
|
|
RequestedEffectiveBinding,
|
|
canonical_evidence_bytes,
|
|
validate_requested_binding,
|
|
validate_result,
|
|
)
|
|
|
|
from scripts.agent_benchmark.lifecycle import (
|
|
COMPLETION_MODES,
|
|
EVENT_FINISH,
|
|
EVENT_IDLE,
|
|
EVENT_QUIET,
|
|
EVENT_SUBMITTED,
|
|
InvocationResult,
|
|
LifecycleRecoveryError,
|
|
RECEIPT_VERSION,
|
|
REASON_CONTROLLER_LOST,
|
|
REASON_RECOVERED_STOP,
|
|
SOCKET_FILENAME,
|
|
SUBMISSION_MODES,
|
|
SupervisorLocator,
|
|
TERMINAL_REASONS,
|
|
recover_invocation,
|
|
)
|
|
from scripts.agent_benchmark.manifest import (
|
|
Manifest,
|
|
MatrixCell,
|
|
Timeout,
|
|
validate_manifest_bytes,
|
|
)
|
|
from scripts.agent_benchmark.measurement import (
|
|
MEASUREMENT_FILENAME,
|
|
MeasurementError,
|
|
WorkspaceWriteObservation,
|
|
WorkspaceWriteObserver,
|
|
build_measurement,
|
|
load_measurement,
|
|
publish_measurement,
|
|
validate_measurement_lifecycle_binding,
|
|
)
|
|
from scripts.agent_benchmark.web_validation import (
|
|
WEB_VALIDATION_FILENAME,
|
|
WebValidationError,
|
|
load_web_validation,
|
|
publish_web_validation,
|
|
validate_web_attempt,
|
|
)
|
|
from scripts.agent_benchmark.workspace import AttemptIdentity, PreparedWorkspace
|
|
|
|
RUN_ID_RE = re.compile(r"^run-[0-9]{8}T[0-9]{6}Z-[0-9a-f]{12}$")
|
|
ATTEMPT_RE = re.compile(r"^attempt-([0-9]{6})$")
|
|
PREFLIGHT_RE = re.compile(r"^preflight-([0-9]{6})\.json$")
|
|
DIGEST_RE = re.compile(r"^sha256:[0-9a-f]{64}$")
|
|
PREFLIGHT_SCHEMA_VERSION = "1"
|
|
PREFLIGHT_STATUSES = ("ready", "registration_required", "implementation_gap")
|
|
TERMINAL_STATES = frozenset(("success", "failed", "timed_out", "cancelled", "interrupted"))
|
|
NONTERMINAL_STATE = "running"
|
|
MEASUREMENT_POLICY_REQUIRED_V1 = "required-v1"
|
|
MEASUREMENT_POLICIES = frozenset((MEASUREMENT_POLICY_REQUIRED_V1,))
|
|
MEASUREMENT_POLICY_FILENAME = "attempt-measurement-policy.json"
|
|
MEASUREMENT_POLICY_RECORD = "attempt-measurement-policy"
|
|
MEASUREMENT_POLICY_VERSION = 1
|
|
WEB_VALIDATION_POLICY_REQUIRED_V1 = "required-v1"
|
|
WEB_VALIDATION_POLICIES = frozenset((WEB_VALIDATION_POLICY_REQUIRED_V1,))
|
|
WEB_VALIDATION_POLICY_FILENAME = "web-validation-policy.json"
|
|
WEB_VALIDATION_POLICY_RECORD = "web-validation-policy"
|
|
WEB_VALIDATION_POLICY_VERSION = 1
|
|
SUCCESS_EVIDENCE_KINDS = (EVENT_SUBMITTED, EVENT_FINISH, EVENT_IDLE, EVENT_QUIET)
|
|
CONTROL_ALIAS_PREFIX = "iop-bench-attempt-"
|
|
CONTROL_DIRECTORY_NAME = "control"
|
|
CONTROL_ALIAS_DIGEST_HEX_LENGTH = 24
|
|
UNIX_SOCKET_PATH_MAX_BYTES = 103
|
|
RECEIPT_FIELDS = {
|
|
"receipt_version", "supervisor_pid", "challenge_digest", "reason", "exit_code",
|
|
"signal", "caller_launched", "cleanup_complete", "process_group_alive", "completed_at",
|
|
}
|
|
RECEIPT_ONLY_TERMINAL_REASONS = frozenset(
|
|
(REASON_CONTROLLER_LOST, REASON_RECOVERED_STOP)
|
|
)
|
|
|
|
|
|
class AttemptError(Exception):
|
|
"""Base error whose message is safe to present to a benchmark caller."""
|
|
|
|
|
|
class RunPathError(AttemptError):
|
|
pass
|
|
|
|
|
|
class RunBusyError(AttemptError):
|
|
pass
|
|
|
|
|
|
class AttemptStateError(AttemptError):
|
|
pass
|
|
|
|
|
|
class CapabilityUnavailable(AttemptError):
|
|
pass
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class RunIdentity:
|
|
run_id: str
|
|
manifest_digest: str
|
|
root: str
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Slot:
|
|
cell_id: str
|
|
repetition: int
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Attempt:
|
|
identity: AttemptIdentity
|
|
root: str
|
|
state: str
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class PreflightObservation:
|
|
"""One adapter result plus opaque identities safe for durable evidence."""
|
|
|
|
result: ConnectivityResult
|
|
endpoint_identity: str
|
|
config_identity: str
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class AttemptControlLease:
|
|
"""One deterministic short pathname bound to an exact attempt root."""
|
|
|
|
alias: str
|
|
control_dir: str
|
|
socket_path: str
|
|
|
|
|
|
class PreflightAdapter(Protocol):
|
|
"""Closed adapter boundary consumed by the public preflight controller."""
|
|
|
|
capability: CallerCapability
|
|
|
|
def preflight(self, cell: "MatrixCell") -> PreflightObservation:
|
|
"""Return one typed observation without exposing raw caller output."""
|
|
|
|
|
|
class ExecutionAdapter(PreflightAdapter, Protocol):
|
|
"""Typed caller boundary for one preflight-approved scored attempt."""
|
|
|
|
def invoke(
|
|
self,
|
|
cell: "MatrixCell",
|
|
prepared: PreparedWorkspace,
|
|
attempt: Attempt,
|
|
control_dir: str,
|
|
task_payload: bytes,
|
|
timeout: Timeout,
|
|
on_started: Callable[[SupervisorLocator, str], None],
|
|
) -> InvocationResult:
|
|
"""Submit the exact task once for the bound cell and prepared identity."""
|
|
|
|
|
|
def _json_bytes(value: Any) -> bytes:
|
|
return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("utf-8") + b"\n"
|
|
|
|
|
|
def _fsync_dir(path: Path) -> None:
|
|
fd = os.open(path, os.O_RDONLY)
|
|
try:
|
|
os.fsync(fd)
|
|
finally:
|
|
os.close(fd)
|
|
|
|
|
|
def _write_new(path: Path, data: bytes) -> None:
|
|
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
|
try:
|
|
os.write(fd, data)
|
|
os.fsync(fd)
|
|
finally:
|
|
os.close(fd)
|
|
_fsync_dir(path.parent)
|
|
|
|
|
|
def _replace(path: Path, data: bytes) -> None:
|
|
fd, tmp = tempfile.mkstemp(prefix=".attempt-", dir=path.parent)
|
|
try:
|
|
with os.fdopen(fd, "wb") as handle:
|
|
handle.write(data)
|
|
handle.flush()
|
|
os.fsync(handle.fileno())
|
|
os.chmod(tmp, 0o600)
|
|
os.replace(tmp, path)
|
|
_fsync_dir(path.parent)
|
|
finally:
|
|
try:
|
|
os.unlink(tmp)
|
|
except FileNotFoundError:
|
|
pass
|
|
|
|
|
|
def _open_regular(path: Path, label: str, flags: int = os.O_RDONLY) -> int:
|
|
"""Open one durable file without following links, blocking or trusting its type."""
|
|
try:
|
|
fd = os.open(path, flags | os.O_NOFOLLOW | os.O_NONBLOCK)
|
|
except OSError as exc:
|
|
raise AttemptStateError(f"{label} is unavailable") from exc
|
|
try:
|
|
opened = os.fstat(fd)
|
|
if not stat.S_ISREG(opened.st_mode):
|
|
raise AttemptStateError(f"{label} must be a regular file")
|
|
except BaseException:
|
|
os.close(fd)
|
|
raise
|
|
return fd
|
|
|
|
|
|
def _read_regular_bytes(path: Path, label: str) -> bytes:
|
|
"""Read only the no-follow descriptor that was verified as regular."""
|
|
fd = _open_regular(path, label)
|
|
try:
|
|
chunks: list[bytes] = []
|
|
while True:
|
|
chunk = os.read(fd, 1 << 20)
|
|
if not chunk:
|
|
return b"".join(chunks)
|
|
chunks.append(chunk)
|
|
finally:
|
|
os.close(fd)
|
|
|
|
|
|
def _utc_timestamp(clock: Callable[[], _datetime.datetime]) -> str:
|
|
return clock().astimezone(_datetime.timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
|
|
|
|
|
def _directory(path: Path, label: str) -> None:
|
|
try:
|
|
mode = os.lstat(path).st_mode
|
|
except OSError as exc:
|
|
raise RunPathError(f"{label} is unavailable") from exc
|
|
if not stat.S_ISDIR(mode):
|
|
raise RunPathError(f"{label} is invalid")
|
|
|
|
|
|
def _contained(path: Path, root: Path) -> bool:
|
|
try:
|
|
path.resolve(strict=False).relative_to(root.resolve())
|
|
return True
|
|
except ValueError:
|
|
return False
|
|
|
|
|
|
def _requested_binding(cell: MatrixCell) -> RequestedEffectiveBinding:
|
|
return RequestedEffectiveBinding(
|
|
cell.id,
|
|
cell.caller,
|
|
cell.iop.route_kind,
|
|
cell.iop.route_id,
|
|
cell.iop.request_model,
|
|
cell.iop.requested_effort,
|
|
)
|
|
|
|
|
|
def _overall_preflight_status(statuses: Iterator[str]) -> str:
|
|
found = tuple(statuses)
|
|
if not found:
|
|
raise AttemptStateError("preflight result set is empty")
|
|
if any(status == "implementation_gap" for status in found):
|
|
return "implementation_gap"
|
|
if any(status == "registration_required" for status in found):
|
|
return "registration_required"
|
|
if all(status == "ready" for status in found):
|
|
return "ready"
|
|
raise AttemptStateError("preflight status is invalid")
|
|
|
|
|
|
def _preflight_counts(results: list[dict[str, Any]]) -> dict[str, int]:
|
|
counts = {status: 0 for status in PREFLIGHT_STATUSES}
|
|
for result in results:
|
|
status = result.get("status")
|
|
if status not in counts:
|
|
raise AttemptStateError("preflight result status is invalid")
|
|
counts[str(status)] += 1
|
|
return counts
|
|
|
|
|
|
def _connectivity_result_from_payload(
|
|
payload: dict[str, Any], cell: MatrixCell
|
|
) -> tuple[ConnectivityResult, str, str]:
|
|
"""Rebuild one closed result so durable reads re-run semantic validation."""
|
|
if not isinstance(payload, dict) or set(payload) != {
|
|
"schema_version", "cell", "status", "binding", "issues",
|
|
"endpoint_identity", "config_identity",
|
|
}:
|
|
raise AttemptStateError("preflight result schema is invalid")
|
|
binding_raw = payload.get("binding")
|
|
if not isinstance(binding_raw, dict) or set(binding_raw) != {
|
|
"cell_id", "caller", "requested_route_kind", "requested_route_id",
|
|
"requested_model", "requested_effort", "effective_route_kind",
|
|
"effective_route_id", "effective_model", "effective_effort",
|
|
"effective_bindings",
|
|
}:
|
|
raise AttemptStateError("preflight result binding is invalid")
|
|
stages_raw = binding_raw.get("effective_bindings")
|
|
if not isinstance(stages_raw, list):
|
|
raise AttemptStateError("preflight result binding is invalid")
|
|
stages: list[EffectiveBinding] = []
|
|
for stage in stages_raw:
|
|
if not isinstance(stage, dict) or set(stage) != {"stage", "model", "effort"}:
|
|
raise AttemptStateError("preflight result binding is invalid")
|
|
stages.append(EffectiveBinding(stage["stage"], stage["model"], stage["effort"]))
|
|
issues_raw = payload.get("issues")
|
|
if not isinstance(issues_raw, list):
|
|
raise AttemptStateError("preflight result issues are invalid")
|
|
issues: list[ConnectivityIssue] = []
|
|
for issue in issues_raw:
|
|
if not isinstance(issue, dict) or set(issue) != {"code", "resume_code"}:
|
|
raise AttemptStateError("preflight result issues are invalid")
|
|
issues.append(ConnectivityIssue(issue["code"], issue["resume_code"]))
|
|
try:
|
|
binding = RequestedEffectiveBinding(
|
|
binding_raw["cell_id"],
|
|
binding_raw["caller"],
|
|
binding_raw["requested_route_kind"],
|
|
binding_raw["requested_route_id"],
|
|
binding_raw["requested_model"],
|
|
binding_raw["requested_effort"],
|
|
binding_raw["effective_route_kind"],
|
|
binding_raw["effective_route_id"],
|
|
binding_raw["effective_model"],
|
|
binding_raw["effective_effort"],
|
|
tuple(stages),
|
|
)
|
|
# Capability is deliberately not serialized. The persisted proof is
|
|
# revalidated against the exact immutable cell and its one direct route.
|
|
capability = CallerCapability(
|
|
cell.caller, (cell.iop.route_kind,), (cell.iop.requested_effort,)
|
|
)
|
|
result = ConnectivityResult(capability, binding, tuple(issues), payload["status"])
|
|
validate_result(cell, result)
|
|
endpoint_identity = payload["endpoint_identity"]
|
|
config_identity = payload["config_identity"]
|
|
canonical = json.loads(
|
|
canonical_evidence_bytes(
|
|
cell, result, endpoint_identity, config_identity
|
|
).decode("ascii")
|
|
)
|
|
except Exception as exc:
|
|
raise AttemptStateError("preflight result is invalid") from exc
|
|
if payload != canonical or payload.get("cell") != {"id": cell.id, "caller": cell.caller}:
|
|
raise AttemptStateError("preflight result is non-canonical")
|
|
return result, endpoint_identity, config_identity
|
|
|
|
|
|
def collect_preflight_observations(
|
|
manifest: Manifest,
|
|
adapters: Mapping[str, PreflightAdapter],
|
|
) -> dict[str, PreflightObservation]:
|
|
"""Validate the full registry, then probe direct cells in manifest order.
|
|
|
|
Execution-preset cells exercise only the local adapter capability contract in
|
|
this milestone. They never become a synthetic live-ready observation.
|
|
"""
|
|
if not isinstance(adapters, Mapping):
|
|
raise CapabilityUnavailable("capability-unavailable: caller-adapter")
|
|
required_callers = {cell.caller for cell in manifest.matrix}
|
|
if any(caller not in adapters for caller in required_callers):
|
|
raise CapabilityUnavailable("capability-unavailable: caller-adapter")
|
|
|
|
for cell in manifest.matrix:
|
|
adapter = adapters[cell.caller]
|
|
capability = getattr(adapter, "capability", None)
|
|
validate_requested_binding(cell, capability, _requested_binding(cell))
|
|
|
|
observations: dict[str, PreflightObservation] = {}
|
|
for cell in manifest.matrix:
|
|
if cell.iop.route_kind != "direct":
|
|
continue
|
|
observation = adapters[cell.caller].preflight(cell)
|
|
if not isinstance(observation, PreflightObservation):
|
|
raise AttemptStateError("preflight observation is invalid")
|
|
validate_result(cell, observation.result)
|
|
# Materializing the canonical bytes validates both opaque identities and
|
|
# proves that no adapter-specific/raw value can enter the durable record.
|
|
canonical_evidence_bytes(
|
|
cell,
|
|
observation.result,
|
|
observation.endpoint_identity,
|
|
observation.config_identity,
|
|
)
|
|
observations[cell.id] = observation
|
|
return observations
|
|
|
|
|
|
class RunStore:
|
|
"""Filesystem-backed run store rooted at a validated manifest output root."""
|
|
|
|
def __init__(
|
|
self,
|
|
repo_root: str | Path,
|
|
*,
|
|
clock: Callable[[], _datetime.datetime] = lambda: _datetime.datetime.now(_datetime.timezone.utc),
|
|
token_hex: Callable[[int], str] = secrets.token_hex,
|
|
) -> None:
|
|
self.repo_root = Path(repo_root).resolve()
|
|
self._clock = clock
|
|
self._token_hex = token_hex
|
|
|
|
def _output_root(self, manifest: Manifest, *, create: bool) -> Path:
|
|
root = self.repo_root / manifest.output_root
|
|
if not _contained(root, self.repo_root):
|
|
raise RunPathError("output root is invalid")
|
|
if create:
|
|
root.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
_directory(root, "output root")
|
|
if root.is_symlink():
|
|
raise RunPathError("output root is invalid")
|
|
return root.resolve()
|
|
|
|
def _run_path(self, manifest: Manifest, run_id: str, *, create_root: bool) -> Path:
|
|
if not RUN_ID_RE.fullmatch(run_id):
|
|
raise RunPathError("run id is invalid")
|
|
root = self._output_root(manifest, create=create_root)
|
|
path = root / run_id
|
|
if path.parent != root or path.is_symlink():
|
|
raise RunPathError("run path is invalid")
|
|
return path
|
|
|
|
def create(self, manifest: Manifest, manifest_bytes: bytes) -> RunIdentity:
|
|
"""Create a run and atomically persist immutable source bytes first."""
|
|
loaded = validate_manifest_bytes(manifest_bytes, repo_root=self.repo_root)
|
|
if loaded.digest != manifest.digest:
|
|
raise AttemptStateError("manifest snapshot does not match manifest digest")
|
|
run_id = f"run-{_utc_timestamp(self._clock)}-{self._token_hex(6)}"
|
|
if not RUN_ID_RE.fullmatch(run_id):
|
|
raise AttemptStateError("generated run id is invalid")
|
|
path = self._run_path(manifest, run_id, create_root=True)
|
|
try:
|
|
path.mkdir(mode=0o700)
|
|
except FileExistsError as exc:
|
|
raise AttemptStateError("generated run id already exists") from exc
|
|
_write_new(path / "manifest.json", manifest_bytes)
|
|
_write_new(path / "run.json", _json_bytes({"run_id": run_id, "manifest_digest": manifest.digest}))
|
|
_write_new(path / "run.lock", b"")
|
|
_fsync_dir(path)
|
|
return RunIdentity(run_id, manifest.digest, str(path))
|
|
|
|
def open(self, manifest: Manifest, run_id: str, manifest_bytes: bytes | None = None) -> RunIdentity:
|
|
path = self._run_path(manifest, run_id, create_root=False)
|
|
_directory(path, "run")
|
|
try:
|
|
record = json.loads(_read_regular_bytes(path / "run.json", "run record").decode("utf-8"))
|
|
snapshot = _read_regular_bytes(path / "manifest.json", "manifest snapshot")
|
|
_read_regular_bytes(path / "run.lock", "run lock")
|
|
except (OSError, json.JSONDecodeError) as exc:
|
|
raise AttemptStateError("run state is unavailable") from exc
|
|
if record != {"run_id": run_id, "manifest_digest": manifest.digest}:
|
|
raise AttemptStateError("run identity does not match manifest")
|
|
if manifest_bytes is not None and snapshot != manifest_bytes:
|
|
raise AttemptStateError("manifest bytes do not match run snapshot")
|
|
if validate_manifest_bytes(snapshot, repo_root=self.repo_root).digest != manifest.digest:
|
|
raise AttemptStateError("run manifest digest is invalid")
|
|
return RunIdentity(run_id, manifest.digest, str(path))
|
|
|
|
@contextlib.contextmanager
|
|
def writer(self, run: RunIdentity) -> Iterator[None]:
|
|
root = Path(run.root)
|
|
_directory(root, "run")
|
|
lock_path = root / "run.lock"
|
|
fd = _open_regular(lock_path, "run lock", os.O_RDWR)
|
|
try:
|
|
try:
|
|
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except BlockingIOError as exc:
|
|
raise RunBusyError("run-busy") from exc
|
|
yield
|
|
finally:
|
|
try:
|
|
fcntl.flock(fd, fcntl.LOCK_UN)
|
|
finally:
|
|
os.close(fd)
|
|
|
|
@staticmethod
|
|
def _direct_cells(manifest: Manifest) -> tuple[MatrixCell, ...]:
|
|
return tuple(cell for cell in manifest.matrix if cell.iop.route_kind == "direct")
|
|
|
|
def _preflight_root(self, run: RunIdentity, *, create: bool) -> Path:
|
|
root = Path(run.root) / "preflight"
|
|
if root.exists() or root.is_symlink():
|
|
try:
|
|
mode = os.lstat(root).st_mode
|
|
except OSError as exc:
|
|
raise AttemptStateError("preflight state is unavailable") from exc
|
|
if not stat.S_ISDIR(mode) or root.is_symlink():
|
|
raise AttemptStateError("preflight state is invalid")
|
|
return root
|
|
if not create:
|
|
return root
|
|
try:
|
|
root.mkdir(mode=0o700)
|
|
_fsync_dir(root.parent)
|
|
except OSError as exc:
|
|
raise AttemptStateError("preflight state is unavailable") from exc
|
|
return root
|
|
|
|
def _validate_preflight_record(
|
|
self,
|
|
raw: bytes,
|
|
path: Path,
|
|
run: RunIdentity,
|
|
manifest: Manifest,
|
|
expected_sequence: int,
|
|
) -> dict[str, Any]:
|
|
try:
|
|
record = json.loads(raw.decode("ascii"))
|
|
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
|
raise AttemptStateError("preflight record is invalid") from exc
|
|
if not isinstance(record, dict) or set(record) != {
|
|
"schema_version", "run_id", "manifest_digest", "sequence",
|
|
"status", "results",
|
|
}:
|
|
raise AttemptStateError("preflight record schema is invalid")
|
|
if (
|
|
record["schema_version"] != PREFLIGHT_SCHEMA_VERSION
|
|
or record["run_id"] != run.run_id
|
|
or record["manifest_digest"] != run.manifest_digest
|
|
or record["sequence"] != expected_sequence
|
|
or record["status"] not in PREFLIGHT_STATUSES
|
|
or not isinstance(record["results"], list)
|
|
):
|
|
raise AttemptStateError("preflight record identity is invalid")
|
|
direct_cells = self._direct_cells(manifest)
|
|
if len(record["results"]) != len(direct_cells):
|
|
raise AttemptStateError("preflight result set is invalid")
|
|
statuses: list[str] = []
|
|
for cell, result_payload in zip(direct_cells, record["results"]):
|
|
result, _, _ = _connectivity_result_from_payload(result_payload, cell)
|
|
statuses.append(result.status)
|
|
if record["status"] != _overall_preflight_status(iter(statuses)):
|
|
raise AttemptStateError("preflight aggregate status is invalid")
|
|
if raw != _json_bytes(record):
|
|
raise AttemptStateError("preflight record is non-canonical")
|
|
expected_name = f"preflight-{expected_sequence:06d}.json"
|
|
if path.name != expected_name:
|
|
raise AttemptStateError("preflight sequence is invalid")
|
|
return record
|
|
|
|
def _preflight_records(
|
|
self, run: RunIdentity, manifest: Manifest
|
|
) -> tuple[dict[str, Any], ...]:
|
|
root = self._preflight_root(run, create=False)
|
|
if not root.exists() and not root.is_symlink():
|
|
return ()
|
|
records: list[dict[str, Any]] = []
|
|
for expected_sequence, child in enumerate(
|
|
sorted(root.iterdir(), key=lambda item: item.name), start=1
|
|
):
|
|
match = PREFLIGHT_RE.fullmatch(child.name)
|
|
if match is None or int(match.group(1)) != expected_sequence:
|
|
raise AttemptStateError("preflight sequence is invalid")
|
|
raw = _read_regular_bytes(child, "preflight record")
|
|
records.append(
|
|
self._validate_preflight_record(
|
|
raw, child, run, manifest, expected_sequence
|
|
)
|
|
)
|
|
return tuple(records)
|
|
|
|
def preflights(
|
|
self, run: RunIdentity, manifest: Manifest
|
|
) -> tuple[dict[str, Any], ...]:
|
|
"""Read append-only preflight records without creating or reconciling state."""
|
|
bound_run = self.open(manifest, run.run_id)
|
|
if bound_run != run:
|
|
raise AttemptStateError("run identity is invalid")
|
|
return self._preflight_records(bound_run, manifest)
|
|
|
|
def record_preflight(
|
|
self,
|
|
run: RunIdentity,
|
|
manifest: Manifest,
|
|
observations: Mapping[str, PreflightObservation],
|
|
) -> dict[str, Any]:
|
|
"""Append one canonical result set while exclusively owning the run writer."""
|
|
bound_run = self.open(manifest, run.run_id)
|
|
if bound_run != run:
|
|
raise AttemptStateError("run identity is invalid")
|
|
with self.writer(bound_run):
|
|
return self._record_preflight_locked(bound_run, manifest, observations)
|
|
|
|
def _record_preflight_locked(
|
|
self,
|
|
run: RunIdentity,
|
|
manifest: Manifest,
|
|
observations: Mapping[str, PreflightObservation],
|
|
) -> dict[str, Any]:
|
|
"""Append one preflight while the caller owns the run writer."""
|
|
bound_run = self.open(manifest, run.run_id)
|
|
if bound_run != run:
|
|
raise AttemptStateError("run identity is invalid")
|
|
direct_cells = self._direct_cells(manifest)
|
|
if set(observations) != {cell.id for cell in direct_cells}:
|
|
raise AttemptStateError("preflight observation set is invalid")
|
|
results: list[dict[str, Any]] = []
|
|
for cell in direct_cells:
|
|
observation = observations[cell.id]
|
|
if not isinstance(observation, PreflightObservation):
|
|
raise AttemptStateError("preflight observation is invalid")
|
|
try:
|
|
encoded = canonical_evidence_bytes(
|
|
cell,
|
|
observation.result,
|
|
observation.endpoint_identity,
|
|
observation.config_identity,
|
|
)
|
|
payload = json.loads(encoded.decode("ascii"))
|
|
except Exception as exc:
|
|
raise AttemptStateError("preflight observation is invalid") from exc
|
|
_connectivity_result_from_payload(payload, cell)
|
|
results.append(payload)
|
|
status = _overall_preflight_status(
|
|
iter(str(result["status"]) for result in results)
|
|
)
|
|
|
|
previous = self._preflight_records(bound_run, manifest)
|
|
sequence = len(previous) + 1
|
|
root = self._preflight_root(bound_run, create=True)
|
|
path = root / f"preflight-{sequence:06d}.json"
|
|
record = {
|
|
"schema_version": PREFLIGHT_SCHEMA_VERSION,
|
|
"run_id": bound_run.run_id,
|
|
"manifest_digest": bound_run.manifest_digest,
|
|
"sequence": sequence,
|
|
"status": status,
|
|
"results": results,
|
|
}
|
|
raw = _json_bytes(record)
|
|
_write_new(path, raw)
|
|
self._validate_preflight_record(
|
|
_read_regular_bytes(path, "preflight record"),
|
|
path,
|
|
bound_run,
|
|
manifest,
|
|
sequence,
|
|
)
|
|
return record
|
|
|
|
@staticmethod
|
|
def slots(manifest: Manifest) -> tuple[Slot, ...]:
|
|
return tuple(Slot(cell.id, repetition) for cell in manifest.matrix for repetition in range(1, manifest.repetitions + 1))
|
|
|
|
def _slot_path(self, run: RunIdentity, slot: Slot) -> Path:
|
|
if not re.fullmatch(r"[a-z0-9][a-z0-9_-]{0,63}", slot.cell_id) or slot.repetition < 1:
|
|
raise AttemptStateError("slot is invalid")
|
|
return Path(run.root) / "cells" / slot.cell_id / f"repetition-{slot.repetition:04d}"
|
|
|
|
def _ensure_slot_parent(self, run: RunIdentity, slot: Slot) -> Path:
|
|
current = Path(run.root)
|
|
for segment in ("cells", slot.cell_id, f"repetition-{slot.repetition:04d}"):
|
|
current = current / segment
|
|
if current.exists() or current.is_symlink():
|
|
_directory(current, "slot state")
|
|
if current.is_symlink():
|
|
raise AttemptStateError("slot state is invalid")
|
|
continue
|
|
try:
|
|
current.mkdir(mode=0o700)
|
|
except FileExistsError:
|
|
_directory(current, "slot state")
|
|
if current.is_symlink():
|
|
raise AttemptStateError("slot state is invalid")
|
|
return current
|
|
|
|
@staticmethod
|
|
def _expected_record(run: RunIdentity, identity: AttemptIdentity, state: str) -> dict[str, Any]:
|
|
return {
|
|
"run_id": run.run_id,
|
|
"manifest_digest": run.manifest_digest,
|
|
"cell_id": identity.cell_id,
|
|
"repetition": identity.repetition,
|
|
"attempt": identity.attempt,
|
|
"state": state,
|
|
}
|
|
|
|
def _attempt_record(self, root: Path, run: RunIdentity, identity: AttemptIdentity, *, absent_ok: bool = False) -> dict[str, Any] | None:
|
|
path = root / "attempt.json"
|
|
try:
|
|
record = json.loads(_read_regular_bytes(path, "attempt record").decode("utf-8"))
|
|
except AttemptStateError:
|
|
if absent_ok:
|
|
try:
|
|
os.lstat(path)
|
|
except FileNotFoundError:
|
|
if self._measurement_policy_start(root, run, identity) is not None:
|
|
raise AttemptStateError(
|
|
"attempt measurement policy provenance is invalid"
|
|
)
|
|
return None
|
|
raise
|
|
except (OSError, json.JSONDecodeError) as exc:
|
|
raise AttemptStateError("attempt record is unavailable") from exc
|
|
if not isinstance(record, dict):
|
|
raise AttemptStateError("attempt record is invalid")
|
|
expected = self._expected_record(run, identity, str(record.get("state", "")))
|
|
if record.get("state") not in TERMINAL_STATES | {NONTERMINAL_STATE}:
|
|
raise AttemptStateError("attempt record is invalid")
|
|
for key, value in expected.items():
|
|
if record.get(key) != value:
|
|
raise AttemptStateError("attempt record identity is invalid")
|
|
allowed = set(expected) | {
|
|
"locator", "spec_digest", "lifecycle", "measurement_policy",
|
|
"web_validation_policy",
|
|
}
|
|
if set(record) - allowed:
|
|
raise AttemptStateError("attempt record schema is invalid")
|
|
policy = record.get("measurement_policy")
|
|
if policy is not None and policy not in MEASUREMENT_POLICIES:
|
|
raise AttemptStateError("attempt measurement policy is invalid")
|
|
self._validate_measurement_policy_start(root, run, identity, policy)
|
|
web_policy = record.get("web_validation_policy")
|
|
if web_policy is not None and web_policy not in WEB_VALIDATION_POLICIES:
|
|
raise AttemptStateError("attempt web validation policy is invalid")
|
|
self._validate_web_validation_policy_start(root, run, identity, web_policy)
|
|
if record["state"] == NONTERMINAL_STATE and "lifecycle" in record:
|
|
raise AttemptStateError("running attempt has terminal evidence")
|
|
if ("locator" in record) != ("spec_digest" in record):
|
|
raise AttemptStateError("attempt invocation identity is invalid")
|
|
if "locator" in record:
|
|
locator = self._locator_from_record(
|
|
root, record["locator"], state=str(record["state"])
|
|
)
|
|
if not isinstance(record["spec_digest"], str) or not DIGEST_RE.fullmatch(record["spec_digest"]):
|
|
raise AttemptStateError("attempt invocation digest is invalid")
|
|
if record["state"] in TERMINAL_STATES:
|
|
lifecycle = record.get("lifecycle")
|
|
expected_receipt_reason = (
|
|
lifecycle.get("terminal_reason")
|
|
if isinstance(lifecycle, dict)
|
|
else None
|
|
)
|
|
self._validate_terminal_invocation_identity(
|
|
root,
|
|
locator,
|
|
record["spec_digest"],
|
|
run=run,
|
|
identity=identity,
|
|
terminal_state=str(record["state"]),
|
|
expected_receipt_reason=expected_receipt_reason,
|
|
measurement_policy=policy,
|
|
)
|
|
if "lifecycle" in record and (not isinstance(record["lifecycle"], dict) or set(record["lifecycle"]) != {"terminal_reason"} or not isinstance(record["lifecycle"]["terminal_reason"], str)):
|
|
raise AttemptStateError("attempt lifecycle is invalid")
|
|
if record["state"] in TERMINAL_STATES:
|
|
self._validate_web_validation(root, run, identity, web_policy)
|
|
elif (root / WEB_VALIDATION_FILENAME).exists() or (
|
|
root / WEB_VALIDATION_FILENAME
|
|
).is_symlink():
|
|
self._validate_web_validation(root, run, identity, web_policy)
|
|
return record
|
|
|
|
def attempts(self, run: RunIdentity, slot: Slot) -> tuple[Attempt, ...]:
|
|
parent = self._slot_path(run, slot)
|
|
if not parent.exists() and not parent.is_symlink():
|
|
return ()
|
|
_directory(parent, "slot state")
|
|
found: list[Attempt] = []
|
|
for child in sorted(parent.iterdir(), key=lambda item: item.name):
|
|
match = ATTEMPT_RE.fullmatch(child.name)
|
|
if not match or child.is_symlink() or not child.is_dir():
|
|
raise AttemptStateError("slot contains invalid state")
|
|
number = int(match.group(1))
|
|
identity = AttemptIdentity(run.run_id, slot.cell_id, slot.repetition, number)
|
|
record = self._attempt_record(child, run, identity, absent_ok=True)
|
|
state = NONTERMINAL_STATE if record is None else str(record["state"])
|
|
found.append(Attempt(identity, str(child), state))
|
|
return tuple(found)
|
|
|
|
def execution_attempts(
|
|
self, run: RunIdentity, manifest: Manifest
|
|
) -> tuple[Attempt, ...]:
|
|
"""Enumerate every retained execution attempt in canonical slot order.
|
|
|
|
Scoring and later reporting use this read-only projection instead of
|
|
reconstructing the private ``cells/`` directory grammar.
|
|
"""
|
|
bound_run = self.open(manifest, run.run_id)
|
|
if bound_run != run:
|
|
raise AttemptStateError("run identity is invalid")
|
|
return tuple(
|
|
attempt
|
|
for slot in self.slots(manifest)
|
|
for attempt in self.attempts(bound_run, slot)
|
|
)
|
|
|
|
def allocate(self, run: RunIdentity, slot: Slot) -> Attempt:
|
|
"""Create the exclusive, deliberately empty attempt root."""
|
|
existing = self.attempts(run, slot)
|
|
if any(item.state not in TERMINAL_STATES for item in existing):
|
|
raise AttemptStateError("slot already has a nonterminal attempt")
|
|
number = max((item.identity.attempt for item in existing), default=0) + 1
|
|
parent = self._ensure_slot_parent(run, slot)
|
|
root = parent / f"attempt-{number:06d}"
|
|
try:
|
|
root.mkdir(mode=0o700)
|
|
_fsync_dir(parent)
|
|
except FileExistsError as exc:
|
|
raise AttemptStateError("attempt allocation collision") from exc
|
|
return Attempt(AttemptIdentity(run.run_id, slot.cell_id, slot.repetition, number), str(root), NONTERMINAL_STATE)
|
|
|
|
def _bound_attempt(self, attempt: Attempt) -> tuple[RunIdentity, Path]:
|
|
root = Path(attempt.root)
|
|
try:
|
|
run_root = root.parents[3]
|
|
except IndexError as exc:
|
|
raise AttemptStateError("attempt path is invalid") from exc
|
|
_directory(run_root, "run")
|
|
try:
|
|
run_record = json.loads(_read_regular_bytes(run_root / "run.json", "run record").decode("utf-8"))
|
|
snapshot = _read_regular_bytes(run_root / "manifest.json", "manifest snapshot")
|
|
_read_regular_bytes(run_root / "run.lock", "run lock")
|
|
manifest = validate_manifest_bytes(snapshot, repo_root=self.repo_root)
|
|
except Exception as exc:
|
|
raise AttemptStateError("attempt run binding is invalid") from exc
|
|
if not isinstance(run_record, dict) or set(run_record) != {"run_id", "manifest_digest"}:
|
|
raise AttemptStateError("attempt run binding is invalid")
|
|
run = RunIdentity(str(run_record["run_id"]), str(run_record["manifest_digest"]), str(run_root))
|
|
if not RUN_ID_RE.fullmatch(run.run_id) or run.manifest_digest != manifest.digest:
|
|
raise AttemptStateError("attempt run binding is invalid")
|
|
if run_root.resolve() != self._run_path(manifest, run.run_id, create_root=False).resolve():
|
|
raise AttemptStateError("attempt run binding is invalid")
|
|
expected = self._slot_path(run, Slot(attempt.identity.cell_id, attempt.identity.repetition)) / f"attempt-{attempt.identity.attempt:06d}"
|
|
if root.resolve() != expected.resolve() or attempt.identity.run_id != run.run_id or root.is_symlink():
|
|
raise AttemptStateError("attempt identity is invalid")
|
|
_directory(root, "attempt root")
|
|
return run, root
|
|
|
|
def _initial_record(
|
|
self, run: RunIdentity, attempt: Attempt, state: str, *,
|
|
reason: str | None = None, measurement_policy: str | None = None,
|
|
web_validation_policy: str | None = None,
|
|
) -> dict[str, Any]:
|
|
record = self._expected_record(run, attempt.identity, state)
|
|
if measurement_policy is not None:
|
|
if measurement_policy not in MEASUREMENT_POLICIES:
|
|
raise AttemptStateError("attempt measurement policy is invalid")
|
|
record["measurement_policy"] = measurement_policy
|
|
if web_validation_policy is not None:
|
|
if web_validation_policy not in WEB_VALIDATION_POLICIES:
|
|
raise AttemptStateError("attempt web validation policy is invalid")
|
|
record["web_validation_policy"] = web_validation_policy
|
|
if reason is not None:
|
|
record["lifecycle"] = {"terminal_reason": reason}
|
|
return record
|
|
|
|
@staticmethod
|
|
def _measurement_policy_start_record(
|
|
run: RunIdentity, identity: AttemptIdentity
|
|
) -> dict[str, Any]:
|
|
"""Return the immutable production measurement-policy start evidence."""
|
|
return {
|
|
"record": MEASUREMENT_POLICY_RECORD,
|
|
"measurement_policy_version": MEASUREMENT_POLICY_VERSION,
|
|
"measurement_policy": MEASUREMENT_POLICY_REQUIRED_V1,
|
|
"run_id": run.run_id,
|
|
"manifest_digest": run.manifest_digest,
|
|
"cell_id": identity.cell_id,
|
|
"repetition": identity.repetition,
|
|
"attempt": identity.attempt,
|
|
}
|
|
|
|
def _measurement_policy_start(
|
|
self, root: Path, run: RunIdentity, identity: AttemptIdentity
|
|
) -> dict[str, Any] | None:
|
|
"""Load one canonical, no-follow policy record, or its explicit absence."""
|
|
path = root / MEASUREMENT_POLICY_FILENAME
|
|
try:
|
|
raw = _read_regular_bytes(path, "measurement policy")
|
|
except AttemptStateError:
|
|
try:
|
|
os.lstat(path)
|
|
except FileNotFoundError:
|
|
return None
|
|
raise
|
|
try:
|
|
record = json.loads(raw.decode("ascii"))
|
|
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
|
raise AttemptStateError("measurement policy is invalid") from exc
|
|
expected = self._measurement_policy_start_record(run, identity)
|
|
if record != expected or raw != _json_bytes(expected):
|
|
raise AttemptStateError("measurement policy is invalid")
|
|
return record
|
|
|
|
def _validate_measurement_policy_start(
|
|
self,
|
|
root: Path,
|
|
run: RunIdentity,
|
|
identity: AttemptIdentity,
|
|
measurement_policy: str | None,
|
|
) -> None:
|
|
"""Require both policy sources for production and neither for legacy paths."""
|
|
start = self._measurement_policy_start(root, run, identity)
|
|
if measurement_policy is None and start is None:
|
|
return
|
|
if (
|
|
measurement_policy == MEASUREMENT_POLICY_REQUIRED_V1
|
|
and start is not None
|
|
):
|
|
return
|
|
raise AttemptStateError("attempt measurement policy provenance is invalid")
|
|
|
|
@staticmethod
|
|
def _web_validation_policy_start_record(
|
|
run: RunIdentity, identity: AttemptIdentity
|
|
) -> dict[str, Any]:
|
|
return {
|
|
"record": WEB_VALIDATION_POLICY_RECORD,
|
|
"web_validation_policy_version": WEB_VALIDATION_POLICY_VERSION,
|
|
"web_validation_policy": WEB_VALIDATION_POLICY_REQUIRED_V1,
|
|
"run_id": run.run_id,
|
|
"manifest_digest": run.manifest_digest,
|
|
"cell_id": identity.cell_id,
|
|
"repetition": identity.repetition,
|
|
"attempt": identity.attempt,
|
|
}
|
|
|
|
def _web_validation_policy_start(
|
|
self, root: Path, run: RunIdentity, identity: AttemptIdentity
|
|
) -> dict[str, Any] | None:
|
|
path = root / WEB_VALIDATION_POLICY_FILENAME
|
|
try:
|
|
raw = _read_regular_bytes(path, "web validation policy")
|
|
except AttemptStateError:
|
|
try:
|
|
os.lstat(path)
|
|
except FileNotFoundError:
|
|
return None
|
|
raise
|
|
try:
|
|
record = json.loads(raw.decode("ascii"))
|
|
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
|
raise AttemptStateError("web validation policy is invalid") from exc
|
|
expected = self._web_validation_policy_start_record(run, identity)
|
|
if record != expected or raw != _json_bytes(expected):
|
|
raise AttemptStateError("web validation policy is invalid")
|
|
return record
|
|
|
|
def _validate_web_validation_policy_start(
|
|
self, root: Path, run: RunIdentity, identity: AttemptIdentity,
|
|
policy: str | None,
|
|
) -> None:
|
|
start = self._web_validation_policy_start(root, run, identity)
|
|
if policy is None and start is None:
|
|
return
|
|
if policy == WEB_VALIDATION_POLICY_REQUIRED_V1 and start is not None:
|
|
return
|
|
raise AttemptStateError("attempt web validation policy provenance is invalid")
|
|
|
|
def _validate_web_validation(
|
|
self, root: Path, run: RunIdentity, identity: AttemptIdentity,
|
|
policy: str | None,
|
|
) -> None:
|
|
path = root / WEB_VALIDATION_FILENAME
|
|
if not path.exists() and not path.is_symlink():
|
|
if policy == WEB_VALIDATION_POLICY_REQUIRED_V1:
|
|
raise AttemptStateError("required web validation is unavailable")
|
|
return
|
|
try:
|
|
manifest = self.open_manifest_snapshot(run)
|
|
web = load_web_validation(root, manifest=manifest)
|
|
measurement = load_measurement(root)
|
|
except (WebValidationError, MeasurementError) as exc:
|
|
raise AttemptStateError("attempt web validation is invalid") from exc
|
|
record = web.record
|
|
if (
|
|
(record["attempt"]["run_id"], record["attempt"]["cell_id"],
|
|
record["attempt"]["repetition"], record["attempt"]["attempt"])
|
|
!= (run.run_id, identity.cell_id, identity.repetition, identity.attempt)
|
|
or record["manifest_digest"] != run.manifest_digest
|
|
or record["fixture_checksum"] != manifest.fixture.checksum
|
|
):
|
|
raise AttemptStateError("attempt web validation identity is invalid")
|
|
digest = "sha256:" + hashlib.sha256(
|
|
_read_regular_bytes(root / MEASUREMENT_FILENAME, "measurement")).hexdigest()
|
|
if record["measurement_digest"] != digest or measurement.run_id != run.run_id:
|
|
raise AttemptStateError("attempt web validation measurement is invalid")
|
|
if (
|
|
(measurement.terminal_reason == "success" and web.status == "not_run")
|
|
or (measurement.terminal_reason != "success" and web.status != "not_run")
|
|
):
|
|
raise AttemptStateError("attempt web validation lifecycle is invalid")
|
|
|
|
def open_manifest_snapshot(self, run: RunIdentity) -> Manifest:
|
|
"""Load the exact immutable run manifest used by recovery and evidence."""
|
|
try:
|
|
return validate_manifest_bytes(
|
|
_read_regular_bytes(
|
|
Path(run.root) / "manifest.json", "manifest snapshot"
|
|
),
|
|
repo_root=self.repo_root,
|
|
)
|
|
except Exception as exc:
|
|
raise AttemptStateError(
|
|
"attempt web validation manifest is invalid"
|
|
) from exc
|
|
|
|
def open_manifest_fixture_checksum(self, run: RunIdentity) -> str:
|
|
"""Read the run-bound manifest only to bind web evidence to its fixture."""
|
|
return self.open_manifest_snapshot(run).fixture.checksum
|
|
|
|
def _ensure_required_web_validation(
|
|
self,
|
|
root: Path,
|
|
run: RunIdentity,
|
|
identity: AttemptIdentity,
|
|
record: Mapping[str, Any],
|
|
terminal: Mapping[str, Any],
|
|
) -> None:
|
|
"""Validate or reconstruct required S12 evidence before terminal commit.
|
|
|
|
An existing invalid/colliding record is never replaced. When the
|
|
sidecar is absent, reconstruction uses only the immutable run manifest,
|
|
conventional attempt workspace, and strict measurement. Any failure
|
|
occurs before ``attempt.json`` is replaced, leaving recovery resumable.
|
|
"""
|
|
policy = record.get("web_validation_policy")
|
|
if policy != WEB_VALIDATION_POLICY_REQUIRED_V1:
|
|
self._validate_web_validation(root, run, identity, policy)
|
|
return
|
|
path = root / WEB_VALIDATION_FILENAME
|
|
if path.exists() or path.is_symlink():
|
|
self._validate_web_validation(root, run, identity, policy)
|
|
return
|
|
try:
|
|
manifest = self.open_manifest_snapshot(run)
|
|
measurement = load_measurement(root)
|
|
prepared = SimpleNamespace(
|
|
workspace_dir=str(root / "workspace"),
|
|
attempt_root=str(root),
|
|
)
|
|
web = validate_web_attempt(
|
|
manifest,
|
|
root,
|
|
prepared,
|
|
measurement,
|
|
terminal,
|
|
)
|
|
publish_web_validation(root, web)
|
|
except (MeasurementError, WebValidationError, OSError) as exc:
|
|
raise AttemptStateError(
|
|
"required web validation reconstruction failed"
|
|
) from exc
|
|
self._validate_web_validation(root, run, identity, policy)
|
|
|
|
@staticmethod
|
|
def _control_lease_for_root(root: Path) -> AttemptControlLease:
|
|
"""Derive the short public alias without reading secret or caller data."""
|
|
canonical_root = root.resolve()
|
|
digest = hashlib.sha256(
|
|
b"iop-benchmark-attempt-control-v1\0"
|
|
+ os.fsencode(str(canonical_root))
|
|
).hexdigest()[:CONTROL_ALIAS_DIGEST_HEX_LENGTH]
|
|
alias = Path(tempfile.gettempdir()).resolve() / f"{CONTROL_ALIAS_PREFIX}{digest}"
|
|
control_dir = alias / CONTROL_DIRECTORY_NAME
|
|
socket_path = control_dir / SOCKET_FILENAME
|
|
if len(os.fsencode(str(socket_path))) > UNIX_SOCKET_PATH_MAX_BYTES:
|
|
raise AttemptStateError("control socket path exceeds platform budget")
|
|
return AttemptControlLease(str(alias), str(control_dir), str(socket_path))
|
|
|
|
@staticmethod
|
|
def _validate_control_alias(root: Path, lease: AttemptControlLease) -> None:
|
|
alias = Path(lease.alias)
|
|
try:
|
|
mode = os.lstat(alias).st_mode
|
|
except OSError as exc:
|
|
raise AttemptStateError("control lease is unavailable") from exc
|
|
if not stat.S_ISLNK(mode):
|
|
raise AttemptStateError("control lease collision")
|
|
try:
|
|
target = os.readlink(alias)
|
|
except OSError as exc:
|
|
raise AttemptStateError("control lease is unavailable") from exc
|
|
canonical_root = root.resolve()
|
|
try:
|
|
resolved_alias = alias.resolve(strict=True)
|
|
except OSError as exc:
|
|
raise AttemptStateError("control lease is unavailable") from exc
|
|
if (
|
|
target != str(canonical_root)
|
|
or not Path(target).is_absolute()
|
|
or resolved_alias != canonical_root
|
|
):
|
|
raise AttemptStateError("control lease target mismatch")
|
|
|
|
def acquire_control_lease(self, attempt: Attempt) -> AttemptControlLease:
|
|
"""Create or authenticate the active attempt's no-overwrite short alias."""
|
|
run, root = self._bound_attempt(attempt)
|
|
record = self._attempt_record(root, run, attempt.identity)
|
|
if record is None or record["state"] != NONTERMINAL_STATE:
|
|
raise AttemptStateError("control lease requires a running attempt")
|
|
lease = self._control_lease_for_root(root)
|
|
alias = Path(lease.alias)
|
|
try:
|
|
os.symlink(str(root.resolve()), alias, target_is_directory=True)
|
|
_fsync_dir(alias.parent)
|
|
except FileExistsError:
|
|
self._validate_control_alias(root, lease)
|
|
except OSError as exc:
|
|
raise AttemptStateError("control lease is unavailable") from exc
|
|
self._validate_control_alias(root, lease)
|
|
return lease
|
|
|
|
def release_control_lease(self, attempt: Attempt) -> None:
|
|
"""Remove only this exact owned alias after durable terminal publication."""
|
|
run, root = self._bound_attempt(attempt)
|
|
record = self._attempt_record(root, run, attempt.identity)
|
|
if record is None or record["state"] not in TERMINAL_STATES:
|
|
raise AttemptStateError("control lease release requires terminal state")
|
|
lease = self._control_lease_for_root(root)
|
|
alias = Path(lease.alias)
|
|
try:
|
|
os.lstat(alias)
|
|
except FileNotFoundError:
|
|
return
|
|
except OSError as exc:
|
|
raise AttemptStateError("control lease is unavailable") from exc
|
|
self._validate_control_alias(root, lease)
|
|
try:
|
|
alias.unlink()
|
|
_fsync_dir(alias.parent)
|
|
except OSError as exc:
|
|
raise AttemptStateError("control lease cleanup failed") from exc
|
|
|
|
def publish_terminal(self, attempt: Attempt, state: str, *, result: dict[str, Any] | None = None) -> Attempt:
|
|
if state not in TERMINAL_STATES:
|
|
raise AttemptStateError("terminal state is invalid")
|
|
run, root = self._bound_attempt(attempt)
|
|
record = self._attempt_record(root, run, attempt.identity, absent_ok=True)
|
|
if record is None:
|
|
reason = str((result or {}).get("terminal_reason") or state)
|
|
_write_new(root / "attempt.json", _json_bytes(self._initial_record(run, attempt, state, reason=reason)))
|
|
return Attempt(attempt.identity, attempt.root, state)
|
|
if record["state"] in TERMINAL_STATES:
|
|
if record["state"] != state:
|
|
raise AttemptStateError("terminal attempt is immutable")
|
|
return Attempt(attempt.identity, attempt.root, state)
|
|
if record["state"] != NONTERMINAL_STATE:
|
|
raise AttemptStateError("attempt transition is invalid")
|
|
terminal_result = result or {"terminal_reason": state}
|
|
self._ensure_required_web_validation(
|
|
root,
|
|
run,
|
|
attempt.identity,
|
|
record,
|
|
terminal_result,
|
|
)
|
|
record["state"] = state
|
|
record["lifecycle"] = {
|
|
"terminal_reason": str(terminal_result.get("terminal_reason") or state)
|
|
}
|
|
_replace(root / "attempt.json", _json_bytes(record))
|
|
return Attempt(attempt.identity, attempt.root, state)
|
|
|
|
def _locator_from_record(
|
|
self, root: Path, raw: Any, *, state: str
|
|
) -> SupervisorLocator:
|
|
fields = {"supervisor_pid", "start_identity", "socket_path", "challenge", "control_dir", "created_at"}
|
|
if not isinstance(raw, dict) or set(raw) != fields:
|
|
raise AttemptStateError("locator is invalid")
|
|
try:
|
|
locator = SupervisorLocator(**raw)
|
|
except TypeError as exc:
|
|
raise AttemptStateError("locator is invalid") from exc
|
|
if not isinstance(locator.supervisor_pid, int) or locator.supervisor_pid < 1 or any(not isinstance(value, str) or not value for value in (locator.start_identity, locator.socket_path, locator.challenge, locator.control_dir, locator.created_at)):
|
|
raise AttemptStateError("locator is invalid")
|
|
control = Path(locator.control_dir)
|
|
socket = Path(locator.socket_path)
|
|
lease = self._control_lease_for_root(root)
|
|
if control != Path(lease.control_dir) or socket != Path(lease.socket_path):
|
|
raise AttemptStateError("locator control binding is invalid")
|
|
alias = Path(lease.alias)
|
|
try:
|
|
os.lstat(alias)
|
|
except FileNotFoundError:
|
|
if state == NONTERMINAL_STATE:
|
|
raise AttemptStateError("running locator control lease is unavailable")
|
|
except OSError as exc:
|
|
raise AttemptStateError("locator control lease is unavailable") from exc
|
|
else:
|
|
self._validate_control_alias(root, lease)
|
|
if not _contained(control, root) or not _contained(socket, control):
|
|
raise AttemptStateError("locator escapes attempt root")
|
|
if socket.parent != control:
|
|
raise AttemptStateError("locator control binding is invalid")
|
|
return locator
|
|
|
|
def _validate_measurement(
|
|
self,
|
|
root: Path,
|
|
run: RunIdentity,
|
|
identity: AttemptIdentity,
|
|
expected_digest: str,
|
|
expected_reason: str | None = None,
|
|
measurement_policy: str | None = None,
|
|
lifecycle: Mapping[str, Any] | None = None,
|
|
) -> None:
|
|
"""Bind the immutable timing/usage sidecar to this exact invocation.
|
|
|
|
The sidecar is optional for historical and lower-level records. A
|
|
marked production attempt must have it, and every present sidecar is
|
|
bound to the terminal lifecycle evidence without rewriting either.
|
|
"""
|
|
path = root / MEASUREMENT_FILENAME
|
|
if not path.exists() and not path.is_symlink():
|
|
if measurement_policy == MEASUREMENT_POLICY_REQUIRED_V1:
|
|
raise AttemptStateError("required attempt measurement is unavailable")
|
|
return
|
|
try:
|
|
measurement = load_measurement(root)
|
|
except MeasurementError as exc:
|
|
raise AttemptStateError("attempt measurement is invalid") from exc
|
|
actual = (
|
|
measurement.run_id, measurement.cell_id,
|
|
measurement.repetition, measurement.attempt,
|
|
)
|
|
expected = (
|
|
run.run_id, identity.cell_id, identity.repetition, identity.attempt,
|
|
)
|
|
if actual != expected or measurement.spec_digest != expected_digest:
|
|
raise AttemptStateError("attempt measurement identity is invalid")
|
|
if expected_reason is not None and measurement.terminal_reason != expected_reason:
|
|
raise AttemptStateError("attempt measurement terminal is invalid")
|
|
if lifecycle is not None:
|
|
try:
|
|
validate_measurement_lifecycle_binding(measurement, lifecycle)
|
|
except MeasurementError as exc:
|
|
raise AttemptStateError("attempt measurement lifecycle is invalid") from exc
|
|
|
|
def _validate_terminal_invocation_identity(
|
|
self,
|
|
root: Path,
|
|
locator: SupervisorLocator,
|
|
expected_digest: str,
|
|
*,
|
|
run: RunIdentity,
|
|
identity: AttemptIdentity,
|
|
terminal_state: str,
|
|
expected_receipt_reason: str | None,
|
|
measurement_policy: str | None,
|
|
) -> None:
|
|
"""Rebind cleaned historical records to their durable invocation digest."""
|
|
result_path = root / "lifecycle-result.json"
|
|
result_exists = result_path.exists() or result_path.is_symlink()
|
|
if result_exists:
|
|
result = self._read_bound_lifecycle_terminal(
|
|
root, locator, expected_digest
|
|
)
|
|
if result is None:
|
|
raise AttemptStateError("terminal invocation identity is invalid")
|
|
self._validate_measurement(
|
|
root, run, identity, expected_digest, expected_receipt_reason,
|
|
measurement_policy, result,
|
|
)
|
|
journal_path = root / "lifecycle-journal.jsonl"
|
|
journal_exists = journal_path.exists() or journal_path.is_symlink()
|
|
if journal_exists:
|
|
try:
|
|
first = _read_regular_bytes(
|
|
journal_path, "lifecycle journal"
|
|
).decode("utf-8").splitlines()[0]
|
|
header = json.loads(first)
|
|
except (IndexError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
|
raise AttemptStateError("terminal invocation identity is invalid") from exc
|
|
if not isinstance(header, dict) or header.get("spec_digest") != expected_digest:
|
|
raise AttemptStateError("terminal invocation identity is invalid")
|
|
if not result_exists and not journal_exists:
|
|
# A receipt-only record has no published result or journal, so its only
|
|
# authenticated reason is recovery; it may project nothing but interrupted.
|
|
if terminal_state != "interrupted" or not isinstance(expected_receipt_reason, str):
|
|
raise AttemptStateError("terminal invocation identity is invalid")
|
|
self._closed_cleanup_receipt(
|
|
root,
|
|
locator,
|
|
expected_reason=expected_receipt_reason,
|
|
required=True,
|
|
)
|
|
self._validate_measurement(
|
|
root, run, identity, expected_digest, expected_receipt_reason,
|
|
measurement_policy,
|
|
)
|
|
|
|
def record_locator(self, attempt: Attempt, locator: SupervisorLocator, invocation_digest: str) -> None:
|
|
run, root = self._bound_attempt(attempt)
|
|
record = self._attempt_record(root, run, attempt.identity)
|
|
if record is None or record["state"] != NONTERMINAL_STATE or "locator" in record:
|
|
raise AttemptStateError("locator transition is invalid")
|
|
if not isinstance(invocation_digest, str) or not DIGEST_RE.fullmatch(invocation_digest):
|
|
raise AttemptStateError("invocation digest is invalid")
|
|
raw = {
|
|
"supervisor_pid": locator.supervisor_pid,
|
|
"start_identity": locator.start_identity,
|
|
"socket_path": locator.socket_path,
|
|
"challenge": locator.challenge,
|
|
"control_dir": locator.control_dir,
|
|
"created_at": locator.created_at,
|
|
}
|
|
self._locator_from_record(root, raw, state=NONTERMINAL_STATE)
|
|
registered = self._read_json_file(Path(locator.control_dir), "locator.json")
|
|
if registered != raw:
|
|
raise AttemptStateError("registered locator is invalid")
|
|
record["locator"] = raw
|
|
record["spec_digest"] = invocation_digest
|
|
_replace(root / "attempt.json", _json_bytes(record))
|
|
|
|
@staticmethod
|
|
def _state_for_reason(reason: str) -> str:
|
|
if reason == "success":
|
|
return "success"
|
|
if reason == "timed_out":
|
|
return "timed_out"
|
|
if reason == "cancelled":
|
|
return "cancelled"
|
|
if reason in RECEIPT_ONLY_TERMINAL_REASONS:
|
|
return "interrupted"
|
|
return "failed"
|
|
|
|
def _read_json_file(self, root: Path, name: str) -> dict[str, Any]:
|
|
path = root / name
|
|
try:
|
|
value = json.loads(_read_regular_bytes(path, name).decode("utf-8"))
|
|
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
|
raise AttemptStateError(f"{name} is invalid") from exc
|
|
if not isinstance(value, dict):
|
|
raise AttemptStateError(f"{name} is invalid")
|
|
return value
|
|
|
|
@staticmethod
|
|
def _exact_fields(value: Any, fields: set[str], label: str) -> dict[str, Any]:
|
|
if not isinstance(value, dict) or set(value) != fields:
|
|
raise AttemptStateError(f"{label} schema is invalid")
|
|
return value
|
|
|
|
@staticmethod
|
|
def _optional_int(value: Any) -> bool:
|
|
return value is None or (isinstance(value, int) and not isinstance(value, bool))
|
|
|
|
def _validate_result_record(self, result: dict[str, Any], locator: SupervisorLocator, expected_digest: str) -> None:
|
|
fields = {
|
|
"record", "success", "terminal_reason", "exit_code", "signal", "submitted",
|
|
"finish_then_idle_then_quiet", "cleanup_complete", "process_group_alive",
|
|
"submission_mode", "completion_mode", "spec_digest", "locator", "started_at",
|
|
"ended_at", "duration_ns", "stdout", "stderr", "events",
|
|
}
|
|
self._exact_fields(result, fields, "lifecycle result")
|
|
required_bools = ("success", "submitted", "finish_then_idle_then_quiet", "cleanup_complete", "process_group_alive")
|
|
if result["record"] != "result" or any(not isinstance(result[key], bool) for key in required_bools):
|
|
raise AttemptStateError("lifecycle result is invalid")
|
|
if result["terminal_reason"] not in TERMINAL_REASONS or result["spec_digest"] != expected_digest:
|
|
raise AttemptStateError("lifecycle result identity is invalid")
|
|
if not self._optional_int(result["exit_code"]) or not self._optional_int(result["signal"]):
|
|
raise AttemptStateError("lifecycle result is invalid")
|
|
if result["submission_mode"] not in SUBMISSION_MODES or result["completion_mode"] not in COMPLETION_MODES:
|
|
raise AttemptStateError("lifecycle result is invalid")
|
|
if not all(isinstance(result[key], str) for key in ("started_at", "ended_at")) or not isinstance(result["duration_ns"], int) or isinstance(result["duration_ns"], bool) or result["duration_ns"] < 0:
|
|
raise AttemptStateError("lifecycle result is invalid")
|
|
expected_public = self._public_locator(locator)
|
|
if result["locator"] != expected_public:
|
|
raise AttemptStateError("lifecycle terminal locator is invalid")
|
|
for name, stream in (("stdout", "stdout"), ("stderr", "stderr")):
|
|
capture = self._exact_fields(result[name], {"stream", "text", "line_count", "byte_count", "truncated"}, f"{name} capture")
|
|
if capture["stream"] != stream or not isinstance(capture["text"], str) or not isinstance(capture["line_count"], int) or not isinstance(capture["byte_count"], int) or isinstance(capture["line_count"], bool) or isinstance(capture["byte_count"], bool) or capture["line_count"] < 0 or capture["byte_count"] < 0 or not isinstance(capture["truncated"], bool):
|
|
raise AttemptStateError("lifecycle capture is invalid")
|
|
if not isinstance(result["events"], list):
|
|
raise AttemptStateError("lifecycle events are invalid")
|
|
for event in result["events"]:
|
|
checked = self._exact_fields(event, {"record", "kind", "source", "stream", "monotonic_ns", "source_monotonic_ns", "observed_at", "detail"}, "lifecycle event")
|
|
if checked["record"] != "event" or not all(isinstance(checked[key], str) for key in ("kind", "source", "stream", "observed_at", "detail")) or not all(isinstance(checked[key], int) and not isinstance(checked[key], bool) and checked[key] >= 0 for key in ("monotonic_ns", "source_monotonic_ns")):
|
|
raise AttemptStateError("lifecycle event is invalid")
|
|
if not result["cleanup_complete"] or result["process_group_alive"] or result["success"] != (result["terminal_reason"] == "success"):
|
|
raise AttemptStateError("lifecycle terminal outcome is invalid")
|
|
if result["success"] and not result["finish_then_idle_then_quiet"]:
|
|
raise AttemptStateError("lifecycle terminal outcome is invalid")
|
|
|
|
@staticmethod
|
|
def _instant(value: Any, label: str) -> _datetime.datetime:
|
|
"""Parse one produced ISO-8601 instant into a comparable UTC value."""
|
|
try:
|
|
parsed = _datetime.datetime.fromisoformat(str(value))
|
|
except (TypeError, ValueError) as exc:
|
|
raise AttemptStateError(f"{label} timestamp is invalid") from exc
|
|
return parsed if parsed.tzinfo is not None else parsed.replace(tzinfo=_datetime.timezone.utc)
|
|
|
|
@staticmethod
|
|
def _validate_terminal_events(events: list[Any]) -> dict[str, int]:
|
|
"""Return the unique ordinal of every success-evidence event kind."""
|
|
positions: dict[str, int] = {}
|
|
for index, event in enumerate(events):
|
|
kind = event["kind"]
|
|
if kind not in SUCCESS_EVIDENCE_KINDS:
|
|
continue
|
|
if kind in positions:
|
|
raise AttemptStateError("lifecycle events are invalid")
|
|
positions[kind] = index
|
|
return positions
|
|
|
|
def _validate_terminal_coherence(self, result: dict[str, Any], receipt: dict[str, Any], positions: dict[str, int]) -> None:
|
|
"""Bind result, ordered events and cleanup receipt to one terminal projection."""
|
|
submitted, finish, idle, quiet = (positions.get(kind) for kind in SUCCESS_EVIDENCE_KINDS)
|
|
ordered = finish is not None and idle is not None and quiet is not None and finish < idle < quiet
|
|
if ordered != result["finish_then_idle_then_quiet"]:
|
|
raise AttemptStateError("lifecycle ordered evidence is invalid")
|
|
if result["exit_code"] != receipt["exit_code"] or result["signal"] != receipt["signal"]:
|
|
raise AttemptStateError("lifecycle terminal outcome is invalid")
|
|
if result["submitted"] and not receipt["caller_launched"]:
|
|
raise AttemptStateError("lifecycle terminal outcome is invalid")
|
|
if result["success"] and (not result["submitted"] or submitted is None or finish is None or submitted > finish):
|
|
raise AttemptStateError("lifecycle success evidence is invalid")
|
|
started = self._instant(result["started_at"], "lifecycle result")
|
|
ended = self._instant(result["ended_at"], "lifecycle result")
|
|
completed = self._instant(receipt["completed_at"], "cleanup receipt")
|
|
if completed < started or ended < completed:
|
|
raise AttemptStateError("lifecycle terminal chronology is invalid")
|
|
|
|
def _validate_receipt_record(self, receipt: dict[str, Any], locator: SupervisorLocator) -> dict[str, Any]:
|
|
"""Validate one cleanup receipt against the registered supervisor identity."""
|
|
self._exact_fields(receipt, RECEIPT_FIELDS, "cleanup receipt")
|
|
if receipt["receipt_version"] != RECEIPT_VERSION or receipt["supervisor_pid"] != locator.supervisor_pid or receipt["challenge_digest"] != self._public_locator(locator)["challenge_digest"] or receipt["reason"] not in TERMINAL_REASONS or not self._optional_int(receipt["exit_code"]) or not self._optional_int(receipt["signal"]) or not isinstance(receipt["caller_launched"], bool) or receipt["cleanup_complete"] is not True or receipt["process_group_alive"] is not False or not isinstance(receipt["completed_at"], str):
|
|
raise AttemptStateError("cleanup receipt is invalid")
|
|
return receipt
|
|
|
|
def _closed_cleanup_receipt(
|
|
self,
|
|
root: Path,
|
|
locator: SupervisorLocator,
|
|
*,
|
|
expected_reason: str = REASON_CONTROLLER_LOST,
|
|
required: bool = False,
|
|
) -> dict[str, Any] | None:
|
|
"""Read one canonical, authenticated receipt after its socket is closed."""
|
|
if expected_reason not in RECEIPT_ONLY_TERMINAL_REASONS:
|
|
raise AttemptStateError("closed cleanup receipt reason is invalid")
|
|
control = root / CONTROL_DIRECTORY_NAME
|
|
try:
|
|
control_mode = os.lstat(control).st_mode
|
|
except OSError as exc:
|
|
raise AttemptStateError("attempt control directory is unavailable") from exc
|
|
if not stat.S_ISDIR(control_mode):
|
|
raise AttemptStateError("attempt control directory is invalid")
|
|
|
|
registered = self._read_json_file(control, "locator.json")
|
|
expected_locator = {
|
|
"supervisor_pid": locator.supervisor_pid,
|
|
"start_identity": locator.start_identity,
|
|
"socket_path": locator.socket_path,
|
|
"challenge": locator.challenge,
|
|
"control_dir": locator.control_dir,
|
|
"created_at": locator.created_at,
|
|
}
|
|
if registered != expected_locator:
|
|
raise AttemptStateError("registered locator is invalid")
|
|
|
|
receipt_path = control / "cleanup-receipt.json"
|
|
try:
|
|
os.lstat(receipt_path)
|
|
except FileNotFoundError:
|
|
if required:
|
|
raise AttemptStateError("closed cleanup receipt is unavailable")
|
|
return None
|
|
except OSError as exc:
|
|
raise AttemptStateError("closed cleanup receipt is unavailable") from exc
|
|
receipt = self._validate_receipt_record(
|
|
self._read_json_file(control, receipt_path.name), locator
|
|
)
|
|
# Result-bound receipts get their instant parsed by terminal coherence; the
|
|
# receipt-only path is the sole authority here, so parse it independently.
|
|
self._instant(receipt["completed_at"], "cleanup receipt")
|
|
if receipt["reason"] != expected_reason:
|
|
raise AttemptStateError("closed cleanup receipt is invalid")
|
|
|
|
socket_path = control / SOCKET_FILENAME
|
|
try:
|
|
socket_mode = os.lstat(socket_path).st_mode
|
|
except FileNotFoundError:
|
|
return receipt
|
|
except OSError as exc:
|
|
raise AttemptStateError("closed cleanup socket is unavailable") from exc
|
|
if not stat.S_ISSOCK(socket_mode):
|
|
raise AttemptStateError("closed cleanup socket is invalid")
|
|
if expected_reason == REASON_CONTROLLER_LOST:
|
|
raise AttemptStateError("closed cleanup socket is still active")
|
|
return receipt
|
|
|
|
@staticmethod
|
|
def _public_locator(locator: SupervisorLocator) -> dict[str, Any]:
|
|
return {
|
|
"supervisor_pid": locator.supervisor_pid,
|
|
"start_identity": locator.start_identity,
|
|
"socket_path": locator.socket_path,
|
|
"control_dir": locator.control_dir,
|
|
"challenge_digest": hashlib.sha256(locator.challenge.encode("utf-8")).hexdigest(),
|
|
"created_at": locator.created_at,
|
|
}
|
|
|
|
def _validate_journal_record(self, root: Path, result: dict[str, Any], expected_digest: str) -> None:
|
|
"""Validate the append-only journal against the published result record."""
|
|
journal = root / "lifecycle-journal.jsonl"
|
|
try:
|
|
lines = [json.loads(line) for line in _read_regular_bytes(journal, "lifecycle journal").decode("utf-8").splitlines()]
|
|
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
|
raise AttemptStateError("lifecycle journal is invalid") from exc
|
|
if len(lines) < 2:
|
|
raise AttemptStateError("lifecycle journal is invalid")
|
|
header, terminal = lines[0], lines[-1]
|
|
self._exact_fields(header, {"record", "journal_version", "spec_digest", "submission_mode", "completion_mode", "started_at"}, "lifecycle journal header")
|
|
self._exact_fields(terminal, {"record", "terminal_reason", "success", "cleanup_complete", "process_group_alive", "ended_at"}, "lifecycle journal terminal")
|
|
if header["record"] != "header" or header["journal_version"] != 1 or header["spec_digest"] != expected_digest or header["submission_mode"] != result["submission_mode"] or header["completion_mode"] != result["completion_mode"] or header["started_at"] != result["started_at"] or not isinstance(header["started_at"], str) or terminal["record"] != "terminal" or terminal["terminal_reason"] != result["terminal_reason"] or terminal["success"] != result["success"] or terminal["cleanup_complete"] is not True or terminal["process_group_alive"] is not False or terminal["ended_at"] != result["ended_at"] or not isinstance(terminal["ended_at"], str):
|
|
raise AttemptStateError("lifecycle journal is invalid")
|
|
for event in lines[1:-1]:
|
|
self._exact_fields(event, {"record", "kind", "source", "stream", "monotonic_ns", "source_monotonic_ns", "observed_at", "detail"}, "lifecycle journal event")
|
|
if lines[1:-1] != result["events"]:
|
|
raise AttemptStateError("lifecycle journal events are invalid")
|
|
|
|
def _read_bound_lifecycle_terminal(self, root: Path, locator: SupervisorLocator, expected_digest: str) -> dict[str, Any] | None:
|
|
result_path = root / "lifecycle-result.json"
|
|
if not result_path.exists() and not result_path.is_symlink():
|
|
return None
|
|
result = self._read_json_file(root, "lifecycle-result.json")
|
|
self._validate_result_record(result, locator, expected_digest)
|
|
self._validate_journal_record(root, result, expected_digest)
|
|
# ``locator.control_dir`` is the live short lease alias. It is
|
|
# intentionally removed after terminal publication, so terminal reads
|
|
# must rebind the receipt through the immutable attempt-owned control
|
|
# directory rather than resolve a stale alias.
|
|
control = root / CONTROL_DIRECTORY_NAME
|
|
_directory(control, "attempt control directory")
|
|
registered = self._read_json_file(control, "locator.json")
|
|
expected_locator = {
|
|
"supervisor_pid": locator.supervisor_pid,
|
|
"start_identity": locator.start_identity,
|
|
"socket_path": locator.socket_path,
|
|
"challenge": locator.challenge,
|
|
"control_dir": locator.control_dir,
|
|
"created_at": locator.created_at,
|
|
}
|
|
if registered != expected_locator:
|
|
raise AttemptStateError("registered locator is invalid")
|
|
receipt_data = self._validate_receipt_record(
|
|
self._read_json_file(control, "cleanup-receipt.json"), locator
|
|
)
|
|
if receipt_data["reason"] != result["terminal_reason"]:
|
|
raise AttemptStateError("cleanup receipt is invalid")
|
|
events = self._validate_terminal_events(result["events"])
|
|
self._validate_terminal_coherence(result, receipt_data, events)
|
|
return result
|
|
|
|
def validate_invocation_terminal(self, attempt: Attempt, invocation: InvocationResult) -> dict[str, Any]:
|
|
"""Validate direct lifecycle output against the identity committed before launch."""
|
|
if not isinstance(invocation, InvocationResult):
|
|
raise AttemptStateError("invocation result is invalid")
|
|
run, root = self._bound_attempt(attempt)
|
|
record = self._attempt_record(root, run, attempt.identity)
|
|
if record is None or record["state"] != NONTERMINAL_STATE:
|
|
raise AttemptStateError("invocation transition is invalid")
|
|
raw_locator = record.get("locator")
|
|
expected_digest = record.get("spec_digest")
|
|
if raw_locator is None or not isinstance(expected_digest, str):
|
|
raise AttemptStateError("invocation identity was not committed")
|
|
locator = self._locator_from_record(
|
|
root, raw_locator, state=NONTERMINAL_STATE
|
|
)
|
|
expected_result_path = root / "lifecycle-result.json"
|
|
expected_journal_path = root / "lifecycle-journal.jsonl"
|
|
if invocation.locator != locator or invocation.spec_digest != expected_digest or Path(invocation.result_path).resolve(strict=False) != expected_result_path.resolve() or Path(invocation.journal_path).resolve(strict=False) != expected_journal_path.resolve():
|
|
raise AttemptStateError("invocation result identity is invalid")
|
|
terminal = self._read_bound_lifecycle_terminal(root, locator, expected_digest)
|
|
if terminal is None:
|
|
raise AttemptStateError("lifecycle terminal is unavailable")
|
|
self._validate_measurement(
|
|
root, run, attempt.identity, expected_digest,
|
|
str(terminal["terminal_reason"]),
|
|
record.get("measurement_policy"), terminal,
|
|
)
|
|
self._validate_web_validation(
|
|
root,
|
|
run,
|
|
attempt.identity,
|
|
record.get("web_validation_policy"),
|
|
)
|
|
fields = ("success", "terminal_reason", "exit_code", "signal", "submitted", "finish_then_idle_then_quiet", "cleanup_complete", "process_group_alive", "spec_digest", "started_at", "ended_at", "duration_ns")
|
|
if any(getattr(invocation, field) != terminal[field] for field in fields):
|
|
raise AttemptStateError("invocation result does not match durable terminal")
|
|
return terminal
|
|
|
|
def publish_attempt_measurement(
|
|
self,
|
|
attempt: Attempt,
|
|
caller: str,
|
|
result: InvocationResult,
|
|
observation: WorkspaceWriteObservation,
|
|
) -> None:
|
|
"""Publish one immutable timing/usage sidecar before terminal commit.
|
|
|
|
Publication happens while the attempt is still running so a collision,
|
|
a corrupt projection or an identity mismatch fails closed before any
|
|
terminal state is written, and never rewrites bytes it does not own.
|
|
"""
|
|
if not isinstance(result, InvocationResult):
|
|
raise AttemptStateError("invocation result is invalid")
|
|
run, root = self._bound_attempt(attempt)
|
|
record = self._attempt_record(root, run, attempt.identity)
|
|
if record is None or record["state"] != NONTERMINAL_STATE:
|
|
raise AttemptStateError("measurement requires a running attempt")
|
|
expected_digest = record.get("spec_digest")
|
|
if not isinstance(expected_digest, str):
|
|
# The invocation identity was never committed, so terminal
|
|
# validation owns this failure and there is nothing to bind to.
|
|
return
|
|
if expected_digest != result.spec_digest:
|
|
raise AttemptStateError("measurement invocation identity is invalid")
|
|
try:
|
|
publish_measurement(root, build_measurement(
|
|
run_id=run.run_id,
|
|
cell_id=attempt.identity.cell_id,
|
|
repetition=attempt.identity.repetition,
|
|
attempt=attempt.identity.attempt,
|
|
caller=caller,
|
|
result=result,
|
|
observation=observation,
|
|
))
|
|
except MeasurementError as exc:
|
|
raise AttemptStateError("attempt measurement is invalid") from exc
|
|
self._validate_measurement(
|
|
root, run, attempt.identity, expected_digest, result.terminal_reason,
|
|
record.get("measurement_policy"),
|
|
)
|
|
|
|
def publish_attempt_web_validation(
|
|
self, attempt: Attempt, web_validation: Any,
|
|
) -> None:
|
|
"""Publish one web sidecar while the attempt is running.
|
|
|
|
The sidecar is deliberately independent from lifecycle success: a page
|
|
may fail a quality gate while its caller lifecycle remains successful.
|
|
It is nevertheless required provenance for every production attempt.
|
|
"""
|
|
run, root = self._bound_attempt(attempt)
|
|
record = self._attempt_record(root, run, attempt.identity)
|
|
if record is None or record["state"] != NONTERMINAL_STATE:
|
|
raise AttemptStateError("web validation requires a running attempt")
|
|
if record.get("web_validation_policy") != WEB_VALIDATION_POLICY_REQUIRED_V1:
|
|
raise AttemptStateError("web validation policy is unavailable")
|
|
try:
|
|
publish_web_validation(root, web_validation)
|
|
except WebValidationError as exc:
|
|
raise AttemptStateError("attempt web validation is invalid") from exc
|
|
self._validate_web_validation(
|
|
root, run, attempt.identity, record.get("web_validation_policy"),
|
|
)
|
|
|
|
def reconcile(self, attempt: Attempt) -> Attempt:
|
|
"""Commit only authenticated terminal evidence before recovery cleanup."""
|
|
run, root = self._bound_attempt(attempt)
|
|
record = self._attempt_record(root, run, attempt.identity, absent_ok=True)
|
|
if record is None:
|
|
terminal = self.publish_terminal(
|
|
attempt, "interrupted", result={"terminal_reason": "interrupted"}
|
|
)
|
|
self.release_control_lease(terminal)
|
|
return terminal
|
|
if record["state"] in TERMINAL_STATES:
|
|
terminal = Attempt(attempt.identity, attempt.root, str(record["state"]))
|
|
self.release_control_lease(terminal)
|
|
return terminal
|
|
raw_locator = record.get("locator")
|
|
if raw_locator is None:
|
|
terminal = self.publish_terminal(
|
|
attempt, "interrupted", result={"terminal_reason": "interrupted"}
|
|
)
|
|
self.release_control_lease(terminal)
|
|
return terminal
|
|
locator = self._locator_from_record(
|
|
root, raw_locator, state=NONTERMINAL_STATE
|
|
)
|
|
expected_digest = record.get("spec_digest")
|
|
if not isinstance(expected_digest, str) or not DIGEST_RE.fullmatch(expected_digest):
|
|
raise AttemptStateError("recovery identity is invalid")
|
|
terminal = self._read_bound_lifecycle_terminal(root, locator, expected_digest)
|
|
if terminal is not None:
|
|
self._validate_measurement(
|
|
root, run, attempt.identity, expected_digest,
|
|
str(terminal["terminal_reason"]),
|
|
record.get("measurement_policy"), terminal,
|
|
)
|
|
self._ensure_required_web_validation(
|
|
root, run, attempt.identity, record, terminal
|
|
)
|
|
published = self.publish_terminal(
|
|
attempt, self._state_for_reason(terminal["terminal_reason"]), result=terminal
|
|
)
|
|
self.release_control_lease(published)
|
|
return published
|
|
closed_receipt = self._closed_cleanup_receipt(root, locator)
|
|
if closed_receipt is not None:
|
|
recovery_terminal = {"terminal_reason": closed_receipt["reason"]}
|
|
self._ensure_required_web_validation(
|
|
root, run, attempt.identity, record, recovery_terminal
|
|
)
|
|
published = self.publish_terminal(
|
|
attempt,
|
|
self._state_for_reason(recovery_terminal["terminal_reason"]),
|
|
result=recovery_terminal,
|
|
)
|
|
self.release_control_lease(published)
|
|
return published
|
|
try:
|
|
outcome = recover_invocation(locator, stop=True)
|
|
except LifecycleRecoveryError as exc:
|
|
# The supervisor may complete cleanup and durably publish its
|
|
# authenticated receipt immediately before the control reply is
|
|
# lost. Re-read that exact receipt instead of treating a missing
|
|
# reply as proof that cleanup did not happen.
|
|
try:
|
|
recovered_receipt = self._closed_cleanup_receipt(
|
|
root,
|
|
locator,
|
|
expected_reason=REASON_RECOVERED_STOP,
|
|
)
|
|
except AttemptStateError:
|
|
raise AttemptStateError("recovery is unverified") from exc
|
|
if recovered_receipt is None:
|
|
raise AttemptStateError("recovery is unverified") from exc
|
|
recovery_terminal = {
|
|
"terminal_reason": recovered_receipt["reason"]
|
|
}
|
|
self._ensure_required_web_validation(
|
|
root, run, attempt.identity, record, recovery_terminal
|
|
)
|
|
terminal = self.publish_terminal(
|
|
attempt,
|
|
self._state_for_reason(recovery_terminal["terminal_reason"]),
|
|
result=recovery_terminal,
|
|
)
|
|
self.release_control_lease(terminal)
|
|
return terminal
|
|
receipt = Path(outcome.receipt_path)
|
|
if not outcome.cleanup_complete or outcome.process_group_alive or receipt.parent != Path(locator.control_dir) or not _contained(receipt, root):
|
|
raise AttemptStateError("recovery cleanup is unverified")
|
|
try:
|
|
self._validate_receipt_record(self._read_json_file(receipt.parent, receipt.name), locator)
|
|
except AttemptStateError as exc:
|
|
raise AttemptStateError("recovery cleanup is unverified") from exc
|
|
recovery_terminal = {"terminal_reason": outcome.reason}
|
|
self._ensure_required_web_validation(
|
|
root, run, attempt.identity, record, recovery_terminal
|
|
)
|
|
terminal = self.publish_terminal(
|
|
attempt,
|
|
self._state_for_reason(recovery_terminal["terminal_reason"]),
|
|
result=recovery_terminal,
|
|
)
|
|
self.release_control_lease(terminal)
|
|
return terminal
|
|
|
|
def execute_attempt(
|
|
self,
|
|
attempt: Attempt,
|
|
*,
|
|
prepare: Callable[[Attempt], Any],
|
|
invoke: Callable[[Attempt, Callable[[SupervisorLocator, str], None]], InvocationResult],
|
|
require_measurement: bool = False,
|
|
require_web_validation: bool = False,
|
|
) -> Attempt:
|
|
"""Prepare once, publish running identity, then invoke lifecycle once."""
|
|
run, root = self._bound_attempt(attempt)
|
|
if self._attempt_record(root, run, attempt.identity, absent_ok=True) is not None:
|
|
raise AttemptStateError("attempt has already been prepared")
|
|
try:
|
|
prepare(attempt)
|
|
except Exception:
|
|
_write_new(root / "attempt.json", _json_bytes(self._initial_record(run, attempt, "failed", reason="preparation_failed")))
|
|
raise
|
|
_write_new(
|
|
root / "attempt.json",
|
|
_json_bytes(self._initial_record(
|
|
run, attempt, NONTERMINAL_STATE,
|
|
measurement_policy=(MEASUREMENT_POLICY_REQUIRED_V1
|
|
if require_measurement else None),
|
|
web_validation_policy=(WEB_VALIDATION_POLICY_REQUIRED_V1
|
|
if require_web_validation else None),
|
|
)),
|
|
)
|
|
if require_measurement:
|
|
_write_new(
|
|
root / MEASUREMENT_POLICY_FILENAME,
|
|
_json_bytes(self._measurement_policy_start_record(run, attempt.identity)),
|
|
)
|
|
if require_web_validation:
|
|
_write_new(
|
|
root / WEB_VALIDATION_POLICY_FILENAME,
|
|
_json_bytes(self._web_validation_policy_start_record(run, attempt.identity)),
|
|
)
|
|
result = invoke(attempt, lambda locator, digest: self.record_locator(attempt, locator, digest))
|
|
terminal = self.validate_invocation_terminal(attempt, result)
|
|
published = self.publish_terminal(
|
|
attempt, self._state_for_reason(terminal["terminal_reason"]), result=terminal
|
|
)
|
|
self.release_control_lease(published)
|
|
return published
|
|
|
|
def status(self, run: RunIdentity, manifest: Manifest) -> dict[str, Any]:
|
|
"""Read-only deterministic status; it neither creates nor reconciles."""
|
|
bound_run = self.open(manifest, run.run_id)
|
|
if bound_run != run:
|
|
raise AttemptStateError("run identity is invalid")
|
|
states = {name: 0 for name in sorted(TERMINAL_STATES | {NONTERMINAL_STATE})}
|
|
for slot in self.slots(manifest):
|
|
for attempt in self.attempts(bound_run, slot):
|
|
states[attempt.state] += 1
|
|
preflights = self._preflight_records(bound_run, manifest)
|
|
latest = preflights[-1] if preflights else None
|
|
projection = {
|
|
"records": len(preflights),
|
|
"latest_sequence": 0 if latest is None else latest["sequence"],
|
|
"latest_status": "unavailable" if latest is None else latest["status"],
|
|
**(
|
|
{status: 0 for status in PREFLIGHT_STATUSES}
|
|
if latest is None
|
|
else _preflight_counts(latest["results"])
|
|
),
|
|
}
|
|
return {
|
|
"run_id": bound_run.run_id,
|
|
"manifest_digest": bound_run.manifest_digest,
|
|
"preflight": projection,
|
|
"attempts": states,
|
|
}
|
|
|
|
|
|
def preflight_manifest(
|
|
store: RunStore,
|
|
manifest: Manifest,
|
|
manifest_bytes: bytes,
|
|
*,
|
|
adapters: Mapping[str, PreflightAdapter],
|
|
) -> tuple[RunIdentity, dict[str, Any]]:
|
|
"""Collect direct observations, then create one run and append one record."""
|
|
observations = collect_preflight_observations(manifest, adapters)
|
|
if not observations:
|
|
raise AttemptStateError("preflight requires a direct cell")
|
|
run = store.create(manifest, manifest_bytes)
|
|
return run, store.record_preflight(run, manifest, observations)
|
|
|
|
|
|
def _validate_prepared_binding(
|
|
store: RunStore,
|
|
manifest: Manifest,
|
|
cell: MatrixCell,
|
|
attempt: Attempt,
|
|
prepared: PreparedWorkspace,
|
|
) -> None:
|
|
"""Reject any caller/cell/workspace identity drift before invocation."""
|
|
if not isinstance(prepared, PreparedWorkspace):
|
|
raise AttemptStateError("prepared workspace is invalid")
|
|
if cell.id != attempt.identity.cell_id or prepared.identity != attempt.identity:
|
|
raise AttemptStateError("prepared workspace identity mismatch")
|
|
|
|
attempt_root = Path(attempt.root).resolve()
|
|
workspace = Path(prepared.workspace_dir)
|
|
session = Path(prepared.session_dir)
|
|
if Path(prepared.attempt_root).resolve() != attempt_root:
|
|
raise AttemptStateError("prepared attempt root mismatch")
|
|
if (
|
|
workspace.is_symlink()
|
|
or session.is_symlink()
|
|
or not workspace.is_dir()
|
|
or not session.is_dir()
|
|
or workspace.resolve() != attempt_root / "workspace"
|
|
or session.resolve() != attempt_root / "session"
|
|
):
|
|
raise AttemptStateError("prepared workspace path mismatch")
|
|
if (
|
|
not prepared.session_is_fresh
|
|
or not isinstance(prepared.session_id, str)
|
|
or not prepared.session_id
|
|
or prepared.workspace_checksum != manifest.fixture.checksum
|
|
or prepared.setup_cache_policy != manifest.setup_cache_policy
|
|
):
|
|
raise AttemptStateError("prepared workspace policy mismatch")
|
|
expected_testbed = (store.repo_root / manifest.testbed).resolve()
|
|
if (
|
|
not prepared.testbed_provenance.clean
|
|
or Path(prepared.testbed_provenance.path).resolve() != expected_testbed
|
|
):
|
|
raise AttemptStateError("prepared testbed provenance mismatch")
|
|
|
|
|
|
def run_slots(
|
|
store: RunStore,
|
|
run: RunIdentity,
|
|
manifest: Manifest,
|
|
*,
|
|
adapters: Mapping[str, ExecutionAdapter],
|
|
prepare: Callable[[Manifest, Attempt], PreparedWorkspace],
|
|
retry_failed: bool = False,
|
|
) -> tuple[Attempt, ...]:
|
|
"""Append fresh preflight, then execute eligible slots under one writer."""
|
|
if not isinstance(adapters, Mapping):
|
|
raise CapabilityUnavailable("capability-unavailable: caller-adapter")
|
|
required_callers = {cell.caller for cell in manifest.matrix}
|
|
if any(
|
|
caller not in adapters or not callable(getattr(adapters[caller], "invoke", None))
|
|
for caller in required_callers
|
|
):
|
|
raise CapabilityUnavailable("capability-unavailable: caller-adapter")
|
|
if not callable(prepare):
|
|
raise AttemptStateError("workspace preparer is invalid")
|
|
|
|
observations = collect_preflight_observations(manifest, adapters)
|
|
if not observations:
|
|
raise AttemptStateError("preflight requires a direct cell")
|
|
bound_run = store.open(manifest, run.run_id)
|
|
if bound_run != run:
|
|
raise AttemptStateError("run identity is invalid")
|
|
cells = {cell.id: cell for cell in manifest.matrix}
|
|
if len(cells) != len(manifest.matrix):
|
|
raise AttemptStateError("manifest cell identity is invalid")
|
|
completed: list[Attempt] = []
|
|
with store.writer(bound_run):
|
|
preflight = store._record_preflight_locked(
|
|
bound_run, manifest, observations
|
|
)
|
|
if preflight["status"] != "ready":
|
|
return ()
|
|
if frozenset(observations) != frozenset(cells):
|
|
return ()
|
|
for slot in store.slots(manifest):
|
|
cell = cells.get(slot.cell_id)
|
|
if cell is None:
|
|
raise AttemptStateError("slot cell identity is invalid")
|
|
existing = store.attempts(bound_run, slot)
|
|
if existing and existing[-1].state == "success":
|
|
continue
|
|
if existing and existing[-1].state in {"failed", "timed_out", "cancelled"} and not retry_failed:
|
|
continue
|
|
if existing and existing[-1].state not in TERMINAL_STATES:
|
|
store.reconcile(existing[-1])
|
|
existing = store.attempts(bound_run, slot)
|
|
if existing[-1].state == "success" or (existing[-1].state in {"failed", "timed_out", "cancelled"} and not retry_failed):
|
|
continue
|
|
attempt = store.allocate(bound_run, slot)
|
|
prepared: PreparedWorkspace | None = None
|
|
|
|
def prepare_bound(current: Attempt) -> PreparedWorkspace:
|
|
nonlocal prepared
|
|
candidate = prepare(manifest, current)
|
|
_validate_prepared_binding(store, manifest, cell, current, candidate)
|
|
prepared = candidate
|
|
return candidate
|
|
|
|
def invoke_bound(
|
|
current: Attempt,
|
|
on_started: Callable[[SupervisorLocator, str], None],
|
|
) -> InvocationResult:
|
|
if prepared is None:
|
|
raise AttemptStateError("prepared workspace is unavailable")
|
|
lease = store.acquire_control_lease(current)
|
|
# The observer's baseline must be older than any caller write,
|
|
# so it starts before the caller can be launched and is joined
|
|
# on every success, error, timeout and cancellation path.
|
|
observer = WorkspaceWriteObserver(prepared.workspace_dir)
|
|
observer.start()
|
|
try:
|
|
result = adapters[cell.caller].invoke(
|
|
cell,
|
|
prepared,
|
|
current,
|
|
lease.control_dir,
|
|
manifest.fixture.prompt_content,
|
|
manifest.timeout,
|
|
on_started,
|
|
)
|
|
finally:
|
|
observation = observer.stop()
|
|
if not observer.stopped:
|
|
raise AttemptStateError("workspace observer did not stop")
|
|
store.publish_attempt_measurement(
|
|
current, cell.caller, result, observation
|
|
)
|
|
measurement = load_measurement(current.root)
|
|
web = validate_web_attempt(
|
|
manifest, current.root, prepared, measurement, result,
|
|
)
|
|
store.publish_attempt_web_validation(current, web)
|
|
return result
|
|
|
|
completed.append(
|
|
store.execute_attempt(
|
|
attempt,
|
|
prepare=prepare_bound,
|
|
invoke=invoke_bound,
|
|
require_measurement=True,
|
|
require_web_validation=True,
|
|
)
|
|
)
|
|
return tuple(completed)
|