iop/scripts/agent_benchmark/attempts.py
toki 029ff0d2c8 feat(benchmark): 비교 파이프라인을 완성한다
동일한 IOP 경유 과업을 caller와 model 설정만 바꿔 재현하고, 실패를 포함한 실행·검증·채점 근거를 보존할 수 있어야 한다.
2026-08-12 01:44:26 +09:00

2035 lines
92 KiB
Python

"""Durable, append-only benchmark run and attempt state.
Attempt-directory creation is the allocation marker. The directory remains
empty until the workspace layer has prepared it; only then is the identity
bound running record published. This preserves the workspace API's
empty-root contract while retaining a crash-recoverable allocation boundary.
"""
from __future__ import annotations
import contextlib
import datetime as _datetime
import fcntl
import hashlib
import json
import os
import re
import secrets
import stat
import tempfile
from dataclasses import dataclass
from pathlib import Path
from types import SimpleNamespace
from typing import Any, Callable, Iterator, Mapping, Protocol
from scripts.agent_benchmark.connectivity import (
CallerCapability,
ConnectivityIssue,
ConnectivityResult,
EffectiveBinding,
RequestedEffectiveBinding,
canonical_evidence_bytes,
validate_requested_binding,
validate_result,
)
from scripts.agent_benchmark.lifecycle import (
COMPLETION_MODES,
EVENT_FINISH,
EVENT_IDLE,
EVENT_QUIET,
EVENT_SUBMITTED,
InvocationResult,
LifecycleRecoveryError,
RECEIPT_VERSION,
REASON_CONTROLLER_LOST,
REASON_RECOVERED_STOP,
SOCKET_FILENAME,
SUBMISSION_MODES,
SupervisorLocator,
TERMINAL_REASONS,
recover_invocation,
)
from scripts.agent_benchmark.manifest import (
Manifest,
MatrixCell,
Timeout,
validate_manifest_bytes,
)
from scripts.agent_benchmark.measurement import (
MEASUREMENT_FILENAME,
MeasurementError,
WorkspaceWriteObservation,
WorkspaceWriteObserver,
build_measurement,
load_measurement,
publish_measurement,
validate_measurement_lifecycle_binding,
)
from scripts.agent_benchmark.web_validation import (
WEB_VALIDATION_FILENAME,
WebValidationError,
load_web_validation,
publish_web_validation,
validate_web_attempt,
)
from scripts.agent_benchmark.workspace import AttemptIdentity, PreparedWorkspace
RUN_ID_RE = re.compile(r"^run-[0-9]{8}T[0-9]{6}Z-[0-9a-f]{12}$")
ATTEMPT_RE = re.compile(r"^attempt-([0-9]{6})$")
PREFLIGHT_RE = re.compile(r"^preflight-([0-9]{6})\.json$")
DIGEST_RE = re.compile(r"^sha256:[0-9a-f]{64}$")
PREFLIGHT_SCHEMA_VERSION = "1"
PREFLIGHT_STATUSES = ("ready", "registration_required", "implementation_gap")
TERMINAL_STATES = frozenset(("success", "failed", "timed_out", "cancelled", "interrupted"))
NONTERMINAL_STATE = "running"
MEASUREMENT_POLICY_REQUIRED_V1 = "required-v1"
MEASUREMENT_POLICIES = frozenset((MEASUREMENT_POLICY_REQUIRED_V1,))
MEASUREMENT_POLICY_FILENAME = "attempt-measurement-policy.json"
MEASUREMENT_POLICY_RECORD = "attempt-measurement-policy"
MEASUREMENT_POLICY_VERSION = 1
WEB_VALIDATION_POLICY_REQUIRED_V1 = "required-v1"
WEB_VALIDATION_POLICIES = frozenset((WEB_VALIDATION_POLICY_REQUIRED_V1,))
WEB_VALIDATION_POLICY_FILENAME = "web-validation-policy.json"
WEB_VALIDATION_POLICY_RECORD = "web-validation-policy"
WEB_VALIDATION_POLICY_VERSION = 1
SUCCESS_EVIDENCE_KINDS = (EVENT_SUBMITTED, EVENT_FINISH, EVENT_IDLE, EVENT_QUIET)
CONTROL_ALIAS_PREFIX = "iop-bench-attempt-"
CONTROL_DIRECTORY_NAME = "control"
CONTROL_ALIAS_DIGEST_HEX_LENGTH = 24
UNIX_SOCKET_PATH_MAX_BYTES = 103
RECEIPT_FIELDS = {
"receipt_version", "supervisor_pid", "challenge_digest", "reason", "exit_code",
"signal", "caller_launched", "cleanup_complete", "process_group_alive", "completed_at",
}
RECEIPT_ONLY_TERMINAL_REASONS = frozenset(
(REASON_CONTROLLER_LOST, REASON_RECOVERED_STOP)
)
class AttemptError(Exception):
"""Base error whose message is safe to present to a benchmark caller."""
class RunPathError(AttemptError):
pass
class RunBusyError(AttemptError):
pass
class AttemptStateError(AttemptError):
pass
class CapabilityUnavailable(AttemptError):
pass
@dataclass(frozen=True)
class RunIdentity:
run_id: str
manifest_digest: str
root: str
@dataclass(frozen=True)
class Slot:
cell_id: str
repetition: int
@dataclass(frozen=True)
class Attempt:
identity: AttemptIdentity
root: str
state: str
@dataclass(frozen=True)
class PreflightObservation:
"""One adapter result plus opaque identities safe for durable evidence."""
result: ConnectivityResult
endpoint_identity: str
config_identity: str
@dataclass(frozen=True)
class AttemptControlLease:
"""One deterministic short pathname bound to an exact attempt root."""
alias: str
control_dir: str
socket_path: str
class PreflightAdapter(Protocol):
"""Closed adapter boundary consumed by the public preflight controller."""
capability: CallerCapability
def preflight(self, cell: "MatrixCell") -> PreflightObservation:
"""Return one typed observation without exposing raw caller output."""
class ExecutionAdapter(PreflightAdapter, Protocol):
"""Typed caller boundary for one preflight-approved scored attempt."""
def invoke(
self,
cell: "MatrixCell",
prepared: PreparedWorkspace,
attempt: Attempt,
control_dir: str,
task_payload: bytes,
timeout: Timeout,
on_started: Callable[[SupervisorLocator, str], None],
) -> InvocationResult:
"""Submit the exact task once for the bound cell and prepared identity."""
def _json_bytes(value: Any) -> bytes:
return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("utf-8") + b"\n"
def _fsync_dir(path: Path) -> None:
fd = os.open(path, os.O_RDONLY)
try:
os.fsync(fd)
finally:
os.close(fd)
def _write_new(path: Path, data: bytes) -> None:
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
try:
os.write(fd, data)
os.fsync(fd)
finally:
os.close(fd)
_fsync_dir(path.parent)
def _replace(path: Path, data: bytes) -> None:
fd, tmp = tempfile.mkstemp(prefix=".attempt-", dir=path.parent)
try:
with os.fdopen(fd, "wb") as handle:
handle.write(data)
handle.flush()
os.fsync(handle.fileno())
os.chmod(tmp, 0o600)
os.replace(tmp, path)
_fsync_dir(path.parent)
finally:
try:
os.unlink(tmp)
except FileNotFoundError:
pass
def _open_regular(path: Path, label: str, flags: int = os.O_RDONLY) -> int:
"""Open one durable file without following links, blocking or trusting its type."""
try:
fd = os.open(path, flags | os.O_NOFOLLOW | os.O_NONBLOCK)
except OSError as exc:
raise AttemptStateError(f"{label} is unavailable") from exc
try:
opened = os.fstat(fd)
if not stat.S_ISREG(opened.st_mode):
raise AttemptStateError(f"{label} must be a regular file")
except BaseException:
os.close(fd)
raise
return fd
def _read_regular_bytes(path: Path, label: str) -> bytes:
"""Read only the no-follow descriptor that was verified as regular."""
fd = _open_regular(path, label)
try:
chunks: list[bytes] = []
while True:
chunk = os.read(fd, 1 << 20)
if not chunk:
return b"".join(chunks)
chunks.append(chunk)
finally:
os.close(fd)
def _utc_timestamp(clock: Callable[[], _datetime.datetime]) -> str:
return clock().astimezone(_datetime.timezone.utc).strftime("%Y%m%dT%H%M%SZ")
def _directory(path: Path, label: str) -> None:
try:
mode = os.lstat(path).st_mode
except OSError as exc:
raise RunPathError(f"{label} is unavailable") from exc
if not stat.S_ISDIR(mode):
raise RunPathError(f"{label} is invalid")
def _contained(path: Path, root: Path) -> bool:
try:
path.resolve(strict=False).relative_to(root.resolve())
return True
except ValueError:
return False
def _requested_binding(cell: MatrixCell) -> RequestedEffectiveBinding:
return RequestedEffectiveBinding(
cell.id,
cell.caller,
cell.iop.route_kind,
cell.iop.route_id,
cell.iop.request_model,
cell.iop.requested_effort,
)
def _overall_preflight_status(statuses: Iterator[str]) -> str:
found = tuple(statuses)
if not found:
raise AttemptStateError("preflight result set is empty")
if any(status == "implementation_gap" for status in found):
return "implementation_gap"
if any(status == "registration_required" for status in found):
return "registration_required"
if all(status == "ready" for status in found):
return "ready"
raise AttemptStateError("preflight status is invalid")
def _preflight_counts(results: list[dict[str, Any]]) -> dict[str, int]:
counts = {status: 0 for status in PREFLIGHT_STATUSES}
for result in results:
status = result.get("status")
if status not in counts:
raise AttemptStateError("preflight result status is invalid")
counts[str(status)] += 1
return counts
def _connectivity_result_from_payload(
payload: dict[str, Any], cell: MatrixCell
) -> tuple[ConnectivityResult, str, str]:
"""Rebuild one closed result so durable reads re-run semantic validation."""
if not isinstance(payload, dict) or set(payload) != {
"schema_version", "cell", "status", "binding", "issues",
"endpoint_identity", "config_identity",
}:
raise AttemptStateError("preflight result schema is invalid")
binding_raw = payload.get("binding")
if not isinstance(binding_raw, dict) or set(binding_raw) != {
"cell_id", "caller", "requested_route_kind", "requested_route_id",
"requested_model", "requested_effort", "effective_route_kind",
"effective_route_id", "effective_model", "effective_effort",
"effective_bindings",
}:
raise AttemptStateError("preflight result binding is invalid")
stages_raw = binding_raw.get("effective_bindings")
if not isinstance(stages_raw, list):
raise AttemptStateError("preflight result binding is invalid")
stages: list[EffectiveBinding] = []
for stage in stages_raw:
if not isinstance(stage, dict) or set(stage) != {"stage", "model", "effort"}:
raise AttemptStateError("preflight result binding is invalid")
stages.append(EffectiveBinding(stage["stage"], stage["model"], stage["effort"]))
issues_raw = payload.get("issues")
if not isinstance(issues_raw, list):
raise AttemptStateError("preflight result issues are invalid")
issues: list[ConnectivityIssue] = []
for issue in issues_raw:
if not isinstance(issue, dict) or set(issue) != {"code", "resume_code"}:
raise AttemptStateError("preflight result issues are invalid")
issues.append(ConnectivityIssue(issue["code"], issue["resume_code"]))
try:
binding = RequestedEffectiveBinding(
binding_raw["cell_id"],
binding_raw["caller"],
binding_raw["requested_route_kind"],
binding_raw["requested_route_id"],
binding_raw["requested_model"],
binding_raw["requested_effort"],
binding_raw["effective_route_kind"],
binding_raw["effective_route_id"],
binding_raw["effective_model"],
binding_raw["effective_effort"],
tuple(stages),
)
# Capability is deliberately not serialized. The persisted proof is
# revalidated against the exact immutable cell and its one direct route.
capability = CallerCapability(
cell.caller, (cell.iop.route_kind,), (cell.iop.requested_effort,)
)
result = ConnectivityResult(capability, binding, tuple(issues), payload["status"])
validate_result(cell, result)
endpoint_identity = payload["endpoint_identity"]
config_identity = payload["config_identity"]
canonical = json.loads(
canonical_evidence_bytes(
cell, result, endpoint_identity, config_identity
).decode("ascii")
)
except Exception as exc:
raise AttemptStateError("preflight result is invalid") from exc
if payload != canonical or payload.get("cell") != {"id": cell.id, "caller": cell.caller}:
raise AttemptStateError("preflight result is non-canonical")
return result, endpoint_identity, config_identity
def collect_preflight_observations(
manifest: Manifest,
adapters: Mapping[str, PreflightAdapter],
) -> dict[str, PreflightObservation]:
"""Validate the full registry, then probe direct cells in manifest order.
Execution-preset cells exercise only the local adapter capability contract in
this milestone. They never become a synthetic live-ready observation.
"""
if not isinstance(adapters, Mapping):
raise CapabilityUnavailable("capability-unavailable: caller-adapter")
required_callers = {cell.caller for cell in manifest.matrix}
if any(caller not in adapters for caller in required_callers):
raise CapabilityUnavailable("capability-unavailable: caller-adapter")
for cell in manifest.matrix:
adapter = adapters[cell.caller]
capability = getattr(adapter, "capability", None)
validate_requested_binding(cell, capability, _requested_binding(cell))
observations: dict[str, PreflightObservation] = {}
for cell in manifest.matrix:
if cell.iop.route_kind != "direct":
continue
observation = adapters[cell.caller].preflight(cell)
if not isinstance(observation, PreflightObservation):
raise AttemptStateError("preflight observation is invalid")
validate_result(cell, observation.result)
# Materializing the canonical bytes validates both opaque identities and
# proves that no adapter-specific/raw value can enter the durable record.
canonical_evidence_bytes(
cell,
observation.result,
observation.endpoint_identity,
observation.config_identity,
)
observations[cell.id] = observation
return observations
class RunStore:
"""Filesystem-backed run store rooted at a validated manifest output root."""
def __init__(
self,
repo_root: str | Path,
*,
clock: Callable[[], _datetime.datetime] = lambda: _datetime.datetime.now(_datetime.timezone.utc),
token_hex: Callable[[int], str] = secrets.token_hex,
) -> None:
self.repo_root = Path(repo_root).resolve()
self._clock = clock
self._token_hex = token_hex
def _output_root(self, manifest: Manifest, *, create: bool) -> Path:
root = self.repo_root / manifest.output_root
if not _contained(root, self.repo_root):
raise RunPathError("output root is invalid")
if create:
root.mkdir(mode=0o700, parents=True, exist_ok=True)
_directory(root, "output root")
if root.is_symlink():
raise RunPathError("output root is invalid")
return root.resolve()
def _run_path(self, manifest: Manifest, run_id: str, *, create_root: bool) -> Path:
if not RUN_ID_RE.fullmatch(run_id):
raise RunPathError("run id is invalid")
root = self._output_root(manifest, create=create_root)
path = root / run_id
if path.parent != root or path.is_symlink():
raise RunPathError("run path is invalid")
return path
def create(self, manifest: Manifest, manifest_bytes: bytes) -> RunIdentity:
"""Create a run and atomically persist immutable source bytes first."""
loaded = validate_manifest_bytes(manifest_bytes, repo_root=self.repo_root)
if loaded.digest != manifest.digest:
raise AttemptStateError("manifest snapshot does not match manifest digest")
run_id = f"run-{_utc_timestamp(self._clock)}-{self._token_hex(6)}"
if not RUN_ID_RE.fullmatch(run_id):
raise AttemptStateError("generated run id is invalid")
path = self._run_path(manifest, run_id, create_root=True)
try:
path.mkdir(mode=0o700)
except FileExistsError as exc:
raise AttemptStateError("generated run id already exists") from exc
_write_new(path / "manifest.json", manifest_bytes)
_write_new(path / "run.json", _json_bytes({"run_id": run_id, "manifest_digest": manifest.digest}))
_write_new(path / "run.lock", b"")
_fsync_dir(path)
return RunIdentity(run_id, manifest.digest, str(path))
def open(self, manifest: Manifest, run_id: str, manifest_bytes: bytes | None = None) -> RunIdentity:
path = self._run_path(manifest, run_id, create_root=False)
_directory(path, "run")
try:
record = json.loads(_read_regular_bytes(path / "run.json", "run record").decode("utf-8"))
snapshot = _read_regular_bytes(path / "manifest.json", "manifest snapshot")
_read_regular_bytes(path / "run.lock", "run lock")
except (OSError, json.JSONDecodeError) as exc:
raise AttemptStateError("run state is unavailable") from exc
if record != {"run_id": run_id, "manifest_digest": manifest.digest}:
raise AttemptStateError("run identity does not match manifest")
if manifest_bytes is not None and snapshot != manifest_bytes:
raise AttemptStateError("manifest bytes do not match run snapshot")
if validate_manifest_bytes(snapshot, repo_root=self.repo_root).digest != manifest.digest:
raise AttemptStateError("run manifest digest is invalid")
return RunIdentity(run_id, manifest.digest, str(path))
@contextlib.contextmanager
def writer(self, run: RunIdentity) -> Iterator[None]:
root = Path(run.root)
_directory(root, "run")
lock_path = root / "run.lock"
fd = _open_regular(lock_path, "run lock", os.O_RDWR)
try:
try:
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError as exc:
raise RunBusyError("run-busy") from exc
yield
finally:
try:
fcntl.flock(fd, fcntl.LOCK_UN)
finally:
os.close(fd)
@staticmethod
def _direct_cells(manifest: Manifest) -> tuple[MatrixCell, ...]:
return tuple(cell for cell in manifest.matrix if cell.iop.route_kind == "direct")
def _preflight_root(self, run: RunIdentity, *, create: bool) -> Path:
root = Path(run.root) / "preflight"
if root.exists() or root.is_symlink():
try:
mode = os.lstat(root).st_mode
except OSError as exc:
raise AttemptStateError("preflight state is unavailable") from exc
if not stat.S_ISDIR(mode) or root.is_symlink():
raise AttemptStateError("preflight state is invalid")
return root
if not create:
return root
try:
root.mkdir(mode=0o700)
_fsync_dir(root.parent)
except OSError as exc:
raise AttemptStateError("preflight state is unavailable") from exc
return root
def _validate_preflight_record(
self,
raw: bytes,
path: Path,
run: RunIdentity,
manifest: Manifest,
expected_sequence: int,
) -> dict[str, Any]:
try:
record = json.loads(raw.decode("ascii"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise AttemptStateError("preflight record is invalid") from exc
if not isinstance(record, dict) or set(record) != {
"schema_version", "run_id", "manifest_digest", "sequence",
"status", "results",
}:
raise AttemptStateError("preflight record schema is invalid")
if (
record["schema_version"] != PREFLIGHT_SCHEMA_VERSION
or record["run_id"] != run.run_id
or record["manifest_digest"] != run.manifest_digest
or record["sequence"] != expected_sequence
or record["status"] not in PREFLIGHT_STATUSES
or not isinstance(record["results"], list)
):
raise AttemptStateError("preflight record identity is invalid")
direct_cells = self._direct_cells(manifest)
if len(record["results"]) != len(direct_cells):
raise AttemptStateError("preflight result set is invalid")
statuses: list[str] = []
for cell, result_payload in zip(direct_cells, record["results"]):
result, _, _ = _connectivity_result_from_payload(result_payload, cell)
statuses.append(result.status)
if record["status"] != _overall_preflight_status(iter(statuses)):
raise AttemptStateError("preflight aggregate status is invalid")
if raw != _json_bytes(record):
raise AttemptStateError("preflight record is non-canonical")
expected_name = f"preflight-{expected_sequence:06d}.json"
if path.name != expected_name:
raise AttemptStateError("preflight sequence is invalid")
return record
def _preflight_records(
self, run: RunIdentity, manifest: Manifest
) -> tuple[dict[str, Any], ...]:
root = self._preflight_root(run, create=False)
if not root.exists() and not root.is_symlink():
return ()
records: list[dict[str, Any]] = []
for expected_sequence, child in enumerate(
sorted(root.iterdir(), key=lambda item: item.name), start=1
):
match = PREFLIGHT_RE.fullmatch(child.name)
if match is None or int(match.group(1)) != expected_sequence:
raise AttemptStateError("preflight sequence is invalid")
raw = _read_regular_bytes(child, "preflight record")
records.append(
self._validate_preflight_record(
raw, child, run, manifest, expected_sequence
)
)
return tuple(records)
def preflights(
self, run: RunIdentity, manifest: Manifest
) -> tuple[dict[str, Any], ...]:
"""Read append-only preflight records without creating or reconciling state."""
bound_run = self.open(manifest, run.run_id)
if bound_run != run:
raise AttemptStateError("run identity is invalid")
return self._preflight_records(bound_run, manifest)
def record_preflight(
self,
run: RunIdentity,
manifest: Manifest,
observations: Mapping[str, PreflightObservation],
) -> dict[str, Any]:
"""Append one canonical result set while exclusively owning the run writer."""
bound_run = self.open(manifest, run.run_id)
if bound_run != run:
raise AttemptStateError("run identity is invalid")
with self.writer(bound_run):
return self._record_preflight_locked(bound_run, manifest, observations)
def _record_preflight_locked(
self,
run: RunIdentity,
manifest: Manifest,
observations: Mapping[str, PreflightObservation],
) -> dict[str, Any]:
"""Append one preflight while the caller owns the run writer."""
bound_run = self.open(manifest, run.run_id)
if bound_run != run:
raise AttemptStateError("run identity is invalid")
direct_cells = self._direct_cells(manifest)
if set(observations) != {cell.id for cell in direct_cells}:
raise AttemptStateError("preflight observation set is invalid")
results: list[dict[str, Any]] = []
for cell in direct_cells:
observation = observations[cell.id]
if not isinstance(observation, PreflightObservation):
raise AttemptStateError("preflight observation is invalid")
try:
encoded = canonical_evidence_bytes(
cell,
observation.result,
observation.endpoint_identity,
observation.config_identity,
)
payload = json.loads(encoded.decode("ascii"))
except Exception as exc:
raise AttemptStateError("preflight observation is invalid") from exc
_connectivity_result_from_payload(payload, cell)
results.append(payload)
status = _overall_preflight_status(
iter(str(result["status"]) for result in results)
)
previous = self._preflight_records(bound_run, manifest)
sequence = len(previous) + 1
root = self._preflight_root(bound_run, create=True)
path = root / f"preflight-{sequence:06d}.json"
record = {
"schema_version": PREFLIGHT_SCHEMA_VERSION,
"run_id": bound_run.run_id,
"manifest_digest": bound_run.manifest_digest,
"sequence": sequence,
"status": status,
"results": results,
}
raw = _json_bytes(record)
_write_new(path, raw)
self._validate_preflight_record(
_read_regular_bytes(path, "preflight record"),
path,
bound_run,
manifest,
sequence,
)
return record
@staticmethod
def slots(manifest: Manifest) -> tuple[Slot, ...]:
return tuple(Slot(cell.id, repetition) for cell in manifest.matrix for repetition in range(1, manifest.repetitions + 1))
def _slot_path(self, run: RunIdentity, slot: Slot) -> Path:
if not re.fullmatch(r"[a-z0-9][a-z0-9_-]{0,63}", slot.cell_id) or slot.repetition < 1:
raise AttemptStateError("slot is invalid")
return Path(run.root) / "cells" / slot.cell_id / f"repetition-{slot.repetition:04d}"
def _ensure_slot_parent(self, run: RunIdentity, slot: Slot) -> Path:
current = Path(run.root)
for segment in ("cells", slot.cell_id, f"repetition-{slot.repetition:04d}"):
current = current / segment
if current.exists() or current.is_symlink():
_directory(current, "slot state")
if current.is_symlink():
raise AttemptStateError("slot state is invalid")
continue
try:
current.mkdir(mode=0o700)
except FileExistsError:
_directory(current, "slot state")
if current.is_symlink():
raise AttemptStateError("slot state is invalid")
return current
@staticmethod
def _expected_record(run: RunIdentity, identity: AttemptIdentity, state: str) -> dict[str, Any]:
return {
"run_id": run.run_id,
"manifest_digest": run.manifest_digest,
"cell_id": identity.cell_id,
"repetition": identity.repetition,
"attempt": identity.attempt,
"state": state,
}
def _attempt_record(self, root: Path, run: RunIdentity, identity: AttemptIdentity, *, absent_ok: bool = False) -> dict[str, Any] | None:
path = root / "attempt.json"
try:
record = json.loads(_read_regular_bytes(path, "attempt record").decode("utf-8"))
except AttemptStateError:
if absent_ok:
try:
os.lstat(path)
except FileNotFoundError:
if self._measurement_policy_start(root, run, identity) is not None:
raise AttemptStateError(
"attempt measurement policy provenance is invalid"
)
return None
raise
except (OSError, json.JSONDecodeError) as exc:
raise AttemptStateError("attempt record is unavailable") from exc
if not isinstance(record, dict):
raise AttemptStateError("attempt record is invalid")
expected = self._expected_record(run, identity, str(record.get("state", "")))
if record.get("state") not in TERMINAL_STATES | {NONTERMINAL_STATE}:
raise AttemptStateError("attempt record is invalid")
for key, value in expected.items():
if record.get(key) != value:
raise AttemptStateError("attempt record identity is invalid")
allowed = set(expected) | {
"locator", "spec_digest", "lifecycle", "measurement_policy",
"web_validation_policy",
}
if set(record) - allowed:
raise AttemptStateError("attempt record schema is invalid")
policy = record.get("measurement_policy")
if policy is not None and policy not in MEASUREMENT_POLICIES:
raise AttemptStateError("attempt measurement policy is invalid")
self._validate_measurement_policy_start(root, run, identity, policy)
web_policy = record.get("web_validation_policy")
if web_policy is not None and web_policy not in WEB_VALIDATION_POLICIES:
raise AttemptStateError("attempt web validation policy is invalid")
self._validate_web_validation_policy_start(root, run, identity, web_policy)
if record["state"] == NONTERMINAL_STATE and "lifecycle" in record:
raise AttemptStateError("running attempt has terminal evidence")
if ("locator" in record) != ("spec_digest" in record):
raise AttemptStateError("attempt invocation identity is invalid")
if "locator" in record:
locator = self._locator_from_record(
root, record["locator"], state=str(record["state"])
)
if not isinstance(record["spec_digest"], str) or not DIGEST_RE.fullmatch(record["spec_digest"]):
raise AttemptStateError("attempt invocation digest is invalid")
if record["state"] in TERMINAL_STATES:
lifecycle = record.get("lifecycle")
expected_receipt_reason = (
lifecycle.get("terminal_reason")
if isinstance(lifecycle, dict)
else None
)
self._validate_terminal_invocation_identity(
root,
locator,
record["spec_digest"],
run=run,
identity=identity,
terminal_state=str(record["state"]),
expected_receipt_reason=expected_receipt_reason,
measurement_policy=policy,
)
if "lifecycle" in record and (not isinstance(record["lifecycle"], dict) or set(record["lifecycle"]) != {"terminal_reason"} or not isinstance(record["lifecycle"]["terminal_reason"], str)):
raise AttemptStateError("attempt lifecycle is invalid")
if record["state"] in TERMINAL_STATES:
self._validate_web_validation(root, run, identity, web_policy)
elif (root / WEB_VALIDATION_FILENAME).exists() or (
root / WEB_VALIDATION_FILENAME
).is_symlink():
self._validate_web_validation(root, run, identity, web_policy)
return record
def attempts(self, run: RunIdentity, slot: Slot) -> tuple[Attempt, ...]:
parent = self._slot_path(run, slot)
if not parent.exists() and not parent.is_symlink():
return ()
_directory(parent, "slot state")
found: list[Attempt] = []
for child in sorted(parent.iterdir(), key=lambda item: item.name):
match = ATTEMPT_RE.fullmatch(child.name)
if not match or child.is_symlink() or not child.is_dir():
raise AttemptStateError("slot contains invalid state")
number = int(match.group(1))
identity = AttemptIdentity(run.run_id, slot.cell_id, slot.repetition, number)
record = self._attempt_record(child, run, identity, absent_ok=True)
state = NONTERMINAL_STATE if record is None else str(record["state"])
found.append(Attempt(identity, str(child), state))
return tuple(found)
def execution_attempts(
self, run: RunIdentity, manifest: Manifest
) -> tuple[Attempt, ...]:
"""Enumerate every retained execution attempt in canonical slot order.
Scoring and later reporting use this read-only projection instead of
reconstructing the private ``cells/`` directory grammar.
"""
bound_run = self.open(manifest, run.run_id)
if bound_run != run:
raise AttemptStateError("run identity is invalid")
return tuple(
attempt
for slot in self.slots(manifest)
for attempt in self.attempts(bound_run, slot)
)
def allocate(self, run: RunIdentity, slot: Slot) -> Attempt:
"""Create the exclusive, deliberately empty attempt root."""
existing = self.attempts(run, slot)
if any(item.state not in TERMINAL_STATES for item in existing):
raise AttemptStateError("slot already has a nonterminal attempt")
number = max((item.identity.attempt for item in existing), default=0) + 1
parent = self._ensure_slot_parent(run, slot)
root = parent / f"attempt-{number:06d}"
try:
root.mkdir(mode=0o700)
_fsync_dir(parent)
except FileExistsError as exc:
raise AttemptStateError("attempt allocation collision") from exc
return Attempt(AttemptIdentity(run.run_id, slot.cell_id, slot.repetition, number), str(root), NONTERMINAL_STATE)
def _bound_attempt(self, attempt: Attempt) -> tuple[RunIdentity, Path]:
root = Path(attempt.root)
try:
run_root = root.parents[3]
except IndexError as exc:
raise AttemptStateError("attempt path is invalid") from exc
_directory(run_root, "run")
try:
run_record = json.loads(_read_regular_bytes(run_root / "run.json", "run record").decode("utf-8"))
snapshot = _read_regular_bytes(run_root / "manifest.json", "manifest snapshot")
_read_regular_bytes(run_root / "run.lock", "run lock")
manifest = validate_manifest_bytes(snapshot, repo_root=self.repo_root)
except Exception as exc:
raise AttemptStateError("attempt run binding is invalid") from exc
if not isinstance(run_record, dict) or set(run_record) != {"run_id", "manifest_digest"}:
raise AttemptStateError("attempt run binding is invalid")
run = RunIdentity(str(run_record["run_id"]), str(run_record["manifest_digest"]), str(run_root))
if not RUN_ID_RE.fullmatch(run.run_id) or run.manifest_digest != manifest.digest:
raise AttemptStateError("attempt run binding is invalid")
if run_root.resolve() != self._run_path(manifest, run.run_id, create_root=False).resolve():
raise AttemptStateError("attempt run binding is invalid")
expected = self._slot_path(run, Slot(attempt.identity.cell_id, attempt.identity.repetition)) / f"attempt-{attempt.identity.attempt:06d}"
if root.resolve() != expected.resolve() or attempt.identity.run_id != run.run_id or root.is_symlink():
raise AttemptStateError("attempt identity is invalid")
_directory(root, "attempt root")
return run, root
def _initial_record(
self, run: RunIdentity, attempt: Attempt, state: str, *,
reason: str | None = None, measurement_policy: str | None = None,
web_validation_policy: str | None = None,
) -> dict[str, Any]:
record = self._expected_record(run, attempt.identity, state)
if measurement_policy is not None:
if measurement_policy not in MEASUREMENT_POLICIES:
raise AttemptStateError("attempt measurement policy is invalid")
record["measurement_policy"] = measurement_policy
if web_validation_policy is not None:
if web_validation_policy not in WEB_VALIDATION_POLICIES:
raise AttemptStateError("attempt web validation policy is invalid")
record["web_validation_policy"] = web_validation_policy
if reason is not None:
record["lifecycle"] = {"terminal_reason": reason}
return record
@staticmethod
def _measurement_policy_start_record(
run: RunIdentity, identity: AttemptIdentity
) -> dict[str, Any]:
"""Return the immutable production measurement-policy start evidence."""
return {
"record": MEASUREMENT_POLICY_RECORD,
"measurement_policy_version": MEASUREMENT_POLICY_VERSION,
"measurement_policy": MEASUREMENT_POLICY_REQUIRED_V1,
"run_id": run.run_id,
"manifest_digest": run.manifest_digest,
"cell_id": identity.cell_id,
"repetition": identity.repetition,
"attempt": identity.attempt,
}
def _measurement_policy_start(
self, root: Path, run: RunIdentity, identity: AttemptIdentity
) -> dict[str, Any] | None:
"""Load one canonical, no-follow policy record, or its explicit absence."""
path = root / MEASUREMENT_POLICY_FILENAME
try:
raw = _read_regular_bytes(path, "measurement policy")
except AttemptStateError:
try:
os.lstat(path)
except FileNotFoundError:
return None
raise
try:
record = json.loads(raw.decode("ascii"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise AttemptStateError("measurement policy is invalid") from exc
expected = self._measurement_policy_start_record(run, identity)
if record != expected or raw != _json_bytes(expected):
raise AttemptStateError("measurement policy is invalid")
return record
def _validate_measurement_policy_start(
self,
root: Path,
run: RunIdentity,
identity: AttemptIdentity,
measurement_policy: str | None,
) -> None:
"""Require both policy sources for production and neither for legacy paths."""
start = self._measurement_policy_start(root, run, identity)
if measurement_policy is None and start is None:
return
if (
measurement_policy == MEASUREMENT_POLICY_REQUIRED_V1
and start is not None
):
return
raise AttemptStateError("attempt measurement policy provenance is invalid")
@staticmethod
def _web_validation_policy_start_record(
run: RunIdentity, identity: AttemptIdentity
) -> dict[str, Any]:
return {
"record": WEB_VALIDATION_POLICY_RECORD,
"web_validation_policy_version": WEB_VALIDATION_POLICY_VERSION,
"web_validation_policy": WEB_VALIDATION_POLICY_REQUIRED_V1,
"run_id": run.run_id,
"manifest_digest": run.manifest_digest,
"cell_id": identity.cell_id,
"repetition": identity.repetition,
"attempt": identity.attempt,
}
def _web_validation_policy_start(
self, root: Path, run: RunIdentity, identity: AttemptIdentity
) -> dict[str, Any] | None:
path = root / WEB_VALIDATION_POLICY_FILENAME
try:
raw = _read_regular_bytes(path, "web validation policy")
except AttemptStateError:
try:
os.lstat(path)
except FileNotFoundError:
return None
raise
try:
record = json.loads(raw.decode("ascii"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise AttemptStateError("web validation policy is invalid") from exc
expected = self._web_validation_policy_start_record(run, identity)
if record != expected or raw != _json_bytes(expected):
raise AttemptStateError("web validation policy is invalid")
return record
def _validate_web_validation_policy_start(
self, root: Path, run: RunIdentity, identity: AttemptIdentity,
policy: str | None,
) -> None:
start = self._web_validation_policy_start(root, run, identity)
if policy is None and start is None:
return
if policy == WEB_VALIDATION_POLICY_REQUIRED_V1 and start is not None:
return
raise AttemptStateError("attempt web validation policy provenance is invalid")
def _validate_web_validation(
self, root: Path, run: RunIdentity, identity: AttemptIdentity,
policy: str | None,
) -> None:
path = root / WEB_VALIDATION_FILENAME
if not path.exists() and not path.is_symlink():
if policy == WEB_VALIDATION_POLICY_REQUIRED_V1:
raise AttemptStateError("required web validation is unavailable")
return
try:
manifest = self.open_manifest_snapshot(run)
web = load_web_validation(root, manifest=manifest)
measurement = load_measurement(root)
except (WebValidationError, MeasurementError) as exc:
raise AttemptStateError("attempt web validation is invalid") from exc
record = web.record
if (
(record["attempt"]["run_id"], record["attempt"]["cell_id"],
record["attempt"]["repetition"], record["attempt"]["attempt"])
!= (run.run_id, identity.cell_id, identity.repetition, identity.attempt)
or record["manifest_digest"] != run.manifest_digest
or record["fixture_checksum"] != manifest.fixture.checksum
):
raise AttemptStateError("attempt web validation identity is invalid")
digest = "sha256:" + hashlib.sha256(
_read_regular_bytes(root / MEASUREMENT_FILENAME, "measurement")).hexdigest()
if record["measurement_digest"] != digest or measurement.run_id != run.run_id:
raise AttemptStateError("attempt web validation measurement is invalid")
if (
(measurement.terminal_reason == "success" and web.status == "not_run")
or (measurement.terminal_reason != "success" and web.status != "not_run")
):
raise AttemptStateError("attempt web validation lifecycle is invalid")
def open_manifest_snapshot(self, run: RunIdentity) -> Manifest:
"""Load the exact immutable run manifest used by recovery and evidence."""
try:
return validate_manifest_bytes(
_read_regular_bytes(
Path(run.root) / "manifest.json", "manifest snapshot"
),
repo_root=self.repo_root,
)
except Exception as exc:
raise AttemptStateError(
"attempt web validation manifest is invalid"
) from exc
def open_manifest_fixture_checksum(self, run: RunIdentity) -> str:
"""Read the run-bound manifest only to bind web evidence to its fixture."""
return self.open_manifest_snapshot(run).fixture.checksum
def _ensure_required_web_validation(
self,
root: Path,
run: RunIdentity,
identity: AttemptIdentity,
record: Mapping[str, Any],
terminal: Mapping[str, Any],
) -> None:
"""Validate or reconstruct required S12 evidence before terminal commit.
An existing invalid/colliding record is never replaced. When the
sidecar is absent, reconstruction uses only the immutable run manifest,
conventional attempt workspace, and strict measurement. Any failure
occurs before ``attempt.json`` is replaced, leaving recovery resumable.
"""
policy = record.get("web_validation_policy")
if policy != WEB_VALIDATION_POLICY_REQUIRED_V1:
self._validate_web_validation(root, run, identity, policy)
return
path = root / WEB_VALIDATION_FILENAME
if path.exists() or path.is_symlink():
self._validate_web_validation(root, run, identity, policy)
return
try:
manifest = self.open_manifest_snapshot(run)
measurement = load_measurement(root)
prepared = SimpleNamespace(
workspace_dir=str(root / "workspace"),
attempt_root=str(root),
)
web = validate_web_attempt(
manifest,
root,
prepared,
measurement,
terminal,
)
publish_web_validation(root, web)
except (MeasurementError, WebValidationError, OSError) as exc:
raise AttemptStateError(
"required web validation reconstruction failed"
) from exc
self._validate_web_validation(root, run, identity, policy)
@staticmethod
def _control_lease_for_root(root: Path) -> AttemptControlLease:
"""Derive the short public alias without reading secret or caller data."""
canonical_root = root.resolve()
digest = hashlib.sha256(
b"iop-benchmark-attempt-control-v1\0"
+ os.fsencode(str(canonical_root))
).hexdigest()[:CONTROL_ALIAS_DIGEST_HEX_LENGTH]
alias = Path(tempfile.gettempdir()).resolve() / f"{CONTROL_ALIAS_PREFIX}{digest}"
control_dir = alias / CONTROL_DIRECTORY_NAME
socket_path = control_dir / SOCKET_FILENAME
if len(os.fsencode(str(socket_path))) > UNIX_SOCKET_PATH_MAX_BYTES:
raise AttemptStateError("control socket path exceeds platform budget")
return AttemptControlLease(str(alias), str(control_dir), str(socket_path))
@staticmethod
def _validate_control_alias(root: Path, lease: AttemptControlLease) -> None:
alias = Path(lease.alias)
try:
mode = os.lstat(alias).st_mode
except OSError as exc:
raise AttemptStateError("control lease is unavailable") from exc
if not stat.S_ISLNK(mode):
raise AttemptStateError("control lease collision")
try:
target = os.readlink(alias)
except OSError as exc:
raise AttemptStateError("control lease is unavailable") from exc
canonical_root = root.resolve()
try:
resolved_alias = alias.resolve(strict=True)
except OSError as exc:
raise AttemptStateError("control lease is unavailable") from exc
if (
target != str(canonical_root)
or not Path(target).is_absolute()
or resolved_alias != canonical_root
):
raise AttemptStateError("control lease target mismatch")
def acquire_control_lease(self, attempt: Attempt) -> AttemptControlLease:
"""Create or authenticate the active attempt's no-overwrite short alias."""
run, root = self._bound_attempt(attempt)
record = self._attempt_record(root, run, attempt.identity)
if record is None or record["state"] != NONTERMINAL_STATE:
raise AttemptStateError("control lease requires a running attempt")
lease = self._control_lease_for_root(root)
alias = Path(lease.alias)
try:
os.symlink(str(root.resolve()), alias, target_is_directory=True)
_fsync_dir(alias.parent)
except FileExistsError:
self._validate_control_alias(root, lease)
except OSError as exc:
raise AttemptStateError("control lease is unavailable") from exc
self._validate_control_alias(root, lease)
return lease
def release_control_lease(self, attempt: Attempt) -> None:
"""Remove only this exact owned alias after durable terminal publication."""
run, root = self._bound_attempt(attempt)
record = self._attempt_record(root, run, attempt.identity)
if record is None or record["state"] not in TERMINAL_STATES:
raise AttemptStateError("control lease release requires terminal state")
lease = self._control_lease_for_root(root)
alias = Path(lease.alias)
try:
os.lstat(alias)
except FileNotFoundError:
return
except OSError as exc:
raise AttemptStateError("control lease is unavailable") from exc
self._validate_control_alias(root, lease)
try:
alias.unlink()
_fsync_dir(alias.parent)
except OSError as exc:
raise AttemptStateError("control lease cleanup failed") from exc
def publish_terminal(self, attempt: Attempt, state: str, *, result: dict[str, Any] | None = None) -> Attempt:
if state not in TERMINAL_STATES:
raise AttemptStateError("terminal state is invalid")
run, root = self._bound_attempt(attempt)
record = self._attempt_record(root, run, attempt.identity, absent_ok=True)
if record is None:
reason = str((result or {}).get("terminal_reason") or state)
_write_new(root / "attempt.json", _json_bytes(self._initial_record(run, attempt, state, reason=reason)))
return Attempt(attempt.identity, attempt.root, state)
if record["state"] in TERMINAL_STATES:
if record["state"] != state:
raise AttemptStateError("terminal attempt is immutable")
return Attempt(attempt.identity, attempt.root, state)
if record["state"] != NONTERMINAL_STATE:
raise AttemptStateError("attempt transition is invalid")
terminal_result = result or {"terminal_reason": state}
self._ensure_required_web_validation(
root,
run,
attempt.identity,
record,
terminal_result,
)
record["state"] = state
record["lifecycle"] = {
"terminal_reason": str(terminal_result.get("terminal_reason") or state)
}
_replace(root / "attempt.json", _json_bytes(record))
return Attempt(attempt.identity, attempt.root, state)
def _locator_from_record(
self, root: Path, raw: Any, *, state: str
) -> SupervisorLocator:
fields = {"supervisor_pid", "start_identity", "socket_path", "challenge", "control_dir", "created_at"}
if not isinstance(raw, dict) or set(raw) != fields:
raise AttemptStateError("locator is invalid")
try:
locator = SupervisorLocator(**raw)
except TypeError as exc:
raise AttemptStateError("locator is invalid") from exc
if not isinstance(locator.supervisor_pid, int) or locator.supervisor_pid < 1 or any(not isinstance(value, str) or not value for value in (locator.start_identity, locator.socket_path, locator.challenge, locator.control_dir, locator.created_at)):
raise AttemptStateError("locator is invalid")
control = Path(locator.control_dir)
socket = Path(locator.socket_path)
lease = self._control_lease_for_root(root)
if control != Path(lease.control_dir) or socket != Path(lease.socket_path):
raise AttemptStateError("locator control binding is invalid")
alias = Path(lease.alias)
try:
os.lstat(alias)
except FileNotFoundError:
if state == NONTERMINAL_STATE:
raise AttemptStateError("running locator control lease is unavailable")
except OSError as exc:
raise AttemptStateError("locator control lease is unavailable") from exc
else:
self._validate_control_alias(root, lease)
if not _contained(control, root) or not _contained(socket, control):
raise AttemptStateError("locator escapes attempt root")
if socket.parent != control:
raise AttemptStateError("locator control binding is invalid")
return locator
def _validate_measurement(
self,
root: Path,
run: RunIdentity,
identity: AttemptIdentity,
expected_digest: str,
expected_reason: str | None = None,
measurement_policy: str | None = None,
lifecycle: Mapping[str, Any] | None = None,
) -> None:
"""Bind the immutable timing/usage sidecar to this exact invocation.
The sidecar is optional for historical and lower-level records. A
marked production attempt must have it, and every present sidecar is
bound to the terminal lifecycle evidence without rewriting either.
"""
path = root / MEASUREMENT_FILENAME
if not path.exists() and not path.is_symlink():
if measurement_policy == MEASUREMENT_POLICY_REQUIRED_V1:
raise AttemptStateError("required attempt measurement is unavailable")
return
try:
measurement = load_measurement(root)
except MeasurementError as exc:
raise AttemptStateError("attempt measurement is invalid") from exc
actual = (
measurement.run_id, measurement.cell_id,
measurement.repetition, measurement.attempt,
)
expected = (
run.run_id, identity.cell_id, identity.repetition, identity.attempt,
)
if actual != expected or measurement.spec_digest != expected_digest:
raise AttemptStateError("attempt measurement identity is invalid")
if expected_reason is not None and measurement.terminal_reason != expected_reason:
raise AttemptStateError("attempt measurement terminal is invalid")
if lifecycle is not None:
try:
validate_measurement_lifecycle_binding(measurement, lifecycle)
except MeasurementError as exc:
raise AttemptStateError("attempt measurement lifecycle is invalid") from exc
def _validate_terminal_invocation_identity(
self,
root: Path,
locator: SupervisorLocator,
expected_digest: str,
*,
run: RunIdentity,
identity: AttemptIdentity,
terminal_state: str,
expected_receipt_reason: str | None,
measurement_policy: str | None,
) -> None:
"""Rebind cleaned historical records to their durable invocation digest."""
result_path = root / "lifecycle-result.json"
result_exists = result_path.exists() or result_path.is_symlink()
if result_exists:
result = self._read_bound_lifecycle_terminal(
root, locator, expected_digest
)
if result is None:
raise AttemptStateError("terminal invocation identity is invalid")
self._validate_measurement(
root, run, identity, expected_digest, expected_receipt_reason,
measurement_policy, result,
)
journal_path = root / "lifecycle-journal.jsonl"
journal_exists = journal_path.exists() or journal_path.is_symlink()
if journal_exists:
try:
first = _read_regular_bytes(
journal_path, "lifecycle journal"
).decode("utf-8").splitlines()[0]
header = json.loads(first)
except (IndexError, UnicodeDecodeError, json.JSONDecodeError) as exc:
raise AttemptStateError("terminal invocation identity is invalid") from exc
if not isinstance(header, dict) or header.get("spec_digest") != expected_digest:
raise AttemptStateError("terminal invocation identity is invalid")
if not result_exists and not journal_exists:
# A receipt-only record has no published result or journal, so its only
# authenticated reason is recovery; it may project nothing but interrupted.
if terminal_state != "interrupted" or not isinstance(expected_receipt_reason, str):
raise AttemptStateError("terminal invocation identity is invalid")
self._closed_cleanup_receipt(
root,
locator,
expected_reason=expected_receipt_reason,
required=True,
)
self._validate_measurement(
root, run, identity, expected_digest, expected_receipt_reason,
measurement_policy,
)
def record_locator(self, attempt: Attempt, locator: SupervisorLocator, invocation_digest: str) -> None:
run, root = self._bound_attempt(attempt)
record = self._attempt_record(root, run, attempt.identity)
if record is None or record["state"] != NONTERMINAL_STATE or "locator" in record:
raise AttemptStateError("locator transition is invalid")
if not isinstance(invocation_digest, str) or not DIGEST_RE.fullmatch(invocation_digest):
raise AttemptStateError("invocation digest is invalid")
raw = {
"supervisor_pid": locator.supervisor_pid,
"start_identity": locator.start_identity,
"socket_path": locator.socket_path,
"challenge": locator.challenge,
"control_dir": locator.control_dir,
"created_at": locator.created_at,
}
self._locator_from_record(root, raw, state=NONTERMINAL_STATE)
registered = self._read_json_file(Path(locator.control_dir), "locator.json")
if registered != raw:
raise AttemptStateError("registered locator is invalid")
record["locator"] = raw
record["spec_digest"] = invocation_digest
_replace(root / "attempt.json", _json_bytes(record))
@staticmethod
def _state_for_reason(reason: str) -> str:
if reason == "success":
return "success"
if reason == "timed_out":
return "timed_out"
if reason == "cancelled":
return "cancelled"
if reason in RECEIPT_ONLY_TERMINAL_REASONS:
return "interrupted"
return "failed"
def _read_json_file(self, root: Path, name: str) -> dict[str, Any]:
path = root / name
try:
value = json.loads(_read_regular_bytes(path, name).decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise AttemptStateError(f"{name} is invalid") from exc
if not isinstance(value, dict):
raise AttemptStateError(f"{name} is invalid")
return value
@staticmethod
def _exact_fields(value: Any, fields: set[str], label: str) -> dict[str, Any]:
if not isinstance(value, dict) or set(value) != fields:
raise AttemptStateError(f"{label} schema is invalid")
return value
@staticmethod
def _optional_int(value: Any) -> bool:
return value is None or (isinstance(value, int) and not isinstance(value, bool))
def _validate_result_record(self, result: dict[str, Any], locator: SupervisorLocator, expected_digest: str) -> None:
fields = {
"record", "success", "terminal_reason", "exit_code", "signal", "submitted",
"finish_then_idle_then_quiet", "cleanup_complete", "process_group_alive",
"submission_mode", "completion_mode", "spec_digest", "locator", "started_at",
"ended_at", "duration_ns", "stdout", "stderr", "events",
}
self._exact_fields(result, fields, "lifecycle result")
required_bools = ("success", "submitted", "finish_then_idle_then_quiet", "cleanup_complete", "process_group_alive")
if result["record"] != "result" or any(not isinstance(result[key], bool) for key in required_bools):
raise AttemptStateError("lifecycle result is invalid")
if result["terminal_reason"] not in TERMINAL_REASONS or result["spec_digest"] != expected_digest:
raise AttemptStateError("lifecycle result identity is invalid")
if not self._optional_int(result["exit_code"]) or not self._optional_int(result["signal"]):
raise AttemptStateError("lifecycle result is invalid")
if result["submission_mode"] not in SUBMISSION_MODES or result["completion_mode"] not in COMPLETION_MODES:
raise AttemptStateError("lifecycle result is invalid")
if not all(isinstance(result[key], str) for key in ("started_at", "ended_at")) or not isinstance(result["duration_ns"], int) or isinstance(result["duration_ns"], bool) or result["duration_ns"] < 0:
raise AttemptStateError("lifecycle result is invalid")
expected_public = self._public_locator(locator)
if result["locator"] != expected_public:
raise AttemptStateError("lifecycle terminal locator is invalid")
for name, stream in (("stdout", "stdout"), ("stderr", "stderr")):
capture = self._exact_fields(result[name], {"stream", "text", "line_count", "byte_count", "truncated"}, f"{name} capture")
if capture["stream"] != stream or not isinstance(capture["text"], str) or not isinstance(capture["line_count"], int) or not isinstance(capture["byte_count"], int) or isinstance(capture["line_count"], bool) or isinstance(capture["byte_count"], bool) or capture["line_count"] < 0 or capture["byte_count"] < 0 or not isinstance(capture["truncated"], bool):
raise AttemptStateError("lifecycle capture is invalid")
if not isinstance(result["events"], list):
raise AttemptStateError("lifecycle events are invalid")
for event in result["events"]:
checked = self._exact_fields(event, {"record", "kind", "source", "stream", "monotonic_ns", "source_monotonic_ns", "observed_at", "detail"}, "lifecycle event")
if checked["record"] != "event" or not all(isinstance(checked[key], str) for key in ("kind", "source", "stream", "observed_at", "detail")) or not all(isinstance(checked[key], int) and not isinstance(checked[key], bool) and checked[key] >= 0 for key in ("monotonic_ns", "source_monotonic_ns")):
raise AttemptStateError("lifecycle event is invalid")
if not result["cleanup_complete"] or result["process_group_alive"] or result["success"] != (result["terminal_reason"] == "success"):
raise AttemptStateError("lifecycle terminal outcome is invalid")
if result["success"] and not result["finish_then_idle_then_quiet"]:
raise AttemptStateError("lifecycle terminal outcome is invalid")
@staticmethod
def _instant(value: Any, label: str) -> _datetime.datetime:
"""Parse one produced ISO-8601 instant into a comparable UTC value."""
try:
parsed = _datetime.datetime.fromisoformat(str(value))
except (TypeError, ValueError) as exc:
raise AttemptStateError(f"{label} timestamp is invalid") from exc
return parsed if parsed.tzinfo is not None else parsed.replace(tzinfo=_datetime.timezone.utc)
@staticmethod
def _validate_terminal_events(events: list[Any]) -> dict[str, int]:
"""Return the unique ordinal of every success-evidence event kind."""
positions: dict[str, int] = {}
for index, event in enumerate(events):
kind = event["kind"]
if kind not in SUCCESS_EVIDENCE_KINDS:
continue
if kind in positions:
raise AttemptStateError("lifecycle events are invalid")
positions[kind] = index
return positions
def _validate_terminal_coherence(self, result: dict[str, Any], receipt: dict[str, Any], positions: dict[str, int]) -> None:
"""Bind result, ordered events and cleanup receipt to one terminal projection."""
submitted, finish, idle, quiet = (positions.get(kind) for kind in SUCCESS_EVIDENCE_KINDS)
ordered = finish is not None and idle is not None and quiet is not None and finish < idle < quiet
if ordered != result["finish_then_idle_then_quiet"]:
raise AttemptStateError("lifecycle ordered evidence is invalid")
if result["exit_code"] != receipt["exit_code"] or result["signal"] != receipt["signal"]:
raise AttemptStateError("lifecycle terminal outcome is invalid")
if result["submitted"] and not receipt["caller_launched"]:
raise AttemptStateError("lifecycle terminal outcome is invalid")
if result["success"] and (not result["submitted"] or submitted is None or finish is None or submitted > finish):
raise AttemptStateError("lifecycle success evidence is invalid")
started = self._instant(result["started_at"], "lifecycle result")
ended = self._instant(result["ended_at"], "lifecycle result")
completed = self._instant(receipt["completed_at"], "cleanup receipt")
if completed < started or ended < completed:
raise AttemptStateError("lifecycle terminal chronology is invalid")
def _validate_receipt_record(self, receipt: dict[str, Any], locator: SupervisorLocator) -> dict[str, Any]:
"""Validate one cleanup receipt against the registered supervisor identity."""
self._exact_fields(receipt, RECEIPT_FIELDS, "cleanup receipt")
if receipt["receipt_version"] != RECEIPT_VERSION or receipt["supervisor_pid"] != locator.supervisor_pid or receipt["challenge_digest"] != self._public_locator(locator)["challenge_digest"] or receipt["reason"] not in TERMINAL_REASONS or not self._optional_int(receipt["exit_code"]) or not self._optional_int(receipt["signal"]) or not isinstance(receipt["caller_launched"], bool) or receipt["cleanup_complete"] is not True or receipt["process_group_alive"] is not False or not isinstance(receipt["completed_at"], str):
raise AttemptStateError("cleanup receipt is invalid")
return receipt
def _closed_cleanup_receipt(
self,
root: Path,
locator: SupervisorLocator,
*,
expected_reason: str = REASON_CONTROLLER_LOST,
required: bool = False,
) -> dict[str, Any] | None:
"""Read one canonical, authenticated receipt after its socket is closed."""
if expected_reason not in RECEIPT_ONLY_TERMINAL_REASONS:
raise AttemptStateError("closed cleanup receipt reason is invalid")
control = root / CONTROL_DIRECTORY_NAME
try:
control_mode = os.lstat(control).st_mode
except OSError as exc:
raise AttemptStateError("attempt control directory is unavailable") from exc
if not stat.S_ISDIR(control_mode):
raise AttemptStateError("attempt control directory is invalid")
registered = self._read_json_file(control, "locator.json")
expected_locator = {
"supervisor_pid": locator.supervisor_pid,
"start_identity": locator.start_identity,
"socket_path": locator.socket_path,
"challenge": locator.challenge,
"control_dir": locator.control_dir,
"created_at": locator.created_at,
}
if registered != expected_locator:
raise AttemptStateError("registered locator is invalid")
receipt_path = control / "cleanup-receipt.json"
try:
os.lstat(receipt_path)
except FileNotFoundError:
if required:
raise AttemptStateError("closed cleanup receipt is unavailable")
return None
except OSError as exc:
raise AttemptStateError("closed cleanup receipt is unavailable") from exc
receipt = self._validate_receipt_record(
self._read_json_file(control, receipt_path.name), locator
)
# Result-bound receipts get their instant parsed by terminal coherence; the
# receipt-only path is the sole authority here, so parse it independently.
self._instant(receipt["completed_at"], "cleanup receipt")
if receipt["reason"] != expected_reason:
raise AttemptStateError("closed cleanup receipt is invalid")
socket_path = control / SOCKET_FILENAME
try:
socket_mode = os.lstat(socket_path).st_mode
except FileNotFoundError:
return receipt
except OSError as exc:
raise AttemptStateError("closed cleanup socket is unavailable") from exc
if not stat.S_ISSOCK(socket_mode):
raise AttemptStateError("closed cleanup socket is invalid")
if expected_reason == REASON_CONTROLLER_LOST:
raise AttemptStateError("closed cleanup socket is still active")
return receipt
@staticmethod
def _public_locator(locator: SupervisorLocator) -> dict[str, Any]:
return {
"supervisor_pid": locator.supervisor_pid,
"start_identity": locator.start_identity,
"socket_path": locator.socket_path,
"control_dir": locator.control_dir,
"challenge_digest": hashlib.sha256(locator.challenge.encode("utf-8")).hexdigest(),
"created_at": locator.created_at,
}
def _validate_journal_record(self, root: Path, result: dict[str, Any], expected_digest: str) -> None:
"""Validate the append-only journal against the published result record."""
journal = root / "lifecycle-journal.jsonl"
try:
lines = [json.loads(line) for line in _read_regular_bytes(journal, "lifecycle journal").decode("utf-8").splitlines()]
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise AttemptStateError("lifecycle journal is invalid") from exc
if len(lines) < 2:
raise AttemptStateError("lifecycle journal is invalid")
header, terminal = lines[0], lines[-1]
self._exact_fields(header, {"record", "journal_version", "spec_digest", "submission_mode", "completion_mode", "started_at"}, "lifecycle journal header")
self._exact_fields(terminal, {"record", "terminal_reason", "success", "cleanup_complete", "process_group_alive", "ended_at"}, "lifecycle journal terminal")
if header["record"] != "header" or header["journal_version"] != 1 or header["spec_digest"] != expected_digest or header["submission_mode"] != result["submission_mode"] or header["completion_mode"] != result["completion_mode"] or header["started_at"] != result["started_at"] or not isinstance(header["started_at"], str) or terminal["record"] != "terminal" or terminal["terminal_reason"] != result["terminal_reason"] or terminal["success"] != result["success"] or terminal["cleanup_complete"] is not True or terminal["process_group_alive"] is not False or terminal["ended_at"] != result["ended_at"] or not isinstance(terminal["ended_at"], str):
raise AttemptStateError("lifecycle journal is invalid")
for event in lines[1:-1]:
self._exact_fields(event, {"record", "kind", "source", "stream", "monotonic_ns", "source_monotonic_ns", "observed_at", "detail"}, "lifecycle journal event")
if lines[1:-1] != result["events"]:
raise AttemptStateError("lifecycle journal events are invalid")
def _read_bound_lifecycle_terminal(self, root: Path, locator: SupervisorLocator, expected_digest: str) -> dict[str, Any] | None:
result_path = root / "lifecycle-result.json"
if not result_path.exists() and not result_path.is_symlink():
return None
result = self._read_json_file(root, "lifecycle-result.json")
self._validate_result_record(result, locator, expected_digest)
self._validate_journal_record(root, result, expected_digest)
# ``locator.control_dir`` is the live short lease alias. It is
# intentionally removed after terminal publication, so terminal reads
# must rebind the receipt through the immutable attempt-owned control
# directory rather than resolve a stale alias.
control = root / CONTROL_DIRECTORY_NAME
_directory(control, "attempt control directory")
registered = self._read_json_file(control, "locator.json")
expected_locator = {
"supervisor_pid": locator.supervisor_pid,
"start_identity": locator.start_identity,
"socket_path": locator.socket_path,
"challenge": locator.challenge,
"control_dir": locator.control_dir,
"created_at": locator.created_at,
}
if registered != expected_locator:
raise AttemptStateError("registered locator is invalid")
receipt_data = self._validate_receipt_record(
self._read_json_file(control, "cleanup-receipt.json"), locator
)
if receipt_data["reason"] != result["terminal_reason"]:
raise AttemptStateError("cleanup receipt is invalid")
events = self._validate_terminal_events(result["events"])
self._validate_terminal_coherence(result, receipt_data, events)
return result
def validate_invocation_terminal(self, attempt: Attempt, invocation: InvocationResult) -> dict[str, Any]:
"""Validate direct lifecycle output against the identity committed before launch."""
if not isinstance(invocation, InvocationResult):
raise AttemptStateError("invocation result is invalid")
run, root = self._bound_attempt(attempt)
record = self._attempt_record(root, run, attempt.identity)
if record is None or record["state"] != NONTERMINAL_STATE:
raise AttemptStateError("invocation transition is invalid")
raw_locator = record.get("locator")
expected_digest = record.get("spec_digest")
if raw_locator is None or not isinstance(expected_digest, str):
raise AttemptStateError("invocation identity was not committed")
locator = self._locator_from_record(
root, raw_locator, state=NONTERMINAL_STATE
)
expected_result_path = root / "lifecycle-result.json"
expected_journal_path = root / "lifecycle-journal.jsonl"
if invocation.locator != locator or invocation.spec_digest != expected_digest or Path(invocation.result_path).resolve(strict=False) != expected_result_path.resolve() or Path(invocation.journal_path).resolve(strict=False) != expected_journal_path.resolve():
raise AttemptStateError("invocation result identity is invalid")
terminal = self._read_bound_lifecycle_terminal(root, locator, expected_digest)
if terminal is None:
raise AttemptStateError("lifecycle terminal is unavailable")
self._validate_measurement(
root, run, attempt.identity, expected_digest,
str(terminal["terminal_reason"]),
record.get("measurement_policy"), terminal,
)
self._validate_web_validation(
root,
run,
attempt.identity,
record.get("web_validation_policy"),
)
fields = ("success", "terminal_reason", "exit_code", "signal", "submitted", "finish_then_idle_then_quiet", "cleanup_complete", "process_group_alive", "spec_digest", "started_at", "ended_at", "duration_ns")
if any(getattr(invocation, field) != terminal[field] for field in fields):
raise AttemptStateError("invocation result does not match durable terminal")
return terminal
def publish_attempt_measurement(
self,
attempt: Attempt,
caller: str,
result: InvocationResult,
observation: WorkspaceWriteObservation,
) -> None:
"""Publish one immutable timing/usage sidecar before terminal commit.
Publication happens while the attempt is still running so a collision,
a corrupt projection or an identity mismatch fails closed before any
terminal state is written, and never rewrites bytes it does not own.
"""
if not isinstance(result, InvocationResult):
raise AttemptStateError("invocation result is invalid")
run, root = self._bound_attempt(attempt)
record = self._attempt_record(root, run, attempt.identity)
if record is None or record["state"] != NONTERMINAL_STATE:
raise AttemptStateError("measurement requires a running attempt")
expected_digest = record.get("spec_digest")
if not isinstance(expected_digest, str):
# The invocation identity was never committed, so terminal
# validation owns this failure and there is nothing to bind to.
return
if expected_digest != result.spec_digest:
raise AttemptStateError("measurement invocation identity is invalid")
try:
publish_measurement(root, build_measurement(
run_id=run.run_id,
cell_id=attempt.identity.cell_id,
repetition=attempt.identity.repetition,
attempt=attempt.identity.attempt,
caller=caller,
result=result,
observation=observation,
))
except MeasurementError as exc:
raise AttemptStateError("attempt measurement is invalid") from exc
self._validate_measurement(
root, run, attempt.identity, expected_digest, result.terminal_reason,
record.get("measurement_policy"),
)
def publish_attempt_web_validation(
self, attempt: Attempt, web_validation: Any,
) -> None:
"""Publish one web sidecar while the attempt is running.
The sidecar is deliberately independent from lifecycle success: a page
may fail a quality gate while its caller lifecycle remains successful.
It is nevertheless required provenance for every production attempt.
"""
run, root = self._bound_attempt(attempt)
record = self._attempt_record(root, run, attempt.identity)
if record is None or record["state"] != NONTERMINAL_STATE:
raise AttemptStateError("web validation requires a running attempt")
if record.get("web_validation_policy") != WEB_VALIDATION_POLICY_REQUIRED_V1:
raise AttemptStateError("web validation policy is unavailable")
try:
publish_web_validation(root, web_validation)
except WebValidationError as exc:
raise AttemptStateError("attempt web validation is invalid") from exc
self._validate_web_validation(
root, run, attempt.identity, record.get("web_validation_policy"),
)
def reconcile(self, attempt: Attempt) -> Attempt:
"""Commit only authenticated terminal evidence before recovery cleanup."""
run, root = self._bound_attempt(attempt)
record = self._attempt_record(root, run, attempt.identity, absent_ok=True)
if record is None:
terminal = self.publish_terminal(
attempt, "interrupted", result={"terminal_reason": "interrupted"}
)
self.release_control_lease(terminal)
return terminal
if record["state"] in TERMINAL_STATES:
terminal = Attempt(attempt.identity, attempt.root, str(record["state"]))
self.release_control_lease(terminal)
return terminal
raw_locator = record.get("locator")
if raw_locator is None:
terminal = self.publish_terminal(
attempt, "interrupted", result={"terminal_reason": "interrupted"}
)
self.release_control_lease(terminal)
return terminal
locator = self._locator_from_record(
root, raw_locator, state=NONTERMINAL_STATE
)
expected_digest = record.get("spec_digest")
if not isinstance(expected_digest, str) or not DIGEST_RE.fullmatch(expected_digest):
raise AttemptStateError("recovery identity is invalid")
terminal = self._read_bound_lifecycle_terminal(root, locator, expected_digest)
if terminal is not None:
self._validate_measurement(
root, run, attempt.identity, expected_digest,
str(terminal["terminal_reason"]),
record.get("measurement_policy"), terminal,
)
self._ensure_required_web_validation(
root, run, attempt.identity, record, terminal
)
published = self.publish_terminal(
attempt, self._state_for_reason(terminal["terminal_reason"]), result=terminal
)
self.release_control_lease(published)
return published
closed_receipt = self._closed_cleanup_receipt(root, locator)
if closed_receipt is not None:
recovery_terminal = {"terminal_reason": closed_receipt["reason"]}
self._ensure_required_web_validation(
root, run, attempt.identity, record, recovery_terminal
)
published = self.publish_terminal(
attempt,
self._state_for_reason(recovery_terminal["terminal_reason"]),
result=recovery_terminal,
)
self.release_control_lease(published)
return published
try:
outcome = recover_invocation(locator, stop=True)
except LifecycleRecoveryError as exc:
# The supervisor may complete cleanup and durably publish its
# authenticated receipt immediately before the control reply is
# lost. Re-read that exact receipt instead of treating a missing
# reply as proof that cleanup did not happen.
try:
recovered_receipt = self._closed_cleanup_receipt(
root,
locator,
expected_reason=REASON_RECOVERED_STOP,
)
except AttemptStateError:
raise AttemptStateError("recovery is unverified") from exc
if recovered_receipt is None:
raise AttemptStateError("recovery is unverified") from exc
recovery_terminal = {
"terminal_reason": recovered_receipt["reason"]
}
self._ensure_required_web_validation(
root, run, attempt.identity, record, recovery_terminal
)
terminal = self.publish_terminal(
attempt,
self._state_for_reason(recovery_terminal["terminal_reason"]),
result=recovery_terminal,
)
self.release_control_lease(terminal)
return terminal
receipt = Path(outcome.receipt_path)
if not outcome.cleanup_complete or outcome.process_group_alive or receipt.parent != Path(locator.control_dir) or not _contained(receipt, root):
raise AttemptStateError("recovery cleanup is unverified")
try:
self._validate_receipt_record(self._read_json_file(receipt.parent, receipt.name), locator)
except AttemptStateError as exc:
raise AttemptStateError("recovery cleanup is unverified") from exc
recovery_terminal = {"terminal_reason": outcome.reason}
self._ensure_required_web_validation(
root, run, attempt.identity, record, recovery_terminal
)
terminal = self.publish_terminal(
attempt,
self._state_for_reason(recovery_terminal["terminal_reason"]),
result=recovery_terminal,
)
self.release_control_lease(terminal)
return terminal
def execute_attempt(
self,
attempt: Attempt,
*,
prepare: Callable[[Attempt], Any],
invoke: Callable[[Attempt, Callable[[SupervisorLocator, str], None]], InvocationResult],
require_measurement: bool = False,
require_web_validation: bool = False,
) -> Attempt:
"""Prepare once, publish running identity, then invoke lifecycle once."""
run, root = self._bound_attempt(attempt)
if self._attempt_record(root, run, attempt.identity, absent_ok=True) is not None:
raise AttemptStateError("attempt has already been prepared")
try:
prepare(attempt)
except Exception:
_write_new(root / "attempt.json", _json_bytes(self._initial_record(run, attempt, "failed", reason="preparation_failed")))
raise
_write_new(
root / "attempt.json",
_json_bytes(self._initial_record(
run, attempt, NONTERMINAL_STATE,
measurement_policy=(MEASUREMENT_POLICY_REQUIRED_V1
if require_measurement else None),
web_validation_policy=(WEB_VALIDATION_POLICY_REQUIRED_V1
if require_web_validation else None),
)),
)
if require_measurement:
_write_new(
root / MEASUREMENT_POLICY_FILENAME,
_json_bytes(self._measurement_policy_start_record(run, attempt.identity)),
)
if require_web_validation:
_write_new(
root / WEB_VALIDATION_POLICY_FILENAME,
_json_bytes(self._web_validation_policy_start_record(run, attempt.identity)),
)
result = invoke(attempt, lambda locator, digest: self.record_locator(attempt, locator, digest))
terminal = self.validate_invocation_terminal(attempt, result)
published = self.publish_terminal(
attempt, self._state_for_reason(terminal["terminal_reason"]), result=terminal
)
self.release_control_lease(published)
return published
def status(self, run: RunIdentity, manifest: Manifest) -> dict[str, Any]:
"""Read-only deterministic status; it neither creates nor reconciles."""
bound_run = self.open(manifest, run.run_id)
if bound_run != run:
raise AttemptStateError("run identity is invalid")
states = {name: 0 for name in sorted(TERMINAL_STATES | {NONTERMINAL_STATE})}
for slot in self.slots(manifest):
for attempt in self.attempts(bound_run, slot):
states[attempt.state] += 1
preflights = self._preflight_records(bound_run, manifest)
latest = preflights[-1] if preflights else None
projection = {
"records": len(preflights),
"latest_sequence": 0 if latest is None else latest["sequence"],
"latest_status": "unavailable" if latest is None else latest["status"],
**(
{status: 0 for status in PREFLIGHT_STATUSES}
if latest is None
else _preflight_counts(latest["results"])
),
}
return {
"run_id": bound_run.run_id,
"manifest_digest": bound_run.manifest_digest,
"preflight": projection,
"attempts": states,
}
def preflight_manifest(
store: RunStore,
manifest: Manifest,
manifest_bytes: bytes,
*,
adapters: Mapping[str, PreflightAdapter],
) -> tuple[RunIdentity, dict[str, Any]]:
"""Collect direct observations, then create one run and append one record."""
observations = collect_preflight_observations(manifest, adapters)
if not observations:
raise AttemptStateError("preflight requires a direct cell")
run = store.create(manifest, manifest_bytes)
return run, store.record_preflight(run, manifest, observations)
def _validate_prepared_binding(
store: RunStore,
manifest: Manifest,
cell: MatrixCell,
attempt: Attempt,
prepared: PreparedWorkspace,
) -> None:
"""Reject any caller/cell/workspace identity drift before invocation."""
if not isinstance(prepared, PreparedWorkspace):
raise AttemptStateError("prepared workspace is invalid")
if cell.id != attempt.identity.cell_id or prepared.identity != attempt.identity:
raise AttemptStateError("prepared workspace identity mismatch")
attempt_root = Path(attempt.root).resolve()
workspace = Path(prepared.workspace_dir)
session = Path(prepared.session_dir)
if Path(prepared.attempt_root).resolve() != attempt_root:
raise AttemptStateError("prepared attempt root mismatch")
if (
workspace.is_symlink()
or session.is_symlink()
or not workspace.is_dir()
or not session.is_dir()
or workspace.resolve() != attempt_root / "workspace"
or session.resolve() != attempt_root / "session"
):
raise AttemptStateError("prepared workspace path mismatch")
if (
not prepared.session_is_fresh
or not isinstance(prepared.session_id, str)
or not prepared.session_id
or prepared.workspace_checksum != manifest.fixture.checksum
or prepared.setup_cache_policy != manifest.setup_cache_policy
):
raise AttemptStateError("prepared workspace policy mismatch")
expected_testbed = (store.repo_root / manifest.testbed).resolve()
if (
not prepared.testbed_provenance.clean
or Path(prepared.testbed_provenance.path).resolve() != expected_testbed
):
raise AttemptStateError("prepared testbed provenance mismatch")
def run_slots(
store: RunStore,
run: RunIdentity,
manifest: Manifest,
*,
adapters: Mapping[str, ExecutionAdapter],
prepare: Callable[[Manifest, Attempt], PreparedWorkspace],
retry_failed: bool = False,
) -> tuple[Attempt, ...]:
"""Append fresh preflight, then execute eligible slots under one writer."""
if not isinstance(adapters, Mapping):
raise CapabilityUnavailable("capability-unavailable: caller-adapter")
required_callers = {cell.caller for cell in manifest.matrix}
if any(
caller not in adapters or not callable(getattr(adapters[caller], "invoke", None))
for caller in required_callers
):
raise CapabilityUnavailable("capability-unavailable: caller-adapter")
if not callable(prepare):
raise AttemptStateError("workspace preparer is invalid")
observations = collect_preflight_observations(manifest, adapters)
if not observations:
raise AttemptStateError("preflight requires a direct cell")
bound_run = store.open(manifest, run.run_id)
if bound_run != run:
raise AttemptStateError("run identity is invalid")
cells = {cell.id: cell for cell in manifest.matrix}
if len(cells) != len(manifest.matrix):
raise AttemptStateError("manifest cell identity is invalid")
completed: list[Attempt] = []
with store.writer(bound_run):
preflight = store._record_preflight_locked(
bound_run, manifest, observations
)
if preflight["status"] != "ready":
return ()
if frozenset(observations) != frozenset(cells):
return ()
for slot in store.slots(manifest):
cell = cells.get(slot.cell_id)
if cell is None:
raise AttemptStateError("slot cell identity is invalid")
existing = store.attempts(bound_run, slot)
if existing and existing[-1].state == "success":
continue
if existing and existing[-1].state in {"failed", "timed_out", "cancelled"} and not retry_failed:
continue
if existing and existing[-1].state not in TERMINAL_STATES:
store.reconcile(existing[-1])
existing = store.attempts(bound_run, slot)
if existing[-1].state == "success" or (existing[-1].state in {"failed", "timed_out", "cancelled"} and not retry_failed):
continue
attempt = store.allocate(bound_run, slot)
prepared: PreparedWorkspace | None = None
def prepare_bound(current: Attempt) -> PreparedWorkspace:
nonlocal prepared
candidate = prepare(manifest, current)
_validate_prepared_binding(store, manifest, cell, current, candidate)
prepared = candidate
return candidate
def invoke_bound(
current: Attempt,
on_started: Callable[[SupervisorLocator, str], None],
) -> InvocationResult:
if prepared is None:
raise AttemptStateError("prepared workspace is unavailable")
lease = store.acquire_control_lease(current)
# The observer's baseline must be older than any caller write,
# so it starts before the caller can be launched and is joined
# on every success, error, timeout and cancellation path.
observer = WorkspaceWriteObserver(prepared.workspace_dir)
observer.start()
try:
result = adapters[cell.caller].invoke(
cell,
prepared,
current,
lease.control_dir,
manifest.fixture.prompt_content,
manifest.timeout,
on_started,
)
finally:
observation = observer.stop()
if not observer.stopped:
raise AttemptStateError("workspace observer did not stop")
store.publish_attempt_measurement(
current, cell.caller, result, observation
)
measurement = load_measurement(current.root)
web = validate_web_attempt(
manifest, current.root, prepared, measurement, result,
)
store.publish_attempt_web_validation(current, web)
return result
completed.append(
store.execute_attempt(
attempt,
prepare=prepare_bound,
invoke=invoke_bound,
require_measurement=True,
require_web_validation=True,
)
)
return tuple(completed)