iop/scripts/agent_benchmark/attempts_test.py

"""Credential-free production-path tests for durable benchmark attempts."""

from __future__ import annotations

import contextlib
import datetime
import io
import json
import os
import signal
import socket
import stat
import subprocess
import sys
import tempfile
import threading
import time
import unittest
from pathlib import Path
from unittest import mock

from scripts import agent_comparison_benchmark as benchmark_cli
from scripts.agent_benchmark.attempts import (
    Attempt,
    AttemptStateError,
    CapabilityUnavailable,
    MEASUREMENT_POLICY_FILENAME,
    PreflightObservation,
    RunBusyError,
    RunIdentity,
    RunStore,
    Slot,
    WEB_VALIDATION_POLICY_FILENAME,
    run_slots,
)
from scripts.agent_benchmark.connectivity import (
    ISSUE_RESUME_CODES,
    CallerCapability,
    ConnectivityIssue,
    EffectiveBinding,
    RequestedEffectiveBinding,
    make_result,
)
from scripts.agent_benchmark.lifecycle import (
    COMPLETION_EXIT_AFTER_IDLE,
    SUBMISSION_ARGV_TASK,
    InvocationResult,
    InvocationSpec,
    LifecycleRecoveryError,
    REASON_CONTROLLER_LOST,
    REASON_RECOVERED_STOP,
    SupervisorLocator,
    count_metric,
    duration_metric,
    env_pairs,
    recover_invocation,
    run_invocation,
    spec_digest,
)
from scripts.agent_benchmark.manifest import AssetMapping, Timeout, digest_workspace_inputs, load_manifest
from scripts.agent_benchmark.measurement import (
    MEASUREMENT_FILENAME,
    WorkspaceWriteObserver,
    load_measurement,
    path_digest,
)
from scripts.agent_benchmark.web_validation import (
    WEB_VALIDATION_FILENAME,
    WebValidationError,
    load_web_validation,
)
from scripts.agent_benchmark.workspace import AttemptIdentity, prepare_workspace


def _manifest(root: Path, repetitions: int = 1):
    fixtures = root / "scripts/fixtures"
    fixtures.mkdir(parents=True, exist_ok=True)
    (fixtures / "prompt.md").write_text("prompt", encoding="utf-8")
    (fixtures / "reference.txt").write_text("reference", encoding="utf-8")
    fixture = {
        "version": "v1", "prompt": "scripts/fixtures/prompt.md",
        "assets": [{"source": "scripts/fixtures/reference.txt", "workspace_path": "workspace/reference.txt"}],
        "checksum": digest_workspace_inputs((AssetMapping("scripts/fixtures/reference.txt", "workspace/reference.txt", b"reference"),)),
    }
    data = {
        "pipeline_version": "2", "environment": "dev", "testbed": "../iop-s2",
        "session_policy": "fresh", "setup_cache_policy": "isolated",
        "timeout": {"run_seconds": 1, "idle_seconds": 1, "quiet_seconds": 1, "cleanup_grace_seconds": 1},
        "viewports": [{"id": "desktop", "width": 1, "height": 1}], "rubric_version": "landing-quality-v1",
        "evaluator": {"caller": "codex", "iop": {"request_model": "judge", "requested_effort": "high", "route_kind": "direct", "route_id": "judge", "expected_bindings": [{"stage": "request", "model": "judge", "effort": "high"}]}},
        "output_root": "agent-test/runs/a", "fixture": fixture, "repetitions": repetitions,
        "matrix": [{"id": "a", "caller": "claude", "iop": {"request_model": "model", "requested_effort": "high", "route_kind": "direct", "route_id": "route", "expected_bindings": [{"stage": "request", "model": "model", "effort": "high"}]}}],
    }
    path = root / "manifest.json"
    raw = json.dumps(data, sort_keys=True).encode("utf-8")
    path.write_bytes(raw)
    return load_manifest(path, repo_root=root), raw, path


def _events(_: str, line: str) -> str | None:
    return {"FINISH": "finish", "IDLE": "idle"}.get(line.strip())


def _preflight_observation(cell, issue_code: str | None = None) -> PreflightObservation:
    capability = CallerCapability(
        cell.caller, ("direct", "execution_preset"), (cell.iop.requested_effort,)
    )
    if issue_code is None:
        bindings = tuple(
            EffectiveBinding(item.stage, item.model, item.effort)
            for item in cell.iop.expected_bindings
        )
        binding = RequestedEffectiveBinding(
            cell.id, cell.caller, cell.iop.route_kind, cell.iop.route_id,
            cell.iop.request_model, cell.iop.requested_effort,
            cell.iop.route_kind, cell.iop.route_id, cell.iop.request_model,
            cell.iop.requested_effort, bindings,
        )
        issues = ()
    else:
        binding = RequestedEffectiveBinding(
            cell.id, cell.caller, cell.iop.route_kind, cell.iop.route_id,
            cell.iop.request_model, cell.iop.requested_effort,
        )
        issues = (ConnectivityIssue(issue_code, ISSUE_RESUME_CODES[issue_code]),)
    return PreflightObservation(
        make_result(cell, capability, binding, issues),
        "sha256:" + "1" * 64,
        "sha256:" + "2" * 64,
    )


_PROBE_TIMEOUT_SECONDS = 30.0


def _controller_loss_child(payload_json: str) -> None:
    """Run one real lifecycle controller that the parent regression will kill."""
    payload = json.loads(payload_json)
    store = RunStore(payload["repo"])
    manifest = load_manifest(
        Path(payload["manifest"]), repo_root=Path(payload["repo"])
    )
    run = RunIdentity(payload["run_id"], manifest.digest, payload["run_root"])
    attempt = Attempt(
        AttemptIdentity(
            payload["run_id"],
            payload["cell_id"],
            payload["repetition"],
            payload["attempt_number"],
        ),
        payload["attempt_root"],
        "running",
    )

    def invoke(current, started):
        lease = store.acquire_control_lease(current)
        spec = InvocationSpec(
            argv=(
                sys.executable,
                "-u",
                "-c",
                "import time; print('START', flush=True); time.sleep(30)",
            ),
            cwd=payload["repo"],
            env=env_pairs({"PATH": os.environ.get("PATH", "/usr/bin:/bin")}),
            submission_mode=SUBMISSION_ARGV_TASK,
            completion_mode=COMPLETION_EXIT_AFTER_IDLE,
            timeout=Timeout(60, 1, 1, 1),
            evidence_dir=current.root,
            control_dir=lease.control_dir,
        )
        return run_invocation(
            spec,
            parse_event=_events,
            on_started=lambda locator: started(locator, spec_digest(spec)),
        )

    with store.writer(run):
        store.execute_attempt(attempt, prepare=lambda _: None, invoke=invoke)

# Every durable read runs in a bounded child so a blocking special file cannot
# hang the suite; the child reports whether the store fails closed.
_PROBE_SOURCE = """
import json
import os
import sys
from pathlib import Path

from scripts.agent_benchmark.attempts import Attempt, AttemptStateError, RunIdentity, RunStore, Slot
from scripts.agent_benchmark.lifecycle import SupervisorLocator
from scripts.agent_benchmark.manifest import load_manifest
from scripts.agent_benchmark.workspace import AttemptIdentity

payload = json.loads(sys.argv[1])
store = RunStore(payload["repo"])
manifest = load_manifest(Path(payload["manifest"]), repo_root=Path(payload["repo"]))
run = RunIdentity(payload["run_id"], manifest.digest, payload["run_root"])
attempt = Attempt(
    AttemptIdentity(payload["run_id"], payload["cell_id"], payload["repetition"], payload["attempt_number"]),
    payload["attempt_root"], "running",
)


def lease():
    with store.writer(run):
        pass


def locator():
    store.record_locator(
        attempt,
        SupervisorLocator(**payload["locator"]),
        "sha256:" + "0" * 64,
    )


operations = {
    "open": lambda: store.open(manifest, run.run_id),
    "lease": lease,
    "attempts": lambda: store.attempts(run, Slot("a", 1)),
    "reconcile": lambda: store.reconcile(attempt),
    "locator": locator,
}
try:
    operations[payload["operation"]]()
except AttemptStateError:
    print("rejected")
    sys.exit(0)
print("accepted")
sys.exit(1)
"""


def _reordered(events: list[dict]) -> list[dict]:
    """Return production events with finish and idle transposed."""
    kinds = [event["kind"] for event in events]
    swapped = list(events)
    finish, idle = kinds.index("finish"), kinds.index("idle")
    swapped[finish], swapped[idle] = swapped[idle], swapped[finish]
    return swapped


def _without(kind: str):
    return lambda events: [event for event in events if event["kind"] != kind]


def _duplicated(kind: str):
    return lambda events: events + [event for event in events if event["kind"] == kind]


class ControllerCrash(RuntimeError):
    """Test-only controller loss after lifecycle evidence has been published."""


class FakeExecutionAdapter:
    """Typed fake that exercises the production run_slots boundary."""

    def __init__(
        self,
        owner: "AttemptBase",
        reason: str,
        calls: list[str],
        issue_code: str | None = None,
    ) -> None:
        self.owner = owner
        self.reason = reason
        self.calls = calls
        self.issue_code = issue_code
        self.capability = CallerCapability(
            "claude", ("direct", "execution_preset"), ("high",)
        )

    def preflight(self, cell):
        self.calls.append("preflight")
        return _preflight_observation(cell, self.issue_code)

    def invoke(
        self,
        cell,
        prepared,
        attempt,
        control_dir,
        task_payload,
        timeout,
        on_started,
    ):
        self.calls.append("invoke")
        if cell.id != attempt.identity.cell_id or prepared.identity != attempt.identity:
            raise AssertionError("typed execution identity drift")
        if task_payload != self.owner.manifest.fixture.prompt_content:
            raise AssertionError("task payload drift")
        source = (
            "print('FINISH'); print('IDLE')"
            if self.reason == "success"
            else "import sys; print('FAILED'); sys.exit(3)"
        )
        if Path(control_dir).resolve(strict=False) != Path(attempt.root).resolve() / "control":
            raise AssertionError("controller control binding drift")
        spec = self.owner._spec(attempt, source, control_dir=control_dir)
        return run_invocation(
            spec,
            parse_event=_events,
            on_started=lambda locator: on_started(locator, spec_digest(spec)),
        )

    def __call__(self, attempt, on_started):
        """Retain the lower-level RunStore lifecycle seam for recovery tests."""
        self.calls.append("invoke")
        source = (
            "print('FINISH'); print('IDLE')"
            if self.reason == "success"
            else "import sys; print('FAILED'); sys.exit(3)"
        )
        spec = self.owner._spec(attempt, source)
        return run_invocation(
            spec,
            parse_event=_events,
            on_started=lambda locator: on_started(locator, spec_digest(spec)),
        )


def _measured_events(_stream: str, line: str):
    """Parse the fake caller's terminal lines plus one typed usage report."""
    text = line.strip()
    if text.startswith("USAGE "):
        _, duration, tokens = text.split()
        return (
            duration_metric("total_duration", duration, model="model"),
            count_metric("input_tokens", int(tokens), model="model"),
        )
    return _events(_stream, line)


_MEASURING_SOURCES = {
    "success": (
        "from pathlib import Path\n"
        "Path({workspace!r}).joinpath('answer.txt').write_text('generated')\n"
        "print('USAGE 12.5 11')\nprint('FINISH')\nprint('IDLE')\n"
    ),
    "failed": "import sys\nprint('FAILED')\nsys.exit(3)\n",
    "timeout": "import time\ntime.sleep(30)\n",
}


class MeasuringExecutionAdapter:
    """Typed fake whose caller writes into the prepared workspace and reports usage."""

    def __init__(self, owner: "AttemptBase", mode: str = "success", *, collide: bool = False) -> None:
        self.owner = owner
        self.mode = mode
        self.collide = collide
        self.capability = CallerCapability(
            "claude", ("direct", "execution_preset"), ("high",)
        )

    def preflight(self, cell):
        return _preflight_observation(cell)

    def invoke(self, cell, prepared, attempt, control_dir, task_payload, timeout, on_started):
        spec = InvocationSpec(
            argv=(
                sys.executable, "-u", "-c",
                _MEASURING_SOURCES[self.mode].format(workspace=prepared.workspace_dir),
            ),
            cwd=prepared.workspace_dir,
            env=env_pairs({"PATH": os.environ.get("PATH", "/usr/bin:/bin")}),
            submission_mode=SUBMISSION_ARGV_TASK,
            completion_mode=COMPLETION_EXIT_AFTER_IDLE,
            timeout=Timeout(1, 1, 1, 1) if self.mode == "timeout" else Timeout(5, 1, 1, 1),
            evidence_dir=attempt.root,
            control_dir=control_dir,
        )
        result = run_invocation(
            spec, parse_event=_measured_events,
            on_started=lambda locator: on_started(locator, spec_digest(spec)),
        )
        if self.collide:
            # Simulate a concurrent owner that already published this sidecar.
            (Path(attempt.root) / MEASUREMENT_FILENAME).write_bytes(b'{"record":"prior"}\n')
        return result


class AttemptBase(unittest.TestCase):
    def setUp(self) -> None:
        self.temp = tempfile.TemporaryDirectory(dir="/tmp", prefix="b")
        self.root = Path(self.temp.name) / "r"
        self.root.mkdir()
        self._control_aliases: list[Path] = []
        self.manifest, self.raw, self.manifest_path = _manifest(self.root)
        self.store = RunStore(
            self.root,
            clock=lambda: datetime.datetime(2026, 1, 2, 3, 4, 5, tzinfo=datetime.timezone.utc),
            token_hex=lambda _: "abcdef123456",
        )

    def tearDown(self) -> None:
        for alias in self._control_aliases:
            try:
                alias.unlink()
            except FileNotFoundError:
                pass
        self.temp.cleanup()

    def create_run(self):
        return self.store.create(self.manifest, self.raw)

    def _control_dir(self, attempt) -> str:
        lease = self.store.acquire_control_lease(attempt)
        self._control_aliases.append(Path(lease.alias))
        return lease.control_dir

    def _spec(
        self, attempt, source: str, *, control_dir: str | None = None
    ) -> InvocationSpec:
        bound_control_dir = control_dir or self._control_dir(attempt)
        return InvocationSpec(
            argv=(sys.executable, "-u", "-c", source),
            cwd=str(self.root),
            env=env_pairs({"PATH": os.environ.get("PATH", "/usr/bin:/bin")}),
            submission_mode=SUBMISSION_ARGV_TASK,
            completion_mode=COMPLETION_EXIT_AFTER_IDLE,
            timeout=Timeout(5, 1, 1, 1),
            evidence_dir=attempt.root,
            control_dir=bound_control_dir,
        )

    def adapter(
        self,
        reason: str,
        calls: list[str],
        issue_code: str | None = None,
    ) -> FakeExecutionAdapter:
        return FakeExecutionAdapter(self, reason, calls, issue_code)

    def preparer(self, calls: list[str]):
        def prepare(manifest, attempt):
            calls.append("prepare")
            return prepare_workspace(
                manifest, attempt.root, attempt.identity, repo_root=self.root
            )

        return prepare

    def _init_testbed(self) -> None:
        testbed = self.root.parent / "iop-s2"
        testbed.mkdir()
        (testbed / "README.md").write_text("testbed", encoding="utf-8")
        for command in (
            ("git", "init"), ("git", "config", "user.name", "test"),
            ("git", "config", "user.email", "test@example.invalid"), ("git", "add", "."),
            ("git", "commit", "-m", "testbed"),
        ):
            subprocess.run(command, cwd=testbed, check=True, capture_output=True)


class AttemptStoreTest(AttemptBase):
    def test_slots_and_append_only_terminals(self):
        self.manifest, self.raw, self.manifest_path = _manifest(self.root, repetitions=2)
        run = self.create_run()
        self.assertEqual([slot.repetition for slot in self.store.slots(self.manifest)], [1, 2])
        with self.store.writer(run):
            first = self.store.allocate(run, Slot("a", 1))
            self.assertEqual(list(Path(first.root).iterdir()), [])
            terminal = self.store.publish_terminal(first, "failed")
            with self.assertRaises(AttemptStateError):
                self.store.publish_terminal(terminal, "success")
            second = self.store.allocate(run, Slot("a", 1))
        self.assertEqual(second.identity.attempt, 2)

    def test_writer_is_fail_fast_and_status_is_read_only(self):
        run = self.create_run()
        before = (Path(run.root) / "run.json").read_bytes()
        with self.store.writer(run):
            with self.assertRaises(RunBusyError):
                with self.store.writer(run):
                    pass
            self.assertEqual(self.store.status(run, self.manifest)["attempts"]["running"], 0)
        self.assertEqual(before, (Path(run.root) / "run.json").read_bytes())

    def test_open_rejects_changed_snapshot_and_empty_allocation_reconciles(self):
        run = self.create_run()
        with self.assertRaises(AttemptStateError):
            self.store.open(self.manifest, run.run_id, self.raw + b" ")
        with self.store.writer(run):
            attempt = self.store.allocate(run, Slot("a", 1))
            interrupted = self.store.reconcile(attempt)
        self.assertEqual(interrupted.state, "interrupted")

    def test_foreign_record_and_symlink_fail_closed_without_status_mutation(self):
        run = self.create_run()
        with self.store.writer(run):
            attempt = self.store.allocate(run, Slot("a", 1))
            self.store.publish_terminal(attempt, "failed")
        record = Path(attempt.root) / "attempt.json"
        foreign = json.loads(record.read_text(encoding="utf-8"))
        foreign["run_id"] = "run-20260102T030405Z-ffffffffffff"
        record.write_text(json.dumps(foreign), encoding="utf-8")
        before = record.read_bytes()
        with self.assertRaises(AttemptStateError):
            self.store.status(run, self.manifest)
        self.assertEqual(before, record.read_bytes())
        record.unlink()
        record.symlink_to(Path(attempt.root) / "other.json")
        with self.assertRaises(AttemptStateError):
            self.store.attempts(run, Slot("a", 1))

    def test_preflight_records_append_in_order_and_status_is_read_only(self):
        run = self.create_run()
        cell = self.manifest.matrix[0]
        first = self.store.record_preflight(
            run, self.manifest, {cell.id: _preflight_observation(cell)}
        )
        second = self.store.record_preflight(
            run,
            self.manifest,
            {cell.id: _preflight_observation(cell, "credential_missing")},
        )
        before = {
            path.name: path.read_bytes()
            for path in (Path(run.root) / "preflight").iterdir()
        }

        self.assertEqual((first["sequence"], second["sequence"]), (1, 2))
        self.assertEqual(
            [record["status"] for record in self.store.preflights(run, self.manifest)],
            ["ready", "registration_required"],
        )
        status = self.store.status(run, self.manifest)
        self.assertEqual(
            status["preflight"],
            {
                "records": 2,
                "latest_sequence": 2,
                "latest_status": "registration_required",
                "ready": 0,
                "registration_required": 1,
                "implementation_gap": 0,
            },
        )
        self.assertEqual(status["attempts"]["running"], 0)
        self.assertFalse((Path(run.root) / "cells").exists())
        self.assertEqual(
            before,
            {
                path.name: path.read_bytes()
                for path in (Path(run.root) / "preflight").iterdir()
            },
        )

    def test_preflight_corruption_and_symlink_fail_closed(self):
        run = self.create_run()
        cell = self.manifest.matrix[0]
        self.store.record_preflight(
            run, self.manifest, {cell.id: _preflight_observation(cell)}
        )
        record = Path(run.root) / "preflight/preflight-000001.json"
        original = record.read_bytes()
        record.write_bytes(original + b" ")
        with self.assertRaises(AttemptStateError):
            self.store.preflights(run, self.manifest)
        record.write_bytes(original)
        record.unlink()
        record.symlink_to(Path(run.root) / "run.json")
        with self.assertRaises(AttemptStateError):
            self.store.status(run, self.manifest)

    def test_preflight_rejects_sequence_gap_and_foreign_result(self):
        run = self.create_run()
        cell = self.manifest.matrix[0]
        self.store.record_preflight(
            run, self.manifest, {cell.id: _preflight_observation(cell)}
        )
        first = Path(run.root) / "preflight/preflight-000001.json"
        first.rename(first.with_name("preflight-000002.json"))
        with self.assertRaises(AttemptStateError):
            self.store.preflights(run, self.manifest)

        first.with_name("preflight-000002.json").rename(first)
        raw = json.loads(first.read_text(encoding="ascii"))
        raw["results"][0]["binding"]["requested_model"] = "fallback"
        first.write_bytes(json.dumps(raw, sort_keys=True, separators=(",", ":")).encode("ascii") + b"\n")
        with self.assertRaises(AttemptStateError):
            self.store.preflights(run, self.manifest)

    def test_preflight_evidence_contains_no_unmodeled_adapter_values(self):
        run = self.create_run()
        cell = self.manifest.matrix[0]
        sentinel = "private_endpoint_or_token_must_not_persist"
        observation = _preflight_observation(cell, "stream_incompatible")
        # An adapter may retain runtime-only values on itself, but the writer
        # accepts only the closed PreflightObservation projection above.
        adapter = type("Adapter", (), {"runtime_value": sentinel})()
        self.assertEqual(adapter.runtime_value, sentinel)
        self.store.record_preflight(run, self.manifest, {cell.id: observation})
        durable = b"".join(
            path.read_bytes()
            for path in Path(run.root).rglob("*")
            if path.is_file()
        )
        self.assertNotIn(sentinel.encode("ascii"), durable)


class AttemptOrchestrationTest(AttemptBase):
    def test_run_slots_prepares_workspace_and_invokes_once(self):
        self._init_testbed()
        run = self.create_run()
        calls: list[str] = []

        def prepare(manifest, attempt):
            self.assertTrue((Path(run.root) / "preflight/preflight-000001.json").is_file())
            calls.append("prepare")
            return prepare_workspace(manifest, attempt.root, attempt.identity, repo_root=self.root)

        completed = run_slots(self.store, run, self.manifest, adapters={"claude": self.adapter("success", calls)}, prepare=prepare)
        self.assertEqual([item.state for item in completed], ["success"])
        self.assertEqual(calls, ["preflight", "prepare", "invoke"])
        attempt_root = Path(completed[0].root)
        self.assertTrue((attempt_root / "prepared.json").is_file())
        state = json.loads((attempt_root / "attempt.json").read_text(encoding="utf-8"))
        alias = Path(state["locator"]["control_dir"]).parent
        self.assertFalse(os.path.lexists(alias))
        self.assertTrue((attempt_root / "control/locator.json").is_file())
        self.assertTrue((attempt_root / "control/cleanup-receipt.json").is_file())
        self.assertEqual(self.store.status(run, self.manifest)["attempts"]["success"], 1)
        record_path = attempt_root / "attempt.json"
        original = record_path.read_bytes()
        corruptions = (
            (
                "arbitrary-control-path",
                lambda raw: raw["locator"].update(
                    {
                        "control_dir": "/tmp/iop-bench-attempt-000000000000000000000000/control",
                        "socket_path": "/tmp/iop-bench-attempt-000000000000000000000000/control/control.sock",
                    }
                ),
            ),
            (
                "mismatched-invocation-digest",
                lambda raw: raw.__setitem__("spec_digest", "sha256:" + "0" * 64),
            ),
        )
        for name, corrupt in corruptions:
            with self.subTest(name=name):
                raw = json.loads(original.decode("utf-8"))
                corrupt(raw)
                record_path.write_text(
                    json.dumps(raw, sort_keys=True, separators=(",", ":")) + "\n",
                    encoding="utf-8",
                )
                before = record_path.read_bytes()
                with self.assertRaises(AttemptStateError):
                    self.store.status(run, self.manifest)
                self.assertEqual(before, record_path.read_bytes())
                record_path.write_bytes(original)

    def test_control_lease_rejects_collision_and_mismatched_target(self):
        run = self.create_run()
        with self.store.writer(run):
            attempt = self.store.allocate(run, Slot("a", 1))
            with self.assertRaisesRegex(ControllerCrash, "before lease"):
                self.store.execute_attempt(
                    attempt,
                    prepare=lambda _: None,
                    invoke=lambda _attempt, _started: (_ for _ in ()).throw(
                        ControllerCrash("before lease")
                    ),
                )
        expected = self.store._control_lease_for_root(Path(attempt.root))
        alias = Path(expected.alias)
        try:
            alias.touch(mode=0o600)
            with self.assertRaisesRegex(AttemptStateError, "collision"):
                self.store.acquire_control_lease(attempt)
            alias.unlink()

            alias.symlink_to(self.root, target_is_directory=True)
            with self.assertRaisesRegex(AttemptStateError, "target mismatch"):
                self.store.acquire_control_lease(attempt)
            alias.unlink()

            first = self.store.acquire_control_lease(attempt)
            second = self.store.acquire_control_lease(attempt)
            self._control_aliases.append(Path(first.alias))
            self.assertEqual(first, second)
            self.assertLessEqual(
                len(os.fsencode(first.socket_path)), 103
            )
            with self.store.writer(run):
                terminal = self.store.reconcile(attempt)
            self.assertEqual(terminal.state, "interrupted")
            self.assertFalse(os.path.lexists(alias))
        finally:
            alias.unlink(missing_ok=True)

    def test_preparation_failure_is_sealed_without_launch(self):
        run = self.create_run()
        calls: list[str] = []

        def fail_prepare(_manifest, _attempt):
            calls.append("prepare")
            raise RuntimeError("prepare failure")

        with self.assertRaisesRegex(RuntimeError, "prepare failure"):
            run_slots(self.store, run, self.manifest, adapters={"claude": self.adapter("success", calls)}, prepare=fail_prepare)
        self.assertEqual(calls, ["preflight", "prepare"])
        self.assertEqual(self.store.attempts(run, Slot("a", 1))[-1].state, "failed")

    def test_retry_and_skip_preserve_prior_terminal_bytes(self):
        self._init_testbed()
        run = self.create_run()
        calls: list[str] = []
        run_slots(
            self.store,
            run,
            self.manifest,
            adapters={"claude": self.adapter("failed", calls)},
            prepare=self.preparer(calls),
        )
        first = self.store.attempts(run, Slot("a", 1))[0]
        prior = (Path(first.root) / "attempt.json").read_bytes()
        self.assertEqual(
            run_slots(
                self.store,
                run,
                self.manifest,
                adapters={"claude": self.adapter("success", calls)},
                prepare=self.preparer(calls),
            ),
            (),
        )
        self.assertEqual(prior, (Path(first.root) / "attempt.json").read_bytes())
        retry = run_slots(
            self.store,
            run,
            self.manifest,
            adapters={"claude": self.adapter("success", calls)},
            prepare=self.preparer(calls),
            retry_failed=True,
        )
        self.assertEqual(retry[0].identity.attempt, 2)
        self.assertEqual(prior, (Path(first.root) / "attempt.json").read_bytes())
        self.assertEqual(len(self.store.preflights(run, self.manifest)), 3)

    def test_preflight_blocker_appends_without_attempt_allocation(self):
        run = self.create_run()
        calls: list[str] = []
        completed = run_slots(
            self.store,
            run,
            self.manifest,
            adapters={
                "claude": self.adapter(
                    "success", calls, issue_code="credential_missing"
                )
            },
            prepare=lambda _manifest, _attempt: self.fail("preparer must not run"),
        )
        self.assertEqual(completed, ())
        self.assertEqual(calls, ["preflight"])
        self.assertFalse((Path(run.root) / "cells").exists())
        self.assertEqual(
            self.store.status(run, self.manifest)["preflight"]["latest_status"],
            "registration_required",
        )

    def test_missing_adapter_has_no_output_root_side_effect(self):
        fake_run = RunIdentity("run-20260102T030405Z-abcdef123456", self.manifest.digest, str(self.root / "absent"))
        output = self.root / self.manifest.output_root
        self.assertFalse(output.exists())
        with self.assertRaises(CapabilityUnavailable):
            run_slots(self.store, fake_run, self.manifest, adapters={}, prepare=lambda _manifest, _attempt: None)
        self.assertFalse(output.exists())


class AttemptMeasurementTest(AttemptBase):
    """The immutable timing/usage sidecar around one production invocation."""

    def _run(self, mode: str = "success", *, collide: bool = False):
        self._init_testbed()
        run = self.create_run()
        completed = run_slots(
            self.store, run, self.manifest,
            adapters={"claude": MeasuringExecutionAdapter(self, mode, collide=collide)},
            prepare=self.preparer([]),
        )
        return run, completed

    def test_successful_attempt_publishes_one_bound_measurement(self):
        threads_before = set(threading.enumerate())
        run, completed = self._run()
        self.assertEqual([item.state for item in completed], ["success"])
        attempt_root = Path(completed[0].root)
        measurement = load_measurement(attempt_root)

        self.assertEqual(
            (measurement.run_id, measurement.cell_id, measurement.repetition, measurement.attempt),
            (run.run_id, "a", 1, 1),
        )
        self.assertEqual(measurement.caller, "claude")
        self.assertEqual(measurement.terminal_reason, "success")
        state = json.loads((attempt_root / "attempt.json").read_text(encoding="utf-8"))
        self.assertEqual(measurement.spec_digest, state["spec_digest"])
        self.assertEqual(state["measurement_policy"], "required-v1")
        marker = json.loads(
            (attempt_root / MEASUREMENT_POLICY_FILENAME).read_text(encoding="ascii")
        )
        self.assertEqual(marker["measurement_policy"], "required-v1")
        self.assertEqual(marker["run_id"], run.run_id)
        self.assertEqual(marker["manifest_digest"], run.manifest_digest)
        self.assertEqual(
            (marker["cell_id"], marker["repetition"], marker["attempt"]),
            ("a", 1, 1),
        )

        self.assertEqual(measurement.usage["total_duration"].value, 12_500_000)
        self.assertEqual(measurement.usage["total_duration"].clock, "caller_reported")
        self.assertEqual(measurement.usage["input_tokens"].value, 11)
        # The fake caller reports no provider total, so it stays unavailable.
        self.assertEqual(measurement.usage["total_tokens"].status, "unavailable")
        self.assertIsNone(measurement.usage["total_tokens"].value)

        timeline = measurement.timeline
        self.assertEqual(timeline["submitted_at"].clock, "harness_monotonic")
        self.assertEqual(timeline["first_output_at"].status, "observed")
        self.assertEqual(timeline["first_write_observed_at"].source, "workspace_poll")
        self.assertEqual(timeline["first_write_mtime"].clock, "filesystem_mtime")
        self.assertEqual(measurement.observer.path_digest, path_digest("answer.txt"))
        self.assertGreater(measurement.observer.precision_ns, 0)
        # The caller-chosen filename is digested, never persisted verbatim.
        self.assertNotIn(
            b"answer.txt", (attempt_root / MEASUREMENT_FILENAME).read_bytes()
        )
        self.assertEqual(set(threading.enumerate()) - threads_before, set())

    def _assert_unavailable_measurement(self, mode: str, state: str, reason: str) -> None:
        _run, completed = self._run(mode)
        self.assertEqual(completed[0].state, state)
        measurement = load_measurement(Path(completed[0].root))
        self.assertEqual(measurement.terminal_reason, reason)
        self.assertEqual(measurement.observations, ())
        for name in ("total_duration", "input_tokens", "model_calls"):
            self.assertEqual(measurement.usage[name].status, "unavailable")
            self.assertIsNone(measurement.usage[name].value)
        self.assertFalse(measurement.observer.observed)
        self.assertEqual(
            measurement.timeline["first_write_observed_at"].reason, "not_observed"
        )

    def test_failed_attempt_keeps_unavailable_values(self):
        self._assert_unavailable_measurement("failed", "failed", "nonzero_exit")

    def test_timed_out_attempt_keeps_unavailable_values(self):
        self._assert_unavailable_measurement("timeout", "timed_out", "timed_out")

    def test_measurement_collision_fails_closed_without_touching_prior_bytes(self):
        self._init_testbed()
        run = self.create_run()
        with self.assertRaises(AttemptStateError):
            run_slots(
                self.store, run, self.manifest,
                adapters={"claude": MeasuringExecutionAdapter(self, collide=True)},
                prepare=self.preparer([]),
            )
        attempt = self.store.attempts(run, Slot("a", 1))[-1]
        sidecar = Path(attempt.root) / MEASUREMENT_FILENAME
        self.assertEqual(sidecar.read_bytes(), b'{"record":"prior"}\n')
        self.assertEqual(attempt.state, "running")

    def test_tampered_or_unbound_measurement_fails_closed_and_preserves_bytes(self):
        run, completed = self._run()
        sidecar = Path(completed[0].root) / MEASUREMENT_FILENAME
        original = sidecar.read_bytes()
        record = json.loads(original.decode("ascii"))
        cases = {
            "foreign-attempt": {**record, "attempt": {**record["attempt"], "cell_id": "other"}},
            "foreign-digest": {**record, "spec_digest": "sha256:" + "0" * 64},
            "rewritten-terminal": {**record, "terminal_reason": "timed_out"},
            "invented-total": {
                **record,
                "usage": {
                    **record["usage"],
                    "total_tokens": {
                        "status": "observed", "value": 11, "unit": "tokens",
                        "clock": "none", "source": "caller_output",
                    },
                },
            },
        }
        for name, payload in cases.items():
            with self.subTest(name=name):
                sidecar.write_bytes(
                    json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() + b"\n"
                )
                before = sidecar.read_bytes()
                with self.assertRaises(AttemptStateError):
                    self.store.status(run, self.manifest)
                self.assertEqual(before, sidecar.read_bytes())
        sidecar.write_bytes(original)
        self.assertEqual(self.store.status(run, self.manifest)["attempts"]["success"], 1)

    def test_marked_measurement_is_required_and_bound_to_lifecycle_events(self):
        run, completed = self._run()
        attempt = completed[0]
        sidecar = Path(attempt.root) / MEASUREMENT_FILENAME
        original = sidecar.read_bytes()
        sidecar.unlink()
        with self.assertRaises(AttemptStateError):
            self.store.status(run, self.manifest)
        with self.assertRaises(AttemptStateError):
            self.store.reconcile(attempt)
        sidecar.write_bytes(original)

        record = json.loads(original.decode("ascii"))
        rewritten = {**record}
        rewritten["observations"] = [
            {**record["observations"][0], "value": 999},
            *record["observations"][1:],
        ]
        rewritten["usage"] = {
            **record["usage"],
            "total_duration": {
                **record["usage"]["total_duration"], "value": 999,
            },
        }
        sidecar.write_bytes(
            json.dumps(rewritten, sort_keys=True, separators=(",", ":")).encode() + b"\n"
        )
        with self.assertRaises(AttemptStateError):
            self.store.status(run, self.manifest)
        self.assertEqual(sidecar.read_bytes(), json.dumps(
            rewritten, sort_keys=True, separators=(",", ":")
        ).encode() + b"\n")

    def test_measurement_policy_start_evidence_rejects_downgrade_and_tampering(self):
        run, completed = self._run()
        attempt = completed[0]
        root = Path(attempt.root)
        record = root / "attempt.json"
        marker = root / MEASUREMENT_POLICY_FILENAME
        sidecar = root / MEASUREMENT_FILENAME
        saved_record, saved_marker, saved_sidecar = (
            record.read_bytes(), marker.read_bytes(), sidecar.read_bytes()
        )

        def durable_marker() -> bytes | str | None:
            try:
                mode = os.lstat(marker).st_mode
            except FileNotFoundError:
                return None
            if stat.S_ISREG(mode):
                return marker.read_bytes()
            return f"nonregular:{stat.S_IFMT(mode)}"

        def reject_on_status_and_reconcile() -> None:
            before = (record.read_bytes(), durable_marker())
            with self.assertRaises(AttemptStateError):
                self.store.status(run, self.manifest)
            with self.assertRaises(AttemptStateError):
                self.store.reconcile(attempt)
            self.assertEqual(before, (record.read_bytes(), durable_marker()))

        try:
            downgraded = json.loads(saved_record.decode("utf-8"))
            downgraded.pop("measurement_policy")
            record.write_bytes(json.dumps(
                downgraded, sort_keys=True, separators=(",", ":")
            ).encode("ascii") + b"\n")
            sidecar.unlink()
            reject_on_status_and_reconcile()
            sidecar.write_bytes(saved_sidecar)
            record.write_bytes(saved_record)

            marker.unlink()
            reject_on_status_and_reconcile()
            marker.write_bytes(saved_marker)

            record.unlink()
            with self.assertRaises(AttemptStateError):
                self.store.status(run, self.manifest)
            with self.assertRaises(AttemptStateError):
                self.store.reconcile(attempt)
            self.assertEqual(marker.read_bytes(), saved_marker)
            record.write_bytes(saved_record)

            mismatched = json.loads(saved_marker.decode("ascii"))
            mismatched["cell_id"] = "other"
            marker.write_bytes(json.dumps(
                mismatched, sort_keys=True, separators=(",", ":")
            ).encode("ascii") + b"\n")
            reject_on_status_and_reconcile()
            marker.write_bytes(saved_marker)

            marker.write_bytes(json.dumps(json.loads(saved_marker), indent=2).encode("ascii"))
            reject_on_status_and_reconcile()
        finally:
            record.write_bytes(saved_record)
            if marker.exists() or marker.is_symlink():
                if marker.is_dir() and not marker.is_symlink():
                    marker.rmdir()
                else:
                    marker.unlink()
            marker.write_bytes(saved_marker)

        for kind in ("fifo", "symlink", "directory"):
            with self.subTest(kind=kind):
                AttemptRecoveryTest._substitute(marker, kind, saved_marker)
                try:
                    reject_on_status_and_reconcile()
                finally:
                    AttemptRecoveryTest._restore(marker, saved_marker)
        self.assertEqual(self.store.status(run, self.manifest)["attempts"]["success"], 1)

    def test_explicitly_unmarked_lower_level_attempt_remains_compatible(self):
        run = self.create_run()
        with self.store.writer(run):
            attempt = self.store.allocate(run, Slot("a", 1))
            terminal = self.store.execute_attempt(
                attempt, prepare=lambda _: None,
                invoke=self.adapter("success", []), require_measurement=False,
            )
        root = Path(terminal.root)
        record = json.loads((root / "attempt.json").read_text(encoding="utf-8"))
        self.assertNotIn("measurement_policy", record)
        self.assertFalse((root / MEASUREMENT_POLICY_FILENAME).exists())
        self.assertFalse((root / MEASUREMENT_FILENAME).exists())
        self.assertEqual(self.store.status(run, self.manifest)["attempts"]["success"], 1)

    def test_nonregular_measurement_fails_closed_without_blocking(self):
        run, completed = self._run()
        sidecar = Path(completed[0].root) / MEASUREMENT_FILENAME
        saved = sidecar.read_bytes()
        for kind in ("fifo", "symlink", "directory"):
            with self.subTest(kind=kind):
                AttemptRecoveryTest._substitute(sidecar, kind, saved)
                with self.assertRaises(AttemptStateError):
                    self.store.attempts(run, Slot("a", 1))
                AttemptRecoveryTest._restore(sidecar, saved)
        self.assertEqual(self.store.attempts(run, Slot("a", 1))[-1].state, "success")

    def test_recovery_commits_only_a_valid_bound_sidecar(self):
        self._init_testbed()
        run = self.create_run()
        adapter = MeasuringExecutionAdapter(self)
        prepared: dict[str, object] = {}

        def prepare(attempt):
            prepared["workspace"] = prepare_workspace(
                self.manifest, attempt.root, attempt.identity, repo_root=self.root
            )
            return prepared["workspace"]

        def invoke(attempt, started):
            workspace = prepared["workspace"]
            observer = WorkspaceWriteObserver(workspace.workspace_dir)
            observer.start()
            try:
                result = adapter.invoke(
                    self.manifest.matrix[0], workspace, attempt,
                    self._control_dir(attempt), b"task", self.manifest.timeout, started,
                )
            finally:
                observation = observer.stop()
            self.store.publish_attempt_measurement(attempt, "claude", result, observation)
            raise ControllerCrash("controller crash")

        with self.store.writer(run):
            attempt = self.store.allocate(run, Slot("a", 1))
            with self.assertRaisesRegex(RuntimeError, "controller crash"):
                self.store.execute_attempt(attempt, prepare=prepare, invoke=invoke)

        sidecar = Path(attempt.root) / MEASUREMENT_FILENAME
        original = sidecar.read_bytes()
        sidecar.write_bytes(
            json.dumps(
                {**json.loads(original.decode("ascii")), "caller": ""},
                sort_keys=True, separators=(",", ":"),
            ).encode() + b"\n"
        )
        with self.store.writer(run):
            with self.assertRaises(AttemptStateError):
                self.store.reconcile(attempt)
        self.assertEqual(self.store.attempts(run, Slot("a", 1))[-1].state, "running")

        sidecar.write_bytes(original)
        with self.store.writer(run):
            recovered = self.store.reconcile(attempt)
        self.assertEqual(recovered.state, "success")
        self.assertEqual(load_measurement(Path(attempt.root)).caller, "claude")


class AttemptWebValidationTest(AttemptBase):
    """Required S12 policy, lifecycle mapping, and recovery-before-terminal."""

    def _run(self, mode: str = "success"):
        self._init_testbed()
        run = self.create_run()
        completed = run_slots(
            self.store,
            run,
            self.manifest,
            adapters={"claude": MeasuringExecutionAdapter(self, mode)},
            prepare=self.preparer([]),
        )
        return run, completed

    def _running_required_web(
        self, *, generated: bool = False, return_result: bool = False
    ):
        self._init_testbed()
        run = self.create_run()
        adapter = MeasuringExecutionAdapter(self)
        prepared: dict[str, object] = {}

        def prepare(attempt):
            workspace = prepare_workspace(
                self.manifest,
                attempt.root,
                attempt.identity,
                repo_root=self.root,
            )
            prepared["workspace"] = workspace
            return workspace

        def invoke(attempt, started):
            workspace = prepared["workspace"]
            if generated:
                root = Path(workspace.workspace_dir)
                (root / "index.html").write_text(
                    "<main><h1>ready</h1><a href='#x'>go</a></main>",
                    encoding="utf-8",
                )
                (root / "styles.css").write_text(
                    "body{color:#111;background:#fff}a:focus{outline:2px solid #05f}",
                    encoding="utf-8",
                )
                (root / "script.js").write_text("", encoding="utf-8")
            observer = WorkspaceWriteObserver(workspace.workspace_dir)
            observer.start()
            try:
                result = adapter.invoke(
                    self.manifest.matrix[0],
                    workspace,
                    attempt,
                    self._control_dir(attempt),
                    b"task",
                    self.manifest.timeout,
                    started,
                )
            finally:
                observation = observer.stop()
            self.store.publish_attempt_measurement(
                attempt, "claude", result, observation
            )
            if return_result:
                return result
            raise ControllerCrash("controller crash before web publication")

        with self.store.writer(run):
            attempt = self.store.allocate(run, Slot("a", 1))
            expected = AttemptStateError if return_result else ControllerCrash
            with self.assertRaises(expected):
                self.store.execute_attempt(
                    attempt,
                    prepare=prepare,
                    invoke=invoke,
                    require_measurement=True,
                    require_web_validation=True,
                )
        return run, attempt

    def test_lifecycle_status_matrix_publishes_not_run_for_non_success(self):
        cases = (
            ("success", "success", "failed"),
            ("failed", "failed", "not_run"),
            ("timeout", "timed_out", "not_run"),
        )
        for mode, terminal, web_status in cases:
            with self.subTest(mode=mode):
                self.tearDown()
                self.setUp()
                _run, completed = self._run(mode)
                self.assertEqual(completed[0].state, terminal)
                web = load_web_validation(Path(completed[0].root))
                self.assertEqual(web.status, web_status)
                if web_status == "not_run":
                    self.assertTrue(web.record["reason"].startswith("lifecycle_"))
                    self.assertFalse(any(item["passed"] for item in web.record["gates"]))

    def test_normal_terminal_requires_web_sidecar_before_commit(self):
        _run, attempt = self._running_required_web(return_result=True)
        record = Path(attempt.root) / "attempt.json"
        self.assertEqual(json.loads(record.read_text())["state"], "running")
        self.assertFalse((Path(attempt.root) / WEB_VALIDATION_FILENAME).exists())

    def test_recovery_reconstructs_web_before_terminal_commit(self):
        run, attempt = self._running_required_web()
        root = Path(attempt.root)
        attempt_record = root / "attempt.json"
        before = attempt_record.read_bytes()
        with self.store.writer(run):
            recovered = self.store.reconcile(attempt)
        self.assertEqual(recovered.state, "success")
        self.assertNotEqual(attempt_record.read_bytes(), before)
        self.assertEqual(load_web_validation(root).status, "failed")
        self.assertEqual(self.store.status(run, self.manifest)["attempts"]["success"], 1)

    def test_recovery_browser_start_failure_publishes_blocked(self):
        run, attempt = self._running_required_web(generated=True)
        with mock.patch(
            "scripts.agent_benchmark.web_validation.BrowserRenderer.render",
            side_effect=FileNotFoundError("missing browser"),
        ):
            with self.store.writer(run):
                recovered = self.store.reconcile(attempt)
        self.assertEqual(recovered.state, "success")
        web = load_web_validation(Path(attempt.root))
        self.assertEqual(web.status, "blocked")
        self.assertFalse(web.record["screenshots"])

    def test_recovery_collision_preserves_running_and_prior_bytes(self):
        run, attempt = self._running_required_web()
        root = Path(attempt.root)
        web_path = root / WEB_VALIDATION_FILENAME
        web_path.write_bytes(b'{"record":"prior"}\n')
        attempt_path = root / "attempt.json"
        before = (attempt_path.read_bytes(), web_path.read_bytes())
        with self.store.writer(run):
            with self.assertRaises(AttemptStateError):
                self.store.reconcile(attempt)
        self.assertEqual(before, (attempt_path.read_bytes(), web_path.read_bytes()))
        self.assertEqual(json.loads(attempt_path.read_text())["state"], "running")

    def test_policy_record_and_artifact_faults_fail_closed(self):
        run, completed = self._run()
        attempt = completed[0]
        root = Path(attempt.root)
        attempt_path = root / "attempt.json"
        web_path = root / WEB_VALIDATION_FILENAME
        marker_path = root / WEB_VALIDATION_POLICY_FILENAME
        saved = {
            "attempt": attempt_path.read_bytes(),
            "web": web_path.read_bytes(),
            "marker": marker_path.read_bytes(),
        }

        def rejected() -> None:
            before = attempt_path.read_bytes()
            with self.assertRaises(AttemptStateError):
                self.store.status(run, self.manifest)
            with self.assertRaises(AttemptStateError):
                self.store.reconcile(attempt)
            self.assertEqual(attempt_path.read_bytes(), before)

        downgraded = json.loads(saved["attempt"])
        downgraded.pop("web_validation_policy")
        attempt_path.write_bytes(
            json.dumps(downgraded, sort_keys=True, separators=(",", ":")).encode()
            + b"\n"
        )
        rejected()
        attempt_path.write_bytes(saved["attempt"])

        marker_path.unlink()
        rejected()
        marker_path.write_bytes(saved["marker"])

        for mutation in ("identity", "measurement", "schema"):
            with self.subTest(mutation=mutation):
                record = json.loads(saved["web"])
                if mutation == "identity":
                    record["attempt"]["cell_id"] = "other"
                elif mutation == "measurement":
                    record["measurement_digest"] = "sha256:" + "0" * 64
                else:
                    record["browser"]["unknown"] = True
                web_path.write_bytes(
                    json.dumps(record, sort_keys=True, separators=(",", ":")).encode()
                    + b"\n"
                )
                rejected()
                web_path.write_bytes(saved["web"])

        web_path.unlink()
        web_path.mkdir()
        rejected()
        web_path.rmdir()
        web_path.write_bytes(saved["web"])
        self.assertEqual(self.store.status(run, self.manifest)["attempts"]["success"], 1)


class AttemptRecoveryTest(AttemptBase):
    def _running_with_terminal(self):
        run = self.create_run()
        with self.store.writer(run):
            attempt = self.store.allocate(run, Slot("a", 1))
            with self.assertRaisesRegex(RuntimeError, "controller crash"):
                self.store.execute_attempt(
                    attempt, prepare=lambda _: None,
                    invoke=self._invoke_then_crash,
                )
        return run, attempt

    def _invoke_then_crash(self, attempt, started):
        result = self.adapter("success", [])(attempt, started)
        self.assertTrue((Path(attempt.root) / "lifecycle-result.json").is_file())
        raise ControllerCrash("controller crash")

    def test_real_terminal_first_recovery_commits_once(self):
        run, attempt = self._running_with_terminal()
        running = json.loads(
            (Path(attempt.root) / "attempt.json").read_text(encoding="utf-8")
        )
        alias = Path(running["locator"]["control_dir"]).parent
        self.assertTrue(alias.is_symlink())
        with self.store.writer(run):
            recovered = self.store.reconcile(attempt)
        self.assertEqual(recovered.state, "success")
        self.assertFalse(os.path.lexists(alias))
        self.assertEqual(self.store.status(run, self.manifest)["attempts"]["success"], 1)
        self.assertEqual(self.store.reconcile(recovered).state, "success")

    def test_corrupt_terminal_variants_fail_closed_and_preserve_bytes(self):
        run, attempt = self._running_with_terminal()
        cases = (
            ("contradictory-success", "lifecycle-result.json", lambda raw: raw.__setitem__("success", False)),
            ("extra-result-field", "lifecycle-result.json", lambda raw: raw.__setitem__("unexpected", True)),
            ("mismatched-digest", "lifecycle-result.json", lambda raw: raw.__setitem__("spec_digest", "sha256:" + "0" * 64)),
            ("receipt-cleanup", "cleanup-receipt.json", lambda raw: raw.__setitem__("cleanup_complete", False)),
        )
        for name, filename, corrupt in cases:
            with self.subTest(name=name):
                target = Path(attempt.root) / "control" / filename if filename == "cleanup-receipt.json" else Path(attempt.root) / filename
                original = target.read_bytes()
                raw = json.loads(target.read_text(encoding="utf-8"))
                corrupt(raw)
                target.write_text(json.dumps(raw), encoding="utf-8")
                evidence_before = {path: path.read_bytes() for path in (Path(attempt.root) / "attempt.json", Path(attempt.root) / "lifecycle-result.json", Path(attempt.root) / "lifecycle-journal.jsonl", Path(attempt.root) / "control" / "cleanup-receipt.json")}
                with self.store.writer(run):
                    with self.assertRaises(AttemptStateError):
                        self.store.reconcile(attempt)
                self.assertEqual(evidence_before, {path: path.read_bytes() for path in evidence_before})
                target.write_bytes(original)

    @staticmethod
    def _evidence_bytes(run, attempt) -> dict[str, bytes]:
        """Snapshot every durable run and attempt record published so far."""
        run_root, attempt_root = Path(run.root), Path(attempt.root)
        paths = (
            run_root / "run.json", run_root / "manifest.json", run_root / "run.lock",
            attempt_root / "attempt.json", attempt_root / "lifecycle-result.json",
            attempt_root / "lifecycle-journal.jsonl", attempt_root / "control" / "locator.json",
            attempt_root / "control" / "cleanup-receipt.json",
        )
        return {str(path): path.read_bytes() for path in paths}

    @staticmethod
    def _substitute(target: Path, kind: str, saved: bytes) -> None:
        """Replace one durable file with a non-regular object of the given kind."""
        target.unlink()
        if kind == "fifo":
            os.mkfifo(target, 0o600)
        elif kind == "directory":
            target.mkdir(mode=0o700)
        elif kind == "socket":
            short = Path(tempfile.mkdtemp(dir="/tmp", prefix="s")) / "s"
            with contextlib.closing(socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)) as endpoint:
                endpoint.bind(str(short))  # bound in a short path, then moved into place
            os.replace(short, target)
            short.parent.rmdir()
        else:
            copy = target.with_name(target.name + ".copy")
            copy.write_bytes(saved)
            target.symlink_to(copy)

    @staticmethod
    def _restore(target: Path, saved: bytes) -> None:
        """Discard the substituted object and republish the original bytes."""
        if target.is_symlink() or not target.is_dir():
            target.unlink()
        else:
            target.rmdir()
        target.with_name(target.name + ".copy").unlink(missing_ok=True)
        target.write_bytes(saved)
        os.chmod(target, 0o600)

    def _assert_probe_rejected(self, run, attempt, operation: str, locator: dict | None = None) -> None:
        """Run one store operation in a bounded child and require a closed failure."""
        payload = json.dumps({
            "repo": str(self.root), "manifest": str(self.manifest_path), "run_id": run.run_id,
            "run_root": run.root, "attempt_root": attempt.root, "operation": operation,
            "cell_id": attempt.identity.cell_id, "repetition": attempt.identity.repetition,
            "attempt_number": attempt.identity.attempt, "locator": locator,
        })
        child = subprocess.Popen(
            [sys.executable, "-c", _PROBE_SOURCE, payload],
            stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
            env={**os.environ, "PYTHONPATH": str(Path(__file__).resolve().parents[2])},
        )
        try:
            out, err = child.communicate(timeout=_PROBE_TIMEOUT_SECONDS)
        except subprocess.TimeoutExpired:
            child.kill()
            child.communicate()
            self.fail(f"{operation} did not return within {_PROBE_TIMEOUT_SECONDS:.0f}s")
        self.assertEqual((child.returncode, out.strip()), (0, "rejected"), err)
        self.assertIsNotNone(child.poll())

    def test_nonregular_durable_files_fail_closed_without_blocking(self):
        run, attempt = self._running_with_terminal()
        run_root, attempt_root = Path(run.root), Path(attempt.root)
        surfaces = (
            ("run record", run_root / "run.json", "open"),
            ("manifest snapshot", run_root / "manifest.json", "open"),
            ("run lock read", run_root / "run.lock", "open"),
            ("run lock lease", run_root / "run.lock", "lease"),
            ("attempt record", attempt_root / "attempt.json", "attempts"),
            ("lifecycle result", attempt_root / "lifecycle-result.json", "reconcile"),
            ("lifecycle journal", attempt_root / "lifecycle-journal.jsonl", "reconcile"),
            ("cleanup receipt", attempt_root / "control" / "cleanup-receipt.json", "reconcile"),
        )
        kinds = ("fifo", "directory", "socket", "symlink")
        covered: list[tuple[str, str]] = []
        for label, target, operation in surfaces:
            for kind in kinds:
                with self.subTest(surface=label, kind=kind):
                    before = self._evidence_bytes(run, attempt)
                    saved = before[str(target)]
                    self._substitute(target, kind, saved)
                    try:
                        self._assert_probe_rejected(run, attempt, operation)
                        self.assertFalse(stat.S_ISREG(os.lstat(target).st_mode))
                    finally:
                        self._restore(target, saved)
                    self.assertEqual(before, self._evidence_bytes(run, attempt))
                    covered.append((label, kind))
        with self.store.writer(run):
            locator_attempt = self.store.allocate(run, Slot("a", 2))
        locator_root = Path(locator_attempt.root)

        def stop_before_locator(_attempt, _started):
            raise ControllerCrash("locator setup")

        with self.assertRaisesRegex(ControllerCrash, "locator setup"):
            self.store.execute_attempt(
                locator_attempt, prepare=lambda _: None, invoke=stop_before_locator,
            )
        lease = self.store.acquire_control_lease(locator_attempt)
        self._control_aliases.append(Path(lease.alias))
        control = Path(lease.control_dir)
        control.mkdir(mode=0o700)
        locator = {
            "supervisor_pid": os.getpid(), "start_identity": "probe",
            "socket_path": lease.socket_path, "challenge": "challenge",
            "control_dir": lease.control_dir, "created_at": "created",
        }
        target = control / "locator.json"
        target.write_text(json.dumps(locator), encoding="utf-8")
        running = (locator_root / "attempt.json").read_bytes()

        for kind in kinds:
            with self.subTest(surface="registered locator", kind=kind):
                saved = target.read_bytes()
                self._substitute(target, kind, saved)
                try:
                    self._assert_probe_rejected(run, locator_attempt, "locator", locator)
                    self.assertFalse(stat.S_ISREG(os.lstat(target).st_mode))
                finally:
                    self._restore(target, saved)
                self.assertEqual(saved, target.read_bytes())
                self.assertTrue(stat.S_ISREG(os.lstat(target).st_mode))
                self.assertEqual(running, (locator_root / "attempt.json").read_bytes())
                covered.append(("registered locator", kind))
        digest = "sha256:" + "0" * 64
        self.store.record_locator(
            locator_attempt, SupervisorLocator(**locator), digest,
        )
        record = json.loads((locator_root / "attempt.json").read_text(encoding="utf-8"))
        self.assertEqual(record.get("locator"), locator)
        self.assertEqual(record.get("spec_digest"), digest)
        located = self.store.attempts(run, Slot("a", 2))
        self.assertEqual([(item.identity, item.state) for item in located], [(locator_attempt.identity, "running")])
        self.assertFalse((locator_root / "lifecycle-result.json").exists())
        self.assertFalse((locator_root / "lifecycle-journal.jsonl").exists())
        self.assertEqual(self.store.attempts(run, Slot("a", 3)), ())
        # Passing subtests are silent, so bind the executed matrix explicitly.
        self.assertEqual(len(covered), (len(surfaces) + 1) * len(kinds))

    @staticmethod
    def _mutate_terminal(paths: dict[str, Path], record: str, mutate) -> None:
        """Apply one contradiction to a single record or to the ordered event evidence."""
        if record != "events":
            raw = json.loads(paths[record].read_text(encoding="utf-8"))
            mutate(raw)
            paths[record].write_text(json.dumps(raw), encoding="utf-8")
            return
        result = json.loads(paths["result"].read_text(encoding="utf-8"))
        result["events"] = mutate(result["events"])
        paths["result"].write_text(json.dumps(result), encoding="utf-8")
        lines = [json.loads(line) for line in paths["journal"].read_text(encoding="utf-8").splitlines()]
        rewritten = [lines[0], *result["events"], lines[-1]]
        paths["journal"].write_text("".join(json.dumps(line) + "\n" for line in rewritten), encoding="utf-8")

    def test_cross_record_terminal_corruption_fails_closed_and_preserves_bytes(self):
        run, attempt = self._running_with_terminal()
        root = Path(attempt.root)
        paths = {
            "result": root / "lifecycle-result.json",
            "journal": root / "lifecycle-journal.jsonl",
            "receipt": root / "control" / "cleanup-receipt.json",
        }
        saved = {key: path.read_bytes() for key, path in paths.items()}
        cases = (
            ("receipt-exit-code", "receipt", lambda raw: raw.__setitem__("exit_code", 9)),
            ("receipt-signal", "receipt", lambda raw: raw.__setitem__("signal", 9)),
            ("receipt-reason", "receipt", lambda raw: raw.__setitem__("reason", "failed")),
            ("receipt-caller-launched", "receipt", lambda raw: raw.__setitem__("caller_launched", False)),
            ("receipt-completed-before-start", "receipt", lambda raw: raw.__setitem__("completed_at", "2000-01-01T00:00:00+00:00")),
            ("receipt-completed-after-end", "receipt", lambda raw: raw.__setitem__("completed_at", "2100-01-01T00:00:00+00:00")),
            ("receipt-completed-unparseable", "receipt", lambda raw: raw.__setitem__("completed_at", "not-a-timestamp")),
            ("result-submitted", "result", lambda raw: raw.__setitem__("submitted", False)),
            ("result-exit-code", "result", lambda raw: raw.__setitem__("exit_code", 7)),
            ("events-cleared", "events", lambda events: []),
            ("events-missing-submitted", "events", _without("submitted")),
            ("events-missing-finish", "events", _without("finish")),
            ("events-missing-idle", "events", _without("idle")),
            ("events-missing-quiet", "events", _without("quiet")),
            ("events-out-of-order", "events", _reordered),
            ("events-duplicate-finish", "events", _duplicated("finish")),
        )
        for name, record, mutate in cases:
            with self.subTest(case=name):
                self._mutate_terminal(paths, record, mutate)
                try:
                    before = self._evidence_bytes(run, attempt)
                    with self.store.writer(run):
                        with self.assertRaises(AttemptStateError):
                            self.store.reconcile(attempt)
                    self.assertEqual(before, self._evidence_bytes(run, attempt))
                    self.assertEqual([item.state for item in self.store.attempts(run, Slot("a", 1))], ["running"])
                finally:
                    for key, path in paths.items():
                        path.write_bytes(saved[key])
        record = root / "attempt.json"
        running = record.read_bytes()
        with self.store.writer(run):
            recovered = self.store.reconcile(attempt)
        published = record.read_bytes()
        self.assertEqual(recovered.state, "success")
        self.assertNotEqual(running, published)
        with self.store.writer(run):
            self.assertEqual(self.store.reconcile(recovered).state, "success")
        self.assertEqual(published, record.read_bytes())
        self.assertEqual(len(self.store.attempts(run, Slot("a", 1))), 1)

    def test_symlink_lifecycle_evidence_fails_closed(self):
        run, attempt = self._running_with_terminal()
        journal = Path(attempt.root) / "lifecycle-journal.jsonl"
        saved = journal.read_bytes()
        target = Path(attempt.root) / "journal-copy.jsonl"
        target.write_bytes(saved)
        journal.unlink()
        journal.symlink_to(target)
        record = Path(attempt.root) / "attempt.json"
        before = record.read_bytes()
        with self.store.writer(run):
            with self.assertRaises(AttemptStateError):
                self.store.reconcile(attempt)
        self.assertEqual(before, record.read_bytes())

    def test_direct_result_requires_bound_production_evidence(self):
        run = self.create_run()
        with self.store.writer(run):
            attempt = self.store.allocate(run, Slot("a", 1))

            def contradictory(current, started):
                result = self.adapter("success", [])(current, started)
                path = Path(current.root) / "lifecycle-result.json"
                raw = json.loads(path.read_text(encoding="utf-8"))
                raw["success"] = False
                path.write_text(json.dumps(raw), encoding="utf-8")
                return result

            with self.assertRaises(AttemptStateError):
                self.store.execute_attempt(attempt, prepare=lambda _: None, invoke=contradictory)
        record = Path(attempt.root) / "attempt.json"
        self.assertEqual(json.loads(record.read_text(encoding="utf-8"))["state"], "running")

    def test_controller_process_loss_reconciles_durable_receipt(self):
        run = self.create_run()
        with self.store.writer(run):
            attempt = self.store.allocate(run, Slot("a", 1))
        attempt_root = Path(attempt.root)
        expected_lease = self.store._control_lease_for_root(attempt_root)
        alias = Path(expected_lease.alias)
        self._control_aliases.append(alias)
        payload = json.dumps(
            {
                "repo": str(self.root),
                "manifest": str(self.manifest_path),
                "run_id": run.run_id,
                "run_root": run.root,
                "attempt_root": attempt.root,
                "cell_id": attempt.identity.cell_id,
                "repetition": attempt.identity.repetition,
                "attempt_number": attempt.identity.attempt,
            }
        )
        child = subprocess.Popen(
            [
                sys.executable,
                "-c",
                (
                    "import sys; "
                    "from scripts.agent_benchmark.attempts_test import "
                    "_controller_loss_child; "
                    "_controller_loss_child(sys.argv[1])"
                ),
                payload,
            ],
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            text=True,
            env={
                **os.environ,
                "PYTHONPATH": str(Path(__file__).resolve().parents[2]),
            },
        )
        locator: SupervisorLocator | None = None

        def child_failure(label: str) -> None:
            if child.poll() is None:
                return
            stdout, stderr = child.communicate()
            self.fail(
                f"controller exited before {label}: returncode={child.returncode} "
                f"stdout={stdout!r} stderr={stderr!r}"
            )

        try:
            deadline = time.monotonic() + 10
            attempt_record = attempt_root / "attempt.json"
            registered_path = attempt_root / "control" / "locator.json"
            while locator is None:
                child_failure("locator commit")
                if attempt_record.is_file() and registered_path.is_file():
                    record = json.loads(attempt_record.read_text(encoding="utf-8"))
                    raw_locator = record.get("locator")
                    if raw_locator is not None:
                        registered = json.loads(
                            registered_path.read_text(encoding="utf-8")
                        )
                        if registered == raw_locator:
                            locator = SupervisorLocator(**raw_locator)
                            break
                if time.monotonic() >= deadline:
                    self.fail("controller did not commit its locator")
                threading.Event().wait(0.01)

            deadline = time.monotonic() + 10
            while True:
                child_failure("caller launch")
                try:
                    status = recover_invocation(locator, stop=False)
                except LifecycleRecoveryError:
                    status = None
                if status is not None and status.caller_launched:
                    break
                if time.monotonic() >= deadline:
                    self.fail("controller did not launch its caller")
                threading.Event().wait(0.01)

            child.kill()
            stdout, stderr = child.communicate(timeout=5)
            self.assertEqual(child.returncode, -signal.SIGKILL, (stdout, stderr))

            receipt_path = attempt_root / "control" / "cleanup-receipt.json"
            canonical_socket = attempt_root / "control" / "control.sock"
            deadline = time.monotonic() + 10
            receipt: dict[str, object] | None = None
            while receipt is None:
                if receipt_path.is_file():
                    candidate = json.loads(receipt_path.read_text(encoding="utf-8"))
                    if (
                        candidate.get("reason") == REASON_CONTROLLER_LOST
                        and candidate.get("caller_launched") is True
                        and candidate.get("cleanup_complete") is True
                        and candidate.get("process_group_alive") is False
                        and not os.path.lexists(canonical_socket)
                        and not os.path.lexists(locator.socket_path)
                    ):
                        receipt = candidate
                        break
                if time.monotonic() >= deadline:
                    self.fail("supervisor did not publish closed controller-loss receipt")
                threading.Event().wait(0.01)

            self.assertTrue(alias.is_symlink())
            self.assertFalse((attempt_root / "lifecycle-result.json").exists())
            self.assertFalse((attempt_root / "lifecycle-journal.jsonl").exists())

            def durable_bytes() -> dict[str, bytes]:
                paths = [attempt_record]
                paths.extend(
                    path
                    for path in sorted((attempt_root / "control").iterdir())
                    if stat.S_ISREG(os.lstat(path).st_mode)
                )
                return {
                    str(path.relative_to(attempt_root)): path.read_bytes()
                    for path in paths
                }

            clean_running = durable_bytes()
            tamper_cases = (
                (
                    "registered-locator",
                    registered_path,
                    lambda raw: raw.__setitem__(
                        "challenge", str(raw["challenge"]) + "-tampered"
                    ),
                ),
                (
                    "receipt-identity",
                    receipt_path,
                    lambda raw: raw.__setitem__("challenge_digest", "0" * 64),
                ),
                (
                    "receipt-reason",
                    receipt_path,
                    lambda raw: raw.__setitem__("reason", "recovered_stop"),
                ),
                (
                    "receipt-incomplete-cleanup",
                    receipt_path,
                    lambda raw: raw.__setitem__("cleanup_complete", False),
                ),
                (
                    "receipt-live-process-group",
                    receipt_path,
                    lambda raw: raw.__setitem__("process_group_alive", True),
                ),
                (
                    "receipt-schema",
                    receipt_path,
                    lambda raw: raw.__setitem__("unexpected", True),
                ),
                (
                    "receipt-completed-at",
                    receipt_path,
                    lambda raw: raw.__setitem__("completed_at", "not-a-timestamp"),
                ),
            )
            for name, target, tamper in tamper_cases:
                with self.subTest(phase="running", case=name):
                    original = target.read_bytes()
                    raw = json.loads(original.decode("utf-8"))
                    tamper(raw)
                    target.write_text(json.dumps(raw), encoding="utf-8")
                    before = durable_bytes()
                    with self.store.writer(run):
                        with self.assertRaises(AttemptStateError):
                            self.store.reconcile(attempt)
                    self.assertEqual(before, durable_bytes())
                    self.assertTrue(alias.is_symlink())
                    target.write_bytes(original)
                    self.assertEqual(clean_running, durable_bytes())

            original_release = self.store.release_control_lease
            release_observations: list[str] = []

            def release_after_publication(current) -> None:
                published = json.loads(attempt_record.read_text(encoding="utf-8"))
                self.assertEqual(published["state"], "interrupted")
                self.assertEqual(
                    published["lifecycle"]["terminal_reason"],
                    REASON_CONTROLLER_LOST,
                )
                self.assertTrue(alias.is_symlink())
                release_observations.append(published["state"])
                original_release(current)

            with mock.patch.object(
                self.store,
                "release_control_lease",
                side_effect=release_after_publication,
            ):
                with self.store.writer(run):
                    recovered = self.store.reconcile(attempt)
                    successor = self.store.allocate(run, Slot("a", 1))

            self.assertEqual(recovered.state, "interrupted")
            self.assertEqual(successor.identity.attempt, 2)
            self.assertEqual(release_observations, ["interrupted"])
            self.assertFalse(os.path.lexists(alias))

            clean_terminal = durable_bytes()
            terminal_tamper_cases = (
                *tamper_cases,
                (
                    "terminal-record-reason",
                    attempt_record,
                    lambda raw: raw["lifecycle"].__setitem__(
                        "terminal_reason", "success"
                    ),
                ),
                (
                    "terminal-record-state-success",
                    attempt_record,
                    lambda raw: raw.__setitem__("state", "success"),
                ),
                (
                    "terminal-record-state-failed",
                    attempt_record,
                    lambda raw: raw.__setitem__("state", "failed"),
                ),
            )
            for name, target, tamper in terminal_tamper_cases:
                with self.subTest(phase="terminal-status", case=name):
                    original = target.read_bytes()
                    raw = json.loads(original.decode("utf-8"))
                    tamper(raw)
                    target.write_text(json.dumps(raw), encoding="utf-8")
                    before = durable_bytes()
                    with self.assertRaises(AttemptStateError):
                        self.store.status(run, self.manifest)
                    self.assertEqual(before, durable_bytes())
                    target.write_bytes(original)
                    self.assertEqual(clean_terminal, durable_bytes())

            projected = self.store.status(run, self.manifest)["attempts"]
            self.assertEqual(projected["interrupted"], 1)
            self.assertEqual(projected["running"], 1)
            self.assertFalse(receipt["process_group_alive"])
            self.assertFalse(os.path.lexists(alias))
        finally:
            if child.poll() is None:
                child.kill()
                child.communicate(timeout=5)

    def test_live_survivor_cleanup_precedes_successor(self):
        run = self.create_run()
        with self.store.writer(run):
            attempt = self.store.allocate(run, Slot("a", 1))
        locator_ready = threading.Event()
        worker_outcomes: list[BaseException | Attempt] = []
        reconciler_outcomes: list[BaseException | Attempt] = []

        def long_running(current, started):
            spec = self._spec(current, "import time; print('START', flush=True); time.sleep(30)")

            def commit(locator):
                started(locator, spec_digest(spec))
                locator_ready.set()

            return run_invocation(spec, parse_event=_events, on_started=commit)

        def invoke() -> None:
            try:
                worker_outcomes.append(
                    self.store.execute_attempt(
                        attempt, prepare=lambda _: None, invoke=long_running
                    )
                )
            except BaseException as exc:  # concurrent reconciliation seals this attempt first
                worker_outcomes.append(exc)

        worker = threading.Thread(target=invoke)
        worker.start()
        self.assertTrue(locator_ready.wait(5))
        locator = SupervisorLocator(
            **json.loads(
                (Path(attempt.root) / "attempt.json").read_text(encoding="utf-8")
            )["locator"]
        )
        deadline = time.monotonic() + 5
        while not recover_invocation(locator, stop=False).caller_launched:
            if time.monotonic() >= deadline:
                self.fail("caller did not launch before recovery")
            threading.Event().wait(0.01)
        actual_recover = recover_invocation

        def lose_cleanup_reply(current, stop=True):
            outcome = actual_recover(current, stop=stop)
            if stop:
                raise LifecycleRecoveryError("simulated lost cleanup reply")
            return outcome

        with mock.patch(
            "scripts.agent_benchmark.attempts.recover_invocation",
            side_effect=lose_cleanup_reply,
        ):
            try:
                with self.store.writer(run):
                    reconciler_outcomes.append(self.store.reconcile(attempt))
            except BaseException as exc:
                reconciler_outcomes.append(exc)
        worker.join(10)
        self.assertFalse(worker.is_alive())
        self.assertEqual(len(worker_outcomes), 1)
        self.assertEqual(len(reconciler_outcomes), 1)
        outcomes = (*worker_outcomes, *reconciler_outcomes)
        for outcome in outcomes:
            if isinstance(outcome, BaseException):
                self.assertIsInstance(outcome, AttemptStateError)
            else:
                self.assertEqual(outcome.state, "interrupted")
        self.assertTrue(any(isinstance(outcome, Attempt) for outcome in outcomes))

        attempt_root = Path(attempt.root)
        record = json.loads(
            (attempt_root / "attempt.json").read_text(encoding="utf-8")
        )
        result = json.loads(
            (attempt_root / "lifecycle-result.json").read_text(encoding="utf-8")
        )
        receipt = json.loads(
            (attempt_root / "control/cleanup-receipt.json").read_text(
                encoding="utf-8"
            )
        )
        self.assertEqual(record["state"], "interrupted")
        self.assertEqual(record["lifecycle"]["terminal_reason"], REASON_RECOVERED_STOP)
        self.assertEqual(result["terminal_reason"], REASON_RECOVERED_STOP)
        self.assertEqual(receipt["reason"], REASON_RECOVERED_STOP)
        self.assertTrue(receipt["cleanup_complete"])
        self.assertFalse(receipt["process_group_alive"])

        terminal = Attempt(attempt.identity, attempt.root, "interrupted")
        self.store.release_control_lease(terminal)
        self.store.release_control_lease(terminal)
        alias = Path(self.store._control_lease_for_root(attempt_root).alias)
        self.assertFalse(os.path.lexists(alias))

        with self.store.writer(run):
            successor = self.store.allocate(run, Slot("a", 1))
        self.assertEqual(successor.identity.attempt, 2)

    def test_cross_process_lease_contention_and_crash_release(self):
        run = self.create_run()
        script = "import fcntl, os, sys, time; f=os.open(sys.argv[1], os.O_RDWR); fcntl.flock(f, fcntl.LOCK_EX); print('locked', flush=True); time.sleep(30)"
        child = subprocess.Popen([sys.executable, "-c", script, str(Path(run.root) / "run.lock")], stdout=subprocess.PIPE, text=True)
        self.assertEqual(child.stdout.readline().strip(), "locked")
        with self.assertRaises(RunBusyError):
            with self.store.writer(run):
                pass
        child.kill()
        child.wait(timeout=5)
        child.stdout.close()
        with self.store.writer(run):
            pass


class AttemptCliContractTest(AttemptBase):
    def test_cli_status_is_read_only_and_run_resume_block_before_attempts(self):
        run = self.create_run()
        run_before = {
            path.relative_to(run.root): path.read_bytes()
            for path in Path(run.root).rglob("*")
            if path.is_file()
        }
        output = io.StringIO()
        with mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root), contextlib.redirect_stdout(output):
            self.assertEqual(benchmark_cli.main(["status", "--manifest", str(self.manifest_path), "--run-id", run.run_id]), 0)
        self.assertIn("'running': 0", output.getvalue())
        self.assertEqual(
            run_before,
            {
                path.relative_to(run.root): path.read_bytes()
                for path in Path(run.root).rglob("*")
                if path.is_file()
            },
        )
        with mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root), contextlib.redirect_stderr(io.StringIO()):
            self.assertEqual(benchmark_cli.main(["resume", "--manifest", str(self.manifest_path), "--run-id", run.run_id]), 69)
        self.assertFalse((Path(run.root) / "cells").exists())
        self.assertEqual(len(self.store.preflights(run, self.manifest)), 1)
        absent_root = self.root / "agent-test/runs/absent"
        raw = json.loads(self.raw)
        raw["output_root"] = "agent-test/runs/absent"
        absent = self.root / "absent.json"
        absent.write_text(json.dumps(raw), encoding="utf-8")
        with mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root), contextlib.redirect_stderr(io.StringIO()):
            self.assertEqual(benchmark_cli.main(["run", "--manifest", str(absent)]), 69)
        created = list(absent_root.glob("run-*"))
        self.assertEqual(len(created), 1)
        self.assertTrue((created[0] / "preflight/preflight-000001.json").is_file())
        self.assertFalse((created[0] / "cells").exists())


if __name__ == "__main__":
    unittest.main()