iop/scripts/agent_benchmark/attempts_test.py

"""Credential-free production-path tests for durable benchmark attempts."""

from __future__ import annotations

import contextlib
import datetime
import io
import json
import os
import socket
import stat
import subprocess
import sys
import tempfile
import threading
import unittest
from pathlib import Path
from unittest import mock

from scripts import agent_comparison_benchmark as benchmark_cli
from scripts.agent_benchmark.attempts import (
    AttemptStateError,
    CapabilityUnavailable,
    PreflightObservation,
    RunBusyError,
    RunIdentity,
    RunStore,
    Slot,
    run_slots,
)
from scripts.agent_benchmark.connectivity import (
    ISSUE_RESUME_CODES,
    CallerCapability,
    ConnectivityIssue,
    EffectiveBinding,
    RequestedEffectiveBinding,
    make_result,
)
from scripts.agent_benchmark.lifecycle import (
    COMPLETION_EXIT_AFTER_IDLE,
    SUBMISSION_ARGV_TASK,
    InvocationResult,
    InvocationSpec,
    SupervisorLocator,
    env_pairs,
    run_invocation,
    spec_digest,
)
from scripts.agent_benchmark.manifest import AssetMapping, Timeout, digest_workspace_inputs, load_manifest
from scripts.agent_benchmark.workspace import prepare_workspace


def _manifest(root: Path, repetitions: int = 1):
    fixtures = root / "scripts/fixtures"
    fixtures.mkdir(parents=True, exist_ok=True)
    (fixtures / "prompt.md").write_text("prompt", encoding="utf-8")
    (fixtures / "reference.txt").write_text("reference", encoding="utf-8")
    fixture = {
        "version": "v1", "prompt": "scripts/fixtures/prompt.md",
        "assets": [{"source": "scripts/fixtures/reference.txt", "workspace_path": "workspace/reference.txt"}],
        "checksum": digest_workspace_inputs((AssetMapping("scripts/fixtures/reference.txt", "workspace/reference.txt", b"reference"),)),
    }
    data = {
        "pipeline_version": "1", "environment": "dev", "testbed": "../iop-s2",
        "session_policy": "fresh", "setup_cache_policy": "isolated",
        "timeout": {"run_seconds": 1, "idle_seconds": 1, "quiet_seconds": 1, "cleanup_grace_seconds": 1},
        "viewports": [{"id": "desktop", "width": 1, "height": 1}], "rubric_version": "v1",
        "output_root": "agent-test/runs/a", "fixture": fixture, "repetitions": repetitions,
        "matrix": [{"id": "a", "caller": "claude", "iop": {"request_model": "model", "requested_effort": "high", "route_kind": "direct", "route_id": "route", "expected_bindings": [{"stage": "request", "model": "model", "effort": "high"}]}}],
    }
    path = root / "manifest.json"
    raw = json.dumps(data, sort_keys=True).encode("utf-8")
    path.write_bytes(raw)
    return load_manifest(path, repo_root=root), raw, path


def _events(_: str, line: str) -> str | None:
    return {"FINISH": "finish", "IDLE": "idle"}.get(line.strip())


def _preflight_observation(cell, issue_code: str | None = None) -> PreflightObservation:
    capability = CallerCapability(
        cell.caller, ("direct", "execution_preset"), (cell.iop.requested_effort,)
    )
    if issue_code is None:
        bindings = tuple(
            EffectiveBinding(item.stage, item.model, item.effort)
            for item in cell.iop.expected_bindings
        )
        binding = RequestedEffectiveBinding(
            cell.id, cell.caller, cell.iop.route_kind, cell.iop.route_id,
            cell.iop.request_model, cell.iop.requested_effort,
            cell.iop.route_kind, cell.iop.route_id, cell.iop.request_model,
            cell.iop.requested_effort, bindings,
        )
        issues = ()
    else:
        binding = RequestedEffectiveBinding(
            cell.id, cell.caller, cell.iop.route_kind, cell.iop.route_id,
            cell.iop.request_model, cell.iop.requested_effort,
        )
        issues = (ConnectivityIssue(issue_code, ISSUE_RESUME_CODES[issue_code]),)
    return PreflightObservation(
        make_result(cell, capability, binding, issues),
        "sha256:" + "1" * 64,
        "sha256:" + "2" * 64,
    )


_PROBE_TIMEOUT_SECONDS = 30.0

# Every durable read runs in a bounded child so a blocking special file cannot
# hang the suite; the child reports whether the store fails closed.
_PROBE_SOURCE = """
import json
import os
import sys
from pathlib import Path

from scripts.agent_benchmark.attempts import Attempt, AttemptStateError, RunIdentity, RunStore, Slot
from scripts.agent_benchmark.lifecycle import SupervisorLocator
from scripts.agent_benchmark.manifest import load_manifest
from scripts.agent_benchmark.workspace import AttemptIdentity

payload = json.loads(sys.argv[1])
store = RunStore(payload["repo"])
manifest = load_manifest(Path(payload["manifest"]), repo_root=Path(payload["repo"]))
run = RunIdentity(payload["run_id"], manifest.digest, payload["run_root"])
attempt = Attempt(
    AttemptIdentity(payload["run_id"], payload["cell_id"], payload["repetition"], payload["attempt_number"]),
    payload["attempt_root"], "running",
)


def lease():
    with store.writer(run):
        pass


def locator():
    store.record_locator(
        attempt,
        SupervisorLocator(**payload["locator"]),
        "sha256:" + "0" * 64,
    )


operations = {
    "open": lambda: store.open(manifest, run.run_id),
    "lease": lease,
    "attempts": lambda: store.attempts(run, Slot("a", 1)),
    "reconcile": lambda: store.reconcile(attempt),
    "locator": locator,
}
try:
    operations[payload["operation"]]()
except AttemptStateError:
    print("rejected")
    sys.exit(0)
print("accepted")
sys.exit(1)
"""


def _reordered(events: list[dict]) -> list[dict]:
    """Return production events with finish and idle transposed."""
    kinds = [event["kind"] for event in events]
    swapped = list(events)
    finish, idle = kinds.index("finish"), kinds.index("idle")
    swapped[finish], swapped[idle] = swapped[idle], swapped[finish]
    return swapped


def _without(kind: str):
    return lambda events: [event for event in events if event["kind"] != kind]


def _duplicated(kind: str):
    return lambda events: events + [event for event in events if event["kind"] == kind]


class ControllerCrash(RuntimeError):
    """Test-only controller loss after lifecycle evidence has been published."""


class FakeExecutionAdapter:
    """Typed fake that exercises the production run_slots boundary."""

    def __init__(
        self,
        owner: "AttemptBase",
        reason: str,
        calls: list[str],
        issue_code: str | None = None,
    ) -> None:
        self.owner = owner
        self.reason = reason
        self.calls = calls
        self.issue_code = issue_code
        self.capability = CallerCapability(
            "claude", ("direct", "execution_preset"), ("high",)
        )

    def preflight(self, cell):
        self.calls.append("preflight")
        return _preflight_observation(cell, self.issue_code)

    def invoke(
        self,
        cell,
        prepared,
        attempt,
        task_payload,
        timeout,
        on_started,
    ):
        self.calls.append("invoke")
        if cell.id != attempt.identity.cell_id or prepared.identity != attempt.identity:
            raise AssertionError("typed execution identity drift")
        if task_payload != self.owner.manifest.fixture.prompt_content:
            raise AssertionError("task payload drift")
        source = (
            "print('FINISH'); print('IDLE')"
            if self.reason == "success"
            else "import sys; print('FAILED'); sys.exit(3)"
        )
        spec = self.owner._spec(attempt, source)
        return run_invocation(
            spec,
            parse_event=_events,
            on_started=lambda locator: on_started(locator, spec_digest(spec)),
        )

    def __call__(self, attempt, on_started):
        """Retain the lower-level RunStore lifecycle seam for recovery tests."""
        self.calls.append("invoke")
        source = (
            "print('FINISH'); print('IDLE')"
            if self.reason == "success"
            else "import sys; print('FAILED'); sys.exit(3)"
        )
        spec = self.owner._spec(attempt, source)
        return run_invocation(
            spec,
            parse_event=_events,
            on_started=lambda locator: on_started(locator, spec_digest(spec)),
        )


class AttemptBase(unittest.TestCase):
    def setUp(self) -> None:
        self.temp = tempfile.TemporaryDirectory(dir="/tmp", prefix="b")
        self.root = Path(self.temp.name) / "r"
        self.root.mkdir()
        self._control_aliases: list[Path] = []
        self.manifest, self.raw, self.manifest_path = _manifest(self.root)
        self.store = RunStore(
            self.root,
            clock=lambda: datetime.datetime(2026, 1, 2, 3, 4, 5, tzinfo=datetime.timezone.utc),
            token_hex=lambda _: "abcdef123456",
        )

    def tearDown(self) -> None:
        for alias in self._control_aliases:
            try:
                alias.unlink()
            except FileNotFoundError:
                pass
        self.temp.cleanup()

    def create_run(self):
        return self.store.create(self.manifest, self.raw)

    def _spec(self, attempt, source: str) -> InvocationSpec:
        alias = Path(tempfile.mkdtemp(dir="/tmp", prefix="c"))
        alias.rmdir()
        alias.symlink_to(Path(attempt.root), target_is_directory=True)
        self._control_aliases.append(alias)
        return InvocationSpec(
            argv=(sys.executable, "-u", "-c", source),
            cwd=str(self.root),
            env=env_pairs({"PATH": os.environ.get("PATH", "/usr/bin:/bin")}),
            submission_mode=SUBMISSION_ARGV_TASK,
            completion_mode=COMPLETION_EXIT_AFTER_IDLE,
            timeout=Timeout(5, 1, 1, 1),
            evidence_dir=attempt.root,
            control_dir=str(alias / "c"),
        )

    def adapter(
        self,
        reason: str,
        calls: list[str],
        issue_code: str | None = None,
    ) -> FakeExecutionAdapter:
        return FakeExecutionAdapter(self, reason, calls, issue_code)

    def preparer(self, calls: list[str]):
        def prepare(manifest, attempt):
            calls.append("prepare")
            return prepare_workspace(
                manifest, attempt.root, attempt.identity, repo_root=self.root
            )

        return prepare

    def _init_testbed(self) -> None:
        testbed = self.root.parent / "iop-s2"
        testbed.mkdir()
        (testbed / "README.md").write_text("testbed", encoding="utf-8")
        for command in (
            ("git", "init"), ("git", "config", "user.name", "test"),
            ("git", "config", "user.email", "test@example.invalid"), ("git", "add", "."),
            ("git", "commit", "-m", "testbed"),
        ):
            subprocess.run(command, cwd=testbed, check=True, capture_output=True)


class AttemptStoreTest(AttemptBase):
    def test_slots_and_append_only_terminals(self):
        self.manifest, self.raw, self.manifest_path = _manifest(self.root, repetitions=2)
        run = self.create_run()
        self.assertEqual([slot.repetition for slot in self.store.slots(self.manifest)], [1, 2])
        with self.store.writer(run):
            first = self.store.allocate(run, Slot("a", 1))
            self.assertEqual(list(Path(first.root).iterdir()), [])
            terminal = self.store.publish_terminal(first, "failed")
            with self.assertRaises(AttemptStateError):
                self.store.publish_terminal(terminal, "success")
            second = self.store.allocate(run, Slot("a", 1))
        self.assertEqual(second.identity.attempt, 2)

    def test_writer_is_fail_fast_and_status_is_read_only(self):
        run = self.create_run()
        before = (Path(run.root) / "run.json").read_bytes()
        with self.store.writer(run):
            with self.assertRaises(RunBusyError):
                with self.store.writer(run):
                    pass
            self.assertEqual(self.store.status(run, self.manifest)["attempts"]["running"], 0)
        self.assertEqual(before, (Path(run.root) / "run.json").read_bytes())

    def test_open_rejects_changed_snapshot_and_empty_allocation_reconciles(self):
        run = self.create_run()
        with self.assertRaises(AttemptStateError):
            self.store.open(self.manifest, run.run_id, self.raw + b" ")
        with self.store.writer(run):
            attempt = self.store.allocate(run, Slot("a", 1))
            interrupted = self.store.reconcile(attempt)
        self.assertEqual(interrupted.state, "interrupted")

    def test_foreign_record_and_symlink_fail_closed_without_status_mutation(self):
        run = self.create_run()
        with self.store.writer(run):
            attempt = self.store.allocate(run, Slot("a", 1))
            self.store.publish_terminal(attempt, "failed")
        record = Path(attempt.root) / "attempt.json"
        foreign = json.loads(record.read_text(encoding="utf-8"))
        foreign["run_id"] = "run-20260102T030405Z-ffffffffffff"
        record.write_text(json.dumps(foreign), encoding="utf-8")
        before = record.read_bytes()
        with self.assertRaises(AttemptStateError):
            self.store.status(run, self.manifest)
        self.assertEqual(before, record.read_bytes())
        record.unlink()
        record.symlink_to(Path(attempt.root) / "other.json")
        with self.assertRaises(AttemptStateError):
            self.store.attempts(run, Slot("a", 1))

    def test_preflight_records_append_in_order_and_status_is_read_only(self):
        run = self.create_run()
        cell = self.manifest.matrix[0]
        first = self.store.record_preflight(
            run, self.manifest, {cell.id: _preflight_observation(cell)}
        )
        second = self.store.record_preflight(
            run,
            self.manifest,
            {cell.id: _preflight_observation(cell, "credential_missing")},
        )
        before = {
            path.name: path.read_bytes()
            for path in (Path(run.root) / "preflight").iterdir()
        }

        self.assertEqual((first["sequence"], second["sequence"]), (1, 2))
        self.assertEqual(
            [record["status"] for record in self.store.preflights(run, self.manifest)],
            ["ready", "registration_required"],
        )
        status = self.store.status(run, self.manifest)
        self.assertEqual(
            status["preflight"],
            {
                "records": 2,
                "latest_sequence": 2,
                "latest_status": "registration_required",
                "ready": 0,
                "registration_required": 1,
                "implementation_gap": 0,
            },
        )
        self.assertEqual(status["attempts"]["running"], 0)
        self.assertFalse((Path(run.root) / "cells").exists())
        self.assertEqual(
            before,
            {
                path.name: path.read_bytes()
                for path in (Path(run.root) / "preflight").iterdir()
            },
        )

    def test_preflight_corruption_and_symlink_fail_closed(self):
        run = self.create_run()
        cell = self.manifest.matrix[0]
        self.store.record_preflight(
            run, self.manifest, {cell.id: _preflight_observation(cell)}
        )
        record = Path(run.root) / "preflight/preflight-000001.json"
        original = record.read_bytes()
        record.write_bytes(original + b" ")
        with self.assertRaises(AttemptStateError):
            self.store.preflights(run, self.manifest)
        record.write_bytes(original)
        record.unlink()
        record.symlink_to(Path(run.root) / "run.json")
        with self.assertRaises(AttemptStateError):
            self.store.status(run, self.manifest)

    def test_preflight_rejects_sequence_gap_and_foreign_result(self):
        run = self.create_run()
        cell = self.manifest.matrix[0]
        self.store.record_preflight(
            run, self.manifest, {cell.id: _preflight_observation(cell)}
        )
        first = Path(run.root) / "preflight/preflight-000001.json"
        first.rename(first.with_name("preflight-000002.json"))
        with self.assertRaises(AttemptStateError):
            self.store.preflights(run, self.manifest)

        first.with_name("preflight-000002.json").rename(first)
        raw = json.loads(first.read_text(encoding="ascii"))
        raw["results"][0]["binding"]["requested_model"] = "fallback"
        first.write_bytes(json.dumps(raw, sort_keys=True, separators=(",", ":")).encode("ascii") + b"\n")
        with self.assertRaises(AttemptStateError):
            self.store.preflights(run, self.manifest)

    def test_preflight_evidence_contains_no_unmodeled_adapter_values(self):
        run = self.create_run()
        cell = self.manifest.matrix[0]
        sentinel = "private_endpoint_or_token_must_not_persist"
        observation = _preflight_observation(cell, "stream_incompatible")
        # An adapter may retain runtime-only values on itself, but the writer
        # accepts only the closed PreflightObservation projection above.
        adapter = type("Adapter", (), {"runtime_value": sentinel})()
        self.assertEqual(adapter.runtime_value, sentinel)
        self.store.record_preflight(run, self.manifest, {cell.id: observation})
        durable = b"".join(
            path.read_bytes()
            for path in Path(run.root).rglob("*")
            if path.is_file()
        )
        self.assertNotIn(sentinel.encode("ascii"), durable)


class AttemptOrchestrationTest(AttemptBase):
    def test_run_slots_prepares_workspace_and_invokes_once(self):
        self._init_testbed()
        run = self.create_run()
        calls: list[str] = []

        def prepare(manifest, attempt):
            self.assertTrue((Path(run.root) / "preflight/preflight-000001.json").is_file())
            calls.append("prepare")
            return prepare_workspace(manifest, attempt.root, attempt.identity, repo_root=self.root)

        completed = run_slots(self.store, run, self.manifest, adapters={"claude": self.adapter("success", calls)}, prepare=prepare)
        self.assertEqual([item.state for item in completed], ["success"])
        self.assertEqual(calls, ["preflight", "prepare", "invoke"])
        attempt_root = Path(completed[0].root)
        self.assertTrue((attempt_root / "prepared.json").is_file())

    def test_preparation_failure_is_sealed_without_launch(self):
        run = self.create_run()
        calls: list[str] = []

        def fail_prepare(_manifest, _attempt):
            calls.append("prepare")
            raise RuntimeError("prepare failure")

        with self.assertRaisesRegex(RuntimeError, "prepare failure"):
            run_slots(self.store, run, self.manifest, adapters={"claude": self.adapter("success", calls)}, prepare=fail_prepare)
        self.assertEqual(calls, ["preflight", "prepare"])
        self.assertEqual(self.store.attempts(run, Slot("a", 1))[-1].state, "failed")

    def test_retry_and_skip_preserve_prior_terminal_bytes(self):
        self._init_testbed()
        run = self.create_run()
        calls: list[str] = []
        run_slots(
            self.store,
            run,
            self.manifest,
            adapters={"claude": self.adapter("failed", calls)},
            prepare=self.preparer(calls),
        )
        first = self.store.attempts(run, Slot("a", 1))[0]
        prior = (Path(first.root) / "attempt.json").read_bytes()
        self.assertEqual(
            run_slots(
                self.store,
                run,
                self.manifest,
                adapters={"claude": self.adapter("success", calls)},
                prepare=self.preparer(calls),
            ),
            (),
        )
        self.assertEqual(prior, (Path(first.root) / "attempt.json").read_bytes())
        retry = run_slots(
            self.store,
            run,
            self.manifest,
            adapters={"claude": self.adapter("success", calls)},
            prepare=self.preparer(calls),
            retry_failed=True,
        )
        self.assertEqual(retry[0].identity.attempt, 2)
        self.assertEqual(prior, (Path(first.root) / "attempt.json").read_bytes())
        self.assertEqual(len(self.store.preflights(run, self.manifest)), 3)

    def test_preflight_blocker_appends_without_attempt_allocation(self):
        run = self.create_run()
        calls: list[str] = []
        completed = run_slots(
            self.store,
            run,
            self.manifest,
            adapters={
                "claude": self.adapter(
                    "success", calls, issue_code="credential_missing"
                )
            },
            prepare=lambda _manifest, _attempt: self.fail("preparer must not run"),
        )
        self.assertEqual(completed, ())
        self.assertEqual(calls, ["preflight"])
        self.assertFalse((Path(run.root) / "cells").exists())
        self.assertEqual(
            self.store.status(run, self.manifest)["preflight"]["latest_status"],
            "registration_required",
        )

    def test_missing_adapter_has_no_output_root_side_effect(self):
        fake_run = RunIdentity("run-20260102T030405Z-abcdef123456", self.manifest.digest, str(self.root / "absent"))
        output = self.root / self.manifest.output_root
        self.assertFalse(output.exists())
        with self.assertRaises(CapabilityUnavailable):
            run_slots(self.store, fake_run, self.manifest, adapters={}, prepare=lambda _manifest, _attempt: None)
        self.assertFalse(output.exists())


class AttemptRecoveryTest(AttemptBase):
    def _running_with_terminal(self):
        run = self.create_run()
        with self.store.writer(run):
            attempt = self.store.allocate(run, Slot("a", 1))
            with self.assertRaisesRegex(RuntimeError, "controller crash"):
                self.store.execute_attempt(
                    attempt, prepare=lambda _: None,
                    invoke=self._invoke_then_crash,
                )
        return run, attempt

    def _invoke_then_crash(self, attempt, started):
        result = self.adapter("success", [])(attempt, started)
        self.assertTrue((Path(attempt.root) / "lifecycle-result.json").is_file())
        raise ControllerCrash("controller crash")

    def test_real_terminal_first_recovery_commits_once(self):
        run, attempt = self._running_with_terminal()
        with self.store.writer(run):
            recovered = self.store.reconcile(attempt)
        self.assertEqual(recovered.state, "success")
        self.assertEqual(self.store.reconcile(recovered).state, "success")

    def test_corrupt_terminal_variants_fail_closed_and_preserve_bytes(self):
        run, attempt = self._running_with_terminal()
        cases = (
            ("contradictory-success", "lifecycle-result.json", lambda raw: raw.__setitem__("success", False)),
            ("extra-result-field", "lifecycle-result.json", lambda raw: raw.__setitem__("unexpected", True)),
            ("mismatched-digest", "lifecycle-result.json", lambda raw: raw.__setitem__("spec_digest", "sha256:" + "0" * 64)),
            ("receipt-cleanup", "cleanup-receipt.json", lambda raw: raw.__setitem__("cleanup_complete", False)),
        )
        for name, filename, corrupt in cases:
            with self.subTest(name=name):
                target = Path(attempt.root) / "c" / filename if filename == "cleanup-receipt.json" else Path(attempt.root) / filename
                original = target.read_bytes()
                raw = json.loads(target.read_text(encoding="utf-8"))
                corrupt(raw)
                target.write_text(json.dumps(raw), encoding="utf-8")
                evidence_before = {path: path.read_bytes() for path in (Path(attempt.root) / "attempt.json", Path(attempt.root) / "lifecycle-result.json", Path(attempt.root) / "lifecycle-journal.jsonl", Path(attempt.root) / "c" / "cleanup-receipt.json")}
                with self.store.writer(run):
                    with self.assertRaises(AttemptStateError):
                        self.store.reconcile(attempt)
                self.assertEqual(evidence_before, {path: path.read_bytes() for path in evidence_before})
                target.write_bytes(original)

    @staticmethod
    def _evidence_bytes(run, attempt) -> dict[str, bytes]:
        """Snapshot every durable run and attempt record published so far."""
        run_root, attempt_root = Path(run.root), Path(attempt.root)
        paths = (
            run_root / "run.json", run_root / "manifest.json", run_root / "run.lock",
            attempt_root / "attempt.json", attempt_root / "lifecycle-result.json",
            attempt_root / "lifecycle-journal.jsonl", attempt_root / "c" / "locator.json",
            attempt_root / "c" / "cleanup-receipt.json",
        )
        return {str(path): path.read_bytes() for path in paths}

    @staticmethod
    def _substitute(target: Path, kind: str, saved: bytes) -> None:
        """Replace one durable file with a non-regular object of the given kind."""
        target.unlink()
        if kind == "fifo":
            os.mkfifo(target, 0o600)
        elif kind == "directory":
            target.mkdir(mode=0o700)
        elif kind == "socket":
            short = Path(tempfile.mkdtemp(dir="/tmp", prefix="s")) / "s"
            with contextlib.closing(socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)) as endpoint:
                endpoint.bind(str(short))  # bound in a short path, then moved into place
            os.replace(short, target)
            short.parent.rmdir()
        else:
            copy = target.with_name(target.name + ".copy")
            copy.write_bytes(saved)
            target.symlink_to(copy)

    @staticmethod
    def _restore(target: Path, saved: bytes) -> None:
        """Discard the substituted object and republish the original bytes."""
        if target.is_symlink() or not target.is_dir():
            target.unlink()
        else:
            target.rmdir()
        target.with_name(target.name + ".copy").unlink(missing_ok=True)
        target.write_bytes(saved)
        os.chmod(target, 0o600)

    def _assert_probe_rejected(self, run, attempt, operation: str, locator: dict | None = None) -> None:
        """Run one store operation in a bounded child and require a closed failure."""
        payload = json.dumps({
            "repo": str(self.root), "manifest": str(self.manifest_path), "run_id": run.run_id,
            "run_root": run.root, "attempt_root": attempt.root, "operation": operation,
            "cell_id": attempt.identity.cell_id, "repetition": attempt.identity.repetition,
            "attempt_number": attempt.identity.attempt, "locator": locator,
        })
        child = subprocess.Popen(
            [sys.executable, "-c", _PROBE_SOURCE, payload],
            stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
            env={**os.environ, "PYTHONPATH": str(Path(__file__).resolve().parents[2])},
        )
        try:
            out, err = child.communicate(timeout=_PROBE_TIMEOUT_SECONDS)
        except subprocess.TimeoutExpired:
            child.kill()
            child.communicate()
            self.fail(f"{operation} did not return within {_PROBE_TIMEOUT_SECONDS:.0f}s")
        self.assertEqual((child.returncode, out.strip()), (0, "rejected"), err)
        self.assertIsNotNone(child.poll())

    def test_nonregular_durable_files_fail_closed_without_blocking(self):
        run, attempt = self._running_with_terminal()
        run_root, attempt_root = Path(run.root), Path(attempt.root)
        surfaces = (
            ("run record", run_root / "run.json", "open"),
            ("manifest snapshot", run_root / "manifest.json", "open"),
            ("run lock read", run_root / "run.lock", "open"),
            ("run lock lease", run_root / "run.lock", "lease"),
            ("attempt record", attempt_root / "attempt.json", "attempts"),
            ("lifecycle result", attempt_root / "lifecycle-result.json", "reconcile"),
            ("lifecycle journal", attempt_root / "lifecycle-journal.jsonl", "reconcile"),
            ("cleanup receipt", attempt_root / "c" / "cleanup-receipt.json", "reconcile"),
        )
        kinds = ("fifo", "directory", "socket", "symlink")
        covered: list[tuple[str, str]] = []
        for label, target, operation in surfaces:
            for kind in kinds:
                with self.subTest(surface=label, kind=kind):
                    before = self._evidence_bytes(run, attempt)
                    saved = before[str(target)]
                    self._substitute(target, kind, saved)
                    try:
                        self._assert_probe_rejected(run, attempt, operation)
                        self.assertFalse(stat.S_ISREG(os.lstat(target).st_mode))
                    finally:
                        self._restore(target, saved)
                    self.assertEqual(before, self._evidence_bytes(run, attempt))
                    covered.append((label, kind))
        with self.store.writer(run):
            locator_attempt = self.store.allocate(run, Slot("a", 2))
        locator_root = Path(locator_attempt.root)
        control = locator_root / "c"
        control.mkdir(mode=0o700)
        locator = {
            "supervisor_pid": os.getpid(), "start_identity": "probe",
            "socket_path": str(control / "control.sock"), "challenge": "challenge",
            "control_dir": str(control), "created_at": "created",
        }
        target = control / "locator.json"
        target.write_text(json.dumps(locator), encoding="utf-8")

        def stop_before_locator(_attempt, _started):
            raise ControllerCrash("locator setup")

        with self.assertRaisesRegex(ControllerCrash, "locator setup"):
            self.store.execute_attempt(
                locator_attempt, prepare=lambda _: None, invoke=stop_before_locator,
            )
        running = (locator_root / "attempt.json").read_bytes()

        for kind in kinds:
            with self.subTest(surface="registered locator", kind=kind):
                saved = target.read_bytes()
                self._substitute(target, kind, saved)
                try:
                    self._assert_probe_rejected(run, locator_attempt, "locator", locator)
                    self.assertFalse(stat.S_ISREG(os.lstat(target).st_mode))
                finally:
                    self._restore(target, saved)
                self.assertEqual(saved, target.read_bytes())
                self.assertTrue(stat.S_ISREG(os.lstat(target).st_mode))
                self.assertEqual(running, (locator_root / "attempt.json").read_bytes())
                covered.append(("registered locator", kind))
        digest = "sha256:" + "0" * 64
        self.store.record_locator(
            locator_attempt, SupervisorLocator(**locator), digest,
        )
        record = json.loads((locator_root / "attempt.json").read_text(encoding="utf-8"))
        self.assertEqual(record.get("locator"), locator)
        self.assertEqual(record.get("spec_digest"), digest)
        located = self.store.attempts(run, Slot("a", 2))
        self.assertEqual([(item.identity, item.state) for item in located], [(locator_attempt.identity, "running")])
        self.assertFalse((locator_root / "lifecycle-result.json").exists())
        self.assertFalse((locator_root / "lifecycle-journal.jsonl").exists())
        self.assertEqual(self.store.attempts(run, Slot("a", 3)), ())
        # Passing subtests are silent, so bind the executed matrix explicitly.
        self.assertEqual(len(covered), (len(surfaces) + 1) * len(kinds))

    @staticmethod
    def _mutate_terminal(paths: dict[str, Path], record: str, mutate) -> None:
        """Apply one contradiction to a single record or to the ordered event evidence."""
        if record != "events":
            raw = json.loads(paths[record].read_text(encoding="utf-8"))
            mutate(raw)
            paths[record].write_text(json.dumps(raw), encoding="utf-8")
            return
        result = json.loads(paths["result"].read_text(encoding="utf-8"))
        result["events"] = mutate(result["events"])
        paths["result"].write_text(json.dumps(result), encoding="utf-8")
        lines = [json.loads(line) for line in paths["journal"].read_text(encoding="utf-8").splitlines()]
        rewritten = [lines[0], *result["events"], lines[-1]]
        paths["journal"].write_text("".join(json.dumps(line) + "\n" for line in rewritten), encoding="utf-8")

    def test_cross_record_terminal_corruption_fails_closed_and_preserves_bytes(self):
        run, attempt = self._running_with_terminal()
        root = Path(attempt.root)
        paths = {
            "result": root / "lifecycle-result.json",
            "journal": root / "lifecycle-journal.jsonl",
            "receipt": root / "c" / "cleanup-receipt.json",
        }
        saved = {key: path.read_bytes() for key, path in paths.items()}
        cases = (
            ("receipt-exit-code", "receipt", lambda raw: raw.__setitem__("exit_code", 9)),
            ("receipt-signal", "receipt", lambda raw: raw.__setitem__("signal", 9)),
            ("receipt-reason", "receipt", lambda raw: raw.__setitem__("reason", "failed")),
            ("receipt-caller-launched", "receipt", lambda raw: raw.__setitem__("caller_launched", False)),
            ("receipt-completed-before-start", "receipt", lambda raw: raw.__setitem__("completed_at", "2000-01-01T00:00:00+00:00")),
            ("receipt-completed-after-end", "receipt", lambda raw: raw.__setitem__("completed_at", "2100-01-01T00:00:00+00:00")),
            ("receipt-completed-unparseable", "receipt", lambda raw: raw.__setitem__("completed_at", "not-a-timestamp")),
            ("result-submitted", "result", lambda raw: raw.__setitem__("submitted", False)),
            ("result-exit-code", "result", lambda raw: raw.__setitem__("exit_code", 7)),
            ("events-cleared", "events", lambda events: []),
            ("events-missing-submitted", "events", _without("submitted")),
            ("events-missing-finish", "events", _without("finish")),
            ("events-missing-idle", "events", _without("idle")),
            ("events-missing-quiet", "events", _without("quiet")),
            ("events-out-of-order", "events", _reordered),
            ("events-duplicate-finish", "events", _duplicated("finish")),
        )
        for name, record, mutate in cases:
            with self.subTest(case=name):
                self._mutate_terminal(paths, record, mutate)
                try:
                    before = self._evidence_bytes(run, attempt)
                    with self.store.writer(run):
                        with self.assertRaises(AttemptStateError):
                            self.store.reconcile(attempt)
                    self.assertEqual(before, self._evidence_bytes(run, attempt))
                    self.assertEqual([item.state for item in self.store.attempts(run, Slot("a", 1))], ["running"])
                finally:
                    for key, path in paths.items():
                        path.write_bytes(saved[key])
        record = root / "attempt.json"
        running = record.read_bytes()
        with self.store.writer(run):
            recovered = self.store.reconcile(attempt)
        published = record.read_bytes()
        self.assertEqual(recovered.state, "success")
        self.assertNotEqual(running, published)
        with self.store.writer(run):
            self.assertEqual(self.store.reconcile(recovered).state, "success")
        self.assertEqual(published, record.read_bytes())
        self.assertEqual(len(self.store.attempts(run, Slot("a", 1))), 1)

    def test_symlink_lifecycle_evidence_fails_closed(self):
        run, attempt = self._running_with_terminal()
        journal = Path(attempt.root) / "lifecycle-journal.jsonl"
        saved = journal.read_bytes()
        target = Path(attempt.root) / "journal-copy.jsonl"
        target.write_bytes(saved)
        journal.unlink()
        journal.symlink_to(target)
        record = Path(attempt.root) / "attempt.json"
        before = record.read_bytes()
        with self.store.writer(run):
            with self.assertRaises(AttemptStateError):
                self.store.reconcile(attempt)
        self.assertEqual(before, record.read_bytes())

    def test_direct_result_requires_bound_production_evidence(self):
        run = self.create_run()
        with self.store.writer(run):
            attempt = self.store.allocate(run, Slot("a", 1))

            def contradictory(current, started):
                result = self.adapter("success", [])(current, started)
                path = Path(current.root) / "lifecycle-result.json"
                raw = json.loads(path.read_text(encoding="utf-8"))
                raw["success"] = False
                path.write_text(json.dumps(raw), encoding="utf-8")
                return result

            with self.assertRaises(AttemptStateError):
                self.store.execute_attempt(attempt, prepare=lambda _: None, invoke=contradictory)
        record = Path(attempt.root) / "attempt.json"
        self.assertEqual(json.loads(record.read_text(encoding="utf-8"))["state"], "running")

    def test_live_survivor_cleanup_precedes_successor(self):
        run = self.create_run()
        with self.store.writer(run):
            attempt = self.store.allocate(run, Slot("a", 1))
        locator_ready = threading.Event()
        result_box: list[BaseException | InvocationResult] = []

        def long_running(current, started):
            spec = self._spec(current, "import time; print('START', flush=True); time.sleep(30)")

            def commit(locator):
                started(locator, spec_digest(spec))
                locator_ready.set()

            return run_invocation(spec, parse_event=_events, on_started=commit)

        def invoke() -> None:
            try:
                self.store.execute_attempt(attempt, prepare=lambda _: None, invoke=long_running)
            except BaseException as exc:  # concurrent reconciliation seals this attempt first
                result_box.append(exc)

        worker = threading.Thread(target=invoke)
        worker.start()
        self.assertTrue(locator_ready.wait(5))
        with self.store.writer(run):
            recovered = self.store.reconcile(attempt)
            successor = self.store.allocate(run, Slot("a", 1))
        worker.join(10)
        self.assertFalse(worker.is_alive())
        self.assertEqual(recovered.state, "interrupted")
        self.assertEqual(successor.identity.attempt, 2)
        self.assertTrue(result_box)

    def test_cross_process_lease_contention_and_crash_release(self):
        run = self.create_run()
        script = "import fcntl, os, sys, time; f=os.open(sys.argv[1], os.O_RDWR); fcntl.flock(f, fcntl.LOCK_EX); print('locked', flush=True); time.sleep(30)"
        child = subprocess.Popen([sys.executable, "-c", script, str(Path(run.root) / "run.lock")], stdout=subprocess.PIPE, text=True)
        self.assertEqual(child.stdout.readline().strip(), "locked")
        with self.assertRaises(RunBusyError):
            with self.store.writer(run):
                pass
        child.kill()
        child.wait(timeout=5)
        child.stdout.close()
        with self.store.writer(run):
            pass


class AttemptCliContractTest(AttemptBase):
    def test_cli_status_is_read_only_and_run_resume_block_before_attempts(self):
        run = self.create_run()
        run_before = {
            path.relative_to(run.root): path.read_bytes()
            for path in Path(run.root).rglob("*")
            if path.is_file()
        }
        output = io.StringIO()
        with mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root), contextlib.redirect_stdout(output):
            self.assertEqual(benchmark_cli.main(["status", "--manifest", str(self.manifest_path), "--run-id", run.run_id]), 0)
        self.assertIn("'running': 0", output.getvalue())
        self.assertEqual(
            run_before,
            {
                path.relative_to(run.root): path.read_bytes()
                for path in Path(run.root).rglob("*")
                if path.is_file()
            },
        )
        with mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root), contextlib.redirect_stderr(io.StringIO()):
            self.assertEqual(benchmark_cli.main(["resume", "--manifest", str(self.manifest_path), "--run-id", run.run_id]), 69)
        self.assertFalse((Path(run.root) / "cells").exists())
        self.assertEqual(len(self.store.preflights(run, self.manifest)), 1)
        absent_root = self.root / "agent-test/runs/absent"
        raw = json.loads(self.raw)
        raw["output_root"] = "agent-test/runs/absent"
        absent = self.root / "absent.json"
        absent.write_text(json.dumps(raw), encoding="utf-8")
        with mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root), contextlib.redirect_stderr(io.StringIO()):
            self.assertEqual(benchmark_cli.main(["run", "--manifest", str(absent)]), 69)
        created = list(absent_root.glob("run-*"))
        self.assertEqual(len(created), 1)
        self.assertTrue((created[0] / "preflight/preflight-000001.json").is_file())
        self.assertFalse((created[0] / "cells").exists())


if __name__ == "__main__":
    unittest.main()