toki
91dec43635
caller와 모델 조합을 반복 비교할 때 실행·격리·재개 근거가 흔들리지 않도록 manifest, workspace, lifecycle, append-only attempt 기반과 project-local 진입점을 함께 고정한다.
673 lines
31 KiB
Python
673 lines
31 KiB
Python
"""Credential-free production-path tests for durable benchmark attempts."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import contextlib
|
|
import datetime
|
|
import io
|
|
import json
|
|
import os
|
|
import socket
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
import unittest
|
|
from pathlib import Path
|
|
from unittest import mock
|
|
|
|
from scripts import agent_comparison_benchmark as benchmark_cli
|
|
from scripts.agent_benchmark.attempts import (
|
|
AttemptStateError,
|
|
CapabilityUnavailable,
|
|
RunBusyError,
|
|
RunIdentity,
|
|
RunStore,
|
|
Slot,
|
|
run_slots,
|
|
)
|
|
from scripts.agent_benchmark.lifecycle import (
|
|
COMPLETION_EXIT_AFTER_IDLE,
|
|
SUBMISSION_ARGV_TASK,
|
|
InvocationResult,
|
|
InvocationSpec,
|
|
SupervisorLocator,
|
|
env_pairs,
|
|
run_invocation,
|
|
spec_digest,
|
|
)
|
|
from scripts.agent_benchmark.manifest import AssetMapping, Timeout, digest_workspace_inputs, load_manifest
|
|
from scripts.agent_benchmark.workspace import prepare_workspace
|
|
|
|
|
|
def _manifest(root: Path, repetitions: int = 1):
|
|
fixtures = root / "scripts/fixtures"
|
|
fixtures.mkdir(parents=True, exist_ok=True)
|
|
(fixtures / "prompt.md").write_text("prompt", encoding="utf-8")
|
|
(fixtures / "reference.txt").write_text("reference", encoding="utf-8")
|
|
fixture = {
|
|
"version": "v1", "prompt": "scripts/fixtures/prompt.md",
|
|
"assets": [{"source": "scripts/fixtures/reference.txt", "workspace_path": "workspace/reference.txt"}],
|
|
"checksum": digest_workspace_inputs((AssetMapping("scripts/fixtures/reference.txt", "workspace/reference.txt", b"reference"),)),
|
|
}
|
|
data = {
|
|
"pipeline_version": "1", "environment": "dev", "testbed": "../iop-s2",
|
|
"session_policy": "fresh", "setup_cache_policy": "isolated",
|
|
"timeout": {"run_seconds": 1, "idle_seconds": 1, "quiet_seconds": 1, "cleanup_grace_seconds": 1},
|
|
"viewports": [{"id": "desktop", "width": 1, "height": 1}], "rubric_version": "v1",
|
|
"output_root": "agent-test/runs/a", "fixture": fixture, "repetitions": repetitions,
|
|
"matrix": [{"id": "a", "caller": "claude", "iop": {"request_model": "model", "requested_effort": "high", "route_kind": "direct", "route_id": "route", "expected_bindings": [{"stage": "request", "model": "model", "effort": "high"}]}}],
|
|
}
|
|
path = root / "manifest.json"
|
|
raw = json.dumps(data, sort_keys=True).encode("utf-8")
|
|
path.write_bytes(raw)
|
|
return load_manifest(path, repo_root=root), raw, path
|
|
|
|
|
|
def _events(_: str, line: str) -> str | None:
|
|
return {"FINISH": "finish", "IDLE": "idle"}.get(line.strip())
|
|
|
|
|
|
_PROBE_TIMEOUT_SECONDS = 30.0
|
|
|
|
# Every durable read runs in a bounded child so a blocking special file cannot
|
|
# hang the suite; the child reports whether the store fails closed.
|
|
_PROBE_SOURCE = """
|
|
import json
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
from scripts.agent_benchmark.attempts import Attempt, AttemptStateError, RunIdentity, RunStore, Slot
|
|
from scripts.agent_benchmark.lifecycle import SupervisorLocator
|
|
from scripts.agent_benchmark.manifest import load_manifest
|
|
from scripts.agent_benchmark.workspace import AttemptIdentity
|
|
|
|
payload = json.loads(sys.argv[1])
|
|
store = RunStore(payload["repo"])
|
|
manifest = load_manifest(Path(payload["manifest"]), repo_root=Path(payload["repo"]))
|
|
run = RunIdentity(payload["run_id"], manifest.digest, payload["run_root"])
|
|
attempt = Attempt(
|
|
AttemptIdentity(payload["run_id"], payload["cell_id"], payload["repetition"], payload["attempt_number"]),
|
|
payload["attempt_root"], "running",
|
|
)
|
|
|
|
|
|
def lease():
|
|
with store.writer(run):
|
|
pass
|
|
|
|
|
|
def locator():
|
|
store.record_locator(
|
|
attempt,
|
|
SupervisorLocator(**payload["locator"]),
|
|
"sha256:" + "0" * 64,
|
|
)
|
|
|
|
|
|
operations = {
|
|
"open": lambda: store.open(manifest, run.run_id),
|
|
"lease": lease,
|
|
"attempts": lambda: store.attempts(run, Slot("a", 1)),
|
|
"reconcile": lambda: store.reconcile(attempt),
|
|
"locator": locator,
|
|
}
|
|
try:
|
|
operations[payload["operation"]]()
|
|
except AttemptStateError:
|
|
print("rejected")
|
|
sys.exit(0)
|
|
print("accepted")
|
|
sys.exit(1)
|
|
"""
|
|
|
|
|
|
def _reordered(events: list[dict]) -> list[dict]:
|
|
"""Return production events with finish and idle transposed."""
|
|
kinds = [event["kind"] for event in events]
|
|
swapped = list(events)
|
|
finish, idle = kinds.index("finish"), kinds.index("idle")
|
|
swapped[finish], swapped[idle] = swapped[idle], swapped[finish]
|
|
return swapped
|
|
|
|
|
|
def _without(kind: str):
|
|
return lambda events: [event for event in events if event["kind"] != kind]
|
|
|
|
|
|
def _duplicated(kind: str):
|
|
return lambda events: events + [event for event in events if event["kind"] == kind]
|
|
|
|
|
|
class ControllerCrash(RuntimeError):
|
|
"""Test-only controller loss after lifecycle evidence has been published."""
|
|
|
|
|
|
class AttemptBase(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.temp = tempfile.TemporaryDirectory(dir="/tmp", prefix="b")
|
|
self.root = Path(self.temp.name) / "r"
|
|
self.root.mkdir()
|
|
self._control_aliases: list[Path] = []
|
|
self.manifest, self.raw, self.manifest_path = _manifest(self.root)
|
|
self.store = RunStore(
|
|
self.root,
|
|
clock=lambda: datetime.datetime(2026, 1, 2, 3, 4, 5, tzinfo=datetime.timezone.utc),
|
|
token_hex=lambda _: "abcdef123456",
|
|
)
|
|
|
|
def tearDown(self) -> None:
|
|
for alias in self._control_aliases:
|
|
try:
|
|
alias.unlink()
|
|
except FileNotFoundError:
|
|
pass
|
|
self.temp.cleanup()
|
|
|
|
def create_run(self):
|
|
return self.store.create(self.manifest, self.raw)
|
|
|
|
def _spec(self, attempt, source: str) -> InvocationSpec:
|
|
alias = Path(tempfile.mkdtemp(dir="/tmp", prefix="c"))
|
|
alias.rmdir()
|
|
alias.symlink_to(Path(attempt.root), target_is_directory=True)
|
|
self._control_aliases.append(alias)
|
|
return InvocationSpec(
|
|
argv=(sys.executable, "-u", "-c", source),
|
|
cwd=str(self.root),
|
|
env=env_pairs({"PATH": os.environ.get("PATH", "/usr/bin:/bin")}),
|
|
submission_mode=SUBMISSION_ARGV_TASK,
|
|
completion_mode=COMPLETION_EXIT_AFTER_IDLE,
|
|
timeout=Timeout(5, 1, 1, 1),
|
|
evidence_dir=attempt.root,
|
|
control_dir=str(alias / "c"),
|
|
)
|
|
|
|
def adapter(self, reason: str, calls: list[str]):
|
|
def invoke(attempt, started):
|
|
calls.append("invoke")
|
|
source = "print('FINISH'); print('IDLE')" if reason == "success" else "import sys; print('FAILED'); sys.exit(3)"
|
|
spec = self._spec(attempt, source)
|
|
return run_invocation(
|
|
spec,
|
|
parse_event=_events,
|
|
on_started=lambda locator: started(locator, spec_digest(spec)),
|
|
)
|
|
return invoke
|
|
|
|
def _init_testbed(self) -> None:
|
|
testbed = self.root.parent / "iop-s2"
|
|
testbed.mkdir()
|
|
(testbed / "README.md").write_text("testbed", encoding="utf-8")
|
|
for command in (
|
|
("git", "init"), ("git", "config", "user.name", "test"),
|
|
("git", "config", "user.email", "test@example.invalid"), ("git", "add", "."),
|
|
("git", "commit", "-m", "testbed"),
|
|
):
|
|
subprocess.run(command, cwd=testbed, check=True, capture_output=True)
|
|
|
|
|
|
class AttemptStoreTest(AttemptBase):
|
|
def test_slots_and_append_only_terminals(self):
|
|
self.manifest, self.raw, self.manifest_path = _manifest(self.root, repetitions=2)
|
|
run = self.create_run()
|
|
self.assertEqual([slot.repetition for slot in self.store.slots(self.manifest)], [1, 2])
|
|
with self.store.writer(run):
|
|
first = self.store.allocate(run, Slot("a", 1))
|
|
self.assertEqual(list(Path(first.root).iterdir()), [])
|
|
terminal = self.store.publish_terminal(first, "failed")
|
|
with self.assertRaises(AttemptStateError):
|
|
self.store.publish_terminal(terminal, "success")
|
|
second = self.store.allocate(run, Slot("a", 1))
|
|
self.assertEqual(second.identity.attempt, 2)
|
|
|
|
def test_writer_is_fail_fast_and_status_is_read_only(self):
|
|
run = self.create_run()
|
|
before = (Path(run.root) / "run.json").read_bytes()
|
|
with self.store.writer(run):
|
|
with self.assertRaises(RunBusyError):
|
|
with self.store.writer(run):
|
|
pass
|
|
self.assertEqual(self.store.status(run, self.manifest)["attempts"]["running"], 0)
|
|
self.assertEqual(before, (Path(run.root) / "run.json").read_bytes())
|
|
|
|
def test_open_rejects_changed_snapshot_and_empty_allocation_reconciles(self):
|
|
run = self.create_run()
|
|
with self.assertRaises(AttemptStateError):
|
|
self.store.open(self.manifest, run.run_id, self.raw + b" ")
|
|
with self.store.writer(run):
|
|
attempt = self.store.allocate(run, Slot("a", 1))
|
|
interrupted = self.store.reconcile(attempt)
|
|
self.assertEqual(interrupted.state, "interrupted")
|
|
|
|
def test_foreign_record_and_symlink_fail_closed_without_status_mutation(self):
|
|
run = self.create_run()
|
|
with self.store.writer(run):
|
|
attempt = self.store.allocate(run, Slot("a", 1))
|
|
self.store.publish_terminal(attempt, "failed")
|
|
record = Path(attempt.root) / "attempt.json"
|
|
foreign = json.loads(record.read_text(encoding="utf-8"))
|
|
foreign["run_id"] = "run-20260102T030405Z-ffffffffffff"
|
|
record.write_text(json.dumps(foreign), encoding="utf-8")
|
|
before = record.read_bytes()
|
|
with self.assertRaises(AttemptStateError):
|
|
self.store.status(run, self.manifest)
|
|
self.assertEqual(before, record.read_bytes())
|
|
record.unlink()
|
|
record.symlink_to(Path(attempt.root) / "other.json")
|
|
with self.assertRaises(AttemptStateError):
|
|
self.store.attempts(run, Slot("a", 1))
|
|
|
|
|
|
class AttemptOrchestrationTest(AttemptBase):
|
|
def test_run_slots_prepares_workspace_and_invokes_once(self):
|
|
self._init_testbed()
|
|
run = self.create_run()
|
|
calls: list[str] = []
|
|
|
|
def prepare(attempt):
|
|
calls.append("prepare")
|
|
return prepare_workspace(self.manifest, attempt.root, attempt.identity, repo_root=self.root)
|
|
|
|
completed = run_slots(self.store, run, self.manifest, adapters={"claude": self.adapter("success", calls)}, prepare=prepare)
|
|
self.assertEqual([item.state for item in completed], ["success"])
|
|
self.assertEqual(calls, ["prepare", "invoke"])
|
|
attempt_root = Path(completed[0].root)
|
|
self.assertTrue((attempt_root / "prepared.json").is_file())
|
|
|
|
def test_preparation_failure_is_sealed_without_launch(self):
|
|
run = self.create_run()
|
|
calls: list[str] = []
|
|
|
|
def fail_prepare(_attempt):
|
|
calls.append("prepare")
|
|
raise RuntimeError("prepare failure")
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "prepare failure"):
|
|
run_slots(self.store, run, self.manifest, adapters={"claude": self.adapter("success", calls)}, prepare=fail_prepare)
|
|
self.assertEqual(calls, ["prepare"])
|
|
self.assertEqual(self.store.attempts(run, Slot("a", 1))[-1].state, "failed")
|
|
|
|
def test_retry_and_skip_preserve_prior_terminal_bytes(self):
|
|
run = self.create_run()
|
|
calls: list[str] = []
|
|
run_slots(self.store, run, self.manifest, adapters={"claude": self.adapter("failed", calls)}, prepare=lambda _: calls.append("prepare"))
|
|
first = self.store.attempts(run, Slot("a", 1))[0]
|
|
prior = (Path(first.root) / "attempt.json").read_bytes()
|
|
self.assertEqual(run_slots(self.store, run, self.manifest, adapters={"claude": self.adapter("success", calls)}, prepare=lambda _: calls.append("prepare")), ())
|
|
self.assertEqual(prior, (Path(first.root) / "attempt.json").read_bytes())
|
|
retry = run_slots(self.store, run, self.manifest, adapters={"claude": self.adapter("success", calls)}, prepare=lambda _: calls.append("prepare"), retry_failed=True)
|
|
self.assertEqual(retry[0].identity.attempt, 2)
|
|
self.assertEqual(prior, (Path(first.root) / "attempt.json").read_bytes())
|
|
|
|
def test_missing_adapter_has_no_output_root_side_effect(self):
|
|
fake_run = RunIdentity("run-20260102T030405Z-abcdef123456", self.manifest.digest, str(self.root / "absent"))
|
|
output = self.root / self.manifest.output_root
|
|
self.assertFalse(output.exists())
|
|
with self.assertRaises(CapabilityUnavailable):
|
|
run_slots(self.store, fake_run, self.manifest, adapters={}, prepare=lambda _: None)
|
|
self.assertFalse(output.exists())
|
|
|
|
|
|
class AttemptRecoveryTest(AttemptBase):
|
|
def _running_with_terminal(self):
|
|
run = self.create_run()
|
|
with self.store.writer(run):
|
|
attempt = self.store.allocate(run, Slot("a", 1))
|
|
with self.assertRaisesRegex(RuntimeError, "controller crash"):
|
|
self.store.execute_attempt(
|
|
attempt, prepare=lambda _: None,
|
|
invoke=self._invoke_then_crash,
|
|
)
|
|
return run, attempt
|
|
|
|
def _invoke_then_crash(self, attempt, started):
|
|
result = self.adapter("success", [])(attempt, started)
|
|
self.assertTrue((Path(attempt.root) / "lifecycle-result.json").is_file())
|
|
raise ControllerCrash("controller crash")
|
|
|
|
def test_real_terminal_first_recovery_commits_once(self):
|
|
run, attempt = self._running_with_terminal()
|
|
with self.store.writer(run):
|
|
recovered = self.store.reconcile(attempt)
|
|
self.assertEqual(recovered.state, "success")
|
|
self.assertEqual(self.store.reconcile(recovered).state, "success")
|
|
|
|
def test_corrupt_terminal_variants_fail_closed_and_preserve_bytes(self):
|
|
run, attempt = self._running_with_terminal()
|
|
cases = (
|
|
("contradictory-success", "lifecycle-result.json", lambda raw: raw.__setitem__("success", False)),
|
|
("extra-result-field", "lifecycle-result.json", lambda raw: raw.__setitem__("unexpected", True)),
|
|
("mismatched-digest", "lifecycle-result.json", lambda raw: raw.__setitem__("spec_digest", "sha256:" + "0" * 64)),
|
|
("receipt-cleanup", "cleanup-receipt.json", lambda raw: raw.__setitem__("cleanup_complete", False)),
|
|
)
|
|
for name, filename, corrupt in cases:
|
|
with self.subTest(name=name):
|
|
target = Path(attempt.root) / "c" / filename if filename == "cleanup-receipt.json" else Path(attempt.root) / filename
|
|
original = target.read_bytes()
|
|
raw = json.loads(target.read_text(encoding="utf-8"))
|
|
corrupt(raw)
|
|
target.write_text(json.dumps(raw), encoding="utf-8")
|
|
evidence_before = {path: path.read_bytes() for path in (Path(attempt.root) / "attempt.json", Path(attempt.root) / "lifecycle-result.json", Path(attempt.root) / "lifecycle-journal.jsonl", Path(attempt.root) / "c" / "cleanup-receipt.json")}
|
|
with self.store.writer(run):
|
|
with self.assertRaises(AttemptStateError):
|
|
self.store.reconcile(attempt)
|
|
self.assertEqual(evidence_before, {path: path.read_bytes() for path in evidence_before})
|
|
target.write_bytes(original)
|
|
|
|
@staticmethod
|
|
def _evidence_bytes(run, attempt) -> dict[str, bytes]:
|
|
"""Snapshot every durable run and attempt record published so far."""
|
|
run_root, attempt_root = Path(run.root), Path(attempt.root)
|
|
paths = (
|
|
run_root / "run.json", run_root / "manifest.json", run_root / "run.lock",
|
|
attempt_root / "attempt.json", attempt_root / "lifecycle-result.json",
|
|
attempt_root / "lifecycle-journal.jsonl", attempt_root / "c" / "locator.json",
|
|
attempt_root / "c" / "cleanup-receipt.json",
|
|
)
|
|
return {str(path): path.read_bytes() for path in paths}
|
|
|
|
@staticmethod
|
|
def _substitute(target: Path, kind: str, saved: bytes) -> None:
|
|
"""Replace one durable file with a non-regular object of the given kind."""
|
|
target.unlink()
|
|
if kind == "fifo":
|
|
os.mkfifo(target, 0o600)
|
|
elif kind == "directory":
|
|
target.mkdir(mode=0o700)
|
|
elif kind == "socket":
|
|
short = Path(tempfile.mkdtemp(dir="/tmp", prefix="s")) / "s"
|
|
with contextlib.closing(socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)) as endpoint:
|
|
endpoint.bind(str(short)) # bound in a short path, then moved into place
|
|
os.replace(short, target)
|
|
short.parent.rmdir()
|
|
else:
|
|
copy = target.with_name(target.name + ".copy")
|
|
copy.write_bytes(saved)
|
|
target.symlink_to(copy)
|
|
|
|
@staticmethod
|
|
def _restore(target: Path, saved: bytes) -> None:
|
|
"""Discard the substituted object and republish the original bytes."""
|
|
if target.is_symlink() or not target.is_dir():
|
|
target.unlink()
|
|
else:
|
|
target.rmdir()
|
|
target.with_name(target.name + ".copy").unlink(missing_ok=True)
|
|
target.write_bytes(saved)
|
|
os.chmod(target, 0o600)
|
|
|
|
def _assert_probe_rejected(self, run, attempt, operation: str, locator: dict | None = None) -> None:
|
|
"""Run one store operation in a bounded child and require a closed failure."""
|
|
payload = json.dumps({
|
|
"repo": str(self.root), "manifest": str(self.manifest_path), "run_id": run.run_id,
|
|
"run_root": run.root, "attempt_root": attempt.root, "operation": operation,
|
|
"cell_id": attempt.identity.cell_id, "repetition": attempt.identity.repetition,
|
|
"attempt_number": attempt.identity.attempt, "locator": locator,
|
|
})
|
|
child = subprocess.Popen(
|
|
[sys.executable, "-c", _PROBE_SOURCE, payload],
|
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
|
|
env={**os.environ, "PYTHONPATH": str(Path(__file__).resolve().parents[2])},
|
|
)
|
|
try:
|
|
out, err = child.communicate(timeout=_PROBE_TIMEOUT_SECONDS)
|
|
except subprocess.TimeoutExpired:
|
|
child.kill()
|
|
child.communicate()
|
|
self.fail(f"{operation} did not return within {_PROBE_TIMEOUT_SECONDS:.0f}s")
|
|
self.assertEqual((child.returncode, out.strip()), (0, "rejected"), err)
|
|
self.assertIsNotNone(child.poll())
|
|
|
|
def test_nonregular_durable_files_fail_closed_without_blocking(self):
|
|
run, attempt = self._running_with_terminal()
|
|
run_root, attempt_root = Path(run.root), Path(attempt.root)
|
|
surfaces = (
|
|
("run record", run_root / "run.json", "open"),
|
|
("manifest snapshot", run_root / "manifest.json", "open"),
|
|
("run lock read", run_root / "run.lock", "open"),
|
|
("run lock lease", run_root / "run.lock", "lease"),
|
|
("attempt record", attempt_root / "attempt.json", "attempts"),
|
|
("lifecycle result", attempt_root / "lifecycle-result.json", "reconcile"),
|
|
("lifecycle journal", attempt_root / "lifecycle-journal.jsonl", "reconcile"),
|
|
("cleanup receipt", attempt_root / "c" / "cleanup-receipt.json", "reconcile"),
|
|
)
|
|
kinds = ("fifo", "directory", "socket", "symlink")
|
|
covered: list[tuple[str, str]] = []
|
|
for label, target, operation in surfaces:
|
|
for kind in kinds:
|
|
with self.subTest(surface=label, kind=kind):
|
|
before = self._evidence_bytes(run, attempt)
|
|
saved = before[str(target)]
|
|
self._substitute(target, kind, saved)
|
|
try:
|
|
self._assert_probe_rejected(run, attempt, operation)
|
|
self.assertFalse(stat.S_ISREG(os.lstat(target).st_mode))
|
|
finally:
|
|
self._restore(target, saved)
|
|
self.assertEqual(before, self._evidence_bytes(run, attempt))
|
|
covered.append((label, kind))
|
|
with self.store.writer(run):
|
|
locator_attempt = self.store.allocate(run, Slot("a", 2))
|
|
locator_root = Path(locator_attempt.root)
|
|
control = locator_root / "c"
|
|
control.mkdir(mode=0o700)
|
|
locator = {
|
|
"supervisor_pid": os.getpid(), "start_identity": "probe",
|
|
"socket_path": str(control / "control.sock"), "challenge": "challenge",
|
|
"control_dir": str(control), "created_at": "created",
|
|
}
|
|
target = control / "locator.json"
|
|
target.write_text(json.dumps(locator), encoding="utf-8")
|
|
|
|
def stop_before_locator(_attempt, _started):
|
|
raise ControllerCrash("locator setup")
|
|
|
|
with self.assertRaisesRegex(ControllerCrash, "locator setup"):
|
|
self.store.execute_attempt(
|
|
locator_attempt, prepare=lambda _: None, invoke=stop_before_locator,
|
|
)
|
|
running = (locator_root / "attempt.json").read_bytes()
|
|
|
|
for kind in kinds:
|
|
with self.subTest(surface="registered locator", kind=kind):
|
|
saved = target.read_bytes()
|
|
self._substitute(target, kind, saved)
|
|
try:
|
|
self._assert_probe_rejected(run, locator_attempt, "locator", locator)
|
|
self.assertFalse(stat.S_ISREG(os.lstat(target).st_mode))
|
|
finally:
|
|
self._restore(target, saved)
|
|
self.assertEqual(saved, target.read_bytes())
|
|
self.assertTrue(stat.S_ISREG(os.lstat(target).st_mode))
|
|
self.assertEqual(running, (locator_root / "attempt.json").read_bytes())
|
|
covered.append(("registered locator", kind))
|
|
digest = "sha256:" + "0" * 64
|
|
self.store.record_locator(
|
|
locator_attempt, SupervisorLocator(**locator), digest,
|
|
)
|
|
record = json.loads((locator_root / "attempt.json").read_text(encoding="utf-8"))
|
|
self.assertEqual(record.get("locator"), locator)
|
|
self.assertEqual(record.get("spec_digest"), digest)
|
|
located = self.store.attempts(run, Slot("a", 2))
|
|
self.assertEqual([(item.identity, item.state) for item in located], [(locator_attempt.identity, "running")])
|
|
self.assertFalse((locator_root / "lifecycle-result.json").exists())
|
|
self.assertFalse((locator_root / "lifecycle-journal.jsonl").exists())
|
|
self.assertEqual(self.store.attempts(run, Slot("a", 3)), ())
|
|
# Passing subtests are silent, so bind the executed matrix explicitly.
|
|
self.assertEqual(len(covered), (len(surfaces) + 1) * len(kinds))
|
|
|
|
@staticmethod
|
|
def _mutate_terminal(paths: dict[str, Path], record: str, mutate) -> None:
|
|
"""Apply one contradiction to a single record or to the ordered event evidence."""
|
|
if record != "events":
|
|
raw = json.loads(paths[record].read_text(encoding="utf-8"))
|
|
mutate(raw)
|
|
paths[record].write_text(json.dumps(raw), encoding="utf-8")
|
|
return
|
|
result = json.loads(paths["result"].read_text(encoding="utf-8"))
|
|
result["events"] = mutate(result["events"])
|
|
paths["result"].write_text(json.dumps(result), encoding="utf-8")
|
|
lines = [json.loads(line) for line in paths["journal"].read_text(encoding="utf-8").splitlines()]
|
|
rewritten = [lines[0], *result["events"], lines[-1]]
|
|
paths["journal"].write_text("".join(json.dumps(line) + "\n" for line in rewritten), encoding="utf-8")
|
|
|
|
def test_cross_record_terminal_corruption_fails_closed_and_preserves_bytes(self):
|
|
run, attempt = self._running_with_terminal()
|
|
root = Path(attempt.root)
|
|
paths = {
|
|
"result": root / "lifecycle-result.json",
|
|
"journal": root / "lifecycle-journal.jsonl",
|
|
"receipt": root / "c" / "cleanup-receipt.json",
|
|
}
|
|
saved = {key: path.read_bytes() for key, path in paths.items()}
|
|
cases = (
|
|
("receipt-exit-code", "receipt", lambda raw: raw.__setitem__("exit_code", 9)),
|
|
("receipt-signal", "receipt", lambda raw: raw.__setitem__("signal", 9)),
|
|
("receipt-reason", "receipt", lambda raw: raw.__setitem__("reason", "failed")),
|
|
("receipt-caller-launched", "receipt", lambda raw: raw.__setitem__("caller_launched", False)),
|
|
("receipt-completed-before-start", "receipt", lambda raw: raw.__setitem__("completed_at", "2000-01-01T00:00:00+00:00")),
|
|
("receipt-completed-after-end", "receipt", lambda raw: raw.__setitem__("completed_at", "2100-01-01T00:00:00+00:00")),
|
|
("receipt-completed-unparseable", "receipt", lambda raw: raw.__setitem__("completed_at", "not-a-timestamp")),
|
|
("result-submitted", "result", lambda raw: raw.__setitem__("submitted", False)),
|
|
("result-exit-code", "result", lambda raw: raw.__setitem__("exit_code", 7)),
|
|
("events-cleared", "events", lambda events: []),
|
|
("events-missing-submitted", "events", _without("submitted")),
|
|
("events-missing-finish", "events", _without("finish")),
|
|
("events-missing-idle", "events", _without("idle")),
|
|
("events-missing-quiet", "events", _without("quiet")),
|
|
("events-out-of-order", "events", _reordered),
|
|
("events-duplicate-finish", "events", _duplicated("finish")),
|
|
)
|
|
for name, record, mutate in cases:
|
|
with self.subTest(case=name):
|
|
self._mutate_terminal(paths, record, mutate)
|
|
try:
|
|
before = self._evidence_bytes(run, attempt)
|
|
with self.store.writer(run):
|
|
with self.assertRaises(AttemptStateError):
|
|
self.store.reconcile(attempt)
|
|
self.assertEqual(before, self._evidence_bytes(run, attempt))
|
|
self.assertEqual([item.state for item in self.store.attempts(run, Slot("a", 1))], ["running"])
|
|
finally:
|
|
for key, path in paths.items():
|
|
path.write_bytes(saved[key])
|
|
record = root / "attempt.json"
|
|
running = record.read_bytes()
|
|
with self.store.writer(run):
|
|
recovered = self.store.reconcile(attempt)
|
|
published = record.read_bytes()
|
|
self.assertEqual(recovered.state, "success")
|
|
self.assertNotEqual(running, published)
|
|
with self.store.writer(run):
|
|
self.assertEqual(self.store.reconcile(recovered).state, "success")
|
|
self.assertEqual(published, record.read_bytes())
|
|
self.assertEqual(len(self.store.attempts(run, Slot("a", 1))), 1)
|
|
|
|
def test_symlink_lifecycle_evidence_fails_closed(self):
|
|
run, attempt = self._running_with_terminal()
|
|
journal = Path(attempt.root) / "lifecycle-journal.jsonl"
|
|
saved = journal.read_bytes()
|
|
target = Path(attempt.root) / "journal-copy.jsonl"
|
|
target.write_bytes(saved)
|
|
journal.unlink()
|
|
journal.symlink_to(target)
|
|
record = Path(attempt.root) / "attempt.json"
|
|
before = record.read_bytes()
|
|
with self.store.writer(run):
|
|
with self.assertRaises(AttemptStateError):
|
|
self.store.reconcile(attempt)
|
|
self.assertEqual(before, record.read_bytes())
|
|
|
|
def test_direct_result_requires_bound_production_evidence(self):
|
|
run = self.create_run()
|
|
with self.store.writer(run):
|
|
attempt = self.store.allocate(run, Slot("a", 1))
|
|
|
|
def contradictory(current, started):
|
|
result = self.adapter("success", [])(current, started)
|
|
path = Path(current.root) / "lifecycle-result.json"
|
|
raw = json.loads(path.read_text(encoding="utf-8"))
|
|
raw["success"] = False
|
|
path.write_text(json.dumps(raw), encoding="utf-8")
|
|
return result
|
|
|
|
with self.assertRaises(AttemptStateError):
|
|
self.store.execute_attempt(attempt, prepare=lambda _: None, invoke=contradictory)
|
|
record = Path(attempt.root) / "attempt.json"
|
|
self.assertEqual(json.loads(record.read_text(encoding="utf-8"))["state"], "running")
|
|
|
|
def test_live_survivor_cleanup_precedes_successor(self):
|
|
run = self.create_run()
|
|
with self.store.writer(run):
|
|
attempt = self.store.allocate(run, Slot("a", 1))
|
|
locator_ready = threading.Event()
|
|
result_box: list[BaseException | InvocationResult] = []
|
|
|
|
def long_running(current, started):
|
|
spec = self._spec(current, "import time; print('START', flush=True); time.sleep(30)")
|
|
|
|
def commit(locator):
|
|
started(locator, spec_digest(spec))
|
|
locator_ready.set()
|
|
|
|
return run_invocation(spec, parse_event=_events, on_started=commit)
|
|
|
|
def invoke() -> None:
|
|
try:
|
|
self.store.execute_attempt(attempt, prepare=lambda _: None, invoke=long_running)
|
|
except BaseException as exc: # concurrent reconciliation seals this attempt first
|
|
result_box.append(exc)
|
|
|
|
worker = threading.Thread(target=invoke)
|
|
worker.start()
|
|
self.assertTrue(locator_ready.wait(5))
|
|
with self.store.writer(run):
|
|
recovered = self.store.reconcile(attempt)
|
|
successor = self.store.allocate(run, Slot("a", 1))
|
|
worker.join(10)
|
|
self.assertFalse(worker.is_alive())
|
|
self.assertEqual(recovered.state, "interrupted")
|
|
self.assertEqual(successor.identity.attempt, 2)
|
|
self.assertTrue(result_box)
|
|
|
|
def test_cross_process_lease_contention_and_crash_release(self):
|
|
run = self.create_run()
|
|
script = "import fcntl, os, sys, time; f=os.open(sys.argv[1], os.O_RDWR); fcntl.flock(f, fcntl.LOCK_EX); print('locked', flush=True); time.sleep(30)"
|
|
child = subprocess.Popen([sys.executable, "-c", script, str(Path(run.root) / "run.lock")], stdout=subprocess.PIPE, text=True)
|
|
self.assertEqual(child.stdout.readline().strip(), "locked")
|
|
with self.assertRaises(RunBusyError):
|
|
with self.store.writer(run):
|
|
pass
|
|
child.kill()
|
|
child.wait(timeout=5)
|
|
child.stdout.close()
|
|
with self.store.writer(run):
|
|
pass
|
|
|
|
|
|
class AttemptCliContractTest(AttemptBase):
|
|
def test_cli_run_resume_status_are_side_effect_free_without_adapters(self):
|
|
run = self.create_run()
|
|
run_before = (Path(run.root) / "run.json").read_bytes()
|
|
output = io.StringIO()
|
|
with mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root), contextlib.redirect_stdout(output):
|
|
self.assertEqual(benchmark_cli.main(["status", "--manifest", str(self.manifest_path), "--run-id", run.run_id]), 0)
|
|
self.assertIn("'running': 0", output.getvalue())
|
|
with mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root), contextlib.redirect_stderr(io.StringIO()):
|
|
self.assertEqual(benchmark_cli.main(["resume", "--manifest", str(self.manifest_path), "--run-id", run.run_id]), 69)
|
|
self.assertEqual(run_before, (Path(run.root) / "run.json").read_bytes())
|
|
absent_root = self.root / "agent-test/runs/absent"
|
|
raw = json.loads(self.raw)
|
|
raw["output_root"] = "agent-test/runs/absent"
|
|
absent = self.root / "absent.json"
|
|
absent.write_text(json.dumps(raw), encoding="utf-8")
|
|
with mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root), contextlib.redirect_stderr(io.StringIO()):
|
|
self.assertEqual(benchmark_cli.main(["run", "--manifest", str(absent)]), 69)
|
|
self.assertFalse(absent_root.exists())
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|