from __future__ import annotations import datetime import hashlib import json import os import socket import sys import tempfile import threading import time import unittest from pathlib import Path from unittest import mock from scripts.agent_benchmark.attempts import ( AttemptStateError, PreflightObservation, RunStore, Slot, ) from scripts.agent_benchmark.browser_cdp import RenderObservation, ViewportObservation from scripts.agent_benchmark.connectivity import ( CallerCapability, ConnectivityIssue, EffectiveBinding, ISSUE_RESUME_CODES, RequestedEffectiveBinding, make_result, ) from scripts.agent_benchmark.lifecycle import ( COMPLETION_EXIT_AFTER_IDLE, CLOCK_HARNESS_MONOTONIC, METRIC_NAMES, SOURCE_HARNESS, SOURCE_WORKSPACE_POLL, SUBMISSION_STDIN_ONCE, UNIT_NANOSECONDS, InvocationSpec, SupervisorLocator, env_pairs, recover_invocation, run_invocation, spec_digest, ) from scripts.agent_benchmark.manifest import AssetMapping, digest_workspace_inputs, load_manifest from scripts.agent_benchmark.measurement import ( AttemptMeasurement, REASON_NOT_OBSERVED, REASON_NOT_REPORTED, WorkspaceWriteObservation, observed, publish_measurement, unavailable, ) from scripts.agent_benchmark.rubric import RUBRIC_CATEGORIES from scripts.agent_benchmark import scoring as scoring_module from scripts.agent_benchmark.scoring import ( BlindWorkspace, ScoringEvidenceFinalization, ScoringError, ScoringInvocationResult, score_run, ) from scripts.agent_benchmark.web_validation import ( WEB_GATES, build_web_validation, publish_web_validation, ) def _digest(data: bytes) -> str: return "sha256:" + hashlib.sha256(data).hexdigest() def _worksheet(total_delta: int = 0) -> dict: categories = [] for index, (ident, maximum) in enumerate(RUBRIC_CATEGORIES): score = maximum - (1 if index == 0 else 0) categories.append( { "id": ident, "max_score": maximum, "score": score, "evidence": f"Anonymous evidence for {ident}.", } ) return { "rubric_version": "landing-quality-v1", "categories": categories, "total": sum(item["score"] for item in categories) + total_delta, } class FakeScoringAdapter: capability = CallerCapability( "codex", ("direct", "execution_preset"), ("xhigh",) ) def __init__( self, *, blocked: bool = False, modes: list[str] | None = None, sensitive_value: str = "", ): self.blocked = blocked self.modes = list(modes or ["success"]) self.preflights = 0 self.invocations: list[tuple[BlindWorkspace, bytes]] = [] self.sensitive_value = sensitive_value self.last_mode = "" def preflight(self, cell): self.preflights += 1 iop = cell.iop requested = RequestedEffectiveBinding( cell.id, cell.caller, iop.route_kind, iop.route_id, iop.request_model, iop.requested_effort, ) issues = () if self.blocked: issues = ( ConnectivityIssue( "credential_missing", ISSUE_RESUME_CODES["credential_missing"], ), ) else: requested = RequestedEffectiveBinding( cell.id, cell.caller, iop.route_kind, iop.route_id, iop.request_model, iop.requested_effort, iop.route_kind, iop.route_id, iop.request_model, iop.requested_effort, tuple( EffectiveBinding(item.stage, item.model, item.effort) for item in iop.expected_bindings ), ) result = make_result(cell, self.capability, requested, issues) return PreflightObservation( result, "sha256:" + "1" * 64, "sha256:" + "2" * 64 ) def _publish_lifecycle(self, cell, blind, mode, on_started): output = Path(blind.output_dir) control = output / "fake-control" control.mkdir() locator = SupervisorLocator( os.getpid(), "fake-start-identity", str(control / "control.sock"), "fake-challenge-" + blind.blind_id, str(control), "2026-08-11T00:00:00+00:00", ) locator_payload = { "supervisor_pid": locator.supervisor_pid, "start_identity": locator.start_identity, "socket_path": locator.socket_path, "challenge": locator.challenge, "control_dir": locator.control_dir, "created_at": locator.created_at, } (control / "locator.json").write_text( json.dumps(locator_payload), encoding="utf-8" ) invocation_digest = "sha256:" + "4" * 64 on_started(locator, invocation_digest) terminal_reason = "nonzero_exit" if mode == "raise" else "success" receipt = { "receipt_version": 1, "supervisor_pid": locator.supervisor_pid, "challenge_digest": hashlib.sha256( locator.challenge.encode("utf-8") ).hexdigest(), "reason": terminal_reason, "exit_code": 1 if mode == "raise" else 0, "signal": None, "caller_launched": True, "cleanup_complete": True, "process_group_alive": False, "completed_at": "2026-08-11T00:00:01+00:00", } (control / "cleanup-receipt.json").write_text( json.dumps(receipt), encoding="utf-8" ) public_locator = { key: value for key, value in locator_payload.items() if key != "challenge" } public_locator["challenge_digest"] = receipt["challenge_digest"] lifecycle = { "record": "result", "success": terminal_reason == "success", "terminal_reason": terminal_reason, "cleanup_complete": True, "process_group_alive": False, "spec_digest": invocation_digest, "locator": public_locator, "effective_binding": { "route_kind": cell.iop.route_kind, "route_id": cell.iop.route_id, "model": cell.iop.request_model, "effort": cell.iop.requested_effort, }, } (output / "lifecycle-result.json").write_text( json.dumps(lifecycle), encoding="utf-8" ) journal = ( json.dumps( {"record": "header", "spec_digest": invocation_digest} ) + "\n" + json.dumps( { "record": "terminal", "terminal_reason": terminal_reason, "cleanup_complete": True, "process_group_alive": False, } ) + "\n" ) (output / "lifecycle-journal.jsonl").write_text( journal, encoding="utf-8" ) def invoke(self, cell, blind, task_payload, timeout, on_started): self.invocations.append((blind, task_payload)) mode = self.modes.pop(0) if self.modes else "success" self.last_mode = mode output = Path(blind.output_dir) self._publish_lifecycle(cell, blind, mode, on_started) if mode == "mutate": target = Path(blind.input_dir) / "index.html" target.chmod(0o600) target.write_text("
mutated
", encoding="utf-8") if mode == "raise": raise RuntimeError("fake evaluator failed") if mode == "malformed": (output / "worksheet.json").write_text("{}", encoding="utf-8") else: worksheet = _worksheet() if mode == "secret": worksheet["categories"][0]["evidence"] = self.sensitive_value (output / "worksheet.json").write_text( json.dumps(worksheet, sort_keys=True, separators=(",", ":")) + "\n", encoding="ascii", ) binding = ( cell.iop.route_kind, cell.iop.route_id, cell.iop.request_model, cell.iop.requested_effort, ) if mode == "binding": binding = (binding[0], "substituted", binding[2], binding[3]) return ScoringInvocationResult(mode not in {"failed", "binding"}, "success" if mode not in {"failed", "binding"} else "failed", binding) def finalize_evidence(self, blind): leaked = False if self.sensitive_value: sensitive = self.sensitive_value.encode("utf-8") for path in Path(blind.root).rglob("*"): if path.is_file() and sensitive in path.read_bytes(): path.unlink() leaked = True return ScoringEvidenceFinalization( not leaked, "" if not leaked else "runtime_secret_leak" ) class ScoringTest(unittest.TestCase): def setUp(self): temporary = tempfile.TemporaryDirectory(dir="/tmp", prefix="iop-score-") self.addCleanup(temporary.cleanup) self.root = Path(temporary.name) (self.root / "Makefile").write_text("test:\n\t@true\n") fixture_root = self.root / "scripts" / "fixtures" / "bench" fixture_root.mkdir(parents=True) (fixture_root / "prompt.md").write_text("Build the page.", encoding="utf-8") self.asset_bytes = { "scripts/fixtures/bench/reference.txt": b"anonymous reference\n", "scripts/fixtures/bench/a.svg": b"", "scripts/fixtures/bench/b.svg": b"", } for relative, data in self.asset_bytes.items(): path = self.root / relative path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(data) assets = ( AssetMapping( "scripts/fixtures/bench/reference.txt", "brief/reference.txt", self.asset_bytes["scripts/fixtures/bench/reference.txt"], ), AssetMapping( "scripts/fixtures/bench/a.svg", "assets/a.svg", self.asset_bytes["scripts/fixtures/bench/a.svg"], ), AssetMapping( "scripts/fixtures/bench/b.svg", "assets/b.svg", self.asset_bytes["scripts/fixtures/bench/b.svg"], ), ) raw = { "pipeline_version": "2", "environment": "dev", "testbed": "../iop-s2", "repetitions": 1, "session_policy": "fresh", "setup_cache_policy": "isolated", "timeout": { "run_seconds": 5, "idle_seconds": 1, "quiet_seconds": 1, "cleanup_grace_seconds": 1, }, "viewports": [ {"id": "desktop", "width": 800, "height": 600}, {"id": "mobile", "width": 375, "height": 600}, ], "rubric_version": "landing-quality-v1", "evaluator": { "caller": "codex", "iop": { "request_model": "judge-model", "requested_effort": "xhigh", "route_kind": "direct", "route_id": "judge-route", "expected_bindings": [ { "stage": "request", "model": "judge-model", "effort": "xhigh", } ], }, }, "output_root": "agent-test/runs/anonymous-bench", "fixture": { "version": "landing-v1", "prompt": "scripts/fixtures/bench/prompt.md", "assets": [ { "source": item.source, "workspace_path": item.workspace_path, } for item in assets ], "checksum": digest_workspace_inputs(assets), }, "matrix": [ { "id": "cell-sentinel", "caller": "claude", "iop": { "request_model": "source-model", "requested_effort": "high", "route_kind": "direct", "route_id": "source-route", "expected_bindings": [ { "stage": "request", "model": "source-model", "effort": "high", } ], }, } ], } self.manifest_path = self.root / "manifest.json" self.manifest_path.write_text(json.dumps(raw), encoding="utf-8") self.manifest = load_manifest(self.manifest_path, repo_root=self.root) tokens = iter(("123456abcdef", "234567abcdef", "345678abcdef")) self.store = RunStore( self.root, clock=lambda: datetime.datetime( 2026, 8, 11, 1, 2, 3, tzinfo=datetime.timezone.utc ), token_hex=lambda _n: next(tokens), ) self.run = self.store.create(self.manifest, self.manifest_path.read_bytes()) @staticmethod def _measurement(attempt, terminal_reason: str) -> AttemptMeasurement: timeline = { "submitted_at": unavailable(REASON_NOT_OBSERVED, SOURCE_HARNESS), "first_output_at": unavailable(REASON_NOT_OBSERVED, SOURCE_HARNESS), "first_write_observed_at": unavailable( REASON_NOT_OBSERVED, SOURCE_WORKSPACE_POLL ), "first_write_mtime": unavailable( REASON_NOT_OBSERVED, SOURCE_WORKSPACE_POLL ), "total_duration": observed( 1, UNIT_NANOSECONDS, CLOCK_HARNESS_MONOTONIC, SOURCE_HARNESS ), } usage = { name: unavailable(REASON_NOT_REPORTED, SOURCE_HARNESS) for name in METRIC_NAMES } return AttemptMeasurement( attempt.identity.run_id, attempt.identity.cell_id, attempt.identity.repetition, attempt.identity.attempt, "claude", "sha256:" + "3" * 64, terminal_reason, timeline, usage, WorkspaceWriteObservation( False, None, None, "", 1, 0, REASON_NOT_OBSERVED ), (), ) def _view(self, root: Path, ident: str, width: int) -> ViewportObservation: screenshot = f"screenshot-{ident}.png" png = b"\x89PNG\r\n\x1a\n" + ident.encode("ascii") (root / screenshot).write_bytes(png) images = tuple( { "src": path, "alt": path, "complete": True, "natural_width": 20, "natural_height": 20, "visible": True, "rect": { "x": 0, "y": 0, "width": 20, "height": 20, "right": 20, "bottom": 20, }, } for path in ("assets/a.svg", "assets/b.svg") ) return ViewportObservation( ident, width, 600, screenshot, _digest(png), len(png), images, { "scroll_width": width, "client_width": width, "clipped": 0, "overlaps": 0, }, { "h1_count": 1, "headings": [1], "heading_progression": True, "main_count": 1, "landmarks": 1, "controls": [ { "name": True, "tab_index": 0, "focused": True, "focus_visible": True, "contrast": 7.0, } ], "ax": {"nodes": 4, "non_ignored": 3, "named": 2}, }, ) def _attempt( self, state: str = "success", *, leaked: bool = False, rendered: bool = True, leaked_identity: str | None = None, ): with self.store.writer(self.run): attempt = self.store.allocate(self.run, Slot("cell-sentinel", 1)) workspace = Path(attempt.root) / "workspace" (workspace / "assets").mkdir(parents=True) (workspace / "brief").mkdir() for asset in self.manifest.fixture.assets: (workspace / asset.workspace_path).write_bytes(asset.content) heading = leaked_identity or ("source-route" if leaked else "Ready") (workspace / "index.html").write_text( f"

{heading}

A" "Bgo" "
", encoding="utf-8", ) (workspace / "styles.css").write_text( "body{color:#111;background:#fff}img{width:20px}" "a:focus{outline:2px solid #05f}", encoding="utf-8", ) (workspace / "script.js").write_text( "document.body.dataset.ready='1';", encoding="utf-8" ) terminal_reason = "success" if state == "success" else state measurement = self._measurement(attempt, terminal_reason) publish_measurement(attempt.root, measurement) if state == "success" and rendered: render = RenderObservation( "Chromium/Test", "http://127.0.0.1:12345", ( {"kind": "local", "path": "/index.html", "allowed": True, "status": 200}, {"kind": "local", "path": "/assets/a.svg", "allowed": True, "status": 200}, {"kind": "local", "path": "/assets/b.svg", "allowed": True, "status": 200}, ), (), ( self._view(Path(attempt.root), "desktop", 800), self._view(Path(attempt.root), "mobile", 375), ), ) else: render = None web = build_web_validation( self.manifest, workspace, measurement, render ) publish_web_validation(attempt.root, web) terminal = self.store.publish_terminal( attempt, state, result={"terminal_reason": terminal_reason} ) return terminal def test_scored_attempt_is_blind_exactly_once_and_strict(self): attempt = self._attempt() adapter = FakeScoringAdapter() summary = score_run( self.store, self.run, self.manifest, adapter=adapter ) self.assertEqual( (summary.scored, summary.unscored, summary.scoring_failed, summary.blocked), (1, 0, 0, 0), ) self.assertEqual(adapter.preflights, 1) self.assertEqual(len(adapter.invocations), 1) blind, prompt = adapter.invocations[0] self.assertNotIn("cell-sentinel", blind.root) self.assertNotIn("source-route", blind.root) self.assertNotIn(str(Path(attempt.root).resolve()), blind.root) visible = prompt for path in Path(blind.root).rglob("*"): if path.is_file(): visible += b"\n" + str(path).encode() + b"\n" + path.read_bytes() for sentinel in ( b"cell-sentinel", b"source-route", b"source-model", str(Path(attempt.root).resolve()).encode(), ): self.assertNotIn(sentinel, visible) self.assertEqual( sorted(path.relative_to(blind.input_dir).as_posix() for path in Path(blind.input_dir).rglob("*") if path.is_file()), [ "assets/a.svg", "assets/b.svg", "index.html", "screenshots/screenshot-desktop.png", "screenshots/screenshot-mobile.png", "script.js", "styles.css", ], ) result_path = Path(attempt.root) / "scoring" / "score-000001" / "result.json" result = json.loads(result_path.read_text(encoding="ascii")) self.assertEqual(result["status"], "scored") self.assertEqual(result["worksheet"]["total"], 99) self.assertNotIn("gates", result["worksheet"]) mappings = tuple((Path(self.run.root) / "blind-mappings").glob("*.json")) self.assertEqual(len(mappings), 1) mapping = json.loads(mappings[0].read_text(encoding="ascii")) self.assertEqual(mapping["attempt"]["cell_id"], "cell-sentinel") self.assertFalse(mappings[0].is_relative_to(Path(blind.root))) before = {path: path.read_bytes() for path in Path(attempt.root).rglob("*") if path.is_file()} second = score_run(self.store, self.run, self.manifest, adapter=adapter) self.assertEqual(second.scored, 1) self.assertEqual(len(adapter.invocations), 1) self.assertEqual(before, {path: path.read_bytes() for path in Path(attempt.root).rglob("*") if path.is_file()}) def test_ineligible_attempt_is_unscored_without_preflight_or_zero(self): attempt = self._attempt("failed") adapter = FakeScoringAdapter() summary = score_run(self.store, self.run, self.manifest, adapter=adapter) self.assertEqual((summary.unscored, summary.scored), (1, 0)) self.assertEqual((adapter.preflights, adapter.invocations), (0, [])) path = Path(attempt.root) / "scoring" / "unscored.json" before = path.read_bytes() record = json.loads(before) self.assertEqual(record["status"], "unscored") self.assertEqual(record["reasons"], ["lifecycle_failed"]) self.assertFalse(set(record) & {"score", "total", "worksheet"}) score_run(self.store, self.run, self.manifest, adapter=adapter) self.assertEqual(path.read_bytes(), before) def test_not_run_web_gates_are_all_unscored_without_evaluator(self): attempt = self._attempt(rendered=False) adapter = FakeScoringAdapter() summary = score_run(self.store, self.run, self.manifest, adapter=adapter) self.assertEqual((summary.unscored, summary.scored), (1, 0)) self.assertEqual((adapter.preflights, adapter.invocations), (0, [])) record = json.loads( (Path(attempt.root) / "scoring" / "unscored.json").read_text() ) self.assertEqual(record["status"], "unscored") web = json.loads( (Path(attempt.root) / "web-validation.json").read_text() ) failed_gates = { f"gate_{item['id']}" for item in web["gates"] if not item["passed"] } self.assertTrue(failed_gates) self.assertTrue(failed_gates.issubset(record["reasons"])) self.assertEqual( [item["id"] for item in web["gates"]], list(WEB_GATES) ) self.assertFalse(set(record) & {"score", "total", "worksheet"}) def test_symlinked_generated_input_fails_before_evaluator_invocation(self): attempt = self._attempt() workspace = Path(attempt.root) / "workspace" (workspace / "index.html").unlink() (workspace / "index.html").symlink_to("styles.css") adapter = FakeScoringAdapter() with self.assertRaises(AttemptStateError): score_run(self.store, self.run, self.manifest, adapter=adapter) self.assertEqual(adapter.invocations, []) self.assertFalse((Path(attempt.root) / "scoring").exists()) def test_preflight_blocker_allocates_no_score(self): attempt = self._attempt() adapter = FakeScoringAdapter(blocked=True) summary = score_run(self.store, self.run, self.manifest, adapter=adapter) self.assertEqual(summary.blocked, 1) self.assertEqual(adapter.invocations, []) self.assertFalse((Path(attempt.root) / "scoring").exists()) self.assertEqual( len(list((Path(self.run.root) / "scoring-preflight").iterdir())), 1 ) def test_failed_score_retries_only_with_new_id_and_fresh_session(self): attempt = self._attempt() adapter = FakeScoringAdapter(modes=["malformed", "success"]) first = score_run(self.store, self.run, self.manifest, adapter=adapter) self.assertEqual(first.scoring_failed, 1) failed = Path(attempt.root) / "scoring" / "score-000001" failed_bytes = {path: path.read_bytes() for path in failed.rglob("*") if path.is_file()} second = score_run(self.store, self.run, self.manifest, adapter=adapter) self.assertEqual(second.scoring_failed, 1) self.assertEqual(len(adapter.invocations), 1) self.assertFalse((Path(attempt.root) / "scoring" / "score-000002").exists()) third = score_run( self.store, self.run, self.manifest, adapter=adapter, retry_scoring_failed=True, ) self.assertEqual(third.scored, 1) self.assertEqual(len(adapter.invocations), 2) self.assertEqual( failed_bytes, {path: path.read_bytes() for path in failed.rglob("*") if path.is_file()}, ) allocations = [ json.loads((Path(attempt.root) / "scoring" / f"score-{index:06d}" / "allocation.json").read_text()) for index in (1, 2) ] self.assertNotEqual(allocations[0]["blind_id"], allocations[1]["blind_id"]) self.assertNotEqual( allocations[0]["session_identity"], allocations[1]["session_identity"] ) def test_invalid_binding_and_identity_leak_fail_without_fallback(self): attempt = self._attempt() binding = FakeScoringAdapter(modes=["binding"]) summary = score_run(self.store, self.run, self.manifest, adapter=binding) self.assertEqual(summary.scoring_failed, 1) result = json.loads( (Path(attempt.root) / "scoring" / "score-000001" / "result.json").read_text() ) self.assertEqual(result["status"], "scoring_failed") self.assertNotIn("worksheet", result) # A separate run proves an identity sentinel in retained page bytes is # rejected before the evaluator is invoked. self.run = self.store.create(self.manifest, self.manifest_path.read_bytes()) leaked = self._attempt(leaked=True) adapter = FakeScoringAdapter() leaked_summary = score_run( self.store, self.run, self.manifest, adapter=adapter ) self.assertEqual(leaked_summary.scoring_failed, 1) self.assertEqual(adapter.invocations, []) leaked_result = json.loads( (Path(leaked.root) / "scoring" / "score-000001" / "result.json").read_text() ) self.assertEqual(leaked_result["reason"], "blind_preparation_failed") def test_shared_evaluator_binding_is_allowed_but_short_caller_leak_fails(self): raw = json.loads(self.manifest_path.read_text()) raw["matrix"][0]["caller"] = "agy" raw["matrix"][0]["iop"] = { "request_model": "judge-model", "requested_effort": "xhigh", "route_kind": "direct", "route_id": "judge-route", "expected_bindings": [ { "stage": "request", "model": "judge-model", "effort": "xhigh", } ], } raw["output_root"] = "agent-test/runs/shared-binding" path = self.root / "shared.json" path.write_text(json.dumps(raw), encoding="utf-8") self.manifest_path = path self.manifest = load_manifest(path, repo_root=self.root) self.run = self.store.create(self.manifest, path.read_bytes()) self._attempt() shared = FakeScoringAdapter() summary = score_run( self.store, self.run, self.manifest, adapter=shared ) self.assertEqual((summary.scored, summary.scoring_failed), (1, 0)) self.run = self.store.create(self.manifest, path.read_bytes()) leaked = self._attempt(leaked_identity="agy") rejected = FakeScoringAdapter() summary = score_run( self.store, self.run, self.manifest, adapter=rejected ) self.assertEqual((summary.scored, summary.scoring_failed), (0, 1)) self.assertEqual(rejected.invocations, []) result = json.loads( ( Path(leaked.root) / "scoring" / "score-000001" / "result.json" ).read_text(encoding="ascii") ) self.assertEqual(result["reason"], "blind_preparation_failed") def test_delimited_short_caller_and_cell_identity_leaks_fail(self): identity = scoring_module.ProducerIdentity( exact_tokens=("agy", "cell-sentinel"), path_tokens=(), producer_tokens=(), evaluator_shared_tokens=(), ) for value in ( b"agy", b"caller=agy", b"agy-output", b"agy_output", b"cell-sentinel-output", b"cell-sentinel_output", ): with self.subTest(value=value): self.assertTrue(scoring_module._contains_identity(value, identity)) for value in ( b"strategy", b"agyextended", b"mycell-sentinel", ): with self.subTest(value=value): self.assertFalse(scoring_module._contains_identity(value, identity)) def test_binary_identity_boundaries_do_not_disappear(self): identity = scoring_module.ProducerIdentity( exact_tokens=("agy", "cell-sentinel"), path_tokens=(), producer_tokens=("producer-model", "shared-model"), evaluator_shared_tokens=("shared-model",), ) for value in ( b"\x89PNG\r\n\x1a\nx\xffagy\x00", b"\x89PNG\r\n\x1a\nagy\xffx", b"\x89PNG\r\n\x1a\nx\xffcell-sentinel\x00", b"\x89PNG\r\n\x1a\ncell-sentinel\xffx", b"\x89PNG\r\n\x1a\nx\xffproducer-model\x00", b"\x89PNG\r\n\x1a\nproducer-model\xffx", ): with self.subTest(value=value): self.assertTrue(scoring_module._contains_identity(value, identity)) for value in ( b"strategy", b"xagy", b"agyx", b"xproducer-model", b"producer-modelx", b"x\xffshared-model\x00", ): with self.subTest(value=value): self.assertFalse(scoring_module._contains_identity(value, identity)) def test_invalid_filesystem_bytes_do_not_bypass_identity_scan(self): if os.name != "posix": self.skipTest("raw invalid filesystem bytes require POSIX paths") identity = scoring_module.ProducerIdentity( exact_tokens=("agy", "cell-sentinel"), path_tokens=(), producer_tokens=("source-route", "source-model"), evaluator_shared_tokens=(), ) def write_raw(directory: Path, name: bytes, data: bytes = b"safe") -> None: descriptor = os.open( os.fsencode(directory) + b"/" + name, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600, ) try: os.write(descriptor, data) finally: os.close(descriptor) for index, name in enumerate( ( b"x\xffagy\xfe.png", b"x\xffcell-sentinel\xfe.png", b"x\xffsource-route\xfe.png", b"x\xffsource-model\xfe.png", ) ): with self.subTest(name=name): visible_root = self.root / f"invalid-visible-{index}" visible_root.mkdir() write_raw(visible_root, name) with self.assertRaisesRegex( ScoringError, "evaluator-visible evidence leaks execution identity" ): scoring_module._scan_visible_tree(visible_root, identity) class RawFilenameAdapter(FakeScoringAdapter): def __init__(self, raw_name: bytes): super().__init__() self.raw_name = raw_name def invoke(self, cell, blind, task_payload, timeout, on_started): result = super().invoke( cell, blind, task_payload, timeout, on_started ) write_raw(Path(blind.output_dir), self.raw_name) return result leaked_attempt = self._attempt() leaked_adapter = RawFilenameAdapter( b"x\xffcell-sentinel\xfe-output.bin" ) leaked_summary = score_run( self.store, self.run, self.manifest, adapter=leaked_adapter ) self.assertEqual( (leaked_summary.scored, leaked_summary.scoring_failed), (0, 1) ) leaked_root = ( Path(leaked_attempt.root) / "scoring" / "score-000001" ) leaked_result = json.loads( (leaked_root / "result.json").read_text(encoding="ascii") ) self.assertEqual(leaked_result["status"], "scoring_failed") self.assertEqual(leaked_result["reason"], "evaluator_output_leak") leaked_bytes = { path: path.read_bytes() for path in leaked_root.rglob("*") if path.is_file() } retained = score_run( self.store, self.run, self.manifest, adapter=leaked_adapter ) self.assertEqual(retained.scoring_failed, 1) self.assertEqual(len(leaked_adapter.invocations), 1) self.assertEqual( leaked_bytes, { path: path.read_bytes() for path in leaked_root.rglob("*") if path.is_file() }, ) self.run = self.store.create( self.manifest, self.manifest_path.read_bytes() ) safe_attempt = self._attempt() safe_adapter = RawFilenameAdapter(b"x\xffanonymous\xfe-output.bin") safe_summary = score_run( self.store, self.run, self.manifest, adapter=safe_adapter ) self.assertEqual( (safe_summary.scored, safe_summary.scoring_failed), (1, 0) ) safe_blind = Path(safe_adapter.invocations[0][0].root) first_digest = scoring_module._blind_tree_digest(safe_blind) self.assertEqual( first_digest, scoring_module._blind_tree_digest(safe_blind) ) safe_result = json.loads( ( Path(safe_attempt.root) / "scoring" / "score-000001" / "result.json" ).read_text(encoding="ascii") ) self.assertEqual(safe_result["status"], "scored") self.assertEqual(safe_result["post_tree_digest"], first_digest) ordinary_path = "input/caf\N{LATIN SMALL LETTER E WITH ACUTE}.txt" ordinary_data = b"ordinary" framed = bytearray(b"IOP-BENCH-BLIND-INPUT-V1\0") ordinary_bytes = ordinary_path.encode("utf-8") framed += len(ordinary_bytes).to_bytes(8, "big") + ordinary_bytes framed += len(ordinary_data).to_bytes(8, "big") + ordinary_data self.assertEqual( scoring_module._input_digest([(ordinary_path, ordinary_data)]), _digest(bytes(framed)), ) def unusable_fsencode(value): raise AssertionError("path framing must not use the filesystem codec") with mock.patch.object(scoring_module.os, "fsencode", unusable_fsencode): self.assertEqual( scoring_module._input_digest([(ordinary_path, ordinary_data)]), _digest(bytes(framed)), ) self.assertEqual( scoring_module._path_bytes(ordinary_path), ordinary_path.encode("utf-8"), ) self.assertEqual( scoring_module._path_bytes("input/x\udcff.txt"), b"input/x\xff.txt", ) with self.assertRaisesRegex(ScoringError, "scoring path is invalid"): scoring_module._path_bytes("input/x\ud800.txt") surrogate_path = "input/x\udcff.txt" surrogate_data = b"raw" surrogate_framed = bytearray(b"IOP-BENCH-BLIND-INPUT-V1\0") surrogate_bytes = b"input/x\xff.txt" surrogate_framed += len(surrogate_bytes).to_bytes(8, "big") + surrogate_bytes surrogate_framed += len(surrogate_data).to_bytes(8, "big") + surrogate_data self.assertEqual( scoring_module._input_digest([(surrogate_path, surrogate_data)]), _digest(bytes(surrogate_framed)), ) def test_receipt_only_recovery_waits_for_lifecycle_quiescence(self): blind_root = self.root / "receipt-only" / "blind" / "blind-paused" for name in ("input", "session", "output"): (blind_root / name).mkdir(parents=True, exist_ok=True) output = blind_root / "output" control = output / "paused-control" control.mkdir() alias = Path(tempfile.gettempdir()) / ( "iop-score-paused-" + hashlib.sha256(str(output).encode("utf-8")).hexdigest()[:16] ) os.symlink(output, alias, target_is_directory=True) self.addCleanup( lambda: alias.unlink() if alias.exists() or alias.is_symlink() else None ) locator = SupervisorLocator( os.getpid(), "paused-start-identity", str(alias / "paused-control" / "control.sock"), "paused-challenge", str(alias / "paused-control"), "2026-08-11T00:00:00+00:00", ) locator_payload = { "supervisor_pid": locator.supervisor_pid, "start_identity": locator.start_identity, "socket_path": locator.socket_path, "challenge": locator.challenge, "control_dir": locator.control_dir, "created_at": locator.created_at, } (control / "locator.json").write_text( json.dumps(locator_payload), encoding="utf-8" ) receipt = { "receipt_version": 1, "supervisor_pid": locator.supervisor_pid, "challenge_digest": hashlib.sha256( locator.challenge.encode("utf-8") ).hexdigest(), "reason": "success", "exit_code": 0, "signal": None, "caller_launched": True, "cleanup_complete": True, "process_group_alive": False, "completed_at": "2026-08-11T00:00:01+00:00", } (control / "cleanup-receipt.json").write_text( json.dumps(receipt), encoding="utf-8" ) invocation_digest = "sha256:" + "9" * 64 public_locator = { key: value for key, value in locator_payload.items() if key != "challenge" } public_locator["challenge_digest"] = receipt["challenge_digest"] lifecycle = { "record": "result", "success": True, "terminal_reason": "success", "cleanup_complete": True, "process_group_alive": False, "spec_digest": invocation_digest, "locator": public_locator, } journal = ( json.dumps({"record": "header", "spec_digest": invocation_digest}) + "\n" + json.dumps( { "record": "terminal", "terminal_reason": "success", "cleanup_complete": True, "process_group_alive": False, } ) + "\n" ) prior = { path: path.read_bytes() for path in (control / "locator.json", control / "cleanup-receipt.json") } recovered: list[tuple[str | None, str]] = [] errors: list[BaseException] = [] successor_started = threading.Event() def recover() -> None: try: recovered.append( scoring_module._recover_runner( blind_root, locator, invocation_digest, control_target=control, ) ) scoring_module._release_runner_alias( { "control_alias": str(alias), "control_target": str(control), } ) successor_started.set() except BaseException as exc: errors.append(exc) worker = threading.Thread(target=recover) started = time.monotonic() worker.start() time.sleep(0.35) self.assertTrue(worker.is_alive()) self.assertFalse(successor_started.is_set()) self.assertTrue(alias.is_symlink()) self.assertFalse((output / "lifecycle-result.json").exists()) (output / "lifecycle-journal.jsonl").write_text(journal, encoding="utf-8") (output / "lifecycle-result.json").write_text( json.dumps(lifecycle), encoding="utf-8" ) time.sleep(0.05) self.assertTrue(worker.is_alive()) worker.join(5) self.assertFalse(worker.is_alive()) self.assertGreaterEqual(time.monotonic() - started, 0.5) self.assertEqual(errors, []) self.assertTrue(successor_started.is_set()) self.assertEqual(len(recovered), 1) self.assertRegex(recovered[0][0] or "", r"^sha256:[0-9a-f]{64}$") self.assertRegex(recovered[0][1], r"^sha256:[0-9a-f]{64}$") self.assertFalse(alias.exists() or alias.is_symlink()) self.assertTrue((output / "lifecycle-result.json").is_file()) self.assertTrue((output / "lifecycle-journal.jsonl").is_file()) for path, data in prior.items(): self.assertEqual(path.read_bytes(), data) def prepare_prepublished(name: str): root = self.root / "receipt-only" / "blind" / name for directory in ("input", "session", "output"): (root / directory).mkdir(parents=True, exist_ok=True) local_output = root / "output" local_control = self.root / f"{name}-control" local_control.mkdir() local_alias = Path(tempfile.gettempdir()) / ( f"iop-score-{name}-" + hashlib.sha256(str(local_output).encode("utf-8")).hexdigest()[:16] ) os.symlink( local_control.parent, local_alias, target_is_directory=True ) self.addCleanup( lambda: local_alias.unlink() if local_alias.exists() or local_alias.is_symlink() else None ) local_locator = SupervisorLocator( os.getpid(), f"{name}-start-identity", str(local_alias / local_control.name / "control.sock"), f"{name}-challenge", str(local_alias / local_control.name), "2026-08-11T00:00:00+00:00", ) local_receipt = dict(receipt) local_receipt["challenge_digest"] = hashlib.sha256( local_locator.challenge.encode("utf-8") ).hexdigest() (local_control / "cleanup-receipt.json").write_text( json.dumps(local_receipt), encoding="utf-8" ) local_public_locator = { "supervisor_pid": local_locator.supervisor_pid, "start_identity": local_locator.start_identity, "socket_path": local_locator.socket_path, "control_dir": local_locator.control_dir, "created_at": local_locator.created_at, "challenge_digest": local_receipt["challenge_digest"], } local_lifecycle = dict(lifecycle) local_lifecycle["locator"] = local_public_locator (local_output / "lifecycle-journal.jsonl").write_text( journal, encoding="utf-8" ) (local_output / "lifecycle-result.json").write_text( json.dumps(local_lifecycle), encoding="utf-8" ) control_socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) control_socket.bind(str(local_control / "control.sock")) self.addCleanup(control_socket.close) return ( root, local_output, local_control, local_alias, local_locator, local_lifecycle, ) def start_prepublished_recovery(root, control, alias, local_locator): local_recovered: list[tuple[str | None, str]] = [] local_errors: list[BaseException] = [] local_successor_started = threading.Event() def local_recover() -> None: try: local_recovered.append( scoring_module._recover_runner( root, local_locator, invocation_digest, control_target=control, ) ) scoring_module._release_runner_alias( { "control_alias": str(alias), "control_target": str(control), } ) local_successor_started.set() except BaseException as exc: local_errors.append(exc) local_worker = threading.Thread(target=local_recover) local_worker.start() return ( local_worker, local_recovered, local_errors, local_successor_started, ) ( stable_root, _stable_output, stable_control, stable_alias, stable_locator, _stable_lifecycle, ) = prepare_prepublished("prepublished-stable") stable_quiet_started = threading.Event() original_wait = scoring_module._wait_post_cleanup_quiet def observe_stable_quiet(*args, **kwargs): stable_quiet_started.set() return original_wait(*args, **kwargs) with mock.patch.object( scoring_module, "_POST_CLEANUP_TIMEOUT_SECONDS", 0.5 ), mock.patch.object( scoring_module, "_POST_CLEANUP_QUIET_SECONDS", 0.1 ), mock.patch.object( scoring_module, "_POST_CLEANUP_POLL_SECONDS", 0.01 ), mock.patch.object( scoring_module, "_wait_post_cleanup_quiet", side_effect=observe_stable_quiet, ): ( stable_worker, stable_recovered, stable_errors, stable_successor_started, ) = start_prepublished_recovery( stable_root, stable_control, stable_alias, stable_locator ) self.assertTrue(stable_quiet_started.wait(1)) time.sleep(0.03) self.assertTrue(stable_worker.is_alive()) self.assertTrue(stable_alias.is_symlink()) self.assertTrue((stable_control / "control.sock").exists()) self.assertFalse(stable_successor_started.is_set()) stable_worker.join(2) self.assertFalse(stable_worker.is_alive()) self.assertEqual(stable_errors, []) self.assertEqual(len(stable_recovered), 1) self.assertFalse(stable_alias.exists() or stable_alias.is_symlink()) self.assertFalse((stable_control / "control.sock").exists()) self.assertTrue(stable_successor_started.is_set()) ( changed_root, changed_output, changed_control, changed_alias, changed_locator, changed_lifecycle, ) = prepare_prepublished("prepublished-changed") changed_quiet_started = threading.Event() def observe_changed_quiet(*args, **kwargs): changed_quiet_started.set() return original_wait(*args, **kwargs) with mock.patch.object( scoring_module, "_POST_CLEANUP_TIMEOUT_SECONDS", 0.5 ), mock.patch.object( scoring_module, "_POST_CLEANUP_QUIET_SECONDS", 0.1 ), mock.patch.object( scoring_module, "_POST_CLEANUP_POLL_SECONDS", 0.01 ), mock.patch.object( scoring_module, "_wait_post_cleanup_quiet", side_effect=observe_changed_quiet, ): ( changed_worker, changed_recovered, changed_errors, changed_successor_started, ) = start_prepublished_recovery( changed_root, changed_control, changed_alias, changed_locator ) self.assertTrue(changed_quiet_started.wait(1)) mutated_lifecycle = dict(changed_lifecycle) mutated_lifecycle["publication_revision"] = 2 staged_lifecycle = changed_output / "lifecycle-result.changed.tmp" staged_lifecycle.write_text( json.dumps(mutated_lifecycle), encoding="utf-8" ) os.replace( staged_lifecycle, changed_output / "lifecycle-result.json" ) changed_worker.join(2) self.assertFalse(changed_worker.is_alive()) self.assertEqual(changed_recovered, []) self.assertEqual(len(changed_errors), 1) self.assertIsInstance(changed_errors[0], ScoringError) self.assertIn("lifecycle publication changed", str(changed_errors[0])) self.assertTrue(changed_alias.is_symlink()) self.assertTrue((changed_control / "control.sock").exists()) self.assertFalse(changed_successor_started.is_set()) missing_root = self.root / "receipt-only" / "blind" / "blind-missing" for name in ("input", "session", "output"): (missing_root / name).mkdir(parents=True, exist_ok=True) missing_output = missing_root / "output" missing_control = missing_output / "missing-control" missing_control.mkdir() missing_alias = Path(tempfile.gettempdir()) / ( "iop-score-missing-" + hashlib.sha256(str(missing_output).encode("utf-8")).hexdigest()[:16] ) os.symlink(missing_output, missing_alias, target_is_directory=True) self.addCleanup( lambda: missing_alias.unlink() if missing_alias.exists() or missing_alias.is_symlink() else None ) missing_locator = SupervisorLocator( os.getpid(), "missing-start-identity", str(missing_alias / "missing-control" / "control.sock"), "missing-challenge", str(missing_alias / "missing-control"), "2026-08-11T00:00:00+00:00", ) missing_receipt = dict(receipt) missing_receipt["challenge_digest"] = hashlib.sha256( missing_locator.challenge.encode("utf-8") ).hexdigest() (missing_control / "cleanup-receipt.json").write_text( json.dumps(missing_receipt), encoding="utf-8" ) with mock.patch.object( scoring_module, "_POST_CLEANUP_TIMEOUT_SECONDS", 0.05 ), mock.patch.object( scoring_module, "_POST_CLEANUP_QUIET_SECONDS", 0.01 ): with self.assertRaisesRegex( ScoringError, "lifecycle publication is incomplete" ): scoring_module._recover_runner( missing_root, missing_locator, invocation_digest, control_target=missing_control, ) self.assertTrue(missing_alias.is_symlink()) self.assertFalse((missing_output / "lifecycle-result.json").exists()) def test_interrupted_evaluator_is_stopped_before_retry(self): attempt = self._attempt() class InterruptingAdapter(FakeScoringAdapter): def __init__(self): super().__init__() self.first = True self.worker: threading.Thread | None = None self.worker_result = None self.worker_error: BaseException | None = None self.successor_started = False self.locator: SupervisorLocator | None = None self.callback_error: BaseException | None = None self.control_alias: Path | None = None def invoke(self, cell, blind, task_payload, timeout, on_started): if not self.first: self.successor_started = True if self.worker is not None and self.worker.is_alive(): raise AssertionError( "successor started before survivor cleanup" ) return super().invoke( cell, blind, task_payload, timeout, on_started ) self.first = False self.invocations.append((blind, task_payload)) ready = threading.Event() locator_box: list[SupervisorLocator] = [] alias = Path(tempfile.gettempdir()) / ( "iop-score-test-" + hashlib.sha256(blind.output_dir.encode("utf-8")).hexdigest()[:16] ) os.symlink(blind.output_dir, alias, target_is_directory=True) self.control_alias = alias spec = InvocationSpec( argv=( sys.executable, "-u", "-c", "import sys,time; sys.stdin.buffer.read(); " "print('START', flush=True); time.sleep(30)", ), cwd=blind.root, env=env_pairs( {"PATH": os.environ.get("PATH", "/usr/bin:/bin")} ), submission_mode=SUBMISSION_STDIN_ONCE, completion_mode=COMPLETION_EXIT_AFTER_IDLE, timeout=timeout, evidence_dir=blind.output_dir, task_payload=b"evaluate", control_dir=str(alias / "live-control"), ) def run() -> None: try: def commit(locator): try: on_started(locator, spec_digest(spec)) locator_box.append(locator) except BaseException as exc: self.callback_error = exc finally: ready.set() if self.callback_error is not None: raise self.callback_error self.worker_result = run_invocation( spec, parse_event=lambda _stream, _line: None, on_started=commit, ) except BaseException as exc: self.worker_error = exc self.worker = threading.Thread(target=run) self.worker.start() if not ready.wait(15): raise KeyboardInterrupt( "evaluator locator was not published: " + repr(self.worker_error) ) if self.callback_error is not None: raise KeyboardInterrupt(repr(self.callback_error)) deadline = time.monotonic() + 5 while not recover_invocation( locator_box[0], stop=False ).caller_launched: if time.monotonic() >= deadline: raise AssertionError("evaluator did not launch") time.sleep(0.01) self.locator = locator_box[0] raise KeyboardInterrupt("simulated scoring controller loss") adapter = InterruptingAdapter() def cleanup() -> None: if adapter.worker is not None and adapter.worker.is_alive(): if adapter.locator is not None: try: recover_invocation(adapter.locator, stop=True) except Exception: pass adapter.worker.join(5) if adapter.control_alias is not None and adapter.control_alias.is_symlink(): adapter.control_alias.unlink() self.addCleanup(cleanup) with self.assertRaises(KeyboardInterrupt): score_run(self.store, self.run, self.manifest, adapter=adapter) self.assertIsNone(adapter.callback_error, repr(adapter.callback_error)) self.assertIsNone(adapter.worker_error, repr(adapter.worker_error)) self.assertIsNone(adapter.worker_result, repr(adapter.worker_result)) first_root = Path(attempt.root) / "scoring" / "score-000001" self.assertTrue((first_root / "runner.json").is_file()) self.assertFalse((first_root / "result.json").exists()) prior = { path: path.read_bytes() for path in first_root.rglob("*") if path.is_file() } summary = score_run( self.store, self.run, self.manifest, adapter=adapter, retry_scoring_failed=True, ) self.assertEqual((summary.scored, summary.scoring_failed), (1, 0)) self.assertTrue(adapter.successor_started) self.assertIsNotNone(adapter.worker) adapter.worker.join(5) # type: ignore[union-attr] self.assertFalse(adapter.worker.is_alive()) # type: ignore[union-attr] self.assertIsNone(adapter.worker_error) self.assertIsNotNone(adapter.worker_result) self.assertTrue(adapter.worker_result.cleanup_complete) self.assertFalse(adapter.worker_result.process_group_alive) receipt = json.loads( ( Path(adapter.invocations[0][0].output_dir) / "live-control" / "cleanup-receipt.json" ).read_text() ) self.assertEqual(receipt["reason"], "recovered_stop") self.assertTrue(receipt["cleanup_complete"]) self.assertFalse(receipt["process_group_alive"]) for path, data in prior.items(): self.assertEqual(path.read_bytes(), data) self.assertTrue( (Path(attempt.root) / "scoring" / "score-000002").is_dir() ) def test_mutated_input_and_runtime_secret_fail_before_scored(self): attempt = self._attempt() mutated = FakeScoringAdapter(modes=["mutate", "success"]) first = score_run( self.store, self.run, self.manifest, adapter=mutated ) self.assertEqual((first.scored, first.scoring_failed), (0, 1)) first_root = Path(attempt.root) / "scoring" / "score-000001" first_bytes = { path: path.read_bytes() for path in first_root.rglob("*") if path.is_file() } result = json.loads((first_root / "result.json").read_text()) self.assertEqual(result["reason"], "input_mutated") self.assertNotIn("worksheet", result) retry = score_run( self.store, self.run, self.manifest, adapter=mutated, retry_scoring_failed=True, ) self.assertEqual((retry.scored, retry.scoring_failed), (1, 0)) self.assertEqual( first_bytes, { path: path.read_bytes() for path in first_root.rglob("*") if path.is_file() }, ) secret = "runtime-evaluator-secret-literal" self.run = self.store.create(self.manifest, self.manifest_path.read_bytes()) secret_attempt = self._attempt() leaking = FakeScoringAdapter( modes=["secret"], sensitive_value=secret ) leaked = score_run( self.store, self.run, self.manifest, adapter=leaking ) self.assertEqual((leaked.scored, leaked.scoring_failed), (0, 1)) durable = b"".join( path.read_bytes() for path in Path(self.run.root).rglob("*") if path.is_file() ) self.assertNotIn(secret.encode("utf-8"), durable) secret_result = json.loads( ( Path(secret_attempt.root) / "scoring" / "score-000001" / "result.json" ).read_text() ) self.assertEqual(secret_result["reason"], "runtime_secret_leak") self.assertNotIn("worksheet", secret_result) def test_tampered_score_fails_closed_without_rewrite(self): attempt = self._attempt() score_run(self.store, self.run, self.manifest, adapter=FakeScoringAdapter()) result = Path(attempt.root) / "scoring" / "score-000001" / "result.json" record = json.loads(result.read_text()) record["worksheet"]["total"] = 0 result.write_text( json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n" ) before = result.read_bytes() with self.assertRaises(ScoringError): score_run(self.store, self.run, self.manifest, adapter=FakeScoringAdapter()) self.assertEqual(result.read_bytes(), before) def test_execution_preset_evaluator_uses_manifest_stage_binding(self): raw = json.loads(self.manifest_path.read_text()) raw["evaluator"]["iop"] = { "request_model": "judge-model", "requested_effort": "xhigh", "route_kind": "execution_preset", "route_id": "judge-preset", "expected_bindings": [ {"stage": stage, "model": "judge-model"} for stage in ("selector", "plan", "work", "review") ], } raw["output_root"] = "agent-test/runs/preset-bench" path = self.root / "preset.json" path.write_text(json.dumps(raw)) self.manifest_path = path self.manifest = load_manifest(path, repo_root=self.root) self.run = self.store.create(self.manifest, path.read_bytes()) self._attempt() adapter = FakeScoringAdapter() summary = score_run(self.store, self.run, self.manifest, adapter=adapter) self.assertEqual(summary.scored, 1) self.assertEqual(adapter.preflights, 1) if __name__ == "__main__": unittest.main()