"""Network-free integration tests for public benchmark preflight.""" from __future__ import annotations import contextlib import datetime import hashlib import io import json import os import re import stat import subprocess import sys import tempfile import threading import time import unittest from dataclasses import replace from pathlib import Path from unittest import mock from urllib.error import HTTPError from scripts import agent_comparison_benchmark as benchmark_cli from scripts.agent_benchmark import live_iop from scripts.agent_benchmark.browser_cdp import RenderObservation, ViewportObservation from scripts.agent_benchmark.attempts import ( CapabilityUnavailable, PreflightObservation, RunBusyError, RunStore, Slot, collect_preflight_observations, preflight_manifest, ) from scripts.agent_benchmark.connectivity import ( ISSUE_RESUME_CODES, CallerCapability, ConnectivityIssue, EffectiveBinding, RequestedEffectiveBinding, canonical_evidence_bytes, make_result, ) from scripts.agent_benchmark.codex_iop import CodexInvocationResult from scripts.agent_benchmark.manifest import ( AssetMapping, ExpectedBinding, MatrixCell, digest_workspace_inputs, load_manifest, ) from scripts.agent_benchmark.lifecycle import ( CLOCK_HARNESS_MONOTONIC, COMPLETION_EXIT_AFTER_IDLE, METRIC_NAMES, SOURCE_HARNESS, SOURCE_WORKSPACE_POLL, SUBMISSION_STDIN_ONCE, UNIT_NANOSECONDS, CaptureStream, InvocationSpec, InvocationResult, LifecycleRecoveryError, ParsedMetric, env_pairs, recover_invocation, run_invocation, spec_digest, ) from scripts.agent_benchmark.measurement import ( AttemptMeasurement, REASON_NOT_OBSERVED, REASON_NOT_REPORTED, WorkspaceWriteObservation, load_measurement, observed, path_digest, publish_measurement, unavailable, ) from scripts.agent_benchmark.scoring import BlindWorkspace, ScoringSummary, score_run from scripts.agent_benchmark.web_validation import ( WEB_GATES, build_web_validation, load_web_validation, publish_web_validation, ) def _cell(cell_id: str, caller: str, model: str, effort: str) -> dict: return { "id": cell_id, "caller": caller, "iop": { "request_model": model, "requested_effort": effort, "route_kind": "direct", "route_id": cell_id, "expected_bindings": [ {"stage": "request", "model": model, "effort": effort} ], }, } def _preset(cell_id: str, caller: str, model: str, effort: str) -> dict: return { "id": cell_id, "caller": caller, "iop": { "request_model": model, "requested_effort": effort, "route_kind": "execution_preset", "route_id": cell_id, "expected_bindings": [ {"stage": stage, "model": model} for stage in ("selector", "plan", "work", "review") ], }, } _SENTINEL_CLASSES = ("task", "secret", "endpoint", "config", "provider") _LIVE_BRANCHES = ("claude", "agy", "codex") def _branch_sentinels() -> dict[str, dict[str, str]]: """One distinct sentinel value per caller branch and leak class.""" return { caller: {kind: f"branch-{kind}-sentinel-{caller}" for kind in _SENTINEL_CLASSES} for caller in _LIVE_BRANCHES } # One production-shaped caller executable. It reads only its own argv, its own # environment, the harness-submitted stdin task and its private caller config, # then emits that caller's real stream shape. Every value it can observe is # echoed back through the exact fields a real caller uses for content, so the # published evidence proves the production redactors - not the test - removed # them. The route is taken from the attempt's ``../prepared.json`` because this # suite's matrix binds ``route_id`` to the cell id. _CALLER_FIXTURE_BODY = r'''"""Production-shaped benchmark caller fixture.""" import json import os import sys import uuid from pathlib import Path name = Path(__file__).name config = json.loads( Path(__file__).with_name(name + ".config.json").read_text(encoding="utf-8") ) argv = sys.argv[1:] task = sys.stdin.buffer.read().decode("utf-8", "replace") prepared = json.loads((Path.cwd().parent / "prepared.json").read_text(encoding="utf-8")) route_id = prepared["identity"]["cell_id"] with open(Path(__file__).with_name(name + ".invocations"), "a", encoding="utf-8") as log: log.write(prepared["identity"]["run_id"] + " " + route_id + "\n") def option(flag): return argv[argv.index(flag) + 1] if flag in argv else "" def override(prefix): for item in argv: if item.startswith(prefix): return json.loads(item[len(prefix):]) return "" leak = json.dumps( {"argv": argv, "env": dict(os.environ), "task": task, "config": config}, sort_keys=True, ) sys.stderr.write(name + ": diagnostic " + leak + "\n") sys.stderr.flush() # Production-shaped benchmark output: exactly the three generated root files, # using both fixture images. ``script.js`` is written last so the workspace # observer still has a real caller-produced first-write observation. Path("index.html").write_text("""Orbit

Orbit landing page

Accessible responsive fixture.

Aurora grid artworkOrbit rings artwork
""", encoding="utf-8") Path("styles.css").write_text("""*{box-sizing:border-box}body{margin:0;background:#fff;color:#111;font:18px sans-serif}main{max-width:960px;margin:auto;padding:24px}img{display:block;max-width:100%;width:320px;height:auto;margin:16px 0}a:focus,button:focus{outline:3px solid #05f}@media(max-width:600px){main{padding:16px}img{width:100%}}""", encoding="utf-8") Path("script.js").write_text("document.querySelector('button').addEventListener('click', () => {});", encoding="utf-8") if name == "claude": model = option("--model") session = str(uuid.uuid4()) events = [ {"type": "system", "subtype": "init", "model": model, "session_id": session, "tools": []}, {"type": "assistant", "session_id": session, "message": {"model": model, "stop_reason": "end_turn", "content": [{"type": "text", "text": leak}]}}, {"type": "result", "subtype": "success", "session_id": session, "is_error": False, "duration_ms": 1234, "duration_api_ms": 1000, "usage": {"input_tokens": 11, "output_tokens": 22, "cache_read_input_tokens": 5}, "result": leak}, ] elif name == "agy": binding = {"route_kind": "direct", "route_id": route_id, "model": option("--model"), "effort": option("--effort")} events = [ {"type": "metric", "subtype": "duration_ms", "value": 12.5}, dict(binding, type="iop", subtype="effective_binding", stages=[{"stage": "request", "model": binding["model"], "effort": binding["effort"]}]), dict(binding, type="result", subtype="success", text=leak), dict(binding, type="system", subtype="idle"), ] else: events = [ {"type": "item.completed", "item": {"type": "agent_message", "text": leak}}, {"type": "item.completed", "item": {"id": "call-1", "type": "command_execution", "duration_ms": 7.25, "output": leak}}, {"type": "turn.completed", "status": "completed", "usage": {"input_tokens": 31, "cached_input_tokens": 8, "output_tokens": 12}, "iop_effective_binding": { "route_kind": "direct", "route_id": route_id, "model": option("-m"), "effort": override("model_reasoning_effort=")}}, ] for event in events: sys.stdout.write(json.dumps(event) + "\n") sys.stdout.flush() ''' def _write_manifest( root: Path, matrix: list[dict], output_id: str = "integration", *, output_root: str | None = None, prompt: str = "public prompt fixture", ): fixture_root = root / "scripts/fixtures" fixture_root.mkdir(parents=True, exist_ok=True) (fixture_root / "prompt.md").write_text(prompt, encoding="utf-8") (fixture_root / "reference.txt").write_text("public reference", encoding="utf-8") for image in ("aurora.svg", "orbit.svg"): (fixture_root / image).write_text( "", encoding="utf-8", ) assets = ( AssetMapping( "scripts/fixtures/reference.txt", "brief/reference.txt", b"public reference", ), AssetMapping("scripts/fixtures/aurora.svg", "assets/aurora-grid.svg", (fixture_root / "aurora.svg").read_bytes()), AssetMapping("scripts/fixtures/orbit.svg", "assets/orbit-rings.svg", (fixture_root / "orbit.svg").read_bytes()), ) payload = { "pipeline_version": "2", "environment": "dev", "testbed": "../iop-s2", "repetitions": 1, "session_policy": "fresh", "setup_cache_policy": "isolated", "timeout": { "run_seconds": 5, "idle_seconds": 1, "quiet_seconds": 1, "cleanup_grace_seconds": 1, }, "viewports": [{"id": "desktop", "width": 900, "height": 700}, {"id": "mobile", "width": 375, "height": 700}], "rubric_version": "landing-quality-v1", "evaluator": {"caller": "codex", "iop": {"request_model": "judge", "requested_effort": "high", "route_kind": "direct", "route_id": "judge", "expected_bindings": [{"stage": "request", "model": "judge", "effort": "high"}]}}, "output_root": output_root or f"agent-test/runs/{output_id}", "fixture": { "version": "v1", "prompt": "scripts/fixtures/prompt.md", "assets": [ { "source": "scripts/fixtures/reference.txt", "workspace_path": "brief/reference.txt", } ,{"source": "scripts/fixtures/aurora.svg", "workspace_path": "assets/aurora-grid.svg"} ,{"source": "scripts/fixtures/orbit.svg", "workspace_path": "assets/orbit-rings.svg"} ], "checksum": digest_workspace_inputs(assets), }, "matrix": matrix, } path = root / "manifest.json" raw = json.dumps(payload, sort_keys=True).encode("utf-8") path.write_bytes(raw) return load_manifest(path, repo_root=root), raw, path class FakeAdapter: def __init__( self, caller: str, efforts: tuple[str, ...], issues_by_cell: dict[str, tuple[str, ...]] | None = None, *, sentinel: str = "", ) -> None: self.capability = CallerCapability( caller, ("direct", "execution_preset"), efforts ) self.issues_by_cell = issues_by_cell or {} self.sentinel = sentinel self.calls: list[str] = [] self.invocations: list[tuple[str, str, str, bytes]] = [] self.fail_invocation = False def preflight(self, cell: MatrixCell) -> PreflightObservation: self.calls.append(cell.id) issue_codes = self.issues_by_cell.get(cell.id, ()) if issue_codes: binding = RequestedEffectiveBinding( cell.id, cell.caller, cell.iop.route_kind, cell.iop.route_id, cell.iop.request_model, cell.iop.requested_effort, ) else: binding = RequestedEffectiveBinding( cell.id, cell.caller, cell.iop.route_kind, cell.iop.route_id, cell.iop.request_model, cell.iop.requested_effort, cell.iop.route_kind, cell.iop.route_id, cell.iop.request_model, cell.iop.requested_effort, tuple( EffectiveBinding(item.stage, item.model, item.effort) for item in cell.iop.expected_bindings ), ) issues = tuple( ConnectivityIssue(code, ISSUE_RESUME_CODES[code]) for code in issue_codes ) return PreflightObservation( make_result(cell, self.capability, binding, issues), "sha256:" + "a" * 64, "sha256:" + "b" * 64, ) def invoke( self, cell, prepared, attempt, control_dir, task_payload, timeout, on_started, ): run_root = Path(attempt.root).parents[3] if not (run_root / "preflight/preflight-000001.json").is_file(): raise AssertionError("attempt allocated before preflight publication") if cell.id != attempt.identity.cell_id or prepared.identity != attempt.identity: raise AssertionError("execution identity drift") self.invocations.append( (cell.id, prepared.workspace_dir, prepared.session_id, task_payload) ) if Path(control_dir).resolve(strict=False) != Path(attempt.root).resolve() / "control": raise AssertionError("controller control binding drift") source = ( "import sys; sys.stdin.buffer.read(); print('FAILED'); sys.exit(3)" if self.fail_invocation else "import sys; sys.stdin.buffer.read(); print('FINISH'); print('IDLE')" ) spec = InvocationSpec( argv=(sys.executable, "-u", "-c", source), cwd=prepared.workspace_dir, env=env_pairs({"PATH": os.environ.get("PATH", "/usr/bin:/bin")}), submission_mode=SUBMISSION_STDIN_ONCE, completion_mode=COMPLETION_EXIT_AFTER_IDLE, timeout=timeout, evidence_dir=attempt.root, task_payload=task_payload, control_dir=control_dir, ) return run_invocation( spec, parse_event=lambda _stream, line: { "FINISH": "finish", "IDLE": "idle", }.get(line.strip()), on_started=lambda locator: on_started(locator, spec_digest(spec)), ) def cleanup(self) -> None: pass class ConnectivityIntegrationTest(unittest.TestCase): def setUp(self) -> None: self.temp = tempfile.TemporaryDirectory(dir="/tmp", prefix="benchmark-preflight-") self.root = Path(self.temp.name) / "repo" self.root.mkdir() self.matrix = [ _cell("claude-sonnet-direct", "claude", "claude-sonnet-5", "max"), _cell("claude-gemini-direct", "claude", "gemini-3.6-flash", "high"), _cell("claude-gpt-direct", "claude", "gpt-5.6-luna", "xhigh"), _cell("agy-gemini-direct", "agy", "gemini-3.6-flash", "high"), _cell("codex-gpt-direct", "codex", "gpt-5.6-luna", "xhigh"), ] self.manifest, self.raw, self.path = _write_manifest(self.root, self.matrix) self.store = RunStore( self.root, clock=lambda: datetime.datetime( 2026, 8, 10, 1, 2, 3, tzinfo=datetime.timezone.utc ), token_hex=lambda _: "123456abcdef", ) def tearDown(self) -> None: self.temp.cleanup() def _init_testbed(self) -> None: testbed = self.root.parent / "iop-s2" testbed.mkdir() (testbed / "README.md").write_text("testbed", encoding="utf-8") for command in ( ("git", "init"), ("git", "config", "user.name", "test"), ("git", "config", "user.email", "test@example.invalid"), ("git", "add", "."), ("git", "commit", "-m", "testbed"), ): subprocess.run(command, cwd=testbed, check=True, capture_output=True) def _live_environment(self, *, token: str = "live-token-must-not-persist", manifest=None) -> dict[str, str]: manifest = self.manifest if manifest is None else manifest routes = [] seen: set[tuple[str, str]] = set() evaluator = MatrixCell("evaluator", manifest.evaluator.caller, manifest.evaluator.iop) for cell in (*manifest.matrix, evaluator): key = (cell.iop.route_kind, cell.iop.route_id) if key not in seen: seen.add(key) routes.append( { "route_kind": key[0], "route_id": key[1], "model": cell.iop.request_model, "bindings": [ { "stage": binding.stage, "model": binding.model, "effort": binding.effort, } for binding in cell.iop.expected_bindings ], } ) environment = { "IOP_BENCH_CONFIG_OBSERVATION_ENV": "BENCH_CONFIG", "BENCH_CONFIG": json.dumps({"schema_version": "1", "routes": routes}, sort_keys=True), "BENCH_TOKEN": token, } for caller in ("CLAUDE", "AGY", "CODEX"): environment[f"IOP_BENCH_{caller}_BASE_URL"] = "http://127.0.0.1:18083/v1" environment[f"IOP_BENCH_{caller}_SECRET_ENV"] = "BENCH_TOKEN" return environment @staticmethod def _score_measurement(attempt, caller: str) -> AttemptMeasurement: timeline = { "submitted_at": unavailable(REASON_NOT_OBSERVED, SOURCE_HARNESS), "first_output_at": unavailable(REASON_NOT_OBSERVED, SOURCE_HARNESS), "first_write_observed_at": unavailable( REASON_NOT_OBSERVED, SOURCE_WORKSPACE_POLL ), "first_write_mtime": unavailable( REASON_NOT_OBSERVED, SOURCE_WORKSPACE_POLL ), "total_duration": observed( 1, UNIT_NANOSECONDS, CLOCK_HARNESS_MONOTONIC, SOURCE_HARNESS ), } usage = { name: unavailable(REASON_NOT_REPORTED, SOURCE_HARNESS) for name in METRIC_NAMES } return AttemptMeasurement( attempt.identity.run_id, attempt.identity.cell_id, attempt.identity.repetition, attempt.identity.attempt, caller, "sha256:" + "3" * 64, "success", timeline, usage, WorkspaceWriteObservation( False, None, None, "", 1, 0, REASON_NOT_OBSERVED ), (), ) @staticmethod def _score_view( attempt_root: Path, ident: str, width: int, height: int, image_paths: tuple[str, str], ) -> ViewportObservation: screenshot = f"screenshot-{ident}.png" png = b"\x89PNG\r\n\x1a\n" + ident.encode("ascii") (attempt_root / screenshot).write_bytes(png) images = tuple( { "src": path, "alt": path, "complete": True, "natural_width": 80, "natural_height": 60, "visible": True, "rect": { "x": 0, "y": 0, "width": 80, "height": 60, "right": 80, "bottom": 60, }, } for path in image_paths ) return ViewportObservation( ident, width, height, screenshot, "sha256:" + hashlib.sha256(png).hexdigest(), len(png), images, { "scroll_width": width, "client_width": width, "clipped": 0, "overlaps": 0, }, { "h1_count": 1, "headings": [1], "heading_progression": True, "main_count": 1, "landmarks": 1, "controls": [ { "name": True, "tab_index": 0, "focused": True, "focus_visible": True, "contrast": 7.0, } ], "ax": {"nodes": 4, "non_ignored": 3, "named": 2}, }, ) def _successful_score_attempt(self, manifest, run): cell = manifest.matrix[0] with self.store.writer(run): attempt = self.store.allocate(run, Slot(cell.id, 1)) workspace = Path(attempt.root) / "workspace" for asset in manifest.fixture.assets: target = workspace / asset.workspace_path target.parent.mkdir(parents=True, exist_ok=True) target.write_bytes(asset.content) image_paths = tuple( asset.workspace_path for asset in manifest.fixture.assets if Path(asset.workspace_path).suffix.lower() in (".png", ".svg") ) if len(image_paths) != 2: raise AssertionError("scoring fixture requires two images") (workspace / "index.html").write_text( "

Ready

" + "".join( f"{path}" for path in image_paths ) + "" "
", encoding="utf-8", ) (workspace / "styles.css").write_text( "body{color:#111;background:#fff}img{width:80px}" "button:focus{outline:2px solid #05f}", encoding="utf-8", ) (workspace / "script.js").write_text( "document.body.dataset.ready='1';", encoding="utf-8" ) measurement = self._score_measurement(attempt, cell.caller) publish_measurement(attempt.root, measurement) render = RenderObservation( "Chromium/Test", "http://127.0.0.1:12345", tuple( {"kind": "local", "path": "/" + path, "allowed": True, "status": 200} for path in ("index.html", *image_paths) ), (), tuple( self._score_view( Path(attempt.root), viewport.id, viewport.width, viewport.height, image_paths, ) for viewport in manifest.viewports ), ) publish_web_validation( attempt.root, build_web_validation(manifest, workspace, measurement, render), ) return self.store.publish_terminal( attempt, "success", result={"terminal_reason": "success"} ) def _run_live_scoring_mutation(self, case, mutate): secret = f"live-{case}-secret-exact-value" raw = json.loads(self.raw) raw["evaluator"]["iop"]["requested_effort"] = "xhigh" raw["evaluator"]["iop"]["expected_bindings"] = [ {"stage": "request", "model": "judge", "effort": "xhigh"} ] raw["output_root"] = f"agent-test/runs/{case}" manifest_raw = json.dumps(raw, sort_keys=True).encode("utf-8") manifest_path = self.root / f"{case}.json" manifest_path.write_bytes(manifest_raw) manifest = load_manifest(manifest_path, repo_root=self.root) evaluator = MatrixCell( "evaluator", manifest.evaluator.caller, manifest.evaluator.iop ) run = self.store.create(manifest, manifest_raw) attempt = self._successful_score_attempt(manifest, run) environment = self._live_environment(token=secret, manifest=manifest) base_url = environment["IOP_BENCH_CODEX_BASE_URL"] def invoke(invocation, _on_started): mutate(invocation, secret, base_url) stream = CaptureStream("stdout", "", 0, 0, False) lifecycle = InvocationResult( True, "success", 0, None, True, True, True, False, (), stream, replace(stream, stream="stderr"), "", "", None, "sha256:" + "a" * 64, "2026-08-11T00:00:00+00:00", "2026-08-11T00:00:01+00:00", 1, (), ) binding = ( evaluator.iop.route_kind, evaluator.iop.route_id, evaluator.iop.request_model, evaluator.iop.requested_effort, ) return CodexInvocationResult(lifecycle, binding) adapter = live_iop.build_live_scoring_adapter( environment, observer=lambda _runtime: live_iop._Observation( (evaluator.iop.request_model,), "sha256:" + "b" * 64, True ), invoker=invoke, ) summary = score_run(self.store, run, manifest, adapter=adapter) score_root = Path(attempt.root) / "scoring" / "score-000001" result = json.loads((score_root / "result.json").read_text()) allocation = json.loads((score_root / "allocation.json").read_text()) blind_root = Path(run.root) / allocation["blind_path"] return summary, result, blind_root, secret, base_url def _sentinel_live_environment( self, manifest, sentinels: dict[str, dict[str, str]] ) -> dict[str, str]: """Give every branch its own endpoint and credential sentinel value.""" environment = self._live_environment(manifest=manifest) del environment["BENCH_TOKEN"] for caller, branch in sentinels.items(): prefix = f"IOP_BENCH_{caller.upper()}_" secret_env = f"BENCH_TOKEN_{caller.upper()}" environment[secret_env] = branch["secret"] environment[prefix + "SECRET_ENV"] = secret_env environment[prefix + "BASE_URL"] = f"http://{branch['endpoint']}.invalid:18083/v1" return environment @staticmethod def _registry(issues: dict[str, tuple[str, ...]] | None = None, sentinel: str = ""): issues = issues or {} return { "claude": FakeAdapter( "claude", ("high", "max", "xhigh"), issues, sentinel=sentinel ), "agy": FakeAdapter("agy", ("high",), issues, sentinel=sentinel), "codex": FakeAdapter("codex", ("xhigh",), issues, sentinel=sentinel), } def test_all_three_callers_append_exact_ready_results_without_attempts(self) -> None: registry = self._registry() run, record = preflight_manifest( self.store, self.manifest, self.raw, adapters=registry ) self.assertEqual(record["status"], "ready") self.assertEqual( [result["cell"]["id"] for result in record["results"]], [cell.id for cell in self.manifest.matrix], ) self.assertEqual(len(self.store.preflights(run, self.manifest)), 1) self.assertFalse((Path(run.root) / "cells").exists()) self.assertEqual( {caller: adapter.calls for caller, adapter in registry.items()}, { "claude": [ "claude-gemini-direct", "claude-gpt-direct", "claude-sonnet-direct", ], "agy": ["agy-gemini-direct"], "codex": ["codex-gpt-direct"], }, ) def test_registration_and_implementation_blockers_are_distinct_and_no_attempt_allocates(self) -> None: registry = self._registry( { "claude-sonnet-direct": ("credential_missing",), "agy-gemini-direct": ("stream_incompatible",), } ) run, record = preflight_manifest( self.store, self.manifest, self.raw, adapters=registry ) statuses = [result["status"] for result in record["results"]] self.assertEqual(statuses.count("registration_required"), 1) self.assertEqual(statuses.count("implementation_gap"), 1) self.assertEqual(record["status"], "implementation_gap") self.assertFalse((Path(run.root) / "cells").exists()) self.assertEqual(self.store.status(run, self.manifest)["attempts"]["running"], 0) @contextlib.contextmanager def _production_shaped_callers(self, sentinels: dict[str, dict[str, str]]): """Publish one executable per caller before any invocation is built.""" # This checkout mounts /tmp with noexec. Keep the fixtures temporary # and test-owned, but place their executable directory on the current # executable test filesystem so the real caller adapters can launch # them through their normal subprocess path. bin_dir = tempfile.TemporaryDirectory(dir=Path.cwd(), prefix=".bc-") self.addCleanup(bin_dir.cleanup) fixture_bin = Path(bin_dir.name) for caller, branch in sentinels.items(): executable = fixture_bin / caller executable.write_text( f"#!{sys.executable}\n" + _CALLER_FIXTURE_BODY, encoding="utf-8" ) executable.chmod(0o700) # The caller's private configuration is the only source of its # config/provider sentinels, exactly as a real client config file. (fixture_bin / f"{caller}.config.json").write_text( json.dumps( { "caller": caller, "config_identity": branch["config"], "provider_id": branch["provider"], }, sort_keys=True, ), encoding="utf-8", ) yield fixture_bin @staticmethod def _snapshot_run_root(run_root: Path) -> dict[str, bytes]: return { str(item.relative_to(run_root)): item.read_bytes() for item in sorted(run_root.rglob("*")) if item.is_file() } def _assert_one_published_spec_digest(self, attempt_root: Path) -> None: """Prove the admitted, journalled and published digests are one value.""" state = json.loads((attempt_root / "attempt.json").read_text(encoding="utf-8")) result = json.loads( (attempt_root / "lifecycle-result.json").read_text(encoding="utf-8") ) header = json.loads( (attempt_root / "lifecycle-journal.jsonl") .read_text(encoding="utf-8") .splitlines()[0] ) self.assertIn("locator", state) digests = {state["spec_digest"], result["spec_digest"], header["spec_digest"]} self.assertEqual(len(digests), 1, attempt_root) self.assertRegex(digests.pop(), r"^sha256:[0-9a-f]{64}$") self.assertEqual(state["state"], "success") self.assertIs(result["success"], True) self.assertEqual(result["terminal_reason"], "success") self.assertIs(result["finish_then_idle_then_quiet"], True) self.assertIs(result["cleanup_complete"], True) self.assertIs(result["process_group_alive"], False) control_dir = Path(state["locator"]["control_dir"]) alias = control_dir.parent self.assertEqual(control_dir.name, "control") self.assertEqual( state["locator"]["socket_path"], str(control_dir / "control.sock") ) self.assertTrue(alias.name.startswith("iop-bench-attempt-")) self.assertFalse(os.path.lexists(alias)) self.assertTrue((attempt_root / "control/locator.json").is_file()) self.assertTrue((attempt_root / "control/cleanup-receipt.json").is_file()) # The exact whole-total categories each caller reports through its own # allowlist. Everything else must remain explicitly unavailable. _EXPECTED_TOTALS = { "claude": { "total_duration", "model_duration", "model_calls", "input_tokens", "output_tokens", "cached_input_tokens", }, "agy": {"total_duration"}, "codex": { "model_calls", "tool_calls", "input_tokens", "output_tokens", "cached_input_tokens", }, } def _assert_measurement_evidence( self, attempt_roots: list[Path], sentinels: dict[str, dict[str, str]] ) -> None: """Every attempt publishes one strict, source-aware, digested sidecar.""" callers = set() for attempt_root in attempt_roots: measurement = load_measurement(attempt_root) caller = measurement.caller callers.add(caller) # This matrix binds the cell id to the caller name. self.assertEqual(measurement.cell_id, caller) self.assertEqual(measurement.terminal_reason, "success") observed = { name for name, item in measurement.usage.items() if item.status == "observed" } self.assertEqual(observed, self._EXPECTED_TOTALS[caller], attempt_root) # No caller reports a provider total, and none is reconstructed. self.assertEqual(measurement.usage["total_tokens"].status, "unavailable") self.assertIsNone(measurement.usage["total_tokens"].value) self.assertEqual(measurement.timeline["first_output_at"].status, "observed") self.assertEqual( measurement.timeline["first_write_mtime"].clock, "filesystem_mtime" ) self.assertTrue(measurement.observer.observed) self.assertEqual( measurement.observer.path_digest, path_digest("index.html"), ) web = load_web_validation(attempt_root) self.assertEqual(web.status, "passed", web.record) self.assertEqual( [item["id"] for item in web.record["gates"]], list(WEB_GATES) ) self.assertTrue(all(item["passed"] for item in web.record["gates"])) expected_images = { item["path"] for item in web.record["workspace"]["inputs"] if item["path"].startswith("assets/") } self.assertEqual(len(expected_images), 2) self.assertEqual( [item["id"] for item in web.record["viewports"]], ["desktop", "mobile"], ) self.assertEqual(len(web.record["screenshots"]), 2) for viewport, screenshot in zip( web.record["viewports"], web.record["screenshots"] ): self.assertEqual( {item["src"] for item in viewport["images"]}, expected_images ) self.assertEqual( screenshot, {"id": viewport["id"], **viewport["screenshot"]} ) data = (attempt_root / screenshot["file"]).read_bytes() self.assertEqual(screenshot["size"], len(data)) self.assertEqual( screenshot["digest"], "sha256:" + hashlib.sha256(data).hexdigest(), ) self.assertEqual(callers, set(_LIVE_BRANCHES)) def _assert_sentinels_absent( self, published: dict[str, bytes], sentinels: dict[str, dict[str, str]] ) -> None: for caller, branch in sentinels.items(): for kind, value in branch.items(): encoded = value.encode("ascii") for relative, data in published.items(): self.assertNotIn(encoded, data, f"{caller}/{kind} in {relative}") def test_cli_live_run_invokes_each_direct_cell_once(self) -> None: short = tempfile.TemporaryDirectory(dir="/tmp", prefix="b") self.addCleanup(short.cleanup) root = Path(short.name) / "r" root.mkdir() matrix = [ _cell("claude", "claude", "sonnet", "max"), _cell("agy", "agy", "gemini", "high"), _cell("codex", "codex", "gpt", "xhigh"), ] sentinels = _branch_sentinels() manifest, _raw, path = _write_manifest( root, matrix, output_root="agent-test/runs/r", prompt="public prompt fixture\n" + "\n".join(sentinels[caller]["task"] for caller in _LIVE_BRANCHES), ) testbed = root.parent / "iop-s2" testbed.mkdir() (testbed / "README.md").write_text("testbed", encoding="utf-8") for command in (("git", "init"), ("git", "config", "user.name", "test"), ("git", "config", "user.email", "test@example.invalid"), ("git", "add", "."), ("git", "commit", "-m", "testbed")): subprocess.run(command, cwd=testbed, check=True, capture_output=True) environment = self._sentinel_live_environment(manifest, sentinels) def observed(_runtime): return live_iop._Observation( tuple(sorted(cell.iop.request_model for cell in manifest.matrix)), "sha256:" + "f" * 64, True, "agy 1.1.11", "--print --output-format --sandbox --model --effort AGY_PROVIDER AGY_OPENAI_BASE_URL AGY_OPENAI_API_KEY stream-json", ) stdout = io.StringIO() stderr = io.StringIO() with self._production_shaped_callers(sentinels) as fixture_bin: with ( mock.patch.dict( os.environ, {"PATH": f"{fixture_bin}:{os.environ['PATH']}"} ), mock.patch.object(benchmark_cli, "_REPO_ROOT", root), contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr), ): registry = live_iop.build_live_adapter_registry( environment, observer=observed, binary_resolver=lambda name: str(fixture_bin / name), ) with mock.patch.object( benchmark_cli, "build_adapter_registry", return_value=registry ): exit_code = benchmark_cli.main(["run", "--manifest", str(path)]) self.assertEqual(exit_code, 0, stderr.getvalue()) self.assertIn("ok: run run_id=", stdout.getvalue()) self.assertEqual(stderr.getvalue(), "") calls = { caller: (fixture_bin / f"{caller}.invocations") .read_text(encoding="utf-8") .splitlines() for caller in _LIVE_BRANCHES } self.assertEqual({caller: len(item) for caller, item in calls.items()}, {"claude": 1, "agy": 1, "codex": 1}) run_roots = list((root / manifest.output_root).glob("run-*")) self.assertEqual(len(run_roots), 1) run_root = run_roots[0] self.assertTrue((run_root / "preflight/preflight-000001.json").is_file()) published = self._snapshot_run_root(run_root) attempt_roots = sorted(run_root.glob("cells/*/repetition-*/attempt-*")) self.assertEqual(len(attempt_roots), len(manifest.matrix)) for attempt_root in attempt_roots: self._assert_one_published_spec_digest(attempt_root) self._assert_measurement_evidence(attempt_roots, sentinels) self._assert_sentinels_absent(published, sentinels) self.assertEqual(published, self._snapshot_run_root(run_root)) def test_cli_run_blocker_persists_preflight_and_allocates_zero_attempts(self) -> None: registry = self._registry( {"claude-sonnet-direct": ("credential_missing",)} ) stdout = io.StringIO() stderr = io.StringIO() with ( mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root), mock.patch.object( benchmark_cli, "build_adapter_registry", return_value=registry ), contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr), ): exit_code = benchmark_cli.main(["run", "--manifest", str(self.path)]) self.assertEqual(exit_code, 69) self.assertEqual(stdout.getvalue(), "") self.assertIn("error: preflight blocked", stderr.getvalue()) run_roots = list((self.root / self.manifest.output_root).glob("run-*")) self.assertEqual(len(run_roots), 1) self.assertTrue((run_roots[0] / "preflight/preflight-000001.json").is_file()) self.assertFalse((run_roots[0] / "cells").exists()) self.assertTrue(all(adapter.invocations == [] for adapter in registry.values())) def test_cli_mixed_manifest_never_invokes_unobserved_preset_cells(self) -> None: manifest, _, path = _write_manifest( self.root, [ _cell("direct-ready", "claude", "claude-sonnet-5", "max"), _preset("preset-unobserved", "claude", "claude-sonnet-5", "max"), ], output_id="mixed-unobserved", ) registry = self._registry() stdout = io.StringIO() stderr = io.StringIO() with ( mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root), mock.patch.object( benchmark_cli, "build_adapter_registry", return_value=registry ), contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr), ): exit_code = benchmark_cli.main(["run", "--manifest", str(path)]) self.assertEqual(exit_code, 69) self.assertEqual(stdout.getvalue(), "") self.assertIn("error: benchmark execution failed", stderr.getvalue()) self.assertIn("completed=0 unresolved=2", stderr.getvalue()) run_roots = list((self.root / manifest.output_root).glob("run-*")) self.assertEqual(len(run_roots), 1) preflight = json.loads( (run_roots[0] / "preflight/preflight-000001.json").read_text( encoding="ascii" ) ) self.assertEqual(preflight["status"], "ready") self.assertEqual( [result["cell"]["id"] for result in preflight["results"]], ["direct-ready"], ) self.assertFalse((run_roots[0] / "cells").exists()) self.assertEqual(registry["claude"].calls, ["direct-ready"]) self.assertTrue(all(adapter.invocations == [] for adapter in registry.values())) def test_cli_resume_retries_append_only_and_status_is_read_only(self) -> None: self._init_testbed() manifest, _, path = _write_manifest( self.root, [_cell("claude-only", "claude", "claude-sonnet-5", "max")], output_id="retry", ) failed = FakeAdapter("claude", ("max",)) failed.fail_invocation = True self.addCleanup(failed.cleanup) first_stdout = io.StringIO() first_stderr = io.StringIO() with ( mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root), mock.patch.object( benchmark_cli, "build_adapter_registry", return_value={"claude": failed}, ), contextlib.redirect_stdout(first_stdout), contextlib.redirect_stderr(first_stderr), ): first_exit = benchmark_cli.main( ["run", "--manifest", str(path)] ) self.assertEqual(first_exit, 69) matched = re.search(r"run_id=(run-[0-9A-Za-z-]+)", first_stderr.getvalue()) self.assertIsNotNone(matched) run_id = matched.group(1) # type: ignore[union-attr] run_root = self.root / manifest.output_root / run_id first_attempt = next(run_root.glob("cells/*/repetition-*/attempt-000001")) old_bytes = { item.relative_to(first_attempt): item.read_bytes() for item in first_attempt.rglob("*") if item.is_file() } ready = FakeAdapter("claude", ("max",)) self.addCleanup(ready.cleanup) resume_stdout = io.StringIO() resume_stderr = io.StringIO() with ( mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root), mock.patch.object( benchmark_cli, "build_adapter_registry", return_value={"claude": ready}, ), contextlib.redirect_stdout(resume_stdout), contextlib.redirect_stderr(resume_stderr), ): resume_exit = benchmark_cli.main( [ "resume", "--manifest", str(path), "--run-id", run_id, "--retry-failed", ] ) self.assertEqual(resume_exit, 0, resume_stderr.getvalue()) self.assertIn("ok: resume", resume_stdout.getvalue()) self.assertEqual( old_bytes, { relative: (first_attempt / relative).read_bytes() for relative in old_bytes }, ) self.assertTrue( next(run_root.glob("cells/*/repetition-*/attempt-000002/attempt.json")) .read_text(encoding="utf-8") .find('"state":"success"') >= 0 ) self.assertEqual(len(list((run_root / "preflight").glob("*.json"))), 2) before_status = { item.relative_to(run_root): item.read_bytes() for item in run_root.rglob("*") if item.is_file() } status_stdout = io.StringIO() status_stderr = io.StringIO() with ( mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root), contextlib.redirect_stdout(status_stdout), contextlib.redirect_stderr(status_stderr), ): status_exit = benchmark_cli.main( ["status", "--manifest", str(path), "--run-id", run_id] ) self.assertEqual(status_exit, 0, status_stderr.getvalue()) self.assertIn("'success': 1", status_stdout.getvalue()) self.assertEqual( before_status, { item.relative_to(run_root): item.read_bytes() for item in run_root.rglob("*") if item.is_file() }, ) def test_missing_adapter_is_rejected_before_output_root_mutation(self) -> None: registry = self._registry() del registry["codex"] output_root = self.root / self.manifest.output_root with self.assertRaises(CapabilityUnavailable): preflight_manifest( self.store, self.manifest, self.raw, adapters=registry ) self.assertFalse(output_root.exists()) def test_generic_preset_cells_are_local_contract_only(self) -> None: generic, _, _ = _write_manifest( self.root, [ _preset("claude-generic", "claude", "claude-sonnet-5", "high"), _preset("agy-generic", "agy", "gemini-3.6-flash", "high"), _preset("codex-generic", "codex", "gpt-5.6-luna", "xhigh"), ], output_id="generic", ) registry = self._registry() observations = collect_preflight_observations(generic, registry) self.assertEqual(observations, {}) self.assertTrue(all(adapter.calls == [] for adapter in registry.values())) def test_generic_preset_only_public_preflight_fails_closed_without_run_state(self) -> None: generic, raw, path = _write_manifest( self.root, [ _preset("claude-generic", "claude", "claude-sonnet-5", "high"), _preset("agy-generic", "agy", "gemini-3.6-flash", "high"), _preset("codex-generic", "codex", "gpt-5.6-luna", "xhigh"), ], output_id="generic", ) registry = self._registry() output_root = self.root / generic.output_root with self.assertRaises(Exception) as ctx: preflight_manifest( self.store, generic, raw, adapters=registry ) self.assertIn("preflight requires a direct cell", str(ctx.exception)) self.assertFalse(output_root.exists()) self.assertTrue(all(adapter.calls == [] for adapter in registry.values())) sentinel = "private_endpoint_and_token_must_not_appear" stdout = io.StringIO() stderr = io.StringIO() with ( mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root), mock.patch.object(benchmark_cli, "build_adapter_registry", return_value=registry), contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr), ): exit_code = benchmark_cli.main(["preflight", "--manifest", str(path)]) self.assertEqual(exit_code, 69) self.assertEqual(stdout.getvalue(), "") self.assertNotIn(sentinel, stderr.getvalue()) self.assertFalse(output_root.exists()) def test_live_registry_dereferences_secret_names_without_persisting_values(self) -> None: sentinel = "live-token-must-not-persist" environment = self._live_environment(token=sentinel) environment["ANTHROPIC_BASE_URL"] = "must-not-be-read" def observed(_runtime): return live_iop._Observation( tuple(sorted(cell.iop.request_model for cell in self.manifest.matrix)), "sha256:" + "c" * 64, True, "agy 1.1.11", "--print --output-format --sandbox --model --effort AGY_PROVIDER AGY_OPENAI_BASE_URL AGY_OPENAI_API_KEY stream-json", ) registry = live_iop.build_live_adapter_registry( environment, observer=observed, binary_resolver=lambda _name: "/bin/true" ) self.assertEqual(tuple(registry), ("claude", "agy", "codex")) observations = collect_preflight_observations(self.manifest, registry) self.assertEqual(set(observations), {cell.id for cell in self.manifest.matrix}) for observation in observations.values(): self.assertEqual(observation.result.status, "ready") self.assertEqual(observation.result.binding.effective_model, observation.result.binding.requested_model) durable = json.dumps( {cell_id: item.result.status for cell_id, item in observations.items()}, sort_keys=True, ) self.assertNotIn(sentinel, durable) self.assertNotIn("must-not-be-read", durable) def test_live_registry_ready_binding_requires_catalog_observation(self) -> None: environment = self._live_environment(token="private-token") registry = live_iop.build_live_adapter_registry( environment, observer=lambda _runtime: live_iop._Observation( (), "sha256:" + "d" * 64, True ), ) claude_cell = next(cell for cell in self.manifest.matrix if cell.caller == "claude") observation = registry["claude"].preflight(claude_cell) self.assertEqual(observation.result.status, "registration_required") self.assertEqual([issue.code for issue in observation.result.issues], ["model_missing"]) self.assertIsNone(observation.result.binding.effective_model) def test_live_scoring_adapter_preserves_manifest_route_and_ephemeral_secret(self) -> None: secret = "evaluator-secret-must-not-persist" source_identities = ( "source-cell-sentinel", "source-route-sentinel", "source-model-sentinel", "source-effort-sentinel", ) for route_kind in ("direct", "execution_preset"): with self.subTest(route_kind=route_kind): expected_bindings = ( (ExpectedBinding("request", "judge", "xhigh"),) if route_kind == "direct" else tuple( ExpectedBinding(stage, "judge") for stage in ("selector", "plan", "work", "review") ) ) evaluator = MatrixCell( "evaluator", self.manifest.evaluator.caller, replace( self.manifest.evaluator.iop, requested_effort="xhigh", route_kind=route_kind, expected_bindings=expected_bindings, ), ) scoring_manifest = replace( self.manifest, evaluator=replace(self.manifest.evaluator, iop=evaluator.iop), ) environment = self._live_environment( token=secret, manifest=scoring_manifest ) captured = {} def invoke(invocation, _on_started): captured["spec"] = invocation.spec stream = CaptureStream("stdout", "", 0, 0, False) lifecycle = InvocationResult( True, "success", 0, None, True, True, True, False, (), stream, replace(stream, stream="stderr"), "", "", None, "sha256:" + "a" * 64, "2026-08-11T00:00:00+00:00", "2026-08-11T00:00:01+00:00", 1, (), ) binding = ( evaluator.iop.route_kind, evaluator.iop.route_id, evaluator.iop.request_model, evaluator.iop.requested_effort, ) return CodexInvocationResult(lifecycle, binding) adapter = live_iop.build_live_scoring_adapter( environment, observer=lambda _runtime: live_iop._Observation( (evaluator.iop.request_model,), "sha256:" + "b" * 64, True, ), invoker=invoke, ) observation = adapter.preflight(evaluator) self.assertEqual(observation.result.status, "ready") self.assertEqual( observation.result.binding.effective_bindings, tuple( EffectiveBinding(item.stage, item.model, item.effort) for item in evaluator.iop.expected_bindings ), ) blind_root = self.root / "runs" / f"blind-{route_kind}" for name in ("input", "session", "output"): (blind_root / name).mkdir(parents=True, exist_ok=True) blind = BlindWorkspace( f"blind-{route_kind}", str(blind_root), str(blind_root / "input"), str(blind_root / "session"), str(blind_root / "output"), "sha256:" + "c" * 64, "sha256:" + "d" * 64, ) prompt = b"Evaluate only anonymous files under input/." result = adapter.invoke( evaluator, blind, prompt, self.manifest.timeout, lambda *_args: None, ) self.assertTrue(result.success) self.assertEqual(result.effective_binding[0], route_kind) finalized = adapter.finalize_evidence(blind) self.assertTrue(finalized.safe) spec = captured["spec"] visible = "\n".join( (*spec.argv, spec.cwd, *(value for pair in spec.env for value in pair)) ).casefold() for identity in source_identities: self.assertNotIn(identity, visible) durable = b"".join( path.read_bytes() for path in blind_root.rglob("*") if path.is_file() ) self.assertNotIn(secret.encode("ascii"), durable) evidence = canonical_evidence_bytes( evaluator, observation.result, observation.endpoint_identity, observation.config_identity, ) self.assertNotIn(secret.encode("ascii"), evidence) def test_live_scoring_alias_is_control_only(self) -> None: evaluator = MatrixCell( "evaluator", self.manifest.evaluator.caller, replace( self.manifest.evaluator.iop, requested_effort="xhigh", expected_bindings=(ExpectedBinding("request", "judge", "xhigh"),), ), ) scoring_manifest = replace( self.manifest, evaluator=replace(self.manifest.evaluator, iop=evaluator.iop), ) captured: dict[str, object] = {} def invoke(invocation, _on_started): captured["spec"] = invocation.spec evidence = Path(invocation.spec.evidence_dir) (evidence / "lifecycle-journal.jsonl").write_text( '{"record":"header"}\n{"record":"terminal"}\n', encoding="utf-8", ) (evidence / "lifecycle-result.json").write_text( '{"record":"result"}\n', encoding="utf-8" ) stream = CaptureStream("stdout", "", 0, 0, False) lifecycle = InvocationResult( True, "success", 0, None, True, True, True, False, (), stream, replace(stream, stream="stderr"), "", "", None, "sha256:" + "a" * 64, "2026-08-11T00:00:00+00:00", "2026-08-11T00:00:01+00:00", 1, (), ) binding = ( evaluator.iop.route_kind, evaluator.iop.route_id, evaluator.iop.request_model, evaluator.iop.requested_effort, ) return CodexInvocationResult(lifecycle, binding) adapter = live_iop.build_live_scoring_adapter( self._live_environment(manifest=scoring_manifest), observer=lambda _runtime: live_iop._Observation( (evaluator.iop.request_model,), "sha256:" + "b" * 64, True ), invoker=invoke, ) self.assertEqual(adapter.preflight(evaluator).result.status, "ready") blind_root = self.root / "alias-run" / "blind" / "blind-alias" for name in ("input", "session", "output"): (blind_root / name).mkdir(parents=True, exist_ok=True) blind = BlindWorkspace( "blind-alias", str(blind_root), str(blind_root / "input"), str(blind_root / "session"), str(blind_root / "output"), "sha256:" + "c" * 64, "sha256:" + "d" * 64, ) result = adapter.invoke( evaluator, blind, b"Evaluate anonymous output.", self.manifest.timeout, lambda *_args: None, ) self.assertTrue(result.success) spec = captured["spec"] self.assertIsInstance(spec, InvocationSpec) assert isinstance(spec, InvocationSpec) self.assertEqual(Path(spec.evidence_dir), blind_root / "output") alias = Path(spec.control_dir).parent self.assertTrue(alias.is_symlink()) self.assertEqual(alias.resolve(strict=True), blind_root / "output") sidecars = { path.name: path.read_bytes() for path in ( blind_root / "output" / "lifecycle-journal.jsonl", blind_root / "output" / "lifecycle-result.json", ) } self.assertTrue(adapter.finalize_evidence(blind).safe) self.assertFalse(alias.exists() or alias.is_symlink()) for name, data in sidecars.items(): self.assertEqual((blind_root / "output" / name).read_bytes(), data) def test_live_scoring_scrubs_evaluator_secret_output(self) -> None: secret = "live-evaluator-secret-exact-value" base_url = "http://127.0.0.1:18083/v1" evaluator = MatrixCell( "evaluator", self.manifest.evaluator.caller, replace( self.manifest.evaluator.iop, requested_effort="xhigh", expected_bindings=( ExpectedBinding("request", "judge", "xhigh"), ), ), ) scoring_manifest = replace( self.manifest, evaluator=replace(self.manifest.evaluator, iop=evaluator.iop), ) environment = self._live_environment( token=secret, manifest=scoring_manifest ) def invoke(invocation, _on_started): output = Path(invocation.spec.evidence_dir) worksheet = _worksheet_payload = { "rubric_version": "landing-quality-v1", "categories": [ { "id": ident, "max_score": maximum, "score": maximum, "evidence": secret if index == 0 else "safe evidence", } for index, (ident, maximum) in enumerate( ( ("task_fidelity", 25), ("visual_hierarchy", 25), ("responsive_composition", 20), ("typography_readability", 15), ("polish_consistency", 15), ) ) ], "total": 100, } (output / "worksheet.json").write_text( json.dumps(worksheet), encoding="utf-8" ) (output / "diagnostic.txt").write_text( f"{base_url}\n{secret}\n", encoding="utf-8" ) stream = CaptureStream("stdout", "", 0, 0, False) lifecycle = InvocationResult( True, "success", 0, None, True, True, True, False, (), stream, replace(stream, stream="stderr"), "", "", None, "sha256:" + "a" * 64, "2026-08-11T00:00:00+00:00", "2026-08-11T00:00:01+00:00", 1, (), ) binding = ( evaluator.iop.route_kind, evaluator.iop.route_id, evaluator.iop.request_model, evaluator.iop.requested_effort, ) return CodexInvocationResult(lifecycle, binding) adapter = live_iop.build_live_scoring_adapter( environment, observer=lambda _runtime: live_iop._Observation( (evaluator.iop.request_model,), "sha256:" + "b" * 64, True, ), invoker=invoke, ) self.assertEqual(adapter.preflight(evaluator).result.status, "ready") blind_root = self.root / "secret-run" / "blind" / "blind-secret" for name in ("input", "session", "output"): (blind_root / name).mkdir(parents=True, exist_ok=True) blind = BlindWorkspace( "blind-secret", str(blind_root), str(blind_root / "input"), str(blind_root / "session"), str(blind_root / "output"), "sha256:" + "c" * 64, "sha256:" + "d" * 64, ) result = adapter.invoke( evaluator, blind, b"Evaluate anonymous output.", self.manifest.timeout, lambda *_args: None, ) self.assertTrue(result.success) finalized = adapter.finalize_evidence(blind) self.assertEqual( (finalized.safe, finalized.reason), (False, "runtime_secret_leak"), ) durable = b"".join( path.read_bytes() for path in (self.root / "secret-run").rglob("*") if path.is_file() ) self.assertNotIn(secret.encode("utf-8"), durable) self.assertNotIn(base_url.encode("utf-8"), durable) self.assertFalse((blind_root / "output" / "worksheet.json").exists()) def test_live_scoring_scrubs_permission_denied_secret_paths(self) -> None: retained_modes: dict[str, int] = {} removed_paths: list[Path] = [] def mutate(invocation, secret, base_url): blind_root = Path(invocation.spec.cwd) input_root = blind_root / "input" output_root = Path(invocation.spec.evidence_dir) os.chmod(input_root, 0o700, follow_symlinks=False) locked = input_root / ("locked-" + secret) locked.mkdir() payload = locked / "payload.bin" payload.write_bytes(secret.encode("utf-8")) os.chmod(payload, 0o000, follow_symlinks=False) os.chmod(locked, 0o000, follow_symlinks=False) removed_paths.append(locked) denied = output_root / ("denied-" + secret + ".bin") denied.write_bytes(base_url.encode("utf-8")) os.chmod(denied, 0o000, follow_symlinks=False) removed_paths.append(denied) safe_dir = output_root / "safe-retained" safe_dir.mkdir() safe_file = output_root / "safe-retained.txt" safe_file.write_text("safe evidence", encoding="utf-8") os.chmod(safe_file, 0o000, follow_symlinks=False) os.chmod(safe_dir, 0o000, follow_symlinks=False) retained_modes["directory"] = 0o000 retained_modes["file"] = 0o000 summary, result, blind_root, secret, base_url = ( self._run_live_scoring_mutation("permission-denied-secret", mutate) ) self.assertEqual((summary.scored, summary.scoring_failed), (0, 1)) self.assertEqual(result["reason"], "runtime_secret_leak") self.assertNotIn("worksheet", result) for path in removed_paths: self.assertFalse(path.exists() or path.is_symlink()) safe_dir = blind_root / "output" / "safe-retained" safe_file = blind_root / "output" / "safe-retained.txt" self.assertEqual(stat.S_IMODE(os.lstat(safe_dir).st_mode), retained_modes["directory"]) self.assertEqual(stat.S_IMODE(os.lstat(safe_file).st_mode), retained_modes["file"]) self.assertNotIn(secret, result["reason"]) self.assertNotIn(base_url, result["reason"]) def test_live_scoring_classifies_safe_invalid_links_without_secret_claim(self) -> None: cases = ( ("input", "input_mutated"), ("output", "evaluator_output_leak"), ) for root_kind, expected_reason in cases: with self.subTest(root_kind=root_kind): link_path: list[Path] = [] target_path: list[Path] = [] def mutate(invocation, _secret, _base_url): blind_root = Path(invocation.spec.cwd) selected = ( blind_root / "input" if root_kind == "input" else Path(invocation.spec.evidence_dir) ) if root_kind == "input": os.chmod(selected, 0o700, follow_symlinks=False) target = selected / "index.html" else: target = selected / "safe-target.txt" target.write_text("safe evidence", encoding="utf-8") link = selected / "safe-invalid-link" link.symlink_to(target.name) link_path.append(link) target_path.append(target) summary, result, _blind_root, _secret, _base_url = ( self._run_live_scoring_mutation( f"safe-link-{root_kind}", mutate ) ) self.assertEqual( (summary.scored, summary.scoring_failed), (0, 1) ) self.assertEqual(result["reason"], expected_reason) self.assertNotEqual(result["reason"], "runtime_secret_leak") self.assertFalse(link_path[0].exists() or link_path[0].is_symlink()) self.assertTrue(target_path[0].is_file()) self.assertNotIn("worksheet", result) def test_live_scoring_scrubs_secret_from_mutated_input_before_failure(self) -> None: secret = "live-mutated-input-secret-exact-value" evaluator = MatrixCell( "evaluator", self.manifest.evaluator.caller, replace( self.manifest.evaluator.iop, requested_effort="xhigh", expected_bindings=(ExpectedBinding("request", "judge", "xhigh"),), ), ) manifest_payload = { **json.loads(self.raw), "evaluator": { "caller": evaluator.caller, "iop": { "request_model": evaluator.iop.request_model, "requested_effort": evaluator.iop.requested_effort, "route_kind": evaluator.iop.route_kind, "route_id": evaluator.iop.route_id, "expected_bindings": [ { "stage": item.stage, "model": item.model, "effort": item.effort, } for item in evaluator.iop.expected_bindings ], }, }, "output_root": "agent-test/runs/mutated-input-secret", } manifest_raw = json.dumps(manifest_payload, sort_keys=True).encode("utf-8") manifest_path = self.root / "mutated-input-secret.json" manifest_path.write_bytes(manifest_raw) scoring_manifest = load_manifest(manifest_path, repo_root=self.root) evaluator = MatrixCell( "evaluator", scoring_manifest.evaluator.caller, scoring_manifest.evaluator.iop, ) run = self.store.create(scoring_manifest, manifest_raw) attempt = self._successful_score_attempt(scoring_manifest, run) environment = self._live_environment( token=secret, manifest=scoring_manifest ) base_url = environment["IOP_BENCH_CODEX_BASE_URL"] invocation_count = 0 def invoke(invocation, _on_started): nonlocal invocation_count invocation_count += 1 if invocation_count == 1: input_root = Path(invocation.spec.cwd) / "input" os.chmod(input_root, 0o700, follow_symlinks=False) target = input_root / "index.html" os.chmod(target, 0o600, follow_symlinks=False) target.write_bytes(target.read_bytes() + secret.encode("utf-8")) sensitive_dir = input_root / ("copied-" + secret) sensitive_dir.mkdir() (sensitive_dir / "runtime.txt").write_text( secret + "\n" + base_url, encoding="utf-8" ) stream = CaptureStream("stdout", "", 0, 0, False) lifecycle = InvocationResult( True, "success", 0, None, True, True, True, False, (), stream, replace(stream, stream="stderr"), "", "", None, "sha256:" + "a" * 64, "2026-08-11T00:00:00+00:00", "2026-08-11T00:00:01+00:00", 1, (), ) binding = ( evaluator.iop.route_kind, evaluator.iop.route_id, evaluator.iop.request_model, evaluator.iop.requested_effort, ) return CodexInvocationResult(lifecycle, binding) adapter = live_iop.build_live_scoring_adapter( environment, observer=lambda _runtime: live_iop._Observation( (evaluator.iop.request_model,), "sha256:" + "b" * 64, True ), invoker=invoke, ) first = score_run( self.store, run, scoring_manifest, adapter=adapter ) self.assertEqual((first.scored, first.scoring_failed), (0, 1)) first_score = Path(attempt.root) / "scoring" / "score-000001" first_result = json.loads((first_score / "result.json").read_text()) self.assertEqual(first_result["reason"], "runtime_secret_leak") self.assertNotIn("worksheet", first_result) first_allocation = json.loads((first_score / "allocation.json").read_text()) first_blind = Path(run.root) / first_allocation["blind_path"] self.assertFalse((first_blind / "output" / "worksheet.json").exists()) for path in (first_blind / "input", *(first_blind / "input").rglob("*")): self.assertEqual(os.lstat(path).st_mode & 0o222, 0) prior = { path: path.read_bytes() for path in Path(run.root).rglob("*") if path.is_file() } retry = score_run( self.store, run, scoring_manifest, adapter=adapter, retry_scoring_failed=True, ) self.assertEqual((retry.scored, retry.scoring_failed), (0, 1)) self.assertEqual(invocation_count, 2) for path, data in prior.items(): self.assertEqual(path.read_bytes(), data) second_score = Path(attempt.root) / "scoring" / "score-000002" second_allocation = json.loads( (second_score / "allocation.json").read_text() ) self.assertNotEqual( first_allocation["blind_id"], second_allocation["blind_id"] ) self.assertNotEqual( first_allocation["session_identity"], second_allocation["session_identity"], ) for path in Path(run.root).rglob("*"): relative = path.relative_to(run.root).as_posix().encode("utf-8") self.assertNotIn(secret.encode("utf-8"), relative) if path.is_file(): data = path.read_bytes() self.assertNotIn(secret.encode("utf-8"), data) self.assertNotIn(base_url.encode("utf-8"), data) def test_live_scoring_survivor_cleanup_precedes_retry(self) -> None: evaluator = MatrixCell( "evaluator", self.manifest.evaluator.caller, replace( self.manifest.evaluator.iop, requested_effort="xhigh", expected_bindings=( ExpectedBinding("request", "judge", "xhigh"), ), ), ) scoring_manifest = replace( self.manifest, evaluator=replace(self.manifest.evaluator, iop=evaluator.iop), ) environment = self._live_environment(manifest=scoring_manifest) locator_ready = threading.Event() locators = [] worker_results = [] workers = [] first = True def invoker(invocation, on_started): nonlocal first if not first: self.assertTrue(workers) self.assertFalse(workers[0].is_alive()) stream = CaptureStream("stdout", "", 0, 0, False) lifecycle = InvocationResult( True, "success", 0, None, True, True, True, False, (), stream, replace(stream, stream="stderr"), "", "", None, "sha256:" + "a" * 64, "2026-08-11T00:00:00+00:00", "2026-08-11T00:00:01+00:00", 1, (), ) binding = ( evaluator.iop.route_kind, evaluator.iop.route_id, evaluator.iop.request_model, evaluator.iop.requested_effort, ) return CodexInvocationResult(lifecycle, binding) first = False spec = replace( invocation.spec, argv=( sys.executable, "-u", "-c", "import sys,time; sys.stdin.buffer.read(); " "print('START', flush=True); time.sleep(30)", ), env=env_pairs( {"PATH": os.environ.get("PATH", "/usr/bin:/bin")} ), task_payload=b"evaluate", ) def run(): worker_results.append( run_invocation( spec, parse_event=lambda _stream, _line: None, on_started=lambda locator: ( on_started(locator), locators.append(locator), locator_ready.set(), ), ) ) worker = threading.Thread(target=run) workers.append(worker) worker.start() if not locator_ready.wait(5): self.fail("live evaluator locator was not published") deadline = time.monotonic() + 5 while not recover_invocation(locators[0], stop=False).caller_launched: if time.monotonic() >= deadline: self.fail("live evaluator did not launch") time.sleep(0.01) raise KeyboardInterrupt("simulated live scoring controller loss") adapter = live_iop.build_live_scoring_adapter( environment, observer=lambda _runtime: live_iop._Observation( (evaluator.iop.request_model,), "sha256:" + "b" * 64, True, ), invoker=invoker, ) self.assertEqual(adapter.preflight(evaluator).result.status, "ready") blind_root = self.root / "recovery-run" / "blind" / "blind-recovery" for name in ("input", "session", "output"): (blind_root / name).mkdir(parents=True, exist_ok=True) blind = BlindWorkspace( "blind-recovery", str(blind_root), str(blind_root / "input"), str(blind_root / "session"), str(blind_root / "output"), "sha256:" + "c" * 64, "sha256:" + "d" * 64, ) def cleanup(): if workers and workers[0].is_alive() and locators: try: recover_invocation(locators[0], stop=True) except Exception: pass workers[0].join(5) try: adapter.finalize_evidence(blind) except Exception: pass self.addCleanup(cleanup) with self.assertRaises(KeyboardInterrupt): adapter.invoke( evaluator, blind, b"Evaluate anonymous output.", self.manifest.timeout, lambda locator, digest: self.assertRegex( digest, r"^sha256:[0-9a-f]{64}$" ), ) try: stopped = recover_invocation(locators[0], stop=True) except LifecycleRecoveryError: stopped = None if stopped is not None: self.assertTrue(stopped.cleanup_complete) self.assertFalse(stopped.process_group_alive) workers[0].join(5) self.assertFalse(workers[0].is_alive()) self.assertEqual(len(worker_results), 1) self.assertTrue(worker_results[0].cleanup_complete) self.assertFalse(worker_results[0].process_group_alive) receipt = json.loads( ( blind_root / "output" / "codex-control" / "cleanup-receipt.json" ).read_text() ) self.assertTrue(receipt["cleanup_complete"]) self.assertFalse(receipt["process_group_alive"]) self.assertTrue(adapter.finalize_evidence(blind).safe) retry = adapter.invoke( evaluator, blind, b"Evaluate anonymous output.", self.manifest.timeout, lambda *_args: None, ) self.assertTrue(retry.success) self.assertTrue(adapter.finalize_evidence(blind).safe) def test_catalog_only_never_creates_ready_binding(self) -> None: environment = self._live_environment() del environment["IOP_BENCH_CONFIG_OBSERVATION_ENV"] registry = live_iop.build_live_adapter_registry( environment, observer=lambda _runtime: live_iop._Observation( tuple(sorted(cell.iop.request_model for cell in self.manifest.matrix)), "sha256:" + "d" * 64, True, ), ) cell = next(cell for cell in self.manifest.matrix if cell.caller == "claude") result = registry["claude"].preflight(cell).result self.assertEqual(result.status, "registration_required") self.assertEqual([item.code for item in result.issues], ["route_missing"]) self.assertIsNone(result.binding.effective_model) def test_config_owner_binding_is_passed_without_manifest_synthesis(self) -> None: environment = self._live_environment() cell = next(cell for cell in self.manifest.matrix if cell.id == "claude-sonnet-direct") registry = live_iop.build_live_adapter_registry( environment, observer=lambda _runtime: live_iop._Observation( tuple(sorted(item.iop.request_model for item in self.manifest.matrix)), "sha256:" + "e" * 64, True, ), ) result = registry["claude"].preflight(cell).result self.assertEqual(result.status, "ready") self.assertEqual(result.binding.effective_route_id, cell.iop.route_id) self.assertEqual(result.binding.effective_model, cell.iop.request_model) self.assertEqual( result.binding.effective_bindings, (EffectiveBinding("request", cell.iop.request_model, cell.iop.requested_effort),), ) def test_live_scoring_preset_requires_observed_stage_bindings(self) -> None: raw = json.loads(self.path.read_text()) raw["evaluator"]["iop"] = { "request_model": "judge", "requested_effort": "xhigh", "route_kind": "execution_preset", "route_id": "judge-preset", "expected_bindings": [ {"stage": stage, "model": "judge"} for stage in ("selector", "plan", "work", "review") ], } raw["output_root"] = "agent-test/runs/preset-observation" path = self.root / "preset-observation.json" path.write_text(json.dumps(raw), encoding="utf-8") manifest = load_manifest(path, repo_root=self.root) evaluator = MatrixCell( "evaluator", manifest.evaluator.caller, manifest.evaluator.iop ) environment = self._live_environment(manifest=manifest) observed = lambda _runtime: live_iop._Observation( ("judge",), "sha256:" + "8" * 64, True ) ready = live_iop.build_live_scoring_adapter( environment, observer=observed ).preflight(evaluator) self.assertEqual(ready.result.status, "ready") self.assertEqual( ready.result.binding.effective_bindings, tuple( EffectiveBinding(item.stage, item.model, item.effort) for item in evaluator.iop.expected_bindings ), ) routes = json.loads(environment["BENCH_CONFIG"])["routes"] index = next( i for i, item in enumerate(routes) if item["route_id"] == "judge-preset" ) cases = { "missing": routes[index]["bindings"][:-1], "reordered": list(reversed(routes[index]["bindings"])), "substituted": [ ( {**binding, "model": "other-model"} if binding["stage"] == "work" else dict(binding) ) for binding in routes[index]["bindings"] ], } for name, bindings in cases.items(): with self.subTest(case=name): changed = [dict(item) for item in routes] changed[index] = {**changed[index], "bindings": bindings} candidate = { **environment, "BENCH_CONFIG": json.dumps( {"schema_version": "1", "routes": changed}, sort_keys=True, ), } result = live_iop.build_live_scoring_adapter( candidate, observer=observed ).preflight(evaluator).result self.assertEqual(result.status, "implementation_gap") self.assertEqual( [issue.code for issue in result.issues], ["protocol_incompatible"], ) self.assertIsNone(result.binding.effective_model) def test_live_scoring_metrics_match_any_admitted_stage_model(self) -> None: direct_evaluator = MatrixCell( "evaluator", self.manifest.evaluator.caller, replace( self.manifest.evaluator.iop, requested_effort="xhigh", expected_bindings=(ExpectedBinding("request", "judge", "xhigh"),), ), ) direct_manifest = replace( self.manifest, evaluator=replace(self.manifest.evaluator, iop=direct_evaluator.iop), ) raw = json.loads(self.path.read_text()) raw["evaluator"]["iop"] = { "request_model": "judge-selector", "requested_effort": "xhigh", "route_kind": "execution_preset", "route_id": "judge-heterogeneous", "expected_bindings": [ {"stage": "selector", "model": "judge-selector"}, {"stage": "plan", "model": "judge-plan"}, {"stage": "work", "model": "judge-work"}, {"stage": "review", "model": "judge-review"}, ], } raw["output_root"] = "agent-test/runs/heterogeneous-metrics" path = self.root / "heterogeneous-metrics.json" path.write_text(json.dumps(raw), encoding="utf-8") preset_manifest = load_manifest(path, repo_root=self.root) def exercise( manifest, metric_model: str, suffix: str, metric_stage: str | None = None ): evaluator = MatrixCell( "evaluator", manifest.evaluator.caller, manifest.evaluator.iop ) def invoke(_invocation, _on_started): stream = CaptureStream("stdout", "", 0, 0, False) metric = ParsedMetric( "model_duration", 1, UNIT_NANOSECONDS, CLOCK_HARNESS_MONOTONIC, SOURCE_HARNESS, ( metric_stage if metric_stage is not None else ( "work" if evaluator.iop.route_kind == "execution_preset" else "request" ) ), metric_model, "call-1", ) lifecycle = InvocationResult( True, "success", 0, None, True, True, True, False, (), stream, replace(stream, stream="stderr"), "", "", None, "sha256:" + "a" * 64, "2026-08-11T00:00:00+00:00", "2026-08-11T00:00:01+00:00", 1, (metric,), ) binding = ( evaluator.iop.route_kind, evaluator.iop.route_id, evaluator.iop.request_model, evaluator.iop.requested_effort, ) return CodexInvocationResult(lifecycle, binding) adapter = live_iop.build_live_scoring_adapter( self._live_environment(manifest=manifest), observer=lambda _runtime: live_iop._Observation( (evaluator.iop.request_model,), "sha256:" + "b" * 64, True ), invoker=invoke, ) self.assertEqual(adapter.preflight(evaluator).result.status, "ready") blind_root = self.root / "metric-run" / suffix for name in ("input", "session", "output"): (blind_root / name).mkdir(parents=True, exist_ok=True) blind = BlindWorkspace( "blind-" + suffix, str(blind_root), str(blind_root / "input"), str(blind_root / "session"), str(blind_root / "output"), "sha256:" + "c" * 64, "sha256:" + "d" * 64, ) try: return adapter.invoke( evaluator, blind, b"Evaluate anonymous output.", manifest.timeout, lambda *_args: None, ) finally: adapter.finalize_evidence(blind) self.assertTrue(exercise(direct_manifest, "judge", "direct").success) self.assertTrue( exercise(preset_manifest, "judge-work", "preset-work").success ) self.assertTrue( exercise( preset_manifest, "judge-plan", "preset-unqualified", metric_stage="" ).success ) for stage, model in (("plan", "judge-work"), ("work", "judge-plan")): with self.subTest(stage=stage, model=model): with self.assertRaises(live_iop.LiveIopError) as raised: exercise( preset_manifest, model, f"preset-cross-{stage}", metric_stage=stage, ) self.assertEqual( raised.exception.issue_code, "stream_incompatible" ) with self.assertRaises(live_iop.LiveIopError) as raised: exercise(preset_manifest, "unadmitted-model", "preset-unknown") self.assertEqual(raised.exception.issue_code, "stream_incompatible") def test_catalog_accepts_edge_routing_ids_and_rejects_malformed_records(self) -> None: environment = self._live_environment() runtime = live_iop._runtime_from_environment("claude", environment).runtime self.assertIsNotNone(runtime) class Response: status = 200 def __init__(self, records) -> None: self.body = json.dumps({"object": "list", "data": records}).encode() def __enter__(self): return self def __exit__(self, *_args): return False def read(self): return self.body model_ids = ( "claude-sonnet-5", "gemini-3.6-flash", "gpt-5.6-luna", "qwen3.6:35b", "ornith:35b", "laguna-s:2.1", ) with mock.patch.object( live_iop, "urlopen", return_value=Response([{"id": model_id} for model_id in model_ids]), ): models, _identity = live_iop._catalog(runtime) # type: ignore[arg-type] self.assertEqual(models, tuple(sorted(model_ids))) malformed = { "numeric": [{"id": 1}], "empty": [{"id": ""}], "whitespace": [{"id": " \t"}], "duplicate": [{"id": "claude-sonnet-5"}, {"id": "claude-sonnet-5"}], } for name, records in malformed.items(): with self.subTest(name=name): with mock.patch.object(live_iop, "urlopen", return_value=Response(records)): with self.assertRaises(live_iop.LiveIopError) as raised: live_iop._catalog(runtime) # type: ignore[arg-type] self.assertEqual(raised.exception.issue_code, "protocol_incompatible") def test_live_failure_taxonomy_is_exact(self) -> None: """Every live setup/catalog boundary returns one closed issue/resume pair.""" environment = self._live_environment(token="secret-must-not-appear") runtime_cases = ( ("missing-base", {"IOP_BENCH_CLAUDE_BASE_URL": ""}, "endpoint_incompatible"), ("invalid-base", {"IOP_BENCH_CLAUDE_BASE_URL": "not-a-url"}, "endpoint_incompatible"), ("invalid-secret-ref", {"IOP_BENCH_CLAUDE_SECRET_ENV": "1BAD"}, "credential_missing"), ("missing-secret", {"BENCH_TOKEN": ""}, "credential_missing"), ) for _name, updates, expected in runtime_cases: candidate = {**environment, **updates} resolution = live_iop._runtime_from_environment("claude", candidate) self.assertIsNone(resolution.runtime) self.assertEqual(resolution.issue_code, expected) runtime = live_iop._runtime_from_environment("claude", environment).runtime self.assertIsNotNone(runtime) class Response: def __init__(self, status, body): self.status, self.body = status, body def __enter__(self): return self def __exit__(self, *_args): return False def read(self): return self.body catalog_cases = ( (HTTPError("http://invalid", 401, "", None, None), "auth_incompatible"), (HTTPError("http://invalid", 403, "", None, None), "auth_incompatible"), (OSError("unreachable"), "endpoint_incompatible"), (Response(502, b"{}"), "endpoint_incompatible"), (Response(200, b"not-json"), "protocol_incompatible"), (Response(200, b'{"data":[{"id":1}]}'), "protocol_incompatible"), ) for outcome, expected in catalog_cases: patch_kwargs = {"side_effect": outcome} if isinstance(outcome, BaseException) else {"return_value": outcome} with mock.patch.object(live_iop, "urlopen", **patch_kwargs): with self.assertRaises(live_iop.LiveIopError) as raised: live_iop._catalog(runtime) # type: ignore[arg-type] self.assertEqual(raised.exception.issue_code, expected) claude = next(cell for cell in self.manifest.matrix if cell.caller == "claude") routes = json.loads(environment["BENCH_CONFIG"])["routes"] no_route = {**environment, "BENCH_CONFIG": json.dumps({"schema_version": "1", "routes": [item for item in routes if item["route_id"] != claude.iop.route_id]})} no_model_routes = [ {**item, "model": "other-model"} if item["route_id"] == claude.iop.route_id else dict(item) for item in routes ] no_model = {**environment, "BENCH_CONFIG": json.dumps({"schema_version": "1", "routes": no_model_routes})} unsupported = replace(next(cell for cell in self.manifest.matrix if cell.caller == "agy"), iop=replace(next(cell for cell in self.manifest.matrix if cell.caller == "agy").iop, requested_effort="max")) observed = lambda _runtime: live_iop._Observation(tuple(sorted(cell.iop.request_model for cell in self.manifest.matrix)), "sha256:" + "1" * 64, True, "agy 1.1.11", "--print --output-format --sandbox --model --effort AGY_PROVIDER AGY_OPENAI_BASE_URL AGY_OPENAI_API_KEY stream-json") checks = ( (live_iop.build_live_adapter_registry(no_route, observer=observed)["claude"].preflight(claude).result, "route_missing"), (live_iop.build_live_adapter_registry(no_model, observer=observed)["claude"].preflight(claude).result, "model_missing"), (live_iop.build_live_adapter_registry(environment, observer=lambda _runtime: (_ for _ in ()).throw(live_iop.LiveIopError("stream_incompatible")))["claude"].preflight(claude).result, "stream_incompatible"), ) for result, issue_code in checks: self.assertEqual( (result.status, tuple((item.code, item.resume_code) for item in result.issues)), ("registration_required" if issue_code in ISSUE_RESUME_CODES and issue_code in {"route_missing", "model_missing", "effort_unsupported"} else "implementation_gap", ((issue_code, ISSUE_RESUME_CODES[issue_code]),)), ) _, unsupported_issues = live_iop._binding_from_config( unsupported, CallerCapability("agy", ("direct", "execution_preset"), ("high", "low", "medium")), live_iop._runtime_from_environment("agy", environment).runtime.config, # type: ignore[union-attr] ) self.assertEqual( tuple((item.code, item.resume_code) for item in unsupported_issues), (("effort_unsupported", ISSUE_RESUME_CODES["effort_unsupported"]),), ) def test_live_invocation_rejects_missing_or_mismatched_caller_binding(self) -> None: environment = self._live_environment() observed = lambda _runtime: live_iop._Observation( tuple(sorted(cell.iop.request_model for cell in self.manifest.matrix)), "sha256:" + "2" * 64, True, "agy 1.1.11", "--print --output-format --sandbox --model --effort AGY_PROVIDER AGY_OPENAI_BASE_URL AGY_OPENAI_API_KEY stream-json", ) registry = live_iop.build_live_adapter_registry( environment, observer=observed, binary_resolver=lambda _name: "/bin/true" ) agy = next(cell for cell in self.manifest.matrix if cell.caller == "agy") codex = next(cell for cell in self.manifest.matrix if cell.caller == "codex") self.assertEqual(registry["agy"].preflight(agy).result.status, "ready") self.assertEqual(registry["codex"].preflight(codex).result.status, "ready") agy_adapter = registry["agy"] codex_adapter = registry["codex"] agy_adapter._invokers = live_iop._InvokerSeams( # type: ignore[attr-defined] live_iop._DEFAULT_INVOKERS.claude, lambda *_args: None, live_iop._DEFAULT_INVOKERS.codex, ) codex_adapter._invokers = live_iop._InvokerSeams( # type: ignore[attr-defined] live_iop._DEFAULT_INVOKERS.claude, live_iop._DEFAULT_INVOKERS.agy, lambda *_args: type("Mismatch", (), {"effective_binding": None, "lifecycle": None})(), ) with mock.patch.object(live_iop, "build_agy_invocation", return_value=object()): with self.assertRaises(live_iop.LiveIopError) as raised: agy_adapter.invoke( agy, object(), object(), "/tmp/control", b"task", self.manifest.timeout, lambda *_args: None, ) self.assertEqual(raised.exception.issue_code, "stream_incompatible") with mock.patch.object(live_iop, "build_codex_invocation", return_value=type("Invocation", (), {"spec": object()})()): with self.assertRaises(live_iop.LiveIopError) as raised: codex_adapter.invoke( codex, object(), object(), "/tmp/control", b"task", self.manifest.timeout, lambda *_args: None, ) self.assertEqual(raised.exception.issue_code, "stream_incompatible") def test_cli_missing_live_input_fails_closed_without_secret_or_attempt(self) -> None: sentinel = "missing-input-token" stdout = io.StringIO() stderr = io.StringIO() with ( mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root), contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr), ): exit_code = benchmark_cli.main(["run", "--manifest", str(self.path)]) self.assertEqual(exit_code, 69) self.assertEqual(stdout.getvalue(), "") self.assertIn("error: preflight blocked", stderr.getvalue()) self.assertNotIn(sentinel, stderr.getvalue()) run_roots = list((self.root / self.manifest.output_root).glob("run-*")) self.assertEqual(len(run_roots), 1) self.assertFalse((run_roots[0] / "cells").exists()) def test_concurrent_writer_fails_fast_without_partial_record(self) -> None: registry = self._registry() observations = collect_preflight_observations(self.manifest, registry) run = self.store.create(self.manifest, self.raw) result: list[BaseException] = [] def append() -> None: try: self.store.record_preflight(run, self.manifest, observations) except BaseException as exc: result.append(exc) with self.store.writer(run): worker = threading.Thread(target=append) worker.start() worker.join(5) self.assertFalse(worker.is_alive()) self.assertEqual(len(result), 1) self.assertIsInstance(result[0], RunBusyError) self.assertEqual(self.store.preflights(run, self.manifest), ()) def test_cli_fake_registry_reports_only_closed_summary(self) -> None: sentinel = "private_endpoint_and_token_must_not_appear" registry = self._registry( {"codex-gpt-direct": ("credential_missing",)}, sentinel=sentinel ) stdout = io.StringIO() stderr = io.StringIO() with ( mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root), mock.patch.object(benchmark_cli, "build_adapter_registry", return_value=registry), contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr), ): exit_code = benchmark_cli.main( ["preflight", "--manifest", str(self.path)] ) self.assertEqual(exit_code, 69) self.assertEqual(stdout.getvalue(), "") self.assertIn("status=registration_required", stderr.getvalue()) self.assertIn("registration_required=1", stderr.getvalue()) self.assertNotIn(sentinel, stderr.getvalue()) run_roots = list((self.root / self.manifest.output_root).glob("run-*")) self.assertEqual(len(run_roots), 1) durable = b"".join( path.read_bytes() for path in run_roots[0].rglob("*") if path.is_file() ) self.assertNotIn(sentinel.encode("ascii"), durable) self.assertFalse((run_roots[0] / "cells").exists()) def test_cli_score_missing_run_fails_closed_with_run_id(self) -> None: run_id = "run-20260811T010203Z-000000000000" stdout = io.StringIO() stderr = io.StringIO() with ( mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root), contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr), ): exit_code = benchmark_cli.main( ["score", "--manifest", str(self.path), "--run-id", run_id] ) self.assertEqual(exit_code, 69) self.assertEqual(stdout.getvalue(), "") self.assertEqual( stderr.getvalue(), f"error: benchmark scoring is unavailable run_id={run_id}\n", ) def test_cli_score_prints_only_closed_counts_and_forwards_retry(self) -> None: run = self.store.create(self.manifest, self.raw) cases = ( (ScoringSummary(run.run_id, 2, 1, 0, 0), 0, "ok: score "), (ScoringSummary(run.run_id, 0, 1, 1, 2), 69, "error: benchmark scoring failed "), ) for summary, expected_exit, prefix in cases: with self.subTest(summary=summary): stdout = io.StringIO() stderr = io.StringIO() with ( mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root), mock.patch.object( benchmark_cli, "build_live_scoring_adapter", return_value=object(), ), mock.patch.object( benchmark_cli, "score_run", return_value=summary ) as score, contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr), ): exit_code = benchmark_cli.main( [ "score", "--manifest", str(self.path), "--run-id", run.run_id, "--retry-scoring-failed", ] ) self.assertEqual(exit_code, expected_exit) rendered = stdout.getvalue() or stderr.getvalue() self.assertEqual( rendered, prefix + f"run_id={run.run_id} scored={summary.scored} " + f"unscored={summary.unscored} " + f"scoring_failed={summary.scoring_failed} " + f"blocked={summary.blocked}\n", ) score.assert_called_once() self.assertTrue(score.call_args.kwargs["retry_scoring_failed"]) if __name__ == "__main__": unittest.main()