""" Comprehensive tests for the benchmark manifest loader, validator, and CLI. Covers: valid minimum/example, omitted repetitions, data-only matrix extension, deterministic ordering, duplicate ids/stages, direct vs preset shapes, every enum/bound, unknown members, path escape/symlink/collision, workspace/prompt/asset/manifest digest drift, non-positive bounds, and secret redaction in errors. """ from __future__ import annotations import hashlib import inspect import json import os import shutil import struct import subprocess import sys import tempfile import unittest from dataclasses import replace from pathlib import Path # Ensure repo root is importable _REPO_ROOT = Path(__file__).resolve().parent.parent.parent if str(_REPO_ROOT) not in sys.path: sys.path.insert(0, str(_REPO_ROOT)) from scripts.agent_benchmark.manifest import ( AssetMapping, Fixture, IopCell, Manifest, ManifestDigestError, ManifestError, ManifestPathError, ManifestValidationError, MatrixCell, Timeout, Viewport, digest_manifest_and_resolved_inputs, digest_workspace_inputs, load_manifest, validate_manifest_bytes, ) # Sentinel values used in redaction tests _SENTINEL_SECRET = "SUPER_SECRET_API_KEY_12345" _SENTINEL_ENDPOINT = "https://private.internal.example.com/secret-endpoint" _SENTINEL_PROMPT = "Do not leak this prompt content" def _make_minimal_manifest_dict(**overrides: object) -> dict: """Build a minimal valid manifest dict with optional overrides.""" d = { "pipeline_version": "2", "environment": "dev", "testbed": "../iop-s2", "session_policy": "fresh", "setup_cache_policy": "isolated", "timeout": { "run_seconds": 300, "idle_seconds": 30, "quiet_seconds": 10, "cleanup_grace_seconds": 5, }, "viewports": [{"id": "desktop", "width": 1920, "height": 1080}], "rubric_version": "landing-quality-v1", "evaluator": { "caller": "codex", "iop": { "request_model": "gpt-5.6-luna", "requested_effort": "xhigh", "route_kind": "direct", "route_id": "gpt-5.6-luna", "expected_bindings": [ { "stage": "request", "model": "gpt-5.6-luna", "effort": "xhigh", } ], }, }, "output_root": "agent-test/runs/bench-01", "fixture": { "version": "v1.0", "prompt": "scripts/fixtures/agent-comparison-benchmark/prompt.md", "assets": [ { "source": "scripts/fixtures/agent-comparison-benchmark/reference.txt", "workspace_path": "workspace/reference.txt", } ], "checksum": "sha256:placeholder", }, "matrix": [ { "id": "cell-a", "caller": "claude", "iop": { "request_model": "claude-sonnet-4-20250514", "requested_effort": "high", "route_kind": "direct", "route_id": "claude-direct", "expected_bindings": [ {"stage": "request", "model": "claude-sonnet-4-20250514", "effort": "high"} ], }, } ], } d.update(overrides) return d def _compute_fixture_checksum(repo_root: Path, fixture: dict) -> str: """Compute the fixture checksum for a given fixture dict.""" assets_raw = fixture["assets"] assets = tuple( AssetMapping( source=a["source"], workspace_path=a["workspace_path"], content=(repo_root / a["source"]).read_bytes(), ) for a in assets_raw ) return digest_workspace_inputs(assets) def _write_tmp_manifest( tmp_dir: Path, data: dict, name: str = "manifest.json", ) -> Path: """Write manifest dict to tmp_dir/name and return the path. Attempts to patch the fixture checksum using repo-root asset files. If that fails (e.g. assets were modified), writes the manifest as-is. """ p = tmp_dir / name if "fixture" in data and "checksum" in data["fixture"]: try: data["fixture"]["checksum"] = _compute_fixture_checksum( _REPO_ROOT, data["fixture"] ) except (FileNotFoundError, OSError, ValueError): # Assets were modified; leave checksum as-is (tests should # expect validation to fail for invalid checksums). pass p.write_text(json.dumps(data, indent=2), encoding="utf-8") return p def _load_tmp_manifest(path: Path) -> Manifest: """Load a manifest from a temp path with repo_root set to _REPO_ROOT.""" return load_manifest(path, repo_root=_REPO_ROOT) class TestLoadManifestValid(unittest.TestCase): """Valid manifest loading tests.""" def test_minimal_valid_manifest(self): """Minimal manifest with explicit repetitions=1 loads.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) path = _write_tmp_manifest(tmp_dir, _make_minimal_manifest_dict()) m = _load_tmp_manifest(path) self.assertEqual(m.pipeline_version, "2") self.assertEqual(m.environment, "dev") self.assertEqual(m.repetitions, 1) self.assertEqual(m.session_policy, "fresh") self.assertEqual(m.setup_cache_policy, "isolated") self.assertEqual(len(m.viewports), 1) self.assertEqual(len(m.matrix), 1) self.assertIsInstance(m, Manifest) # Frozen with self.assertRaises(AttributeError): m.repetitions = 99 def test_omitted_repetitions_defaults_to_one(self): """Omitted repetitions defaults to 1.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict() # repetitions is not included (optional field) path = _write_tmp_manifest(tmp_dir, d) m = _load_tmp_manifest(path) self.assertEqual(m.repetitions, 1) def test_omitted_equals_explicit_one(self): """Omitted repetitions and explicit repetitions=1 produce identical manifests.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) # d_no_rep: repetitions is not included (optional field) d_no_rep = _make_minimal_manifest_dict() d_explicit = _make_minimal_manifest_dict(repetitions=1) p1 = _write_tmp_manifest(tmp_dir, d_no_rep, "no_rep.json") p2 = _write_tmp_manifest(tmp_dir, d_explicit, "explicit_rep.json") m1 = _load_tmp_manifest(p1) m2 = _load_tmp_manifest(p2) self.assertEqual(m1.repetitions, m2.repetitions) self.assertEqual(m1, m2) def test_explicit_repetitions_greater_than_one(self): """Explicit repetitions > 1 is preserved.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) path = _write_tmp_manifest( tmp_dir, _make_minimal_manifest_dict(repetitions=5) ) m = _load_tmp_manifest(path) self.assertEqual(m.repetitions, 5) def test_example_manifest_loads(self): """The shipped example manifest loads successfully.""" example_path = ( _REPO_ROOT / "scripts" / "fixtures" / "agent-comparison-benchmark-manifest.example.json" ) if example_path.exists(): m = _load_tmp_manifest(example_path) self.assertEqual(m.pipeline_version, "2") self.assertEqual(len(m.matrix), 3) # Cells sorted by id ids = [c.id for c in m.matrix] self.assertEqual(ids, sorted(ids)) def test_multiple_viewports_unique(self): """Multiple viewports with unique ids load.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict( viewports=[ {"id": "desktop", "width": 1920, "height": 1080}, {"id": "mobile", "width": 375, "height": 812}, ] ) path = _write_tmp_manifest(tmp_dir, d) m = _load_tmp_manifest(path) self.assertEqual(len(m.viewports), 2) def test_execution_preset_cell_loads(self): """Execution-preset cell with all required stages loads.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict( matrix=[ { "id": "preset-cell", "caller": "agy", "iop": { "request_model": "gemini-2.0-flash", "requested_effort": "high", "route_kind": "execution_preset", "route_id": "agy-generic", "expected_bindings": [ {"stage": "plan", "model": "gemini-2.0-flash"}, {"stage": "work", "model": "gemini-2.0-flash"}, {"stage": "review", "model": "gemini-2.0-flash"}, {"stage": "selector", "model": "gemini-2.0-flash"}, ], }, } ] ) path = _write_tmp_manifest(tmp_dir, d) m = _load_tmp_manifest(path) cell = m.matrix[0] # Sorted by canonical stage rank stages = [b.stage for b in cell.iop.expected_bindings] self.assertEqual(stages, ["selector", "plan", "work", "review"]) def test_execution_preset_with_repair(self): """Execution-preset cell with optional repair stage loads.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict( matrix=[ { "id": "preset-repair", "caller": "codex", "iop": { "request_model": "gpt-4.1", "requested_effort": "xhigh", "route_kind": "execution_preset", "route_id": "codex-generic", "expected_bindings": [ {"stage": "selector", "model": "gpt-4.1"}, {"stage": "plan", "model": "gpt-4.1"}, {"stage": "work", "model": "gpt-4.1"}, {"stage": "review", "model": "gpt-4.1"}, {"stage": "repair", "model": "gpt-4.1"}, ], }, } ] ) path = _write_tmp_manifest(tmp_dir, d) m = _load_tmp_manifest(path) stages = [b.stage for b in m.matrix[0].iop.expected_bindings] self.assertEqual(stages, ["selector", "plan", "work", "review", "repair"]) class TestMatrixExtension(unittest.TestCase): """Data-only matrix extension tests.""" def test_data_only_matrix_extension(self): """Adding a new cell to the matrix does not require code changes.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict( matrix=[ { "id": "cell-a", "caller": "claude", "iop": { "request_model": "claude-sonnet-4-20250514", "requested_effort": "high", "route_kind": "direct", "route_id": "claude-direct", "expected_bindings": [ {"stage": "request", "model": "claude-sonnet-4-20250514"} ], }, }, { "id": "cell-b", "caller": "agy", "iop": { "request_model": "gemini-2.0-flash", "requested_effort": "high", "route_kind": "execution_preset", "route_id": "agy-generic", "expected_bindings": [ {"stage": "selector", "model": "gemini-2.0-flash"}, {"stage": "plan", "model": "gemini-2.0-flash"}, {"stage": "work", "model": "gemini-2.0-flash"}, {"stage": "review", "model": "gemini-2.0-flash"}, ], }, }, { "id": "cell-c", "caller": "codex", "iop": { "request_model": "gpt-4.1", "requested_effort": "xhigh", "route_kind": "execution_preset", "route_id": "codex-generic", "expected_bindings": [ {"stage": "selector", "model": "gpt-4.1"}, {"stage": "plan", "model": "gpt-4.1"}, {"stage": "work", "model": "gpt-4.1"}, {"stage": "review", "model": "gpt-4.1"}, ], }, }, ] ) path = _write_tmp_manifest(tmp_dir, d) m = _load_tmp_manifest(path) self.assertEqual(len(m.matrix), 3) # Sorted by id ids = [c.id for c in m.matrix] self.assertEqual(ids, sorted(ids)) self.assertEqual(ids, ["cell-a", "cell-b", "cell-c"]) class TestDeterministicOrdering(unittest.TestCase): """Deterministic cell and binding ordering tests.""" def test_cells_sorted_by_id(self): """Cells are sorted by id regardless of input order.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict( matrix=[ { "id": "z-cell", "caller": "claude", "iop": { "request_model": "claude-sonnet-4-20250514", "requested_effort": "high", "route_kind": "direct", "route_id": "claude-direct", "expected_bindings": [ {"stage": "request", "model": "claude-sonnet-4-20250514"} ], }, }, { "id": "a-cell", "caller": "agy", "iop": { "request_model": "gemini-2.0-flash", "requested_effort": "high", "route_kind": "execution_preset", "route_id": "agy-generic", "expected_bindings": [ {"stage": "work", "model": "gemini-2.0-flash"}, {"stage": "plan", "model": "gemini-2.0-flash"}, {"stage": "review", "model": "gemini-2.0-flash"}, {"stage": "selector", "model": "gemini-2.0-flash"}, ], }, }, ] ) path = _write_tmp_manifest(tmp_dir, d) m = _load_tmp_manifest(path) ids = [c.id for c in m.matrix] self.assertEqual(ids, ["a-cell", "z-cell"]) def test_bindings_sorted_by_canonical_rank(self): """Bindings are sorted by fixed stage rank, not lexical order.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict( matrix=[ { "id": "rank-test", "caller": "agy", "iop": { "request_model": "gemini-2.0-flash", "requested_effort": "high", "route_kind": "execution_preset", "route_id": "agy-generic", "expected_bindings": [ {"stage": "review", "model": "gemini-2.0-flash"}, {"stage": "work", "model": "gemini-2.0-flash"}, {"stage": "selector", "model": "gemini-2.0-flash"}, {"stage": "plan", "model": "gemini-2.0-flash"}, ], }, } ] ) path = _write_tmp_manifest(tmp_dir, d) m = _load_tmp_manifest(path) stages = [b.stage for b in m.matrix[0].iop.expected_bindings] self.assertEqual(stages, ["selector", "plan", "work", "review"]) def test_canonical_rank_full_order(self): """Full canonical rank order for preset: selector, plan, work, review, repair.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict( matrix=[ { "id": "full-rank", "caller": "agy", "iop": { "request_model": "gemini-2.0-flash", "requested_effort": "high", "route_kind": "execution_preset", "route_id": "agy-generic", "expected_bindings": [ {"stage": "repair", "model": "gemini-2.0-flash"}, {"stage": "review", "model": "gemini-2.0-flash"}, {"stage": "plan", "model": "gemini-2.0-flash"}, {"stage": "work", "model": "gemini-2.0-flash"}, {"stage": "selector", "model": "gemini-2.0-flash"}, ], }, } ] ) path = _write_tmp_manifest(tmp_dir, d) m = _load_tmp_manifest(path) stages = [b.stage for b in m.matrix[0].iop.expected_bindings] self.assertEqual( stages, ["selector", "plan", "work", "review", "repair"] ) class TestDuplicateDetection(unittest.TestCase): """Duplicate id and stage detection tests.""" def test_duplicate_cell_ids_rejected(self): """Two cells with the same id are rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict( matrix=[ { "id": "dup", "caller": "claude", "iop": { "request_model": "claude-sonnet-4-20250514", "requested_effort": "high", "route_kind": "direct", "route_id": "r1", "expected_bindings": [ {"stage": "request", "model": "claude-sonnet-4-20250514"} ], }, }, { "id": "dup", "caller": "agy", "iop": { "request_model": "gemini-2.0-flash", "requested_effort": "high", "route_kind": "execution_preset", "route_id": "r2", "expected_bindings": [ {"stage": "selector", "model": "gemini-2.0-flash"}, {"stage": "plan", "model": "gemini-2.0-flash"}, {"stage": "work", "model": "gemini-2.0-flash"}, {"stage": "review", "model": "gemini-2.0-flash"}, ], }, }, ] ) path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestValidationError): _load_tmp_manifest(path) def test_duplicate_binding_stages_rejected(self): """Two bindings with the same stage in one cell are rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict( matrix=[ { "id": "dup-stage", "caller": "agy", "iop": { "request_model": "gemini-2.0-flash", "requested_effort": "high", "route_kind": "execution_preset", "route_id": "r1", "expected_bindings": [ {"stage": "work", "model": "gemini-2.0-flash"}, {"stage": "work", "model": "gemini-2.0-flash"}, ], }, } ] ) path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestValidationError): _load_tmp_manifest(path) def test_duplicate_viewport_ids_rejected(self): """Two viewports with the same id are rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict( viewports=[ {"id": "dup", "width": 1920, "height": 1080}, {"id": "dup", "width": 375, "height": 812}, ] ) path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestValidationError): _load_tmp_manifest(path) class TestDirectVsPresetShapes(unittest.TestCase): """Direct vs execution-preset shape validation tests.""" def test_direct_requires_exactly_one_request_binding(self): """Direct route with no bindings is rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict( matrix=[ { "id": "direct-empty", "caller": "claude", "iop": { "request_model": "claude-sonnet-4-20250514", "requested_effort": "high", "route_kind": "direct", "route_id": "r1", "expected_bindings": [], }, } ] ) path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestValidationError): _load_tmp_manifest(path) def test_direct_with_non_request_binding_rejected(self): """Direct route with a non-request binding is rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict( matrix=[ { "id": "direct-wrong", "caller": "claude", "iop": { "request_model": "claude-sonnet-4-20250514", "requested_effort": "high", "route_kind": "direct", "route_id": "r1", "expected_bindings": [ {"stage": "plan", "model": "claude-sonnet-4-20250514"} ], }, } ] ) path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestValidationError): _load_tmp_manifest(path) def test_preset_missing_required_stages_rejected(self): """Execution-preset missing selector/plan/work/review is rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict( matrix=[ { "id": "preset-incomplete", "caller": "agy", "iop": { "request_model": "gemini-2.0-flash", "requested_effort": "high", "route_kind": "execution_preset", "route_id": "r1", "expected_bindings": [ {"stage": "work", "model": "gemini-2.0-flash"}, ], }, } ] ) path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestValidationError): _load_tmp_manifest(path) def test_preset_two_repair_bindings_rejected(self): """Execution-preset with two repair bindings is rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict( matrix=[ { "id": "preset-double-repair", "caller": "agy", "iop": { "request_model": "gemini-2.0-flash", "requested_effort": "high", "route_kind": "execution_preset", "route_id": "r1", "expected_bindings": [ {"stage": "selector", "model": "gemini-2.0-flash"}, {"stage": "plan", "model": "gemini-2.0-flash"}, {"stage": "work", "model": "gemini-2.0-flash"}, {"stage": "review", "model": "gemini-2.0-flash"}, {"stage": "repair", "model": "gemini-2.0-flash"}, {"stage": "repair", "model": "gemini-2.0-flash"}, ], }, } ] ) path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestValidationError): _load_tmp_manifest(path) class TestEnumsAndBounds(unittest.TestCase): """Enum and numeric bound validation tests.""" def test_invalid_caller_rejected(self): """Invalid caller value is rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict( matrix=[ { "id": "bad-caller", "caller": "invalid_caller", "iop": { "request_model": "claude-sonnet-4-20250514", "requested_effort": "high", "route_kind": "direct", "route_id": "r1", "expected_bindings": [ {"stage": "request", "model": "claude-sonnet-4-20250514"} ], }, } ] ) path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestValidationError): _load_tmp_manifest(path) def test_invalid_route_kind_rejected(self): """Invalid route_kind is rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict( matrix=[ { "id": "bad-route", "caller": "claude", "iop": { "request_model": "claude-sonnet-4-20250514", "requested_effort": "high", "route_kind": "invalid_route", "route_id": "r1", "expected_bindings": [ {"stage": "request", "model": "claude-sonnet-4-20250514"} ], }, } ] ) path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestValidationError): _load_tmp_manifest(path) def test_invalid_cell_id_pattern_rejected(self): """Cell id with uppercase is rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict( matrix=[ { "id": "Bad-Id", "caller": "claude", "iop": { "request_model": "claude-sonnet-4-20250514", "requested_effort": "high", "route_kind": "direct", "route_id": "r1", "expected_bindings": [ {"stage": "request", "model": "claude-sonnet-4-20250514"} ], }, } ] ) path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestValidationError): _load_tmp_manifest(path) def test_cell_id_too_long_rejected(self): """Cell id exceeding 64 chars is rejected.""" long_id = "a" * 65 with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict( matrix=[ { "id": long_id, "caller": "claude", "iop": { "request_model": "claude-sonnet-4-20250514", "requested_effort": "high", "route_kind": "direct", "route_id": "r1", "expected_bindings": [ {"stage": "request", "model": "claude-sonnet-4-20250514"} ], }, } ] ) path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestValidationError): _load_tmp_manifest(path) def test_run_seconds_zero_rejected(self): """timeout.run_seconds of 0 is rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict( timeout={ "run_seconds": 0, "idle_seconds": 30, "quiet_seconds": 10, "cleanup_grace_seconds": 5, } ) path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestValidationError): _load_tmp_manifest(path) def test_run_seconds_too_large_rejected(self): """timeout.run_seconds > 86400 is rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict( timeout={ "run_seconds": 86401, "idle_seconds": 30, "quiet_seconds": 10, "cleanup_grace_seconds": 5, } ) path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestValidationError): _load_tmp_manifest(path) def test_idle_seconds_zero_rejected(self): """timeout.idle_seconds of 0 is rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict( timeout={ "run_seconds": 300, "idle_seconds": 0, "quiet_seconds": 10, "cleanup_grace_seconds": 5, } ) path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestValidationError): _load_tmp_manifest(path) def test_quiet_seconds_zero_rejected(self): """timeout.quiet_seconds of 0 is rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict( timeout={ "run_seconds": 300, "idle_seconds": 30, "quiet_seconds": 0, "cleanup_grace_seconds": 5, } ) path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestValidationError): _load_tmp_manifest(path) def test_cleanup_grace_seconds_zero_rejected(self): """timeout.cleanup_grace_seconds of 0 is rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict( timeout={ "run_seconds": 300, "idle_seconds": 30, "quiet_seconds": 10, "cleanup_grace_seconds": 0, } ) path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestValidationError): _load_tmp_manifest(path) def test_repetitions_zero_rejected(self): """repetitions of 0 is rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict(repetitions=0) path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestValidationError): _load_tmp_manifest(path) def test_repetitions_negative_rejected(self): """Negative repetitions is rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict(repetitions=-1) path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestValidationError): _load_tmp_manifest(path) def test_viewport_width_zero_rejected(self): """Viewport width of 0 is rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict( viewports=[{"id": "vp", "width": 0, "height": 1080}] ) path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestValidationError): _load_tmp_manifest(path) def test_viewport_width_too_large_rejected(self): """Viewport width > 8192 is rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict( viewports=[{"id": "vp", "width": 8193, "height": 1080}] ) path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestValidationError): _load_tmp_manifest(path) def test_viewport_height_too_large_rejected(self): """Viewport height > 8192 is rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict( viewports=[{"id": "vp", "width": 1920, "height": 8193}] ) path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestValidationError): _load_tmp_manifest(path) def test_empty_viewports_rejected(self): """Empty viewports array is rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict(viewports=[]) path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestValidationError): _load_tmp_manifest(path) def test_invalid_pipeline_version_rejected(self): """Invalid pipeline_version is rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict(pipeline_version="1") path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestValidationError): _load_tmp_manifest(path) def test_invalid_environment_rejected(self): """Invalid environment is rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict(environment="prod") path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestValidationError): _load_tmp_manifest(path) def test_invalid_session_policy_rejected(self): """Invalid session_policy is rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict(session_policy="persistent") path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestValidationError): _load_tmp_manifest(path) def test_invalid_setup_cache_policy_rejected(self): """Invalid setup_cache_policy is rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict(setup_cache_policy="shared") path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestValidationError): _load_tmp_manifest(path) def test_invalid_rubric_version_rejected(self): """Invalid rubric_version pattern is rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict(rubric_version="Invalid Version!") path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestValidationError): _load_tmp_manifest(path) class TestUnknownMembers(unittest.TestCase): """Unknown member rejection tests.""" def test_unknown_top_level_field_rejected(self): """Unknown top-level field is rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict() d["unknown_field"] = "should_fail" path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestValidationError): _load_tmp_manifest(path) def test_unknown_timeout_field_rejected(self): """Unknown timeout field is rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict( timeout={ "run_seconds": 300, "idle_seconds": 30, "quiet_seconds": 10, "cleanup_grace_seconds": 5, "extra_field": 42, } ) path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestValidationError): _load_tmp_manifest(path) def test_unknown_fixture_field_rejected(self): """Unknown fixture field is rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict() d["fixture"]["extra_fixture_field"] = "should_fail" path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestValidationError): _load_tmp_manifest(path) def test_unknown_cell_field_rejected(self): """Unknown cell field is rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict() d["matrix"][0]["extra_cell_field"] = "should_fail" path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestValidationError): _load_tmp_manifest(path) def test_unknown_iop_field_rejected(self): """Unknown iop field is rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict() d["matrix"][0]["iop"]["extra_iop_field"] = "should_fail" path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestValidationError): _load_tmp_manifest(path) def test_unknown_binding_field_rejected(self): """Unknown binding field is rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict() d["matrix"][0]["iop"]["expected_bindings"][0]["extra_binding"] = "should_fail" path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestValidationError): _load_tmp_manifest(path) def test_unknown_viewport_field_rejected(self): """Unknown viewport field is rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict( viewports=[{"id": "vp", "width": 1920, "height": 1080, "extra": True}] ) path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestValidationError): _load_tmp_manifest(path) def test_unknown_asset_field_rejected(self): """Unknown asset field is rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict() d["fixture"]["assets"][0]["extra_asset"] = "should_fail" path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestValidationError): _load_tmp_manifest(path) class TestPathRules(unittest.TestCase): """Path escape, symlink, collision, and containment tests.""" def test_absolute_prompt_path_rejected(self): """Absolute prompt path is rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict() d["fixture"]["prompt"] = "/absolute/path/prompt.md" path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestPathError): _load_tmp_manifest(path) def test_absolute_asset_source_rejected(self): """Absolute asset source path is rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict() d["fixture"]["assets"][0]["source"] = "/absolute/source.txt" path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestPathError): _load_tmp_manifest(path) def test_absolute_workspace_path_rejected(self): """Absolute workspace_path is rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict() d["fixture"]["assets"][0]["workspace_path"] = "/absolute/workspace.txt" path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestPathError): _load_tmp_manifest(path) def test_dotdot_escape_in_prompt_rejected(self): """Prompt path with .. escape is rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict() d["fixture"]["prompt"] = "../escape/prompt.md" path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestPathError): _load_tmp_manifest(path) def test_dotdot_escape_in_asset_source_rejected(self): """Asset source with .. escape is rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict() d["fixture"]["assets"][0]["source"] = "../escape/source.txt" path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestPathError): _load_tmp_manifest(path) def test_dotdot_escape_in_workspace_path_rejected(self): """Asset workspace_path with .. escape is rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict() d["fixture"]["assets"][0]["workspace_path"] = "../escape/workspace.txt" path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestPathError): _load_tmp_manifest(path) def test_colon_in_path_rejected(self): """Path with colon is rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict() d["fixture"]["prompt"] = "path:with:colons.md" path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestPathError): _load_tmp_manifest(path) def test_symlink_source_rejected(self): """Symlink as asset source is rejected.""" with tempfile.TemporaryDirectory(dir=_REPO_ROOT) as tmp_sub: tmp_sub_dir = Path(tmp_sub) rel_sub = tmp_sub_dir.relative_to(_REPO_ROOT) real_file = tmp_sub_dir / "real.txt" real_file.write_text("content") link_file = tmp_sub_dir / "link.txt" link_file.symlink_to(real_file) with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict() d["fixture"]["assets"] = [ { "source": str(rel_sub / "link.txt"), "workspace_path": "workspace/link_dest.txt", } ] manifest_path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestPathError): _load_tmp_manifest(manifest_path) def test_destination_collision_rejected(self): """Two assets with the same workspace_path are rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict() d["fixture"]["assets"] = [ { "source": "scripts/fixtures/agent-comparison-benchmark/prompt.md", "workspace_path": "workspace/dup.txt", }, { "source": "scripts/fixtures/agent-comparison-benchmark/reference.txt", "workspace_path": "workspace/dup.txt", }, ] path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestPathError): _load_tmp_manifest(path) def test_output_root_not_under_runs_rejected(self): """output_root not under agent-test/runs/ is rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict() d["output_root"] = "other/path/bench-01" path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestValidationError): _load_tmp_manifest(path) def test_output_root_with_subpath_rejected(self): """output_root with sub-path segments is rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict() d["output_root"] = "agent-test/runs/sub/path" path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestValidationError): _load_tmp_manifest(path) def test_testbed_pattern_rejected(self): r"""testbed not matching ^\.\./[^/]+$ is rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict() d["testbed"] = "../../double-escape" path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestValidationError): _load_tmp_manifest(path) class TestChecksumAndDigest(unittest.TestCase): """Checksum and digest drift tests.""" def test_wrong_fixture_checksum_rejected(self): """Wrong fixture checksum is rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict() d["fixture"]["checksum"] = "sha256:" + "0" * 64 p = tmp_dir / "manifest.json" p.write_text(json.dumps(d, indent=2), encoding="utf-8") with self.assertRaises(ManifestDigestError): _load_tmp_manifest(p) def test_computed_checksum_matches(self): """Computed checksum equals declared checksum for valid manifest.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict() path = _write_tmp_manifest(tmp_dir, d) m = _load_tmp_manifest(path) expected = digest_workspace_inputs(m.fixture.assets) self.assertEqual(m.fixture.checksum, expected) def test_manifest_digest_computed(self): """Manifest digest is computed deterministically.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict() path = _write_tmp_manifest(tmp_dir, d) m = _load_tmp_manifest(path) digest = digest_manifest_and_resolved_inputs(m) self.assertTrue(digest.startswith("sha256:")) self.assertEqual(len(digest), 7 + 64) def test_manifest_digest_deterministic(self): """Same manifest produces the same digest on repeated calls.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict() path = _write_tmp_manifest(tmp_dir, d) m = _load_tmp_manifest(path) d1 = digest_manifest_and_resolved_inputs(m) d2 = digest_manifest_and_resolved_inputs(m) self.assertEqual(d1, d2) class TestSecretRedaction(unittest.TestCase): """Secret and private-endpoint redaction in error messages.""" def test_secret_not_in_validation_error(self): """Secret values do not appear in validation errors.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict() d["pipeline_version"] = _SENTINEL_SECRET path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestValidationError) as caught: _load_tmp_manifest(path) self.assertNotIn(_SENTINEL_SECRET, str(caught.exception)) def test_secret_not_in_path_error(self): """Secret values do not appear in path errors.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict() d["fixture"]["prompt"] = f"nonexistent/{_SENTINEL_SECRET}/prompt.md" path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestPathError) as caught: _load_tmp_manifest(path) self.assertNotIn(_SENTINEL_SECRET, str(caught.exception)) def test_secret_not_in_digest_error(self): """Secret values do not appear in digest errors.""" with tempfile.TemporaryDirectory(dir=_REPO_ROOT) as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict() d["output_root"] = f"agent-test/runs/{_SENTINEL_SECRET}" d["fixture"]["checksum"] = "sha256:" + "0" * 64 p = tmp_dir / "manifest.json" p.write_text(json.dumps(d, indent=2), encoding="utf-8") with self.assertRaises(ManifestDigestError) as caught: _load_tmp_manifest(p) self.assertNotIn(_SENTINEL_SECRET, str(caught.exception)) def test_prompt_content_not_in_any_error(self): """Prompt content does not appear in any error.""" with tempfile.TemporaryDirectory(dir=_REPO_ROOT) as tmp: tmp_dir = Path(tmp) prompt_path = tmp_dir / "prompt.md" prompt_path.write_text(_SENTINEL_PROMPT, encoding="utf-8") d = _make_minimal_manifest_dict() d["fixture"]["prompt"] = prompt_path.relative_to(_REPO_ROOT).as_posix() d["fixture"]["checksum"] = "sha256:" + "0" * 64 manifest_path = tmp_dir / "manifest.json" manifest_path.write_text(json.dumps(d, indent=2), encoding="utf-8") with self.assertRaises(ManifestDigestError) as caught: _load_tmp_manifest(manifest_path) self.assertNotIn(_SENTINEL_PROMPT, str(caught.exception)) class TestSchemaLoaderParity(unittest.TestCase): """Schema and loader parity tests for types, bounds, enums, and grammar.""" def test_tracked_example_parity(self): """Tracked example loads cleanly.""" example_path = ( _REPO_ROOT / "scripts" / "fixtures" / "agent-comparison-benchmark-manifest.example.json" ) m = load_manifest(example_path) self.assertEqual(m.pipeline_version, "2") self.assertEqual(m.testbed, "../iop-s2") def test_tracked_fixtures_separate_generic_contract_from_direct_preflight(self): fixtures = _REPO_ROOT / "scripts" / "fixtures" generic = load_manifest( fixtures / "agent-comparison-benchmark-manifest.example.json" ) direct = load_manifest( fixtures / "agent-comparison-benchmark-direct-preflight.example.json" ) self.assertEqual( [(cell.caller, cell.iop.route_kind) for cell in generic.matrix], [ ("agy", "execution_preset"), ("claude", "execution_preset"), ("codex", "execution_preset"), ], ) self.assertEqual(len(direct.matrix), 5) self.assertTrue(all(cell.iop.route_kind == "direct" for cell in direct.matrix)) self.assertEqual( [ (cell.caller, cell.iop.request_model, cell.iop.requested_effort) for cell in direct.matrix ], [ ("agy", "gemini-3.6-flash", "high"), ("claude", "gemini-3.6-flash", "high"), ("claude", "gpt-5.6-luna", "xhigh"), ("claude", "claude-sonnet-5", "max"), ("codex", "gpt-5.6-luna", "xhigh"), ], ) public_aliases = { cell.iop.request_model for cell in generic.matrix + direct.matrix } self.assertEqual( public_aliases, {"claude-sonnet-5", "gemini-3.6-flash", "gpt-5.6-luna"}, ) def test_booleans_rejected_in_numeric_fields(self): """Booleans in numeric fields raise ManifestValidationError.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) for field_name in ("run_seconds", "idle_seconds", "quiet_seconds", "cleanup_grace_seconds"): d = _make_minimal_manifest_dict() d["timeout"][field_name] = True p = _write_tmp_manifest(tmp_dir, d, f"{field_name}_bool.json") with self.assertRaises(ManifestValidationError): _load_tmp_manifest(p) d = _make_minimal_manifest_dict(repetitions=True) p = _write_tmp_manifest(tmp_dir, d, "repetitions_bool.json") with self.assertRaises(ManifestValidationError): _load_tmp_manifest(p) d = _make_minimal_manifest_dict( viewports=[{"id": "vp", "width": True, "height": 1080}] ) p = _write_tmp_manifest(tmp_dir, d, "width_bool.json") with self.assertRaises(ManifestValidationError): _load_tmp_manifest(p) def test_preset_with_request_stage_rejected(self): """Execution preset cell with extra request stage raises ManifestValidationError.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict( matrix=[ { "id": "bad-preset", "caller": "agy", "iop": { "request_model": "gemini-2.0-flash", "requested_effort": "high", "route_kind": "execution_preset", "route_id": "agy-generic", "expected_bindings": [ {"stage": "request", "model": "gemini-2.0-flash"}, {"stage": "selector", "model": "gemini-2.0-flash"}, {"stage": "plan", "model": "gemini-2.0-flash"}, {"stage": "work", "model": "gemini-2.0-flash"}, {"stage": "review", "model": "gemini-2.0-flash"}, ], }, } ] ) p = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestValidationError): _load_tmp_manifest(p) def test_testbed_must_be_exact(self): """Testbed other than ../iop-s2 raises ManifestValidationError.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict(testbed="../another-repo") p = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestValidationError): _load_tmp_manifest(p) def test_dotted_tokens_accepted(self): """Tokens with dots like v1.0 and gemini-2.0-flash load without error.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict(rubric_version="landing-quality-v1") d["matrix"][0]["iop"]["request_model"] = "claude-sonnet-4-20250514" p = _write_tmp_manifest(tmp_dir, d) m = _load_tmp_manifest(p) self.assertEqual(m.rubric_version, "landing-quality-v1") self.assertEqual(m.matrix[0].iop.request_model, "claude-sonnet-4-20250514") def _evaluate_schema_execution_preset_bindings( self, schema_path: Path, expected_bindings: list[dict] ) -> bool: """Evaluate candidate expected_bindings against declared execution_preset schema constraints.""" raw_schema = json.loads(schema_path.read_text(encoding="utf-8")) iop_cell_schema = raw_schema["$defs"]["iop_cell"] preset_branch = None for cond in iop_cell_schema.get("allOf", []): if cond.get("if", {}).get("properties", {}).get("route_kind", {}).get("const") == "execution_preset": preset_branch = cond.get("then", {}).get("properties", {}).get("expected_bindings", {}) break if not preset_branch: return False min_items = preset_branch.get("minItems", 0) max_items = preset_branch.get("maxItems", float("inf")) if not (min_items <= len(expected_bindings) <= max_items): return False allowed_enum = preset_branch.get("items", {}).get("properties", {}).get("stage", {}).get("enum", []) for binding in expected_bindings: if not isinstance(binding, dict) or "stage" not in binding or binding["stage"] not in allowed_enum: return False all_of = preset_branch.get("allOf", []) for sub in all_of: contains = sub.get("contains", {}) target_stage = contains.get("properties", {}).get("stage", {}).get("const") min_c = sub.get("minContains", 0) max_c = sub.get("maxContains", float("inf")) matches = sum(1 for b in expected_bindings if isinstance(b, dict) and b.get("stage") == target_stage) if not (min_c <= matches <= max_c): return False return True def test_schema_and_loader_share_route_shape_corpus(self): """Schema-backed evaluator and loader agree on all valid and malformed route shapes.""" schema_path = ( _REPO_ROOT / "scripts" / "fixtures" / "agent-comparison-benchmark-manifest.schema.json" ) example_path = ( _REPO_ROOT / "scripts" / "fixtures" / "agent-comparison-benchmark-manifest.example.json" ) # 1. Tracked example preset cells example_raw = json.loads(example_path.read_text(encoding="utf-8")) for cell in example_raw["matrix"]: iop = cell["iop"] if iop["route_kind"] == "execution_preset": bindings = iop["expected_bindings"] self.assertTrue(self._evaluate_schema_execution_preset_bindings(schema_path, bindings)) # 2. Valid four-stage and five-stage presets valid_4 = [ {"stage": "selector", "model": "m1"}, {"stage": "plan", "model": "m1"}, {"stage": "work", "model": "m1"}, {"stage": "review", "model": "m1"}, ] valid_5 = [ {"stage": "selector", "model": "m1"}, {"stage": "plan", "model": "m1"}, {"stage": "work", "model": "m1"}, {"stage": "review", "model": "m1"}, {"stage": "repair", "model": "m1"}, ] for valid_b in (valid_4, valid_5): self.assertTrue(self._evaluate_schema_execution_preset_bindings(schema_path, valid_b)) with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict( matrix=[ { "id": "valid-preset", "caller": "agy", "iop": { "request_model": "gemini-2.0-flash", "requested_effort": "high", "route_kind": "execution_preset", "route_id": "r1", "expected_bindings": valid_b, }, } ] ) path = _write_tmp_manifest(tmp_dir, d) m = _load_tmp_manifest(path) self.assertEqual(len(m.matrix[0].iop.expected_bindings), len(valid_b)) # 3. Malformed preset stage variants malformed_corpus = [ # missing selector [{"stage": "plan", "model": "m1"}, {"stage": "work", "model": "m1"}, {"stage": "review", "model": "m1"}, {"stage": "repair", "model": "m1"}], # missing plan [{"stage": "selector", "model": "m1"}, {"stage": "work", "model": "m1"}, {"stage": "review", "model": "m1"}, {"stage": "repair", "model": "m1"}], # missing work [{"stage": "selector", "model": "m1"}, {"stage": "plan", "model": "m1"}, {"stage": "review", "model": "m1"}, {"stage": "repair", "model": "m1"}], # missing review [{"stage": "selector", "model": "m1"}, {"stage": "plan", "model": "m1"}, {"stage": "work", "model": "m1"}, {"stage": "repair", "model": "m1"}], # duplicate repair [{"stage": "selector", "model": "m1"}, {"stage": "plan", "model": "m1"}, {"stage": "work", "model": "m1"}, {"stage": "review", "model": "m1"}, {"stage": "repair", "model": "m1"}, {"stage": "repair", "model": "m2"}], # duplicate work [{"stage": "selector", "model": "m1"}, {"stage": "plan", "model": "m1"}, {"stage": "work", "model": "m1"}, {"stage": "work", "model": "m2"}], # extra request stage [{"stage": "request", "model": "m1"}, {"stage": "selector", "model": "m1"}, {"stage": "plan", "model": "m1"}, {"stage": "work", "model": "m1"}, {"stage": "review", "model": "m1"}], ] for bad_b in malformed_corpus: self.assertFalse(self._evaluate_schema_execution_preset_bindings(schema_path, bad_b)) with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict( matrix=[ { "id": "bad-preset", "caller": "agy", "iop": { "request_model": "gemini-2.0-flash", "requested_effort": "high", "route_kind": "execution_preset", "route_id": "r1", "expected_bindings": bad_b, }, } ] ) path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestValidationError): _load_tmp_manifest(path) class TestCanonicalPaths(unittest.TestCase): """Canonical path normalization, containment, and collision tests.""" def test_non_normal_asset_source_rejected(self): """Asset source with ./ is rejected as non-canonical.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict() d["fixture"]["assets"][0]["source"] = ( "scripts/fixtures/agent-comparison-benchmark/./reference.txt" ) p = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestPathError): _load_tmp_manifest(p) def test_non_normal_workspace_path_rejected(self): """Asset workspace_path with ./ is rejected as non-canonical.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict() d["fixture"]["assets"][0]["workspace_path"] = "workspace/./reference.txt" p = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestPathError): _load_tmp_manifest(p) def test_output_root_containment_and_normalization(self): """output_root escaping agent-test/runs via .. or non-normal segment is rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict() d["output_root"] = "agent-test/runs/../escape" p = _write_tmp_manifest(tmp_dir, d) with self.assertRaises((ManifestValidationError, ManifestPathError)): _load_tmp_manifest(p) class TestCanonicalDigestAPI(unittest.TestCase): """Public digest exposure, content immutability, and drift tests.""" def test_loaded_manifest_digest_property(self): """Manifest object exposes digest property matching sha256: format.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) p = _write_tmp_manifest(tmp_dir, _make_minimal_manifest_dict()) m = _load_tmp_manifest(p) self.assertTrue(hasattr(m, "digest")) self.assertTrue(m.digest.startswith("sha256:")) self.assertEqual(len(m.digest), 7 + 64) def test_digest_helpers_match_loaded_manifest(self): """digest helpers reproduce loaded checksum and digest.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) p = _write_tmp_manifest(tmp_dir, _make_minimal_manifest_dict()) m = _load_tmp_manifest(p) self.assertEqual(digest_workspace_inputs(m.fixture.assets), m.fixture.checksum) self.assertEqual(digest_manifest_and_resolved_inputs(m), m.digest) def test_repr_omits_content_bytes(self): """repr of Manifest, Fixture, AssetMapping does not include raw prompt/asset bytes.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) p = _write_tmp_manifest(tmp_dir, _make_minimal_manifest_dict()) m = _load_tmp_manifest(p) r_manifest = repr(m) r_fixture = repr(m.fixture) r_asset = repr(m.fixture.assets[0]) self.assertNotIn(_SENTINEL_PROMPT, r_manifest) self.assertNotIn(_SENTINEL_PROMPT, r_fixture) self.assertNotIn(_SENTINEL_PROMPT, r_asset) self.assertNotIn("content=", r_asset) def test_digest_signatures_exact(self): """digest helpers reject legacy override arguments.""" sig_ws = inspect.signature(digest_workspace_inputs) self.assertEqual(list(sig_ws.parameters.keys()), ["assets"]) sig_manifest = inspect.signature(digest_manifest_and_resolved_inputs) self.assertEqual(list(sig_manifest.parameters.keys()), ["manifest"]) with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) p = _write_tmp_manifest(tmp_dir, _make_minimal_manifest_dict()) m = _load_tmp_manifest(p) with self.assertRaises(TypeError): digest_workspace_inputs(m.fixture.assets, read_content=True) # type: ignore with self.assertRaises(TypeError): digest_manifest_and_resolved_inputs(m, prompt_content=b"test") # type: ignore def test_asset_input_order_equivalence_and_canonicalization(self): """Assets passed in different order produce identical sorted assets, checksum, and digest.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d_order1 = _make_minimal_manifest_dict() d_order1["fixture"]["assets"] = [ {"source": "scripts/fixtures/agent-comparison-benchmark/reference.txt", "workspace_path": "workspace/z_ref.txt"}, {"source": "scripts/fixtures/agent-comparison-benchmark/prompt.md", "workspace_path": "workspace/a_prompt.md"}, ] d_order2 = _make_minimal_manifest_dict() d_order2["fixture"]["assets"] = [ {"source": "scripts/fixtures/agent-comparison-benchmark/prompt.md", "workspace_path": "workspace/a_prompt.md"}, {"source": "scripts/fixtures/agent-comparison-benchmark/reference.txt", "workspace_path": "workspace/z_ref.txt"}, ] p1 = _write_tmp_manifest(tmp_dir, d_order1, "order1.json") p2 = _write_tmp_manifest(tmp_dir, d_order2, "order2.json") m1 = _load_tmp_manifest(p1) m2 = _load_tmp_manifest(p2) self.assertEqual(m1.fixture.assets[0].workspace_path, "workspace/a_prompt.md") self.assertEqual(m1.fixture.assets[1].workspace_path, "workspace/z_ref.txt") self.assertEqual(m1.fixture.assets, m2.fixture.assets) self.assertEqual(m1.fixture.checksum, m2.fixture.checksum) self.assertEqual(m1.digest, m2.digest) def test_input_drift_changes_digest(self): """Altering manifest, prompt content, asset path, or asset content changes m.digest.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d1 = _make_minimal_manifest_dict() p1 = _write_tmp_manifest(tmp_dir, d1, "m1.json") m1 = _load_tmp_manifest(p1) # A valid evaluator binding drift remains loadable and changes the # digest even though the rubric revision itself is closed. d2 = _make_minimal_manifest_dict() d2["evaluator"]["iop"]["route_id"] = "gpt-5.6-luna-alt" p2 = _write_tmp_manifest(tmp_dir, d2, "m2.json") m2 = _load_tmp_manifest(p2) self.assertNotEqual(m1.digest, m2.digest) # Prompt content drift prompt_changed = replace( m1, fixture=replace(m1.fixture, prompt_content=m1.fixture.prompt_content + b" changed"), ) self.assertNotEqual(digest_manifest_and_resolved_inputs(prompt_changed), m1.digest) # Asset source drift asset0 = m1.fixture.assets[0] source_changed = replace( m1, fixture=replace( m1.fixture, assets=(replace(asset0, source="changed/source.txt"),) + m1.fixture.assets[1:], ), ) self.assertNotEqual(digest_manifest_and_resolved_inputs(source_changed), m1.digest) # Asset workspace_path drift d3 = _make_minimal_manifest_dict() d3["fixture"]["assets"][0]["workspace_path"] = "workspace/other_ref.txt" p3 = _write_tmp_manifest(tmp_dir, d3, "m3.json") m3 = _load_tmp_manifest(p3) self.assertNotEqual(m1.fixture.checksum, m3.fixture.checksum) self.assertNotEqual(m1.digest, m3.digest) dest_changed = replace( m1, fixture=replace( m1.fixture, assets=(replace(asset0, workspace_path="workspace/other_ref.txt"),) + m1.fixture.assets[1:], ), ) self.assertNotEqual(digest_manifest_and_resolved_inputs(dest_changed), m1.digest) # Asset content drift content_changed = replace( m1, fixture=replace( m1.fixture, assets=(replace(asset0, content=b"changed asset bytes"),) + m1.fixture.assets[1:], ), ) self.assertNotEqual(digest_manifest_and_resolved_inputs(content_changed), m1.digest) asset_a = AssetMapping(source="src.txt", workspace_path="w.txt", content=b"content A") asset_b = AssetMapping(source="src.txt", workspace_path="w.txt", content=b"content B") self.assertNotEqual(digest_workspace_inputs([asset_a]), digest_workspace_inputs([asset_b])) class TestCLI(unittest.TestCase): """Public CLI tests.""" def _run_cli(self, *args: str) -> subprocess.CompletedProcess: cli = str(_REPO_ROOT / "scripts" / "agent_comparison_benchmark.py") return subprocess.run( [sys.executable, cli, *args], capture_output=True, text=True, cwd=str(_REPO_ROOT), ) def test_cli_validate_valid_manifest(self): """Valid manifest exits 0 with sanitized single success line.""" example = str( _REPO_ROOT / "scripts" / "fixtures" / "agent-comparison-benchmark-manifest.example.json" ) if not Path(example).exists(): self.skipTest("example manifest not found") result = self._run_cli("validate", "--manifest", example) self.assertEqual(result.returncode, 0) stdout_lines = [line for line in result.stdout.splitlines() if line.strip()] stderr_lines = [line for line in result.stderr.splitlines() if line.strip()] self.assertEqual(len(stdout_lines), 1) self.assertEqual(stdout_lines[0], "ok: manifest is valid") self.assertEqual(len(stderr_lines), 0) def test_cli_validate_missing_file(self): """Missing manifest file exits 69 with single sanitized line.""" result = self._run_cli("validate", "--manifest", "/nonexistent/path.json") self.assertEqual(result.returncode, 69) stdout_lines = [line for line in result.stdout.splitlines() if line.strip()] stderr_lines = [line for line in result.stderr.splitlines() if line.strip()] self.assertEqual(len(stdout_lines), 0) self.assertEqual(len(stderr_lines), 1) self.assertIn("error:", stderr_lines[0]) self.assertNotIn("Traceback", result.stderr) def test_cli_validate_malformed_json(self): """Malformed JSON exits 69 with single sanitized line.""" with tempfile.NamedTemporaryFile( mode="w", suffix=".json", delete=False ) as f: f.write("{invalid json") tmp_path = f.name try: result = self._run_cli("validate", "--manifest", tmp_path) self.assertEqual(result.returncode, 69) stdout_lines = [line for line in result.stdout.splitlines() if line.strip()] stderr_lines = [line for line in result.stderr.splitlines() if line.strip()] self.assertEqual(len(stdout_lines), 0) self.assertEqual(len(stderr_lines), 1) self.assertIn("error:", stderr_lines[0]) self.assertNotIn("Traceback", result.stderr) finally: os.unlink(tmp_path) def test_cli_validate_secret_manifest(self): """Manifest with secret values exits 69 without echoing secrets.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict() d["pipeline_version"] = _SENTINEL_SECRET path = _write_tmp_manifest(tmp_dir, d) result = self._run_cli("validate", "--manifest", str(path)) self.assertEqual(result.returncode, 69) self.assertNotIn(_SENTINEL_SECRET, result.stdout) self.assertNotIn(_SENTINEL_SECRET, result.stderr) self.assertNotIn("Traceback", result.stderr) def test_cli_usage_error(self): """Missing subcommand exits 64 with single sanitized line.""" result = self._run_cli() self.assertEqual(result.returncode, 64) stdout_lines = [line for line in result.stdout.splitlines() if line.strip()] stderr_lines = [line for line in result.stderr.splitlines() if line.strip()] self.assertEqual(len(stdout_lines), 0) self.assertEqual(len(stderr_lines), 1) self.assertEqual(stderr_lines[0], "error: invalid usage") def test_cli_validate_no_manifest_flag(self): """Missing --manifest flag exits 64 with single sanitized line.""" result = self._run_cli("validate") self.assertEqual(result.returncode, 64) stdout_lines = [line for line in result.stdout.splitlines() if line.strip()] stderr_lines = [line for line in result.stderr.splitlines() if line.strip()] self.assertEqual(len(stdout_lines), 0) self.assertEqual(len(stderr_lines), 1) self.assertEqual(stderr_lines[0], "error: invalid usage") def test_cli_validate_invalid_utf8(self): """Invalid UTF-8 manifest file exits 69 with single sanitized error line.""" with tempfile.NamedTemporaryFile(mode="wb", suffix=".json", delete=False) as f: f.write(b"\x80\xff\xfe") tmp_path = f.name try: result = self._run_cli("validate", "--manifest", tmp_path) self.assertEqual(result.returncode, 69) stdout_lines = [line for line in result.stdout.splitlines() if line.strip()] stderr_lines = [line for line in result.stderr.splitlines() if line.strip()] self.assertEqual(len(stdout_lines), 0) self.assertEqual(len(stderr_lines), 1) self.assertEqual(stderr_lines[0], "error: invalid JSON format") self.assertNotIn("Traceback", result.stderr) finally: os.unlink(tmp_path) def test_cli_validate_checksum_mismatch(self): """Manifest with checksum mismatch exits 69 with single sanitized error line.""" with tempfile.TemporaryDirectory(dir=_REPO_ROOT) as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict() d["fixture"]["checksum"] = "sha256:" + "0" * 64 p = tmp_dir / "manifest.json" p.write_text(json.dumps(d, indent=2), encoding="utf-8") result = self._run_cli("validate", "--manifest", str(p)) self.assertEqual(result.returncode, 69) stdout_lines = [line for line in result.stdout.splitlines() if line.strip()] stderr_lines = [line for line in result.stderr.splitlines() if line.strip()] self.assertEqual(len(stdout_lines), 0) self.assertEqual(len(stderr_lines), 1) self.assertEqual(stderr_lines[0], "error: fixture.checksum mismatch") self.assertNotIn("Traceback", result.stderr) def test_cli_validate_secret_missing_path(self): """Secret in missing path exits 69 with single sanitized error line without echoing secret.""" with tempfile.TemporaryDirectory(dir=_REPO_ROOT) as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict() d["fixture"]["prompt"] = f"nonexistent/{_SENTINEL_SECRET}/prompt.md" p = tmp_dir / "manifest.json" p.write_text(json.dumps(d, indent=2), encoding="utf-8") result = self._run_cli("validate", "--manifest", str(p)) self.assertEqual(result.returncode, 69) stdout_lines = [line for line in result.stdout.splitlines() if line.strip()] stderr_lines = [line for line in result.stderr.splitlines() if line.strip()] self.assertEqual(stdout_lines, []) self.assertEqual(len(stderr_lines), 1) self.assertNotIn(_SENTINEL_SECRET, result.stdout) self.assertNotIn(_SENTINEL_SECRET, result.stderr) self.assertNotIn("Traceback", result.stderr) def test_cli_validate_secret_unknown_argument(self): """Secret in unknown CLI flag exits 64 without echoing secret.""" example = str( _REPO_ROOT / "scripts" / "fixtures" / "agent-comparison-benchmark-manifest.example.json" ) result = self._run_cli("validate", "--manifest", example, f"--secret={_SENTINEL_SECRET}") self.assertEqual(result.returncode, 64) stdout_lines = [line for line in result.stdout.splitlines() if line.strip()] stderr_lines = [line for line in result.stderr.splitlines() if line.strip()] self.assertEqual(len(stdout_lines), 0) self.assertEqual(len(stderr_lines), 1) self.assertEqual(stderr_lines[0], "error: invalid usage") self.assertNotIn(_SENTINEL_SECRET, result.stdout) self.assertNotIn(_SENTINEL_SECRET, result.stderr) self.assertNotIn("Traceback", result.stderr) class TestValidateManifestBytes(unittest.TestCase): """validate_manifest_bytes tests.""" def test_validate_bytes_valid(self): """Valid bytes validate without disk write.""" d = _make_minimal_manifest_dict() # Compute correct checksum for the fixture d["fixture"]["checksum"] = _compute_fixture_checksum(_REPO_ROOT, d["fixture"]) data = json.dumps(d).encode("utf-8") m = validate_manifest_bytes(data, repo_root=_REPO_ROOT) self.assertEqual(m.pipeline_version, "2") def test_validate_bytes_invalid(self): """Invalid bytes raise error.""" data = b"{invalid" with self.assertRaises(ManifestValidationError): validate_manifest_bytes(data) class TestFrozenReturnTypes(unittest.TestCase): """Return type immutability tests.""" def test_manifest_is_frozen(self): """Manifest is a frozen dataclass.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) path = _write_tmp_manifest(tmp_dir, _make_minimal_manifest_dict()) m = _load_tmp_manifest(path) self.assertTrue(hasattr(m, "__dataclass_fields__")) with self.assertRaises(AttributeError): m.repetitions = 99 def test_timeout_is_frozen(self): """Timeout is frozen.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) path = _write_tmp_manifest(tmp_dir, _make_minimal_manifest_dict()) m = _load_tmp_manifest(path) with self.assertRaises(AttributeError): m.timeout.run_seconds = 999 def test_viewport_is_frozen(self): """Viewport is frozen.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) path = _write_tmp_manifest(tmp_dir, _make_minimal_manifest_dict()) m = _load_tmp_manifest(path) with self.assertRaises(AttributeError): m.viewports[0].width = 9999 def test_cell_is_frozen(self): """MatrixCell is frozen.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) path = _write_tmp_manifest(tmp_dir, _make_minimal_manifest_dict()) m = _load_tmp_manifest(path) with self.assertRaises(AttributeError): m.matrix[0].id = "changed" def test_tuple_fields_are_tuples(self): """tuple fields are actual tuples, not lists.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) path = _write_tmp_manifest(tmp_dir, _make_minimal_manifest_dict()) m = _load_tmp_manifest(path) self.assertIsInstance(m.viewports, tuple) self.assertIsInstance(m.matrix, tuple) self.assertIsInstance(m.fixture.assets, tuple) self.assertIsInstance(m.matrix[0].iop.expected_bindings, tuple) class TestEdgeCases(unittest.TestCase): """Additional edge cases.""" def test_non_object_top_level_rejected(self): """Top-level JSON array is rejected.""" with tempfile.NamedTemporaryFile( mode="w", suffix=".json", delete=False ) as f: f.write("[]") tmp_path = f.name try: with self.assertRaises(ManifestValidationError): _load_tmp_manifest(tmp_path) finally: os.unlink(tmp_path) def test_missing_required_field_rejected(self): """Missing required top-level field is rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict() del d["timeout"] path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestValidationError): _load_tmp_manifest(path) def test_fixture_missing_required_field_rejected(self): """Missing fixture.version is rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict() del d["fixture"]["version"] path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestValidationError): _load_tmp_manifest(path) def test_fixture_missing_prompt_file_rejected(self): """Prompt file that does not exist is rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict() d["fixture"]["prompt"] = "nonexistent_prompt.md" path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestPathError): _load_tmp_manifest(path) def test_fixture_missing_asset_file_rejected(self): """Asset source file that does not exist is rejected.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict() d["fixture"]["assets"] = [ { "source": "nonexistent_source.txt", "workspace_path": "workspace/dst.txt", } ] path = _write_tmp_manifest(tmp_dir, d) with self.assertRaises(ManifestPathError): _load_tmp_manifest(path) def test_multiple_assets_loaded(self): """Manifest with multiple assets loads correctly.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict() d["fixture"]["assets"] = [ { "source": "scripts/fixtures/agent-comparison-benchmark/prompt.md", "workspace_path": "workspace/prompt.md", }, { "source": "scripts/fixtures/agent-comparison-benchmark/reference.txt", "workspace_path": "workspace/reference.txt", }, ] path = _write_tmp_manifest(tmp_dir, d) m = _load_tmp_manifest(path) self.assertEqual(len(m.fixture.assets), 2) def test_file_not_found(self): """Non-existent manifest file raises ManifestValidationError.""" with self.assertRaises(ManifestValidationError): load_manifest("/nonexistent/manifest.json") def test_caller_request_vs_evidence_separation(self): """request_model/requested_effort are separate from route/binding evidence.""" with tempfile.TemporaryDirectory() as tmp: tmp_dir = Path(tmp) d = _make_minimal_manifest_dict( matrix=[ { "id": "separation-test", "caller": "claude", "iop": { "request_model": "claude-sonnet-4-20250514", "requested_effort": "high", "route_kind": "direct", "route_id": "claude-direct", "expected_bindings": [ { "stage": "request", "model": "claude-sonnet-4-20250514", "effort": "high", } ], }, } ] ) path = _write_tmp_manifest(tmp_dir, d) m = _load_tmp_manifest(path) cell = m.matrix[0] # request_model and requested_effort are on iop, not in bindings self.assertEqual(cell.iop.request_model, "claude-sonnet-4-20250514") self.assertEqual(cell.iop.requested_effort, "high") # route_kind/route_id are preflight evidence, not adapter inputs self.assertEqual(cell.iop.route_kind, "direct") self.assertEqual(cell.iop.route_id, "claude-direct") # bindings contain the expected evidence binding = cell.iop.expected_bindings[0] self.assertEqual(binding.stage, "request") self.assertEqual(binding.model, "claude-sonnet-4-20250514") if __name__ == "__main__": unittest.main()