""" Closed manifest loader, validator, and digest calculator. Standard-library-only. Returns frozen dataclasses; never mutable raw dicts. """ from __future__ import annotations import hashlib import json import os import posixpath import re import struct from collections.abc import Iterable from dataclasses import dataclass, field from pathlib import Path from typing import Any, Optional # --------------------------------------------------------------------------- # Constants # --------------------------------------------------------------------------- PIPELINE_VERSION = "2" ENVIRONMENT = "dev" TESTBED_REQUIRED = "../iop-s2" SESSION_POLICY = "fresh" SETUP_CACHE_POLICY = "isolated" DEFAULT_REPETITIONS = 1 RUBRIC_VERSION = "landing-quality-v1" CALLER_ENUM = ("claude", "agy", "codex") ROUTE_KIND_ENUM = ("direct", "execution_preset") STAGE_ENUM = ("request", "selector", "plan", "work", "review", "repair") # Canonical rank for sorting bindings: lower rank = earlier position STAGE_RANK: dict[str, int] = {s: i for i, s in enumerate(STAGE_ENUM)} CELL_ID_RE = re.compile(r"^[a-z0-9][a-z0-9_-]{0,63}$") TOKEN_RE = re.compile(r"^[a-z0-9][a-z0-9_.+-]{0,31}$") VIEWPORT_ID_RE = re.compile(r"^[a-z0-9][a-z0-9_.+-]{0,31}$") CHECKSUM_RE = re.compile(r"^sha256:[0-9a-f]{64}$") WORKSPACE_PREFIX = b"IOP-BENCH-WORKSPACE\x00" MANIFEST_PREFIX = b"IOP-BENCH-MANIFEST\x00" # --------------------------------------------------------------------------- # Frozen data containers # --------------------------------------------------------------------------- @dataclass(frozen=True) class Timeout: run_seconds: int idle_seconds: int quiet_seconds: int cleanup_grace_seconds: int @dataclass(frozen=True) class Viewport: id: str width: int height: int @dataclass(frozen=True) class AssetMapping: source: str workspace_path: str content: bytes = field(default=b"", repr=False) @dataclass(frozen=True) class ExpectedBinding: stage: str model: str effort: Optional[str] = None @dataclass(frozen=True) class IopCell: request_model: str requested_effort: str route_kind: str route_id: str expected_bindings: tuple[ExpectedBinding, ...] @dataclass(frozen=True) class MatrixCell: id: str caller: str iop: IopCell @dataclass(frozen=True) class Evaluator: caller: str iop: IopCell @dataclass(frozen=True) class Fixture: version: str prompt: str assets: tuple[AssetMapping, ...] checksum: str prompt_content: bytes = field(default=b"", repr=False) @dataclass(frozen=True) class Manifest: pipeline_version: str environment: str testbed: str repetitions: int session_policy: str setup_cache_policy: str timeout: Timeout viewports: tuple[Viewport, ...] rubric_version: str evaluator: Evaluator output_root: str fixture: Fixture matrix: tuple[MatrixCell, ...] digest: str # --------------------------------------------------------------------------- # Errors # --------------------------------------------------------------------------- class ManifestError(Exception): """Base error for manifest validation failures.""" class ManifestValidationError(ManifestError): """Raised when the manifest JSON fails schema validation.""" class ManifestPathError(ManifestError): """Raised when a path rule is violated (escape, symlink, collision, etc.).""" class ManifestDigestError(ManifestError): """Raised when a declared digest does not match the computed value.""" # --------------------------------------------------------------------------- # Internal helpers # --------------------------------------------------------------------------- def _require_bool(cond: bool, msg: str) -> None: if not cond: raise ManifestValidationError(msg) def _require_str(value: Any, field_name: str) -> str: if not isinstance(value, str): raise ManifestValidationError(f"field '{field_name}' must be a string") return value def _require_int(value: Any, field_name: str) -> int: if isinstance(value, bool) or not isinstance(value, int): raise ManifestValidationError(f"field '{field_name}' must be an integer") return value def _require_object(value: Any, field_name: str) -> dict[str, Any]: if not isinstance(value, dict): raise ManifestValidationError(f"field '{field_name}' must be an object") return value def _require_array(value: Any, field_name: str) -> list[Any]: if not isinstance(value, list): raise ManifestValidationError(f"field '{field_name}' must be an array") return value def _require_enum(value: Any, field_name: str, allowed: tuple[str, ...]) -> str: s = _require_str(value, field_name) if s not in allowed: raise ManifestValidationError( f"field '{field_name}' must be one of {allowed}" ) return s def _require_pattern(value: Any, field_name: str, pattern: re.Pattern) -> str: s = _require_str(value, field_name) if not pattern.match(s): raise ManifestValidationError( f"field '{field_name}' does not match required pattern" ) return s def _require_positive_int(value: Any, field_name: str) -> int: val = _require_int(value, field_name) if val <= 0: raise ManifestValidationError( f"field '{field_name}' must be a positive integer" ) return val def _require_bounded_int(value: Any, field_name: str, lo: int, hi: int) -> int: val = _require_int(value, field_name) if not (lo <= val <= hi): raise ManifestValidationError( f"field '{field_name}' must be between {lo} and {hi}" ) return val def _require_sha256(value: Any, field_name: str) -> str: s = _require_str(value, field_name) if not CHECKSUM_RE.match(s): raise ManifestValidationError( f"field '{field_name}' must match sha256:<64 hex chars>" ) return s def _normalize_posix_relative_path(path_str: Any, context: str) -> str: """Validate and return a normalized POSIX relative path. Rejects non-string, empty, absolute, colon, backslash, non-normal, or escaping paths ('.', '..', starting with '../'). """ if not isinstance(path_str, str) or not path_str: raise ManifestPathError(f"field '{context}' must be a non-empty string path") if "\\" in path_str or ":" in path_str: raise ManifestPathError(f"field '{context}' contains invalid path characters") if path_str.startswith("/"): raise ManifestPathError(f"field '{context}' must be a relative path") norm = posixpath.normpath(path_str) if path_str != norm: raise ManifestPathError(f"field '{context}' is not in canonical relative form") if norm in (".", "..") or norm.startswith("../"): raise ManifestPathError(f"field '{context}' escapes root or is empty") return norm def _require_regular_file(path: Path, context: str) -> None: """Require that path exists and is a regular file (no symlinks).""" if not path.exists(): raise ManifestPathError(f"field '{context}' target file does not exist") if path.is_symlink(): raise ManifestPathError(f"field '{context}' target must be a regular file, not a symlink") if not path.is_file(): raise ManifestPathError(f"field '{context}' target is not a regular file") # --------------------------------------------------------------------------- # Digest computation # --------------------------------------------------------------------------- def digest_workspace_inputs(assets: Iterable[AssetMapping]) -> str: """Compute the declared fixture checksum from asset workspace paths and content. sha256(b"IOP-BENCH-WORKSPACE\x00" + length-framed assets sorted by workspace_path, each asset framed as: uint64 BE len(workspace_path) + workspace_path UTF-8 bytes + uint64 BE len(file_content) + file_content bytes """ sorted_assets = sorted(assets, key=lambda a: a.workspace_path) data = bytearray(WORKSPACE_PREFIX) for asset in sorted_assets: wp_bytes = asset.workspace_path.encode("utf-8") data += struct.pack(">Q", len(wp_bytes)) + wp_bytes data += struct.pack(">Q", len(asset.content)) + asset.content return "sha256:" + hashlib.sha256(bytes(data)).hexdigest() def digest_manifest_and_resolved_inputs(manifest: Manifest) -> str: """Compute the self-contained manifest digest. sha256(b"IOP-BENCH-MANIFEST\x00" + canonical JSON + length-framed prompt path/content + length-framed every asset source/destination/content). """ prompt_content = manifest.fixture.prompt_content canonical = json.dumps( _manifest_to_dict(manifest), sort_keys=True, separators=(",", ":"), ensure_ascii=False, ).encode("utf-8") data = bytearray(MANIFEST_PREFIX) data += struct.pack(">Q", len(canonical)) + canonical prompt_path_bytes = manifest.fixture.prompt.encode("utf-8") data += struct.pack(">Q", len(prompt_path_bytes)) + prompt_path_bytes data += struct.pack(">Q", len(prompt_content)) + prompt_content for asset in sorted(manifest.fixture.assets, key=lambda a: a.workspace_path): src_bytes = asset.source.encode("utf-8") data += struct.pack(">Q", len(src_bytes)) + src_bytes wp_bytes = asset.workspace_path.encode("utf-8") data += struct.pack(">Q", len(wp_bytes)) + wp_bytes data += struct.pack(">Q", len(asset.content)) + asset.content return "sha256:" + hashlib.sha256(bytes(data)).hexdigest() def _manifest_to_dict(manifest: Manifest) -> dict[str, Any]: """Convert a Manifest to a plain dict for canonical JSON serialization.""" return { "pipeline_version": manifest.pipeline_version, "environment": manifest.environment, "testbed": manifest.testbed, "repetitions": manifest.repetitions, "session_policy": manifest.session_policy, "setup_cache_policy": manifest.setup_cache_policy, "timeout": { "run_seconds": manifest.timeout.run_seconds, "idle_seconds": manifest.timeout.idle_seconds, "quiet_seconds": manifest.timeout.quiet_seconds, "cleanup_grace_seconds": manifest.timeout.cleanup_grace_seconds, }, "viewports": [ {"id": v.id, "width": v.width, "height": v.height} for v in manifest.viewports ], "rubric_version": manifest.rubric_version, "evaluator": { "caller": manifest.evaluator.caller, "iop": { "request_model": manifest.evaluator.iop.request_model, "requested_effort": manifest.evaluator.iop.requested_effort, "route_kind": manifest.evaluator.iop.route_kind, "route_id": manifest.evaluator.iop.route_id, "expected_bindings": [ { "stage": binding.stage, "model": binding.model, **({"effort": binding.effort} if binding.effort else {}), } for binding in manifest.evaluator.iop.expected_bindings ], }, }, "output_root": manifest.output_root, "fixture": { "version": manifest.fixture.version, "prompt": manifest.fixture.prompt, "assets": [ {"source": a.source, "workspace_path": a.workspace_path} for a in manifest.fixture.assets ], "checksum": manifest.fixture.checksum, }, "matrix": [ { "id": c.id, "caller": c.caller, "iop": { "request_model": c.iop.request_model, "requested_effort": c.iop.requested_effort, "route_kind": c.iop.route_kind, "route_id": c.iop.route_id, "expected_bindings": [ { "stage": b.stage, "model": b.model, **({"effort": b.effort} if b.effort else {}), } for b in c.iop.expected_bindings ], }, } for c in manifest.matrix ], } # --------------------------------------------------------------------------- # Validation # --------------------------------------------------------------------------- def _validate_timeout(data: dict[str, Any]) -> Timeout: obj = _require_object(data, "timeout") expected_keys = {"run_seconds", "idle_seconds", "quiet_seconds", "cleanup_grace_seconds"} if set(obj.keys()) != expected_keys: raise ManifestValidationError("timeout has invalid schema") return Timeout( run_seconds=_require_bounded_int(obj["run_seconds"], "timeout.run_seconds", 1, 86400), idle_seconds=_require_bounded_int(obj["idle_seconds"], "timeout.idle_seconds", 1, 600), quiet_seconds=_require_bounded_int(obj["quiet_seconds"], "timeout.quiet_seconds", 1, 60), cleanup_grace_seconds=_require_bounded_int( obj["cleanup_grace_seconds"], "timeout.cleanup_grace_seconds", 1, 60 ), ) def _validate_viewports(data: list[Any]) -> tuple[Viewport, ...]: arr = _require_array(data, "viewports") if len(arr) < 1: raise ManifestValidationError("viewports must have at least 1 item") seen_ids: set[str] = set() viewports: list[Viewport] = [] for i, item in enumerate(arr): obj = _require_object(item, f"viewports[{i}]") expected_keys = {"id", "width", "height"} if set(obj.keys()) != expected_keys: raise ManifestValidationError(f"viewports[{i}] has invalid schema") vid = _require_pattern(obj["id"], f"viewports[{i}].id", VIEWPORT_ID_RE) if vid in seen_ids: raise ManifestValidationError(f"viewports[{i}].id is duplicate") seen_ids.add(vid) w = _require_bounded_int(obj["width"], f"viewports[{i}].width", 1, 8192) h = _require_bounded_int(obj["height"], f"viewports[{i}].height", 1, 8192) viewports.append(Viewport(id=vid, width=w, height=h)) return tuple(viewports) def _validate_output_root(data: Any, repo_root: Path) -> str: s = _require_str(data, "output_root") if "\\" in s or ":" in s: raise ManifestPathError("field 'output_root' contains invalid path characters") norm = posixpath.normpath(s) if s != norm: raise ManifestPathError("field 'output_root' is not in canonical form") if not s.startswith("agent-test/runs/"): raise ManifestValidationError( "field 'output_root' must be relative to agent-test/runs/" ) tail = s[len("agent-test/runs/"):] if not tail or "/" in tail: raise ManifestValidationError( "field 'output_root' must be a single non-empty segment after agent-test/runs/" ) runs_dir = (repo_root / "agent-test" / "runs").resolve() output_path = (repo_root / s).resolve() try: output_path.relative_to(runs_dir) except ValueError: raise ManifestPathError("field 'output_root' resolves outside agent-test/runs") return s def _validate_asset_mapping( data: dict[str, Any], i: int ) -> tuple[str, str]: obj = _require_object(data, f"assets[{i}]") expected_keys = {"source", "workspace_path"} if set(obj.keys()) != expected_keys: raise ManifestValidationError(f"assets[{i}] has invalid schema") source = _normalize_posix_relative_path(obj["source"], f"assets[{i}].source") workspace_path = _normalize_posix_relative_path( obj["workspace_path"], f"assets[{i}].workspace_path" ) return source, workspace_path def _validate_fixture( data: dict[str, Any], repo_root: Path ) -> Fixture: obj = _require_object(data, "fixture") expected_keys = {"version", "prompt", "assets", "checksum"} if set(obj.keys()) != expected_keys: raise ManifestValidationError("fixture has invalid schema") version = _require_pattern(obj["version"], "fixture.version", TOKEN_RE) prompt = _normalize_posix_relative_path(obj["prompt"], "fixture.prompt") assets_raw = _require_array(obj["assets"], "fixture.assets") if len(assets_raw) < 1: raise ManifestValidationError("fixture.assets must have at least 1 item") assets_list: list[AssetMapping] = [] workspace_destinations: set[str] = set() for i, item in enumerate(assets_raw): source, workspace_path = _validate_asset_mapping(item, i) if workspace_path in workspace_destinations: raise ManifestPathError( "fixture.assets workspace_path is duplicate" ) workspace_destinations.add(workspace_path) src_path = repo_root / source _require_regular_file(src_path, f"fixture asset source ({i})") try: src_path.resolve().relative_to(repo_root.resolve()) except ValueError: raise ManifestPathError("fixture asset source resolves outside repo root") content = src_path.read_bytes() assets_list.append(AssetMapping(source=source, workspace_path=workspace_path, content=content)) assets_list.sort(key=lambda a: a.workspace_path) # Validate prompt file prompt_path = repo_root / prompt _require_regular_file(prompt_path, "fixture prompt") try: prompt_path.resolve().relative_to(repo_root.resolve()) except ValueError: raise ManifestPathError("fixture prompt resolves outside repo root") prompt_content = prompt_path.read_bytes() checksum = _require_sha256(obj["checksum"], "fixture.checksum") computed = digest_workspace_inputs(assets_list) if checksum != computed: raise ManifestDigestError("fixture.checksum mismatch") return Fixture( version=version, prompt=prompt, assets=tuple(assets_list), checksum=checksum, prompt_content=prompt_content, ) def _validate_expected_binding( data: dict[str, Any], i: int ) -> ExpectedBinding: obj = _require_object(data, f"expected_bindings[{i}]") expected_keys = {"stage", "model"} if not expected_keys.issubset(set(obj.keys())): raise ManifestValidationError(f"expected_bindings[{i}] missing required keys") extra = set(obj.keys()) - {"stage", "model", "effort"} if extra: raise ManifestValidationError(f"expected_bindings[{i}] has unexpected keys") stage = _require_enum(obj["stage"], f"expected_bindings[{i}].stage", STAGE_ENUM) model = _require_pattern(obj["model"], f"expected_bindings[{i}].model", TOKEN_RE) effort = None if "effort" in obj: effort = _require_pattern(obj["effort"], f"expected_bindings[{i}].effort", TOKEN_RE) return ExpectedBinding(stage=stage, model=model, effort=effort) def _validate_iop_cell(data: dict[str, Any]) -> IopCell: obj = _require_object(data, "iop") expected_keys = {"request_model", "requested_effort", "route_kind", "route_id", "expected_bindings"} if set(obj.keys()) != expected_keys: raise ManifestValidationError("iop has invalid schema") request_model = _require_pattern(obj["request_model"], "iop.request_model", TOKEN_RE) requested_effort = _require_pattern(obj["requested_effort"], "iop.requested_effort", TOKEN_RE) route_kind = _require_enum(obj["route_kind"], "iop.route_kind", ROUTE_KIND_ENUM) route_id = _require_pattern(obj["route_id"], "iop.route_id", TOKEN_RE) bindings_raw = _require_array(obj["expected_bindings"], "iop.expected_bindings") if len(bindings_raw) < 1: raise ManifestValidationError("iop.expected_bindings must have at least 1 item") bindings: list[ExpectedBinding] = [] seen_stages: set[str] = set() for i, item in enumerate(bindings_raw): b = _validate_expected_binding(item, i) if b.stage in seen_stages: raise ManifestValidationError("iop.expected_bindings contains duplicate stage") seen_stages.add(b.stage) bindings.append(b) # Sort bindings by canonical stage rank bindings.sort(key=lambda b: STAGE_RANK[b.stage]) # Validate direct vs execution_preset constraints if route_kind == "direct": if set(b.stage for b in bindings) != {"request"}: raise ManifestValidationError( "direct route requires exactly one binding with stage=request" ) elif route_kind == "execution_preset": required_stages = {"selector", "plan", "work", "review"} allowed_stages = required_stages | {"repair"} actual_stages = set(b.stage for b in bindings) if actual_stages not in (required_stages, allowed_stages): raise ManifestValidationError( "execution_preset has an invalid stage set" ) return IopCell( request_model=request_model, requested_effort=requested_effort, route_kind=route_kind, route_id=route_id, expected_bindings=tuple(bindings), ) def _validate_cell(data: dict[str, Any], i: int) -> MatrixCell: obj = _require_object(data, f"matrix[{i}]") expected_keys = {"id", "caller", "iop"} if set(obj.keys()) != expected_keys: raise ManifestValidationError(f"matrix[{i}] has invalid schema") cell_id = _require_pattern(obj["id"], f"matrix[{i}].id", CELL_ID_RE) caller = _require_enum(obj["caller"], f"matrix[{i}].caller", CALLER_ENUM) iop = _validate_iop_cell(obj["iop"]) return MatrixCell(id=cell_id, caller=caller, iop=iop) def _validate_evaluator(data: dict[str, Any]) -> Evaluator: obj = _require_object(data, "evaluator") if set(obj) != {"caller", "iop"}: raise ManifestValidationError("evaluator has invalid schema") caller = _require_enum(obj["caller"], "evaluator.caller", ("codex",)) return Evaluator(caller=caller, iop=_validate_iop_cell(obj["iop"])) def _validate_matrix(data: list[Any]) -> tuple[MatrixCell, ...]: arr = _require_array(data, "matrix") if len(arr) < 1: raise ManifestValidationError("matrix must have at least 1 item") cells: list[MatrixCell] = [] seen_ids: set[str] = set() for i, item in enumerate(arr): cell = _validate_cell(item, i) if cell.id in seen_ids: raise ManifestValidationError("matrix contains duplicate cell id") seen_ids.add(cell.id) cells.append(cell) # Sort cells by id for canonical ordering cells.sort(key=lambda c: c.id) return tuple(cells) def _default_repo_root(manifest_path: Path) -> Path: """Walk up from manifest_path to find the repository root. Looks for a directory containing ``Makefile`` or ``.git``. Falls back to the manifest file's parent directory. """ candidate = manifest_path.resolve() for _ in range(20): # safety limit if (candidate / "Makefile").is_file() or (candidate / ".git").exists(): return candidate parent = candidate.parent if parent == candidate: break candidate = parent return manifest_path.resolve().parent # --------------------------------------------------------------------------- # Public API # --------------------------------------------------------------------------- def load_manifest(path: str | Path, repo_root: str | Path | None = None) -> Manifest: """Load and validate a benchmark manifest JSON file. Args: path: Path to the manifest JSON file. repo_root: Repository root for resolving relative paths. Returns: A frozen Manifest object with computed digest. Raises: ManifestValidationError: If the JSON fails schema validation. ManifestPathError: If a path rule is violated. ManifestDigestError: If a declared digest does not match. """ path = Path(path) if repo_root is None: repo_root = _default_repo_root(path) repo_root = Path(repo_root).resolve() try: raw = json.loads(path.read_text(encoding="utf-8")) except FileNotFoundError: raise ManifestValidationError("manifest file not found") except (json.JSONDecodeError, UnicodeDecodeError) as exc: raise ManifestValidationError("invalid JSON format") if not isinstance(raw, dict): raise ManifestValidationError("manifest must be a JSON object") # Top-level field validation expected_top = { "pipeline_version", "environment", "testbed", "fixture", "matrix", "session_policy", "setup_cache_policy", "timeout", "viewports", "rubric_version", "evaluator", "output_root", } optional_top = {"repetitions"} declared_keys = set(raw.keys()) required_present = expected_top.issubset(declared_keys) extra = declared_keys - expected_top - optional_top if not required_present: raise ManifestValidationError("manifest missing required top-level fields") if extra: raise ManifestValidationError("manifest has unexpected top-level fields") pipeline_version = _require_enum(raw["pipeline_version"], "pipeline_version", (PIPELINE_VERSION,)) environment = _require_enum(raw["environment"], "environment", (ENVIRONMENT,)) testbed = _require_enum(raw["testbed"], "testbed", (TESTBED_REQUIRED,)) repetitions = DEFAULT_REPETITIONS if "repetitions" in raw: repetitions = _require_positive_int(raw["repetitions"], "repetitions") session_policy = _require_enum(raw["session_policy"], "session_policy", (SESSION_POLICY,)) setup_cache_policy = _require_enum( raw["setup_cache_policy"], "setup_cache_policy", (SETUP_CACHE_POLICY,) ) timeout = _validate_timeout(raw["timeout"]) viewports = _validate_viewports(raw["viewports"]) rubric_version = _require_enum( raw["rubric_version"], "rubric_version", (RUBRIC_VERSION,) ) evaluator = _validate_evaluator(raw["evaluator"]) output_root = _validate_output_root(raw["output_root"], repo_root) fixture = _validate_fixture(raw["fixture"], repo_root) matrix = _validate_matrix(raw["matrix"]) # Compute manifest digest during load dummy_manifest = Manifest( pipeline_version=pipeline_version, environment=environment, testbed=testbed, repetitions=repetitions, session_policy=session_policy, setup_cache_policy=setup_cache_policy, timeout=timeout, viewports=viewports, rubric_version=rubric_version, evaluator=evaluator, output_root=output_root, fixture=fixture, matrix=matrix, digest="", ) computed_digest = digest_manifest_and_resolved_inputs(dummy_manifest) return Manifest( pipeline_version=pipeline_version, environment=environment, testbed=testbed, repetitions=repetitions, session_policy=session_policy, setup_cache_policy=setup_cache_policy, timeout=timeout, viewports=viewports, rubric_version=rubric_version, evaluator=evaluator, output_root=output_root, fixture=fixture, matrix=matrix, digest=computed_digest, ) def validate_manifest_bytes( data: bytes, path_hint: str = "", repo_root: str | Path | None = None, ) -> Manifest: """Validate manifest JSON bytes without writing to persistent disk.""" import tempfile with tempfile.NamedTemporaryFile( mode="wb", suffix=".json", delete=False ) as tmp: tmp.write(data) tmp_path = tmp.name try: return load_manifest(tmp_path, repo_root=repo_root) finally: try: os.unlink(tmp_path) except OSError: pass