774 lines
27 KiB
Python
774 lines
27 KiB
Python
"""
|
|
Closed manifest loader, validator, and digest calculator.
|
|
|
|
Standard-library-only. Returns frozen dataclasses; never mutable raw dicts.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import posixpath
|
|
import re
|
|
import struct
|
|
from collections.abc import Iterable
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from typing import Any, Optional
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Constants
|
|
# ---------------------------------------------------------------------------
|
|
|
|
PIPELINE_VERSION = "2"
|
|
ENVIRONMENT = "dev"
|
|
TESTBED_REQUIRED = "../iop-s2"
|
|
SESSION_POLICY = "fresh"
|
|
SETUP_CACHE_POLICY = "isolated"
|
|
DEFAULT_REPETITIONS = 1
|
|
RUBRIC_VERSION = "landing-quality-v1"
|
|
|
|
CALLER_ENUM = ("claude", "agy", "codex")
|
|
ROUTE_KIND_ENUM = ("direct", "execution_preset")
|
|
STAGE_ENUM = ("request", "selector", "plan", "work", "review", "repair")
|
|
# Canonical rank for sorting bindings: lower rank = earlier position
|
|
STAGE_RANK: dict[str, int] = {s: i for i, s in enumerate(STAGE_ENUM)}
|
|
|
|
CELL_ID_RE = re.compile(r"^[a-z0-9][a-z0-9_-]{0,63}$")
|
|
TOKEN_RE = re.compile(r"^[a-z0-9][a-z0-9_.+-]{0,31}$")
|
|
VIEWPORT_ID_RE = re.compile(r"^[a-z0-9][a-z0-9_.+-]{0,31}$")
|
|
CHECKSUM_RE = re.compile(r"^sha256:[0-9a-f]{64}$")
|
|
|
|
WORKSPACE_PREFIX = b"IOP-BENCH-WORKSPACE\x00"
|
|
MANIFEST_PREFIX = b"IOP-BENCH-MANIFEST\x00"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Frozen data containers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@dataclass(frozen=True)
|
|
class Timeout:
|
|
run_seconds: int
|
|
idle_seconds: int
|
|
quiet_seconds: int
|
|
cleanup_grace_seconds: int
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Viewport:
|
|
id: str
|
|
width: int
|
|
height: int
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class AssetMapping:
|
|
source: str
|
|
workspace_path: str
|
|
content: bytes = field(default=b"", repr=False)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ExpectedBinding:
|
|
stage: str
|
|
model: str
|
|
effort: Optional[str] = None
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class IopCell:
|
|
request_model: str
|
|
requested_effort: str
|
|
route_kind: str
|
|
route_id: str
|
|
expected_bindings: tuple[ExpectedBinding, ...]
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class MatrixCell:
|
|
id: str
|
|
caller: str
|
|
iop: IopCell
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Evaluator:
|
|
caller: str
|
|
iop: IopCell
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Fixture:
|
|
version: str
|
|
prompt: str
|
|
assets: tuple[AssetMapping, ...]
|
|
checksum: str
|
|
prompt_content: bytes = field(default=b"", repr=False)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Manifest:
|
|
pipeline_version: str
|
|
environment: str
|
|
testbed: str
|
|
repetitions: int
|
|
session_policy: str
|
|
setup_cache_policy: str
|
|
timeout: Timeout
|
|
viewports: tuple[Viewport, ...]
|
|
rubric_version: str
|
|
evaluator: Evaluator
|
|
output_root: str
|
|
fixture: Fixture
|
|
matrix: tuple[MatrixCell, ...]
|
|
digest: str
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Errors
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class ManifestError(Exception):
|
|
"""Base error for manifest validation failures."""
|
|
|
|
|
|
class ManifestValidationError(ManifestError):
|
|
"""Raised when the manifest JSON fails schema validation."""
|
|
|
|
|
|
class ManifestPathError(ManifestError):
|
|
"""Raised when a path rule is violated (escape, symlink, collision, etc.)."""
|
|
|
|
|
|
class ManifestDigestError(ManifestError):
|
|
"""Raised when a declared digest does not match the computed value."""
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Internal helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _require_bool(cond: bool, msg: str) -> None:
|
|
if not cond:
|
|
raise ManifestValidationError(msg)
|
|
|
|
|
|
def _require_str(value: Any, field_name: str) -> str:
|
|
if not isinstance(value, str):
|
|
raise ManifestValidationError(f"field '{field_name}' must be a string")
|
|
return value
|
|
|
|
|
|
def _require_int(value: Any, field_name: str) -> int:
|
|
if isinstance(value, bool) or not isinstance(value, int):
|
|
raise ManifestValidationError(f"field '{field_name}' must be an integer")
|
|
return value
|
|
|
|
|
|
def _require_object(value: Any, field_name: str) -> dict[str, Any]:
|
|
if not isinstance(value, dict):
|
|
raise ManifestValidationError(f"field '{field_name}' must be an object")
|
|
return value
|
|
|
|
|
|
def _require_array(value: Any, field_name: str) -> list[Any]:
|
|
if not isinstance(value, list):
|
|
raise ManifestValidationError(f"field '{field_name}' must be an array")
|
|
return value
|
|
|
|
|
|
def _require_enum(value: Any, field_name: str, allowed: tuple[str, ...]) -> str:
|
|
s = _require_str(value, field_name)
|
|
if s not in allowed:
|
|
raise ManifestValidationError(
|
|
f"field '{field_name}' must be one of {allowed}"
|
|
)
|
|
return s
|
|
|
|
|
|
def _require_pattern(value: Any, field_name: str, pattern: re.Pattern) -> str:
|
|
s = _require_str(value, field_name)
|
|
if not pattern.match(s):
|
|
raise ManifestValidationError(
|
|
f"field '{field_name}' does not match required pattern"
|
|
)
|
|
return s
|
|
|
|
|
|
def _require_positive_int(value: Any, field_name: str) -> int:
|
|
val = _require_int(value, field_name)
|
|
if val <= 0:
|
|
raise ManifestValidationError(
|
|
f"field '{field_name}' must be a positive integer"
|
|
)
|
|
return val
|
|
|
|
|
|
def _require_bounded_int(value: Any, field_name: str, lo: int, hi: int) -> int:
|
|
val = _require_int(value, field_name)
|
|
if not (lo <= val <= hi):
|
|
raise ManifestValidationError(
|
|
f"field '{field_name}' must be between {lo} and {hi}"
|
|
)
|
|
return val
|
|
|
|
|
|
def _require_sha256(value: Any, field_name: str) -> str:
|
|
s = _require_str(value, field_name)
|
|
if not CHECKSUM_RE.match(s):
|
|
raise ManifestValidationError(
|
|
f"field '{field_name}' must match sha256:<64 hex chars>"
|
|
)
|
|
return s
|
|
|
|
|
|
def _normalize_posix_relative_path(path_str: Any, context: str) -> str:
|
|
"""Validate and return a normalized POSIX relative path.
|
|
|
|
Rejects non-string, empty, absolute, colon, backslash, non-normal,
|
|
or escaping paths ('.', '..', starting with '../').
|
|
"""
|
|
if not isinstance(path_str, str) or not path_str:
|
|
raise ManifestPathError(f"field '{context}' must be a non-empty string path")
|
|
if "\\" in path_str or ":" in path_str:
|
|
raise ManifestPathError(f"field '{context}' contains invalid path characters")
|
|
if path_str.startswith("/"):
|
|
raise ManifestPathError(f"field '{context}' must be a relative path")
|
|
|
|
norm = posixpath.normpath(path_str)
|
|
if path_str != norm:
|
|
raise ManifestPathError(f"field '{context}' is not in canonical relative form")
|
|
if norm in (".", "..") or norm.startswith("../"):
|
|
raise ManifestPathError(f"field '{context}' escapes root or is empty")
|
|
return norm
|
|
|
|
|
|
def _require_regular_file(path: Path, context: str) -> None:
|
|
"""Require that path exists and is a regular file (no symlinks)."""
|
|
if not path.exists():
|
|
raise ManifestPathError(f"field '{context}' target file does not exist")
|
|
if path.is_symlink():
|
|
raise ManifestPathError(f"field '{context}' target must be a regular file, not a symlink")
|
|
if not path.is_file():
|
|
raise ManifestPathError(f"field '{context}' target is not a regular file")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Digest computation
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def digest_workspace_inputs(assets: Iterable[AssetMapping]) -> str:
|
|
"""Compute the declared fixture checksum from asset workspace paths and content.
|
|
|
|
sha256(b"IOP-BENCH-WORKSPACE\x00" + length-framed assets sorted by
|
|
workspace_path, each asset framed as:
|
|
uint64 BE len(workspace_path) + workspace_path UTF-8 bytes +
|
|
uint64 BE len(file_content) + file_content bytes
|
|
"""
|
|
sorted_assets = sorted(assets, key=lambda a: a.workspace_path)
|
|
data = bytearray(WORKSPACE_PREFIX)
|
|
for asset in sorted_assets:
|
|
wp_bytes = asset.workspace_path.encode("utf-8")
|
|
data += struct.pack(">Q", len(wp_bytes)) + wp_bytes
|
|
data += struct.pack(">Q", len(asset.content)) + asset.content
|
|
return "sha256:" + hashlib.sha256(bytes(data)).hexdigest()
|
|
|
|
|
|
def digest_manifest_and_resolved_inputs(manifest: Manifest) -> str:
|
|
"""Compute the self-contained manifest digest.
|
|
|
|
sha256(b"IOP-BENCH-MANIFEST\x00" + canonical JSON + length-framed
|
|
prompt path/content + length-framed every asset source/destination/content).
|
|
"""
|
|
prompt_content = manifest.fixture.prompt_content
|
|
|
|
canonical = json.dumps(
|
|
_manifest_to_dict(manifest),
|
|
sort_keys=True,
|
|
separators=(",", ":"),
|
|
ensure_ascii=False,
|
|
).encode("utf-8")
|
|
|
|
data = bytearray(MANIFEST_PREFIX)
|
|
data += struct.pack(">Q", len(canonical)) + canonical
|
|
|
|
prompt_path_bytes = manifest.fixture.prompt.encode("utf-8")
|
|
data += struct.pack(">Q", len(prompt_path_bytes)) + prompt_path_bytes
|
|
data += struct.pack(">Q", len(prompt_content)) + prompt_content
|
|
|
|
for asset in sorted(manifest.fixture.assets, key=lambda a: a.workspace_path):
|
|
src_bytes = asset.source.encode("utf-8")
|
|
data += struct.pack(">Q", len(src_bytes)) + src_bytes
|
|
wp_bytes = asset.workspace_path.encode("utf-8")
|
|
data += struct.pack(">Q", len(wp_bytes)) + wp_bytes
|
|
data += struct.pack(">Q", len(asset.content)) + asset.content
|
|
|
|
return "sha256:" + hashlib.sha256(bytes(data)).hexdigest()
|
|
|
|
|
|
def _manifest_to_dict(manifest: Manifest) -> dict[str, Any]:
|
|
"""Convert a Manifest to a plain dict for canonical JSON serialization."""
|
|
return {
|
|
"pipeline_version": manifest.pipeline_version,
|
|
"environment": manifest.environment,
|
|
"testbed": manifest.testbed,
|
|
"repetitions": manifest.repetitions,
|
|
"session_policy": manifest.session_policy,
|
|
"setup_cache_policy": manifest.setup_cache_policy,
|
|
"timeout": {
|
|
"run_seconds": manifest.timeout.run_seconds,
|
|
"idle_seconds": manifest.timeout.idle_seconds,
|
|
"quiet_seconds": manifest.timeout.quiet_seconds,
|
|
"cleanup_grace_seconds": manifest.timeout.cleanup_grace_seconds,
|
|
},
|
|
"viewports": [
|
|
{"id": v.id, "width": v.width, "height": v.height}
|
|
for v in manifest.viewports
|
|
],
|
|
"rubric_version": manifest.rubric_version,
|
|
"evaluator": {
|
|
"caller": manifest.evaluator.caller,
|
|
"iop": {
|
|
"request_model": manifest.evaluator.iop.request_model,
|
|
"requested_effort": manifest.evaluator.iop.requested_effort,
|
|
"route_kind": manifest.evaluator.iop.route_kind,
|
|
"route_id": manifest.evaluator.iop.route_id,
|
|
"expected_bindings": [
|
|
{
|
|
"stage": binding.stage,
|
|
"model": binding.model,
|
|
**({"effort": binding.effort} if binding.effort else {}),
|
|
}
|
|
for binding in manifest.evaluator.iop.expected_bindings
|
|
],
|
|
},
|
|
},
|
|
"output_root": manifest.output_root,
|
|
"fixture": {
|
|
"version": manifest.fixture.version,
|
|
"prompt": manifest.fixture.prompt,
|
|
"assets": [
|
|
{"source": a.source, "workspace_path": a.workspace_path}
|
|
for a in manifest.fixture.assets
|
|
],
|
|
"checksum": manifest.fixture.checksum,
|
|
},
|
|
"matrix": [
|
|
{
|
|
"id": c.id,
|
|
"caller": c.caller,
|
|
"iop": {
|
|
"request_model": c.iop.request_model,
|
|
"requested_effort": c.iop.requested_effort,
|
|
"route_kind": c.iop.route_kind,
|
|
"route_id": c.iop.route_id,
|
|
"expected_bindings": [
|
|
{
|
|
"stage": b.stage,
|
|
"model": b.model,
|
|
**({"effort": b.effort} if b.effort else {}),
|
|
}
|
|
for b in c.iop.expected_bindings
|
|
],
|
|
},
|
|
}
|
|
for c in manifest.matrix
|
|
],
|
|
}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Validation
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _validate_timeout(data: dict[str, Any]) -> Timeout:
|
|
obj = _require_object(data, "timeout")
|
|
expected_keys = {"run_seconds", "idle_seconds", "quiet_seconds", "cleanup_grace_seconds"}
|
|
if set(obj.keys()) != expected_keys:
|
|
raise ManifestValidationError("timeout has invalid schema")
|
|
return Timeout(
|
|
run_seconds=_require_bounded_int(obj["run_seconds"], "timeout.run_seconds", 1, 86400),
|
|
idle_seconds=_require_bounded_int(obj["idle_seconds"], "timeout.idle_seconds", 1, 600),
|
|
quiet_seconds=_require_bounded_int(obj["quiet_seconds"], "timeout.quiet_seconds", 1, 60),
|
|
cleanup_grace_seconds=_require_bounded_int(
|
|
obj["cleanup_grace_seconds"], "timeout.cleanup_grace_seconds", 1, 60
|
|
),
|
|
)
|
|
|
|
|
|
def _validate_viewports(data: list[Any]) -> tuple[Viewport, ...]:
|
|
arr = _require_array(data, "viewports")
|
|
if len(arr) < 1:
|
|
raise ManifestValidationError("viewports must have at least 1 item")
|
|
seen_ids: set[str] = set()
|
|
viewports: list[Viewport] = []
|
|
for i, item in enumerate(arr):
|
|
obj = _require_object(item, f"viewports[{i}]")
|
|
expected_keys = {"id", "width", "height"}
|
|
if set(obj.keys()) != expected_keys:
|
|
raise ManifestValidationError(f"viewports[{i}] has invalid schema")
|
|
vid = _require_pattern(obj["id"], f"viewports[{i}].id", VIEWPORT_ID_RE)
|
|
if vid in seen_ids:
|
|
raise ManifestValidationError(f"viewports[{i}].id is duplicate")
|
|
seen_ids.add(vid)
|
|
w = _require_bounded_int(obj["width"], f"viewports[{i}].width", 1, 8192)
|
|
h = _require_bounded_int(obj["height"], f"viewports[{i}].height", 1, 8192)
|
|
viewports.append(Viewport(id=vid, width=w, height=h))
|
|
return tuple(viewports)
|
|
|
|
|
|
def _validate_output_root(data: Any, repo_root: Path) -> str:
|
|
s = _require_str(data, "output_root")
|
|
if "\\" in s or ":" in s:
|
|
raise ManifestPathError("field 'output_root' contains invalid path characters")
|
|
norm = posixpath.normpath(s)
|
|
if s != norm:
|
|
raise ManifestPathError("field 'output_root' is not in canonical form")
|
|
if not s.startswith("agent-test/runs/"):
|
|
raise ManifestValidationError(
|
|
"field 'output_root' must be relative to agent-test/runs/"
|
|
)
|
|
tail = s[len("agent-test/runs/"):]
|
|
if not tail or "/" in tail:
|
|
raise ManifestValidationError(
|
|
"field 'output_root' must be a single non-empty segment after agent-test/runs/"
|
|
)
|
|
|
|
runs_dir = (repo_root / "agent-test" / "runs").resolve()
|
|
output_path = (repo_root / s).resolve()
|
|
try:
|
|
output_path.relative_to(runs_dir)
|
|
except ValueError:
|
|
raise ManifestPathError("field 'output_root' resolves outside agent-test/runs")
|
|
return s
|
|
|
|
|
|
def _validate_asset_mapping(
|
|
data: dict[str, Any], i: int
|
|
) -> tuple[str, str]:
|
|
obj = _require_object(data, f"assets[{i}]")
|
|
expected_keys = {"source", "workspace_path"}
|
|
if set(obj.keys()) != expected_keys:
|
|
raise ManifestValidationError(f"assets[{i}] has invalid schema")
|
|
source = _normalize_posix_relative_path(obj["source"], f"assets[{i}].source")
|
|
workspace_path = _normalize_posix_relative_path(
|
|
obj["workspace_path"], f"assets[{i}].workspace_path"
|
|
)
|
|
return source, workspace_path
|
|
|
|
|
|
def _validate_fixture(
|
|
data: dict[str, Any], repo_root: Path
|
|
) -> Fixture:
|
|
obj = _require_object(data, "fixture")
|
|
expected_keys = {"version", "prompt", "assets", "checksum"}
|
|
if set(obj.keys()) != expected_keys:
|
|
raise ManifestValidationError("fixture has invalid schema")
|
|
|
|
version = _require_pattern(obj["version"], "fixture.version", TOKEN_RE)
|
|
prompt = _normalize_posix_relative_path(obj["prompt"], "fixture.prompt")
|
|
|
|
assets_raw = _require_array(obj["assets"], "fixture.assets")
|
|
if len(assets_raw) < 1:
|
|
raise ManifestValidationError("fixture.assets must have at least 1 item")
|
|
|
|
assets_list: list[AssetMapping] = []
|
|
workspace_destinations: set[str] = set()
|
|
for i, item in enumerate(assets_raw):
|
|
source, workspace_path = _validate_asset_mapping(item, i)
|
|
if workspace_path in workspace_destinations:
|
|
raise ManifestPathError(
|
|
"fixture.assets workspace_path is duplicate"
|
|
)
|
|
workspace_destinations.add(workspace_path)
|
|
|
|
src_path = repo_root / source
|
|
_require_regular_file(src_path, f"fixture asset source ({i})")
|
|
try:
|
|
src_path.resolve().relative_to(repo_root.resolve())
|
|
except ValueError:
|
|
raise ManifestPathError("fixture asset source resolves outside repo root")
|
|
|
|
content = src_path.read_bytes()
|
|
assets_list.append(AssetMapping(source=source, workspace_path=workspace_path, content=content))
|
|
|
|
assets_list.sort(key=lambda a: a.workspace_path)
|
|
|
|
# Validate prompt file
|
|
prompt_path = repo_root / prompt
|
|
_require_regular_file(prompt_path, "fixture prompt")
|
|
try:
|
|
prompt_path.resolve().relative_to(repo_root.resolve())
|
|
except ValueError:
|
|
raise ManifestPathError("fixture prompt resolves outside repo root")
|
|
prompt_content = prompt_path.read_bytes()
|
|
|
|
checksum = _require_sha256(obj["checksum"], "fixture.checksum")
|
|
computed = digest_workspace_inputs(assets_list)
|
|
if checksum != computed:
|
|
raise ManifestDigestError("fixture.checksum mismatch")
|
|
|
|
return Fixture(
|
|
version=version,
|
|
prompt=prompt,
|
|
assets=tuple(assets_list),
|
|
checksum=checksum,
|
|
prompt_content=prompt_content,
|
|
)
|
|
|
|
|
|
def _validate_expected_binding(
|
|
data: dict[str, Any], i: int
|
|
) -> ExpectedBinding:
|
|
obj = _require_object(data, f"expected_bindings[{i}]")
|
|
expected_keys = {"stage", "model"}
|
|
if not expected_keys.issubset(set(obj.keys())):
|
|
raise ManifestValidationError(f"expected_bindings[{i}] missing required keys")
|
|
extra = set(obj.keys()) - {"stage", "model", "effort"}
|
|
if extra:
|
|
raise ManifestValidationError(f"expected_bindings[{i}] has unexpected keys")
|
|
stage = _require_enum(obj["stage"], f"expected_bindings[{i}].stage", STAGE_ENUM)
|
|
model = _require_pattern(obj["model"], f"expected_bindings[{i}].model", TOKEN_RE)
|
|
effort = None
|
|
if "effort" in obj:
|
|
effort = _require_pattern(obj["effort"], f"expected_bindings[{i}].effort", TOKEN_RE)
|
|
return ExpectedBinding(stage=stage, model=model, effort=effort)
|
|
|
|
|
|
def _validate_iop_cell(data: dict[str, Any]) -> IopCell:
|
|
obj = _require_object(data, "iop")
|
|
expected_keys = {"request_model", "requested_effort", "route_kind", "route_id", "expected_bindings"}
|
|
if set(obj.keys()) != expected_keys:
|
|
raise ManifestValidationError("iop has invalid schema")
|
|
|
|
request_model = _require_pattern(obj["request_model"], "iop.request_model", TOKEN_RE)
|
|
requested_effort = _require_pattern(obj["requested_effort"], "iop.requested_effort", TOKEN_RE)
|
|
route_kind = _require_enum(obj["route_kind"], "iop.route_kind", ROUTE_KIND_ENUM)
|
|
route_id = _require_pattern(obj["route_id"], "iop.route_id", TOKEN_RE)
|
|
|
|
bindings_raw = _require_array(obj["expected_bindings"], "iop.expected_bindings")
|
|
if len(bindings_raw) < 1:
|
|
raise ManifestValidationError("iop.expected_bindings must have at least 1 item")
|
|
|
|
bindings: list[ExpectedBinding] = []
|
|
seen_stages: set[str] = set()
|
|
for i, item in enumerate(bindings_raw):
|
|
b = _validate_expected_binding(item, i)
|
|
if b.stage in seen_stages:
|
|
raise ManifestValidationError("iop.expected_bindings contains duplicate stage")
|
|
seen_stages.add(b.stage)
|
|
bindings.append(b)
|
|
|
|
# Sort bindings by canonical stage rank
|
|
bindings.sort(key=lambda b: STAGE_RANK[b.stage])
|
|
|
|
# Validate direct vs execution_preset constraints
|
|
if route_kind == "direct":
|
|
if set(b.stage for b in bindings) != {"request"}:
|
|
raise ManifestValidationError(
|
|
"direct route requires exactly one binding with stage=request"
|
|
)
|
|
elif route_kind == "execution_preset":
|
|
required_stages = {"selector", "plan", "work", "review"}
|
|
allowed_stages = required_stages | {"repair"}
|
|
actual_stages = set(b.stage for b in bindings)
|
|
if actual_stages not in (required_stages, allowed_stages):
|
|
raise ManifestValidationError(
|
|
"execution_preset has an invalid stage set"
|
|
)
|
|
|
|
return IopCell(
|
|
request_model=request_model,
|
|
requested_effort=requested_effort,
|
|
route_kind=route_kind,
|
|
route_id=route_id,
|
|
expected_bindings=tuple(bindings),
|
|
)
|
|
|
|
|
|
def _validate_cell(data: dict[str, Any], i: int) -> MatrixCell:
|
|
obj = _require_object(data, f"matrix[{i}]")
|
|
expected_keys = {"id", "caller", "iop"}
|
|
if set(obj.keys()) != expected_keys:
|
|
raise ManifestValidationError(f"matrix[{i}] has invalid schema")
|
|
|
|
cell_id = _require_pattern(obj["id"], f"matrix[{i}].id", CELL_ID_RE)
|
|
caller = _require_enum(obj["caller"], f"matrix[{i}].caller", CALLER_ENUM)
|
|
iop = _validate_iop_cell(obj["iop"])
|
|
return MatrixCell(id=cell_id, caller=caller, iop=iop)
|
|
|
|
|
|
def _validate_evaluator(data: dict[str, Any]) -> Evaluator:
|
|
obj = _require_object(data, "evaluator")
|
|
if set(obj) != {"caller", "iop"}:
|
|
raise ManifestValidationError("evaluator has invalid schema")
|
|
caller = _require_enum(obj["caller"], "evaluator.caller", ("codex",))
|
|
return Evaluator(caller=caller, iop=_validate_iop_cell(obj["iop"]))
|
|
|
|
|
|
def _validate_matrix(data: list[Any]) -> tuple[MatrixCell, ...]:
|
|
arr = _require_array(data, "matrix")
|
|
if len(arr) < 1:
|
|
raise ManifestValidationError("matrix must have at least 1 item")
|
|
|
|
cells: list[MatrixCell] = []
|
|
seen_ids: set[str] = set()
|
|
for i, item in enumerate(arr):
|
|
cell = _validate_cell(item, i)
|
|
if cell.id in seen_ids:
|
|
raise ManifestValidationError("matrix contains duplicate cell id")
|
|
seen_ids.add(cell.id)
|
|
cells.append(cell)
|
|
|
|
# Sort cells by id for canonical ordering
|
|
cells.sort(key=lambda c: c.id)
|
|
return tuple(cells)
|
|
|
|
|
|
def _default_repo_root(manifest_path: Path) -> Path:
|
|
"""Walk up from manifest_path to find the repository root.
|
|
|
|
Looks for a directory containing ``Makefile`` or ``.git``.
|
|
Falls back to the manifest file's parent directory.
|
|
"""
|
|
candidate = manifest_path.resolve()
|
|
for _ in range(20): # safety limit
|
|
if (candidate / "Makefile").is_file() or (candidate / ".git").exists():
|
|
return candidate
|
|
parent = candidate.parent
|
|
if parent == candidate:
|
|
break
|
|
candidate = parent
|
|
return manifest_path.resolve().parent
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Public API
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def load_manifest(path: str | Path, repo_root: str | Path | None = None) -> Manifest:
|
|
"""Load and validate a benchmark manifest JSON file.
|
|
|
|
Args:
|
|
path: Path to the manifest JSON file.
|
|
repo_root: Repository root for resolving relative paths.
|
|
|
|
Returns:
|
|
A frozen Manifest object with computed digest.
|
|
|
|
Raises:
|
|
ManifestValidationError: If the JSON fails schema validation.
|
|
ManifestPathError: If a path rule is violated.
|
|
ManifestDigestError: If a declared digest does not match.
|
|
"""
|
|
path = Path(path)
|
|
if repo_root is None:
|
|
repo_root = _default_repo_root(path)
|
|
repo_root = Path(repo_root).resolve()
|
|
|
|
try:
|
|
raw = json.loads(path.read_text(encoding="utf-8"))
|
|
except FileNotFoundError:
|
|
raise ManifestValidationError("manifest file not found")
|
|
except (json.JSONDecodeError, UnicodeDecodeError) as exc:
|
|
raise ManifestValidationError("invalid JSON format")
|
|
|
|
if not isinstance(raw, dict):
|
|
raise ManifestValidationError("manifest must be a JSON object")
|
|
|
|
# Top-level field validation
|
|
expected_top = {
|
|
"pipeline_version", "environment", "testbed", "fixture", "matrix",
|
|
"session_policy", "setup_cache_policy", "timeout", "viewports",
|
|
"rubric_version", "evaluator", "output_root",
|
|
}
|
|
optional_top = {"repetitions"}
|
|
declared_keys = set(raw.keys())
|
|
required_present = expected_top.issubset(declared_keys)
|
|
extra = declared_keys - expected_top - optional_top
|
|
if not required_present:
|
|
raise ManifestValidationError("manifest missing required top-level fields")
|
|
if extra:
|
|
raise ManifestValidationError("manifest has unexpected top-level fields")
|
|
|
|
pipeline_version = _require_enum(raw["pipeline_version"], "pipeline_version", (PIPELINE_VERSION,))
|
|
environment = _require_enum(raw["environment"], "environment", (ENVIRONMENT,))
|
|
testbed = _require_enum(raw["testbed"], "testbed", (TESTBED_REQUIRED,))
|
|
|
|
repetitions = DEFAULT_REPETITIONS
|
|
if "repetitions" in raw:
|
|
repetitions = _require_positive_int(raw["repetitions"], "repetitions")
|
|
|
|
session_policy = _require_enum(raw["session_policy"], "session_policy", (SESSION_POLICY,))
|
|
setup_cache_policy = _require_enum(
|
|
raw["setup_cache_policy"], "setup_cache_policy", (SETUP_CACHE_POLICY,)
|
|
)
|
|
timeout = _validate_timeout(raw["timeout"])
|
|
viewports = _validate_viewports(raw["viewports"])
|
|
rubric_version = _require_enum(
|
|
raw["rubric_version"], "rubric_version", (RUBRIC_VERSION,)
|
|
)
|
|
evaluator = _validate_evaluator(raw["evaluator"])
|
|
output_root = _validate_output_root(raw["output_root"], repo_root)
|
|
|
|
fixture = _validate_fixture(raw["fixture"], repo_root)
|
|
matrix = _validate_matrix(raw["matrix"])
|
|
|
|
# Compute manifest digest during load
|
|
dummy_manifest = Manifest(
|
|
pipeline_version=pipeline_version,
|
|
environment=environment,
|
|
testbed=testbed,
|
|
repetitions=repetitions,
|
|
session_policy=session_policy,
|
|
setup_cache_policy=setup_cache_policy,
|
|
timeout=timeout,
|
|
viewports=viewports,
|
|
rubric_version=rubric_version,
|
|
evaluator=evaluator,
|
|
output_root=output_root,
|
|
fixture=fixture,
|
|
matrix=matrix,
|
|
digest="",
|
|
)
|
|
computed_digest = digest_manifest_and_resolved_inputs(dummy_manifest)
|
|
|
|
return Manifest(
|
|
pipeline_version=pipeline_version,
|
|
environment=environment,
|
|
testbed=testbed,
|
|
repetitions=repetitions,
|
|
session_policy=session_policy,
|
|
setup_cache_policy=setup_cache_policy,
|
|
timeout=timeout,
|
|
viewports=viewports,
|
|
rubric_version=rubric_version,
|
|
evaluator=evaluator,
|
|
output_root=output_root,
|
|
fixture=fixture,
|
|
matrix=matrix,
|
|
digest=computed_digest,
|
|
)
|
|
|
|
|
|
def validate_manifest_bytes(
|
|
data: bytes,
|
|
path_hint: str = "<bytes>",
|
|
repo_root: str | Path | None = None,
|
|
) -> Manifest:
|
|
"""Validate manifest JSON bytes without writing to persistent disk."""
|
|
import tempfile
|
|
with tempfile.NamedTemporaryFile(
|
|
mode="wb", suffix=".json", delete=False
|
|
) as tmp:
|
|
tmp.write(data)
|
|
tmp_path = tmp.name
|
|
try:
|
|
return load_manifest(tmp_path, repo_root=repo_root)
|
|
finally:
|
|
try:
|
|
os.unlink(tmp_path)
|
|
except OSError:
|
|
pass
|