2111 lines
91 KiB
Python
2111 lines
91 KiB
Python
"""
|
|
Comprehensive tests for the benchmark manifest loader, validator, and CLI.
|
|
|
|
Covers: valid minimum/example, omitted repetitions, data-only matrix extension,
|
|
deterministic ordering, duplicate ids/stages, direct vs preset shapes,
|
|
every enum/bound, unknown members, path escape/symlink/collision,
|
|
workspace/prompt/asset/manifest digest drift, non-positive bounds, and
|
|
secret redaction in errors.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import inspect
|
|
import json
|
|
import os
|
|
import shutil
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from dataclasses import replace
|
|
from pathlib import Path
|
|
|
|
# Ensure repo root is importable
|
|
_REPO_ROOT = Path(__file__).resolve().parent.parent.parent
|
|
if str(_REPO_ROOT) not in sys.path:
|
|
sys.path.insert(0, str(_REPO_ROOT))
|
|
|
|
from scripts.agent_benchmark.manifest import (
|
|
AssetMapping,
|
|
Fixture,
|
|
IopCell,
|
|
Manifest,
|
|
ManifestDigestError,
|
|
ManifestError,
|
|
ManifestPathError,
|
|
ManifestValidationError,
|
|
MatrixCell,
|
|
Timeout,
|
|
Viewport,
|
|
digest_manifest_and_resolved_inputs,
|
|
digest_workspace_inputs,
|
|
load_manifest,
|
|
validate_manifest_bytes,
|
|
)
|
|
|
|
# Sentinel values used in redaction tests
|
|
_SENTINEL_SECRET = "SUPER_SECRET_API_KEY_12345"
|
|
_SENTINEL_ENDPOINT = "https://private.internal.example.com/secret-endpoint"
|
|
_SENTINEL_PROMPT = "Do not leak this prompt content"
|
|
|
|
|
|
def _make_minimal_manifest_dict(**overrides: object) -> dict:
|
|
"""Build a minimal valid manifest dict with optional overrides."""
|
|
d = {
|
|
"pipeline_version": "2",
|
|
"environment": "dev",
|
|
"testbed": "../iop-s2",
|
|
"session_policy": "fresh",
|
|
"setup_cache_policy": "isolated",
|
|
"timeout": {
|
|
"run_seconds": 300,
|
|
"idle_seconds": 30,
|
|
"quiet_seconds": 10,
|
|
"cleanup_grace_seconds": 5,
|
|
},
|
|
"viewports": [{"id": "desktop", "width": 1920, "height": 1080}],
|
|
"rubric_version": "landing-quality-v1",
|
|
"evaluator": {
|
|
"caller": "codex",
|
|
"iop": {
|
|
"request_model": "gpt-5.6-luna",
|
|
"requested_effort": "xhigh",
|
|
"route_kind": "direct",
|
|
"route_id": "gpt-5.6-luna",
|
|
"expected_bindings": [
|
|
{
|
|
"stage": "request",
|
|
"model": "gpt-5.6-luna",
|
|
"effort": "xhigh",
|
|
}
|
|
],
|
|
},
|
|
},
|
|
"output_root": "agent-test/runs/bench-01",
|
|
"fixture": {
|
|
"version": "v1.0",
|
|
"prompt": "scripts/fixtures/agent-comparison-benchmark/prompt.md",
|
|
"assets": [
|
|
{
|
|
"source": "scripts/fixtures/agent-comparison-benchmark/reference.txt",
|
|
"workspace_path": "workspace/reference.txt",
|
|
}
|
|
],
|
|
"checksum": "sha256:placeholder",
|
|
},
|
|
"matrix": [
|
|
{
|
|
"id": "cell-a",
|
|
"caller": "claude",
|
|
"iop": {
|
|
"request_model": "claude-sonnet-4-20250514",
|
|
"requested_effort": "high",
|
|
"route_kind": "direct",
|
|
"route_id": "claude-direct",
|
|
"expected_bindings": [
|
|
{"stage": "request", "model": "claude-sonnet-4-20250514", "effort": "high"}
|
|
],
|
|
},
|
|
}
|
|
],
|
|
}
|
|
d.update(overrides)
|
|
return d
|
|
|
|
|
|
def _compute_fixture_checksum(repo_root: Path, fixture: dict) -> str:
|
|
"""Compute the fixture checksum for a given fixture dict."""
|
|
assets_raw = fixture["assets"]
|
|
assets = tuple(
|
|
AssetMapping(
|
|
source=a["source"],
|
|
workspace_path=a["workspace_path"],
|
|
content=(repo_root / a["source"]).read_bytes(),
|
|
)
|
|
for a in assets_raw
|
|
)
|
|
return digest_workspace_inputs(assets)
|
|
|
|
|
|
def _write_tmp_manifest(
|
|
tmp_dir: Path,
|
|
data: dict,
|
|
name: str = "manifest.json",
|
|
) -> Path:
|
|
"""Write manifest dict to tmp_dir/name and return the path.
|
|
|
|
Attempts to patch the fixture checksum using repo-root asset files.
|
|
If that fails (e.g. assets were modified), writes the manifest as-is.
|
|
"""
|
|
p = tmp_dir / name
|
|
if "fixture" in data and "checksum" in data["fixture"]:
|
|
try:
|
|
data["fixture"]["checksum"] = _compute_fixture_checksum(
|
|
_REPO_ROOT, data["fixture"]
|
|
)
|
|
except (FileNotFoundError, OSError, ValueError):
|
|
# Assets were modified; leave checksum as-is (tests should
|
|
# expect validation to fail for invalid checksums).
|
|
pass
|
|
p.write_text(json.dumps(data, indent=2), encoding="utf-8")
|
|
return p
|
|
|
|
|
|
def _load_tmp_manifest(path: Path) -> Manifest:
|
|
"""Load a manifest from a temp path with repo_root set to _REPO_ROOT."""
|
|
return load_manifest(path, repo_root=_REPO_ROOT)
|
|
|
|
|
|
class TestLoadManifestValid(unittest.TestCase):
|
|
"""Valid manifest loading tests."""
|
|
|
|
def test_minimal_valid_manifest(self):
|
|
"""Minimal manifest with explicit repetitions=1 loads."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
path = _write_tmp_manifest(tmp_dir, _make_minimal_manifest_dict())
|
|
m = _load_tmp_manifest(path)
|
|
self.assertEqual(m.pipeline_version, "2")
|
|
self.assertEqual(m.environment, "dev")
|
|
self.assertEqual(m.repetitions, 1)
|
|
self.assertEqual(m.session_policy, "fresh")
|
|
self.assertEqual(m.setup_cache_policy, "isolated")
|
|
self.assertEqual(len(m.viewports), 1)
|
|
self.assertEqual(len(m.matrix), 1)
|
|
self.assertIsInstance(m, Manifest)
|
|
# Frozen
|
|
with self.assertRaises(AttributeError):
|
|
m.repetitions = 99
|
|
|
|
def test_omitted_repetitions_defaults_to_one(self):
|
|
"""Omitted repetitions defaults to 1."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict()
|
|
# repetitions is not included (optional field)
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
m = _load_tmp_manifest(path)
|
|
self.assertEqual(m.repetitions, 1)
|
|
|
|
def test_omitted_equals_explicit_one(self):
|
|
"""Omitted repetitions and explicit repetitions=1 produce identical manifests."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
# d_no_rep: repetitions is not included (optional field)
|
|
d_no_rep = _make_minimal_manifest_dict()
|
|
d_explicit = _make_minimal_manifest_dict(repetitions=1)
|
|
p1 = _write_tmp_manifest(tmp_dir, d_no_rep, "no_rep.json")
|
|
p2 = _write_tmp_manifest(tmp_dir, d_explicit, "explicit_rep.json")
|
|
m1 = _load_tmp_manifest(p1)
|
|
m2 = _load_tmp_manifest(p2)
|
|
self.assertEqual(m1.repetitions, m2.repetitions)
|
|
self.assertEqual(m1, m2)
|
|
|
|
def test_explicit_repetitions_greater_than_one(self):
|
|
"""Explicit repetitions > 1 is preserved."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
path = _write_tmp_manifest(
|
|
tmp_dir, _make_minimal_manifest_dict(repetitions=5)
|
|
)
|
|
m = _load_tmp_manifest(path)
|
|
self.assertEqual(m.repetitions, 5)
|
|
|
|
def test_example_manifest_loads(self):
|
|
"""The shipped example manifest loads successfully."""
|
|
example_path = (
|
|
_REPO_ROOT
|
|
/ "scripts"
|
|
/ "fixtures"
|
|
/ "agent-comparison-benchmark-manifest.example.json"
|
|
)
|
|
if example_path.exists():
|
|
m = _load_tmp_manifest(example_path)
|
|
self.assertEqual(m.pipeline_version, "2")
|
|
self.assertEqual(len(m.matrix), 3)
|
|
# Cells sorted by id
|
|
ids = [c.id for c in m.matrix]
|
|
self.assertEqual(ids, sorted(ids))
|
|
|
|
def test_multiple_viewports_unique(self):
|
|
"""Multiple viewports with unique ids load."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict(
|
|
viewports=[
|
|
{"id": "desktop", "width": 1920, "height": 1080},
|
|
{"id": "mobile", "width": 375, "height": 812},
|
|
]
|
|
)
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
m = _load_tmp_manifest(path)
|
|
self.assertEqual(len(m.viewports), 2)
|
|
|
|
def test_execution_preset_cell_loads(self):
|
|
"""Execution-preset cell with all required stages loads."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict(
|
|
matrix=[
|
|
{
|
|
"id": "preset-cell",
|
|
"caller": "agy",
|
|
"iop": {
|
|
"request_model": "gemini-2.0-flash",
|
|
"requested_effort": "high",
|
|
"route_kind": "execution_preset",
|
|
"route_id": "agy-generic",
|
|
"expected_bindings": [
|
|
{"stage": "plan", "model": "gemini-2.0-flash"},
|
|
{"stage": "work", "model": "gemini-2.0-flash"},
|
|
{"stage": "review", "model": "gemini-2.0-flash"},
|
|
{"stage": "selector", "model": "gemini-2.0-flash"},
|
|
],
|
|
},
|
|
}
|
|
]
|
|
)
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
m = _load_tmp_manifest(path)
|
|
cell = m.matrix[0]
|
|
# Sorted by canonical stage rank
|
|
stages = [b.stage for b in cell.iop.expected_bindings]
|
|
self.assertEqual(stages, ["selector", "plan", "work", "review"])
|
|
|
|
def test_execution_preset_with_repair(self):
|
|
"""Execution-preset cell with optional repair stage loads."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict(
|
|
matrix=[
|
|
{
|
|
"id": "preset-repair",
|
|
"caller": "codex",
|
|
"iop": {
|
|
"request_model": "gpt-4.1",
|
|
"requested_effort": "xhigh",
|
|
"route_kind": "execution_preset",
|
|
"route_id": "codex-generic",
|
|
"expected_bindings": [
|
|
{"stage": "selector", "model": "gpt-4.1"},
|
|
{"stage": "plan", "model": "gpt-4.1"},
|
|
{"stage": "work", "model": "gpt-4.1"},
|
|
{"stage": "review", "model": "gpt-4.1"},
|
|
{"stage": "repair", "model": "gpt-4.1"},
|
|
],
|
|
},
|
|
}
|
|
]
|
|
)
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
m = _load_tmp_manifest(path)
|
|
stages = [b.stage for b in m.matrix[0].iop.expected_bindings]
|
|
self.assertEqual(stages, ["selector", "plan", "work", "review", "repair"])
|
|
|
|
|
|
class TestMatrixExtension(unittest.TestCase):
|
|
"""Data-only matrix extension tests."""
|
|
|
|
def test_data_only_matrix_extension(self):
|
|
"""Adding a new cell to the matrix does not require code changes."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict(
|
|
matrix=[
|
|
{
|
|
"id": "cell-a",
|
|
"caller": "claude",
|
|
"iop": {
|
|
"request_model": "claude-sonnet-4-20250514",
|
|
"requested_effort": "high",
|
|
"route_kind": "direct",
|
|
"route_id": "claude-direct",
|
|
"expected_bindings": [
|
|
{"stage": "request", "model": "claude-sonnet-4-20250514"}
|
|
],
|
|
},
|
|
},
|
|
{
|
|
"id": "cell-b",
|
|
"caller": "agy",
|
|
"iop": {
|
|
"request_model": "gemini-2.0-flash",
|
|
"requested_effort": "high",
|
|
"route_kind": "execution_preset",
|
|
"route_id": "agy-generic",
|
|
"expected_bindings": [
|
|
{"stage": "selector", "model": "gemini-2.0-flash"},
|
|
{"stage": "plan", "model": "gemini-2.0-flash"},
|
|
{"stage": "work", "model": "gemini-2.0-flash"},
|
|
{"stage": "review", "model": "gemini-2.0-flash"},
|
|
],
|
|
},
|
|
},
|
|
{
|
|
"id": "cell-c",
|
|
"caller": "codex",
|
|
"iop": {
|
|
"request_model": "gpt-4.1",
|
|
"requested_effort": "xhigh",
|
|
"route_kind": "execution_preset",
|
|
"route_id": "codex-generic",
|
|
"expected_bindings": [
|
|
{"stage": "selector", "model": "gpt-4.1"},
|
|
{"stage": "plan", "model": "gpt-4.1"},
|
|
{"stage": "work", "model": "gpt-4.1"},
|
|
{"stage": "review", "model": "gpt-4.1"},
|
|
],
|
|
},
|
|
},
|
|
]
|
|
)
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
m = _load_tmp_manifest(path)
|
|
self.assertEqual(len(m.matrix), 3)
|
|
# Sorted by id
|
|
ids = [c.id for c in m.matrix]
|
|
self.assertEqual(ids, sorted(ids))
|
|
self.assertEqual(ids, ["cell-a", "cell-b", "cell-c"])
|
|
|
|
|
|
class TestDeterministicOrdering(unittest.TestCase):
|
|
"""Deterministic cell and binding ordering tests."""
|
|
|
|
def test_cells_sorted_by_id(self):
|
|
"""Cells are sorted by id regardless of input order."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict(
|
|
matrix=[
|
|
{
|
|
"id": "z-cell",
|
|
"caller": "claude",
|
|
"iop": {
|
|
"request_model": "claude-sonnet-4-20250514",
|
|
"requested_effort": "high",
|
|
"route_kind": "direct",
|
|
"route_id": "claude-direct",
|
|
"expected_bindings": [
|
|
{"stage": "request", "model": "claude-sonnet-4-20250514"}
|
|
],
|
|
},
|
|
},
|
|
{
|
|
"id": "a-cell",
|
|
"caller": "agy",
|
|
"iop": {
|
|
"request_model": "gemini-2.0-flash",
|
|
"requested_effort": "high",
|
|
"route_kind": "execution_preset",
|
|
"route_id": "agy-generic",
|
|
"expected_bindings": [
|
|
{"stage": "work", "model": "gemini-2.0-flash"},
|
|
{"stage": "plan", "model": "gemini-2.0-flash"},
|
|
{"stage": "review", "model": "gemini-2.0-flash"},
|
|
{"stage": "selector", "model": "gemini-2.0-flash"},
|
|
],
|
|
},
|
|
},
|
|
]
|
|
)
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
m = _load_tmp_manifest(path)
|
|
ids = [c.id for c in m.matrix]
|
|
self.assertEqual(ids, ["a-cell", "z-cell"])
|
|
|
|
def test_bindings_sorted_by_canonical_rank(self):
|
|
"""Bindings are sorted by fixed stage rank, not lexical order."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict(
|
|
matrix=[
|
|
{
|
|
"id": "rank-test",
|
|
"caller": "agy",
|
|
"iop": {
|
|
"request_model": "gemini-2.0-flash",
|
|
"requested_effort": "high",
|
|
"route_kind": "execution_preset",
|
|
"route_id": "agy-generic",
|
|
"expected_bindings": [
|
|
{"stage": "review", "model": "gemini-2.0-flash"},
|
|
{"stage": "work", "model": "gemini-2.0-flash"},
|
|
{"stage": "selector", "model": "gemini-2.0-flash"},
|
|
{"stage": "plan", "model": "gemini-2.0-flash"},
|
|
],
|
|
},
|
|
}
|
|
]
|
|
)
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
m = _load_tmp_manifest(path)
|
|
stages = [b.stage for b in m.matrix[0].iop.expected_bindings]
|
|
self.assertEqual(stages, ["selector", "plan", "work", "review"])
|
|
|
|
def test_canonical_rank_full_order(self):
|
|
"""Full canonical rank order for preset: selector, plan, work, review, repair."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict(
|
|
matrix=[
|
|
{
|
|
"id": "full-rank",
|
|
"caller": "agy",
|
|
"iop": {
|
|
"request_model": "gemini-2.0-flash",
|
|
"requested_effort": "high",
|
|
"route_kind": "execution_preset",
|
|
"route_id": "agy-generic",
|
|
"expected_bindings": [
|
|
{"stage": "repair", "model": "gemini-2.0-flash"},
|
|
{"stage": "review", "model": "gemini-2.0-flash"},
|
|
{"stage": "plan", "model": "gemini-2.0-flash"},
|
|
{"stage": "work", "model": "gemini-2.0-flash"},
|
|
{"stage": "selector", "model": "gemini-2.0-flash"},
|
|
],
|
|
},
|
|
}
|
|
]
|
|
)
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
m = _load_tmp_manifest(path)
|
|
stages = [b.stage for b in m.matrix[0].iop.expected_bindings]
|
|
self.assertEqual(
|
|
stages, ["selector", "plan", "work", "review", "repair"]
|
|
)
|
|
|
|
|
|
class TestDuplicateDetection(unittest.TestCase):
|
|
"""Duplicate id and stage detection tests."""
|
|
|
|
def test_duplicate_cell_ids_rejected(self):
|
|
"""Two cells with the same id are rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict(
|
|
matrix=[
|
|
{
|
|
"id": "dup",
|
|
"caller": "claude",
|
|
"iop": {
|
|
"request_model": "claude-sonnet-4-20250514",
|
|
"requested_effort": "high",
|
|
"route_kind": "direct",
|
|
"route_id": "r1",
|
|
"expected_bindings": [
|
|
{"stage": "request", "model": "claude-sonnet-4-20250514"}
|
|
],
|
|
},
|
|
},
|
|
{
|
|
"id": "dup",
|
|
"caller": "agy",
|
|
"iop": {
|
|
"request_model": "gemini-2.0-flash",
|
|
"requested_effort": "high",
|
|
"route_kind": "execution_preset",
|
|
"route_id": "r2",
|
|
"expected_bindings": [
|
|
{"stage": "selector", "model": "gemini-2.0-flash"},
|
|
{"stage": "plan", "model": "gemini-2.0-flash"},
|
|
{"stage": "work", "model": "gemini-2.0-flash"},
|
|
{"stage": "review", "model": "gemini-2.0-flash"},
|
|
],
|
|
},
|
|
},
|
|
]
|
|
)
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestValidationError):
|
|
_load_tmp_manifest(path)
|
|
|
|
def test_duplicate_binding_stages_rejected(self):
|
|
"""Two bindings with the same stage in one cell are rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict(
|
|
matrix=[
|
|
{
|
|
"id": "dup-stage",
|
|
"caller": "agy",
|
|
"iop": {
|
|
"request_model": "gemini-2.0-flash",
|
|
"requested_effort": "high",
|
|
"route_kind": "execution_preset",
|
|
"route_id": "r1",
|
|
"expected_bindings": [
|
|
{"stage": "work", "model": "gemini-2.0-flash"},
|
|
{"stage": "work", "model": "gemini-2.0-flash"},
|
|
],
|
|
},
|
|
}
|
|
]
|
|
)
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestValidationError):
|
|
_load_tmp_manifest(path)
|
|
|
|
def test_duplicate_viewport_ids_rejected(self):
|
|
"""Two viewports with the same id are rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict(
|
|
viewports=[
|
|
{"id": "dup", "width": 1920, "height": 1080},
|
|
{"id": "dup", "width": 375, "height": 812},
|
|
]
|
|
)
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestValidationError):
|
|
_load_tmp_manifest(path)
|
|
|
|
|
|
class TestDirectVsPresetShapes(unittest.TestCase):
|
|
"""Direct vs execution-preset shape validation tests."""
|
|
|
|
def test_direct_requires_exactly_one_request_binding(self):
|
|
"""Direct route with no bindings is rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict(
|
|
matrix=[
|
|
{
|
|
"id": "direct-empty",
|
|
"caller": "claude",
|
|
"iop": {
|
|
"request_model": "claude-sonnet-4-20250514",
|
|
"requested_effort": "high",
|
|
"route_kind": "direct",
|
|
"route_id": "r1",
|
|
"expected_bindings": [],
|
|
},
|
|
}
|
|
]
|
|
)
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestValidationError):
|
|
_load_tmp_manifest(path)
|
|
|
|
def test_direct_with_non_request_binding_rejected(self):
|
|
"""Direct route with a non-request binding is rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict(
|
|
matrix=[
|
|
{
|
|
"id": "direct-wrong",
|
|
"caller": "claude",
|
|
"iop": {
|
|
"request_model": "claude-sonnet-4-20250514",
|
|
"requested_effort": "high",
|
|
"route_kind": "direct",
|
|
"route_id": "r1",
|
|
"expected_bindings": [
|
|
{"stage": "plan", "model": "claude-sonnet-4-20250514"}
|
|
],
|
|
},
|
|
}
|
|
]
|
|
)
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestValidationError):
|
|
_load_tmp_manifest(path)
|
|
|
|
def test_preset_missing_required_stages_rejected(self):
|
|
"""Execution-preset missing selector/plan/work/review is rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict(
|
|
matrix=[
|
|
{
|
|
"id": "preset-incomplete",
|
|
"caller": "agy",
|
|
"iop": {
|
|
"request_model": "gemini-2.0-flash",
|
|
"requested_effort": "high",
|
|
"route_kind": "execution_preset",
|
|
"route_id": "r1",
|
|
"expected_bindings": [
|
|
{"stage": "work", "model": "gemini-2.0-flash"},
|
|
],
|
|
},
|
|
}
|
|
]
|
|
)
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestValidationError):
|
|
_load_tmp_manifest(path)
|
|
|
|
def test_preset_two_repair_bindings_rejected(self):
|
|
"""Execution-preset with two repair bindings is rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict(
|
|
matrix=[
|
|
{
|
|
"id": "preset-double-repair",
|
|
"caller": "agy",
|
|
"iop": {
|
|
"request_model": "gemini-2.0-flash",
|
|
"requested_effort": "high",
|
|
"route_kind": "execution_preset",
|
|
"route_id": "r1",
|
|
"expected_bindings": [
|
|
{"stage": "selector", "model": "gemini-2.0-flash"},
|
|
{"stage": "plan", "model": "gemini-2.0-flash"},
|
|
{"stage": "work", "model": "gemini-2.0-flash"},
|
|
{"stage": "review", "model": "gemini-2.0-flash"},
|
|
{"stage": "repair", "model": "gemini-2.0-flash"},
|
|
{"stage": "repair", "model": "gemini-2.0-flash"},
|
|
],
|
|
},
|
|
}
|
|
]
|
|
)
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestValidationError):
|
|
_load_tmp_manifest(path)
|
|
|
|
|
|
class TestEnumsAndBounds(unittest.TestCase):
|
|
"""Enum and numeric bound validation tests."""
|
|
|
|
def test_invalid_caller_rejected(self):
|
|
"""Invalid caller value is rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict(
|
|
matrix=[
|
|
{
|
|
"id": "bad-caller",
|
|
"caller": "invalid_caller",
|
|
"iop": {
|
|
"request_model": "claude-sonnet-4-20250514",
|
|
"requested_effort": "high",
|
|
"route_kind": "direct",
|
|
"route_id": "r1",
|
|
"expected_bindings": [
|
|
{"stage": "request", "model": "claude-sonnet-4-20250514"}
|
|
],
|
|
},
|
|
}
|
|
]
|
|
)
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestValidationError):
|
|
_load_tmp_manifest(path)
|
|
|
|
def test_invalid_route_kind_rejected(self):
|
|
"""Invalid route_kind is rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict(
|
|
matrix=[
|
|
{
|
|
"id": "bad-route",
|
|
"caller": "claude",
|
|
"iop": {
|
|
"request_model": "claude-sonnet-4-20250514",
|
|
"requested_effort": "high",
|
|
"route_kind": "invalid_route",
|
|
"route_id": "r1",
|
|
"expected_bindings": [
|
|
{"stage": "request", "model": "claude-sonnet-4-20250514"}
|
|
],
|
|
},
|
|
}
|
|
]
|
|
)
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestValidationError):
|
|
_load_tmp_manifest(path)
|
|
|
|
def test_invalid_cell_id_pattern_rejected(self):
|
|
"""Cell id with uppercase is rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict(
|
|
matrix=[
|
|
{
|
|
"id": "Bad-Id",
|
|
"caller": "claude",
|
|
"iop": {
|
|
"request_model": "claude-sonnet-4-20250514",
|
|
"requested_effort": "high",
|
|
"route_kind": "direct",
|
|
"route_id": "r1",
|
|
"expected_bindings": [
|
|
{"stage": "request", "model": "claude-sonnet-4-20250514"}
|
|
],
|
|
},
|
|
}
|
|
]
|
|
)
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestValidationError):
|
|
_load_tmp_manifest(path)
|
|
|
|
def test_cell_id_too_long_rejected(self):
|
|
"""Cell id exceeding 64 chars is rejected."""
|
|
long_id = "a" * 65
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict(
|
|
matrix=[
|
|
{
|
|
"id": long_id,
|
|
"caller": "claude",
|
|
"iop": {
|
|
"request_model": "claude-sonnet-4-20250514",
|
|
"requested_effort": "high",
|
|
"route_kind": "direct",
|
|
"route_id": "r1",
|
|
"expected_bindings": [
|
|
{"stage": "request", "model": "claude-sonnet-4-20250514"}
|
|
],
|
|
},
|
|
}
|
|
]
|
|
)
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestValidationError):
|
|
_load_tmp_manifest(path)
|
|
|
|
def test_run_seconds_zero_rejected(self):
|
|
"""timeout.run_seconds of 0 is rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict(
|
|
timeout={
|
|
"run_seconds": 0,
|
|
"idle_seconds": 30,
|
|
"quiet_seconds": 10,
|
|
"cleanup_grace_seconds": 5,
|
|
}
|
|
)
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestValidationError):
|
|
_load_tmp_manifest(path)
|
|
|
|
def test_run_seconds_too_large_rejected(self):
|
|
"""timeout.run_seconds > 86400 is rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict(
|
|
timeout={
|
|
"run_seconds": 86401,
|
|
"idle_seconds": 30,
|
|
"quiet_seconds": 10,
|
|
"cleanup_grace_seconds": 5,
|
|
}
|
|
)
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestValidationError):
|
|
_load_tmp_manifest(path)
|
|
|
|
def test_idle_seconds_zero_rejected(self):
|
|
"""timeout.idle_seconds of 0 is rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict(
|
|
timeout={
|
|
"run_seconds": 300,
|
|
"idle_seconds": 0,
|
|
"quiet_seconds": 10,
|
|
"cleanup_grace_seconds": 5,
|
|
}
|
|
)
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestValidationError):
|
|
_load_tmp_manifest(path)
|
|
|
|
def test_quiet_seconds_zero_rejected(self):
|
|
"""timeout.quiet_seconds of 0 is rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict(
|
|
timeout={
|
|
"run_seconds": 300,
|
|
"idle_seconds": 30,
|
|
"quiet_seconds": 0,
|
|
"cleanup_grace_seconds": 5,
|
|
}
|
|
)
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestValidationError):
|
|
_load_tmp_manifest(path)
|
|
|
|
def test_cleanup_grace_seconds_zero_rejected(self):
|
|
"""timeout.cleanup_grace_seconds of 0 is rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict(
|
|
timeout={
|
|
"run_seconds": 300,
|
|
"idle_seconds": 30,
|
|
"quiet_seconds": 10,
|
|
"cleanup_grace_seconds": 0,
|
|
}
|
|
)
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestValidationError):
|
|
_load_tmp_manifest(path)
|
|
|
|
def test_repetitions_zero_rejected(self):
|
|
"""repetitions of 0 is rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict(repetitions=0)
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestValidationError):
|
|
_load_tmp_manifest(path)
|
|
|
|
def test_repetitions_negative_rejected(self):
|
|
"""Negative repetitions is rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict(repetitions=-1)
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestValidationError):
|
|
_load_tmp_manifest(path)
|
|
|
|
def test_viewport_width_zero_rejected(self):
|
|
"""Viewport width of 0 is rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict(
|
|
viewports=[{"id": "vp", "width": 0, "height": 1080}]
|
|
)
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestValidationError):
|
|
_load_tmp_manifest(path)
|
|
|
|
def test_viewport_width_too_large_rejected(self):
|
|
"""Viewport width > 8192 is rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict(
|
|
viewports=[{"id": "vp", "width": 8193, "height": 1080}]
|
|
)
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestValidationError):
|
|
_load_tmp_manifest(path)
|
|
|
|
def test_viewport_height_too_large_rejected(self):
|
|
"""Viewport height > 8192 is rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict(
|
|
viewports=[{"id": "vp", "width": 1920, "height": 8193}]
|
|
)
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestValidationError):
|
|
_load_tmp_manifest(path)
|
|
|
|
def test_empty_viewports_rejected(self):
|
|
"""Empty viewports array is rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict(viewports=[])
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestValidationError):
|
|
_load_tmp_manifest(path)
|
|
|
|
def test_invalid_pipeline_version_rejected(self):
|
|
"""Invalid pipeline_version is rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict(pipeline_version="1")
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestValidationError):
|
|
_load_tmp_manifest(path)
|
|
|
|
def test_invalid_environment_rejected(self):
|
|
"""Invalid environment is rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict(environment="prod")
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestValidationError):
|
|
_load_tmp_manifest(path)
|
|
|
|
def test_invalid_session_policy_rejected(self):
|
|
"""Invalid session_policy is rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict(session_policy="persistent")
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestValidationError):
|
|
_load_tmp_manifest(path)
|
|
|
|
def test_invalid_setup_cache_policy_rejected(self):
|
|
"""Invalid setup_cache_policy is rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict(setup_cache_policy="shared")
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestValidationError):
|
|
_load_tmp_manifest(path)
|
|
|
|
def test_invalid_rubric_version_rejected(self):
|
|
"""Invalid rubric_version pattern is rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict(rubric_version="Invalid Version!")
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestValidationError):
|
|
_load_tmp_manifest(path)
|
|
|
|
|
|
class TestUnknownMembers(unittest.TestCase):
|
|
"""Unknown member rejection tests."""
|
|
|
|
def test_unknown_top_level_field_rejected(self):
|
|
"""Unknown top-level field is rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict()
|
|
d["unknown_field"] = "should_fail"
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestValidationError):
|
|
_load_tmp_manifest(path)
|
|
|
|
def test_unknown_timeout_field_rejected(self):
|
|
"""Unknown timeout field is rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict(
|
|
timeout={
|
|
"run_seconds": 300,
|
|
"idle_seconds": 30,
|
|
"quiet_seconds": 10,
|
|
"cleanup_grace_seconds": 5,
|
|
"extra_field": 42,
|
|
}
|
|
)
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestValidationError):
|
|
_load_tmp_manifest(path)
|
|
|
|
def test_unknown_fixture_field_rejected(self):
|
|
"""Unknown fixture field is rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict()
|
|
d["fixture"]["extra_fixture_field"] = "should_fail"
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestValidationError):
|
|
_load_tmp_manifest(path)
|
|
|
|
def test_unknown_cell_field_rejected(self):
|
|
"""Unknown cell field is rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict()
|
|
d["matrix"][0]["extra_cell_field"] = "should_fail"
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestValidationError):
|
|
_load_tmp_manifest(path)
|
|
|
|
def test_unknown_iop_field_rejected(self):
|
|
"""Unknown iop field is rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict()
|
|
d["matrix"][0]["iop"]["extra_iop_field"] = "should_fail"
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestValidationError):
|
|
_load_tmp_manifest(path)
|
|
|
|
def test_unknown_binding_field_rejected(self):
|
|
"""Unknown binding field is rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict()
|
|
d["matrix"][0]["iop"]["expected_bindings"][0]["extra_binding"] = "should_fail"
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestValidationError):
|
|
_load_tmp_manifest(path)
|
|
|
|
def test_unknown_viewport_field_rejected(self):
|
|
"""Unknown viewport field is rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict(
|
|
viewports=[{"id": "vp", "width": 1920, "height": 1080, "extra": True}]
|
|
)
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestValidationError):
|
|
_load_tmp_manifest(path)
|
|
|
|
def test_unknown_asset_field_rejected(self):
|
|
"""Unknown asset field is rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict()
|
|
d["fixture"]["assets"][0]["extra_asset"] = "should_fail"
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestValidationError):
|
|
_load_tmp_manifest(path)
|
|
|
|
|
|
class TestPathRules(unittest.TestCase):
|
|
"""Path escape, symlink, collision, and containment tests."""
|
|
|
|
def test_absolute_prompt_path_rejected(self):
|
|
"""Absolute prompt path is rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict()
|
|
d["fixture"]["prompt"] = "/absolute/path/prompt.md"
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestPathError):
|
|
_load_tmp_manifest(path)
|
|
|
|
def test_absolute_asset_source_rejected(self):
|
|
"""Absolute asset source path is rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict()
|
|
d["fixture"]["assets"][0]["source"] = "/absolute/source.txt"
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestPathError):
|
|
_load_tmp_manifest(path)
|
|
|
|
def test_absolute_workspace_path_rejected(self):
|
|
"""Absolute workspace_path is rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict()
|
|
d["fixture"]["assets"][0]["workspace_path"] = "/absolute/workspace.txt"
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestPathError):
|
|
_load_tmp_manifest(path)
|
|
|
|
def test_dotdot_escape_in_prompt_rejected(self):
|
|
"""Prompt path with .. escape is rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict()
|
|
d["fixture"]["prompt"] = "../escape/prompt.md"
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestPathError):
|
|
_load_tmp_manifest(path)
|
|
|
|
def test_dotdot_escape_in_asset_source_rejected(self):
|
|
"""Asset source with .. escape is rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict()
|
|
d["fixture"]["assets"][0]["source"] = "../escape/source.txt"
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestPathError):
|
|
_load_tmp_manifest(path)
|
|
|
|
def test_dotdot_escape_in_workspace_path_rejected(self):
|
|
"""Asset workspace_path with .. escape is rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict()
|
|
d["fixture"]["assets"][0]["workspace_path"] = "../escape/workspace.txt"
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestPathError):
|
|
_load_tmp_manifest(path)
|
|
|
|
def test_colon_in_path_rejected(self):
|
|
"""Path with colon is rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict()
|
|
d["fixture"]["prompt"] = "path:with:colons.md"
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestPathError):
|
|
_load_tmp_manifest(path)
|
|
|
|
def test_symlink_source_rejected(self):
|
|
"""Symlink as asset source is rejected."""
|
|
with tempfile.TemporaryDirectory(dir=_REPO_ROOT) as tmp_sub:
|
|
tmp_sub_dir = Path(tmp_sub)
|
|
rel_sub = tmp_sub_dir.relative_to(_REPO_ROOT)
|
|
real_file = tmp_sub_dir / "real.txt"
|
|
real_file.write_text("content")
|
|
link_file = tmp_sub_dir / "link.txt"
|
|
link_file.symlink_to(real_file)
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict()
|
|
d["fixture"]["assets"] = [
|
|
{
|
|
"source": str(rel_sub / "link.txt"),
|
|
"workspace_path": "workspace/link_dest.txt",
|
|
}
|
|
]
|
|
manifest_path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestPathError):
|
|
_load_tmp_manifest(manifest_path)
|
|
|
|
def test_destination_collision_rejected(self):
|
|
"""Two assets with the same workspace_path are rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict()
|
|
d["fixture"]["assets"] = [
|
|
{
|
|
"source": "scripts/fixtures/agent-comparison-benchmark/prompt.md",
|
|
"workspace_path": "workspace/dup.txt",
|
|
},
|
|
{
|
|
"source": "scripts/fixtures/agent-comparison-benchmark/reference.txt",
|
|
"workspace_path": "workspace/dup.txt",
|
|
},
|
|
]
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestPathError):
|
|
_load_tmp_manifest(path)
|
|
|
|
def test_output_root_not_under_runs_rejected(self):
|
|
"""output_root not under agent-test/runs/ is rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict()
|
|
d["output_root"] = "other/path/bench-01"
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestValidationError):
|
|
_load_tmp_manifest(path)
|
|
|
|
def test_output_root_with_subpath_rejected(self):
|
|
"""output_root with sub-path segments is rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict()
|
|
d["output_root"] = "agent-test/runs/sub/path"
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestValidationError):
|
|
_load_tmp_manifest(path)
|
|
|
|
def test_testbed_pattern_rejected(self):
|
|
r"""testbed not matching ^\.\./[^/]+$ is rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict()
|
|
d["testbed"] = "../../double-escape"
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestValidationError):
|
|
_load_tmp_manifest(path)
|
|
|
|
|
|
class TestChecksumAndDigest(unittest.TestCase):
|
|
"""Checksum and digest drift tests."""
|
|
|
|
def test_wrong_fixture_checksum_rejected(self):
|
|
"""Wrong fixture checksum is rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict()
|
|
d["fixture"]["checksum"] = "sha256:" + "0" * 64
|
|
p = tmp_dir / "manifest.json"
|
|
p.write_text(json.dumps(d, indent=2), encoding="utf-8")
|
|
with self.assertRaises(ManifestDigestError):
|
|
_load_tmp_manifest(p)
|
|
|
|
def test_computed_checksum_matches(self):
|
|
"""Computed checksum equals declared checksum for valid manifest."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict()
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
m = _load_tmp_manifest(path)
|
|
expected = digest_workspace_inputs(m.fixture.assets)
|
|
self.assertEqual(m.fixture.checksum, expected)
|
|
|
|
def test_manifest_digest_computed(self):
|
|
"""Manifest digest is computed deterministically."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict()
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
m = _load_tmp_manifest(path)
|
|
digest = digest_manifest_and_resolved_inputs(m)
|
|
self.assertTrue(digest.startswith("sha256:"))
|
|
self.assertEqual(len(digest), 7 + 64)
|
|
|
|
def test_manifest_digest_deterministic(self):
|
|
"""Same manifest produces the same digest on repeated calls."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict()
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
m = _load_tmp_manifest(path)
|
|
d1 = digest_manifest_and_resolved_inputs(m)
|
|
d2 = digest_manifest_and_resolved_inputs(m)
|
|
self.assertEqual(d1, d2)
|
|
|
|
|
|
class TestSecretRedaction(unittest.TestCase):
|
|
"""Secret and private-endpoint redaction in error messages."""
|
|
|
|
def test_secret_not_in_validation_error(self):
|
|
"""Secret values do not appear in validation errors."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict()
|
|
d["pipeline_version"] = _SENTINEL_SECRET
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestValidationError) as caught:
|
|
_load_tmp_manifest(path)
|
|
self.assertNotIn(_SENTINEL_SECRET, str(caught.exception))
|
|
|
|
def test_secret_not_in_path_error(self):
|
|
"""Secret values do not appear in path errors."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict()
|
|
d["fixture"]["prompt"] = f"nonexistent/{_SENTINEL_SECRET}/prompt.md"
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestPathError) as caught:
|
|
_load_tmp_manifest(path)
|
|
self.assertNotIn(_SENTINEL_SECRET, str(caught.exception))
|
|
|
|
def test_secret_not_in_digest_error(self):
|
|
"""Secret values do not appear in digest errors."""
|
|
with tempfile.TemporaryDirectory(dir=_REPO_ROOT) as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict()
|
|
d["output_root"] = f"agent-test/runs/{_SENTINEL_SECRET}"
|
|
d["fixture"]["checksum"] = "sha256:" + "0" * 64
|
|
p = tmp_dir / "manifest.json"
|
|
p.write_text(json.dumps(d, indent=2), encoding="utf-8")
|
|
with self.assertRaises(ManifestDigestError) as caught:
|
|
_load_tmp_manifest(p)
|
|
self.assertNotIn(_SENTINEL_SECRET, str(caught.exception))
|
|
|
|
def test_prompt_content_not_in_any_error(self):
|
|
"""Prompt content does not appear in any error."""
|
|
with tempfile.TemporaryDirectory(dir=_REPO_ROOT) as tmp:
|
|
tmp_dir = Path(tmp)
|
|
prompt_path = tmp_dir / "prompt.md"
|
|
prompt_path.write_text(_SENTINEL_PROMPT, encoding="utf-8")
|
|
d = _make_minimal_manifest_dict()
|
|
d["fixture"]["prompt"] = prompt_path.relative_to(_REPO_ROOT).as_posix()
|
|
d["fixture"]["checksum"] = "sha256:" + "0" * 64
|
|
manifest_path = tmp_dir / "manifest.json"
|
|
manifest_path.write_text(json.dumps(d, indent=2), encoding="utf-8")
|
|
with self.assertRaises(ManifestDigestError) as caught:
|
|
_load_tmp_manifest(manifest_path)
|
|
self.assertNotIn(_SENTINEL_PROMPT, str(caught.exception))
|
|
|
|
|
|
class TestSchemaLoaderParity(unittest.TestCase):
|
|
"""Schema and loader parity tests for types, bounds, enums, and grammar."""
|
|
|
|
def test_tracked_example_parity(self):
|
|
"""Tracked example loads cleanly."""
|
|
example_path = (
|
|
_REPO_ROOT
|
|
/ "scripts"
|
|
/ "fixtures"
|
|
/ "agent-comparison-benchmark-manifest.example.json"
|
|
)
|
|
m = load_manifest(example_path)
|
|
self.assertEqual(m.pipeline_version, "2")
|
|
self.assertEqual(m.testbed, "../iop-s2")
|
|
|
|
def test_tracked_fixtures_separate_generic_contract_from_direct_preflight(self):
|
|
fixtures = _REPO_ROOT / "scripts" / "fixtures"
|
|
generic = load_manifest(
|
|
fixtures / "agent-comparison-benchmark-manifest.example.json"
|
|
)
|
|
direct = load_manifest(
|
|
fixtures / "agent-comparison-benchmark-direct-preflight.example.json"
|
|
)
|
|
|
|
self.assertEqual(
|
|
[(cell.caller, cell.iop.route_kind) for cell in generic.matrix],
|
|
[
|
|
("agy", "execution_preset"),
|
|
("claude", "execution_preset"),
|
|
("codex", "execution_preset"),
|
|
],
|
|
)
|
|
self.assertEqual(len(direct.matrix), 5)
|
|
self.assertTrue(all(cell.iop.route_kind == "direct" for cell in direct.matrix))
|
|
self.assertEqual(
|
|
[
|
|
(cell.caller, cell.iop.request_model, cell.iop.requested_effort)
|
|
for cell in direct.matrix
|
|
],
|
|
[
|
|
("agy", "gemini-3.6-flash", "high"),
|
|
("claude", "gemini-3.6-flash", "high"),
|
|
("claude", "gpt-5.6-luna", "xhigh"),
|
|
("claude", "claude-sonnet-5", "max"),
|
|
("codex", "gpt-5.6-luna", "xhigh"),
|
|
],
|
|
)
|
|
public_aliases = {
|
|
cell.iop.request_model for cell in generic.matrix + direct.matrix
|
|
}
|
|
self.assertEqual(
|
|
public_aliases,
|
|
{"claude-sonnet-5", "gemini-3.6-flash", "gpt-5.6-luna"},
|
|
)
|
|
|
|
def test_booleans_rejected_in_numeric_fields(self):
|
|
"""Booleans in numeric fields raise ManifestValidationError."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
for field_name in ("run_seconds", "idle_seconds", "quiet_seconds", "cleanup_grace_seconds"):
|
|
d = _make_minimal_manifest_dict()
|
|
d["timeout"][field_name] = True
|
|
p = _write_tmp_manifest(tmp_dir, d, f"{field_name}_bool.json")
|
|
with self.assertRaises(ManifestValidationError):
|
|
_load_tmp_manifest(p)
|
|
|
|
d = _make_minimal_manifest_dict(repetitions=True)
|
|
p = _write_tmp_manifest(tmp_dir, d, "repetitions_bool.json")
|
|
with self.assertRaises(ManifestValidationError):
|
|
_load_tmp_manifest(p)
|
|
|
|
d = _make_minimal_manifest_dict(
|
|
viewports=[{"id": "vp", "width": True, "height": 1080}]
|
|
)
|
|
p = _write_tmp_manifest(tmp_dir, d, "width_bool.json")
|
|
with self.assertRaises(ManifestValidationError):
|
|
_load_tmp_manifest(p)
|
|
|
|
def test_preset_with_request_stage_rejected(self):
|
|
"""Execution preset cell with extra request stage raises ManifestValidationError."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict(
|
|
matrix=[
|
|
{
|
|
"id": "bad-preset",
|
|
"caller": "agy",
|
|
"iop": {
|
|
"request_model": "gemini-2.0-flash",
|
|
"requested_effort": "high",
|
|
"route_kind": "execution_preset",
|
|
"route_id": "agy-generic",
|
|
"expected_bindings": [
|
|
{"stage": "request", "model": "gemini-2.0-flash"},
|
|
{"stage": "selector", "model": "gemini-2.0-flash"},
|
|
{"stage": "plan", "model": "gemini-2.0-flash"},
|
|
{"stage": "work", "model": "gemini-2.0-flash"},
|
|
{"stage": "review", "model": "gemini-2.0-flash"},
|
|
],
|
|
},
|
|
}
|
|
]
|
|
)
|
|
p = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestValidationError):
|
|
_load_tmp_manifest(p)
|
|
|
|
def test_testbed_must_be_exact(self):
|
|
"""Testbed other than ../iop-s2 raises ManifestValidationError."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict(testbed="../another-repo")
|
|
p = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestValidationError):
|
|
_load_tmp_manifest(p)
|
|
|
|
def test_dotted_tokens_accepted(self):
|
|
"""Tokens with dots like v1.0 and gemini-2.0-flash load without error."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict(rubric_version="landing-quality-v1")
|
|
d["matrix"][0]["iop"]["request_model"] = "claude-sonnet-4-20250514"
|
|
p = _write_tmp_manifest(tmp_dir, d)
|
|
m = _load_tmp_manifest(p)
|
|
self.assertEqual(m.rubric_version, "landing-quality-v1")
|
|
self.assertEqual(m.matrix[0].iop.request_model, "claude-sonnet-4-20250514")
|
|
|
|
def _evaluate_schema_execution_preset_bindings(
|
|
self, schema_path: Path, expected_bindings: list[dict]
|
|
) -> bool:
|
|
"""Evaluate candidate expected_bindings against declared execution_preset schema constraints."""
|
|
raw_schema = json.loads(schema_path.read_text(encoding="utf-8"))
|
|
iop_cell_schema = raw_schema["$defs"]["iop_cell"]
|
|
preset_branch = None
|
|
for cond in iop_cell_schema.get("allOf", []):
|
|
if cond.get("if", {}).get("properties", {}).get("route_kind", {}).get("const") == "execution_preset":
|
|
preset_branch = cond.get("then", {}).get("properties", {}).get("expected_bindings", {})
|
|
break
|
|
if not preset_branch:
|
|
return False
|
|
|
|
min_items = preset_branch.get("minItems", 0)
|
|
max_items = preset_branch.get("maxItems", float("inf"))
|
|
if not (min_items <= len(expected_bindings) <= max_items):
|
|
return False
|
|
|
|
allowed_enum = preset_branch.get("items", {}).get("properties", {}).get("stage", {}).get("enum", [])
|
|
for binding in expected_bindings:
|
|
if not isinstance(binding, dict) or "stage" not in binding or binding["stage"] not in allowed_enum:
|
|
return False
|
|
|
|
all_of = preset_branch.get("allOf", [])
|
|
for sub in all_of:
|
|
contains = sub.get("contains", {})
|
|
target_stage = contains.get("properties", {}).get("stage", {}).get("const")
|
|
min_c = sub.get("minContains", 0)
|
|
max_c = sub.get("maxContains", float("inf"))
|
|
matches = sum(1 for b in expected_bindings if isinstance(b, dict) and b.get("stage") == target_stage)
|
|
if not (min_c <= matches <= max_c):
|
|
return False
|
|
|
|
return True
|
|
|
|
def test_schema_and_loader_share_route_shape_corpus(self):
|
|
"""Schema-backed evaluator and loader agree on all valid and malformed route shapes."""
|
|
schema_path = (
|
|
_REPO_ROOT
|
|
/ "scripts"
|
|
/ "fixtures"
|
|
/ "agent-comparison-benchmark-manifest.schema.json"
|
|
)
|
|
example_path = (
|
|
_REPO_ROOT
|
|
/ "scripts"
|
|
/ "fixtures"
|
|
/ "agent-comparison-benchmark-manifest.example.json"
|
|
)
|
|
|
|
# 1. Tracked example preset cells
|
|
example_raw = json.loads(example_path.read_text(encoding="utf-8"))
|
|
for cell in example_raw["matrix"]:
|
|
iop = cell["iop"]
|
|
if iop["route_kind"] == "execution_preset":
|
|
bindings = iop["expected_bindings"]
|
|
self.assertTrue(self._evaluate_schema_execution_preset_bindings(schema_path, bindings))
|
|
|
|
# 2. Valid four-stage and five-stage presets
|
|
valid_4 = [
|
|
{"stage": "selector", "model": "m1"},
|
|
{"stage": "plan", "model": "m1"},
|
|
{"stage": "work", "model": "m1"},
|
|
{"stage": "review", "model": "m1"},
|
|
]
|
|
valid_5 = [
|
|
{"stage": "selector", "model": "m1"},
|
|
{"stage": "plan", "model": "m1"},
|
|
{"stage": "work", "model": "m1"},
|
|
{"stage": "review", "model": "m1"},
|
|
{"stage": "repair", "model": "m1"},
|
|
]
|
|
|
|
for valid_b in (valid_4, valid_5):
|
|
self.assertTrue(self._evaluate_schema_execution_preset_bindings(schema_path, valid_b))
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict(
|
|
matrix=[
|
|
{
|
|
"id": "valid-preset",
|
|
"caller": "agy",
|
|
"iop": {
|
|
"request_model": "gemini-2.0-flash",
|
|
"requested_effort": "high",
|
|
"route_kind": "execution_preset",
|
|
"route_id": "r1",
|
|
"expected_bindings": valid_b,
|
|
},
|
|
}
|
|
]
|
|
)
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
m = _load_tmp_manifest(path)
|
|
self.assertEqual(len(m.matrix[0].iop.expected_bindings), len(valid_b))
|
|
|
|
# 3. Malformed preset stage variants
|
|
malformed_corpus = [
|
|
# missing selector
|
|
[{"stage": "plan", "model": "m1"}, {"stage": "work", "model": "m1"}, {"stage": "review", "model": "m1"}, {"stage": "repair", "model": "m1"}],
|
|
# missing plan
|
|
[{"stage": "selector", "model": "m1"}, {"stage": "work", "model": "m1"}, {"stage": "review", "model": "m1"}, {"stage": "repair", "model": "m1"}],
|
|
# missing work
|
|
[{"stage": "selector", "model": "m1"}, {"stage": "plan", "model": "m1"}, {"stage": "review", "model": "m1"}, {"stage": "repair", "model": "m1"}],
|
|
# missing review
|
|
[{"stage": "selector", "model": "m1"}, {"stage": "plan", "model": "m1"}, {"stage": "work", "model": "m1"}, {"stage": "repair", "model": "m1"}],
|
|
# duplicate repair
|
|
[{"stage": "selector", "model": "m1"}, {"stage": "plan", "model": "m1"}, {"stage": "work", "model": "m1"}, {"stage": "review", "model": "m1"}, {"stage": "repair", "model": "m1"}, {"stage": "repair", "model": "m2"}],
|
|
# duplicate work
|
|
[{"stage": "selector", "model": "m1"}, {"stage": "plan", "model": "m1"}, {"stage": "work", "model": "m1"}, {"stage": "work", "model": "m2"}],
|
|
# extra request stage
|
|
[{"stage": "request", "model": "m1"}, {"stage": "selector", "model": "m1"}, {"stage": "plan", "model": "m1"}, {"stage": "work", "model": "m1"}, {"stage": "review", "model": "m1"}],
|
|
]
|
|
|
|
for bad_b in malformed_corpus:
|
|
self.assertFalse(self._evaluate_schema_execution_preset_bindings(schema_path, bad_b))
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict(
|
|
matrix=[
|
|
{
|
|
"id": "bad-preset",
|
|
"caller": "agy",
|
|
"iop": {
|
|
"request_model": "gemini-2.0-flash",
|
|
"requested_effort": "high",
|
|
"route_kind": "execution_preset",
|
|
"route_id": "r1",
|
|
"expected_bindings": bad_b,
|
|
},
|
|
}
|
|
]
|
|
)
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestValidationError):
|
|
_load_tmp_manifest(path)
|
|
|
|
|
|
class TestCanonicalPaths(unittest.TestCase):
|
|
"""Canonical path normalization, containment, and collision tests."""
|
|
|
|
def test_non_normal_asset_source_rejected(self):
|
|
"""Asset source with ./ is rejected as non-canonical."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict()
|
|
d["fixture"]["assets"][0]["source"] = (
|
|
"scripts/fixtures/agent-comparison-benchmark/./reference.txt"
|
|
)
|
|
p = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestPathError):
|
|
_load_tmp_manifest(p)
|
|
|
|
def test_non_normal_workspace_path_rejected(self):
|
|
"""Asset workspace_path with ./ is rejected as non-canonical."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict()
|
|
d["fixture"]["assets"][0]["workspace_path"] = "workspace/./reference.txt"
|
|
p = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestPathError):
|
|
_load_tmp_manifest(p)
|
|
|
|
def test_output_root_containment_and_normalization(self):
|
|
"""output_root escaping agent-test/runs via .. or non-normal segment is rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict()
|
|
d["output_root"] = "agent-test/runs/../escape"
|
|
p = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises((ManifestValidationError, ManifestPathError)):
|
|
_load_tmp_manifest(p)
|
|
|
|
|
|
class TestCanonicalDigestAPI(unittest.TestCase):
|
|
"""Public digest exposure, content immutability, and drift tests."""
|
|
|
|
def test_loaded_manifest_digest_property(self):
|
|
"""Manifest object exposes digest property matching sha256: format."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
p = _write_tmp_manifest(tmp_dir, _make_minimal_manifest_dict())
|
|
m = _load_tmp_manifest(p)
|
|
self.assertTrue(hasattr(m, "digest"))
|
|
self.assertTrue(m.digest.startswith("sha256:"))
|
|
self.assertEqual(len(m.digest), 7 + 64)
|
|
|
|
def test_digest_helpers_match_loaded_manifest(self):
|
|
"""digest helpers reproduce loaded checksum and digest."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
p = _write_tmp_manifest(tmp_dir, _make_minimal_manifest_dict())
|
|
m = _load_tmp_manifest(p)
|
|
self.assertEqual(digest_workspace_inputs(m.fixture.assets), m.fixture.checksum)
|
|
self.assertEqual(digest_manifest_and_resolved_inputs(m), m.digest)
|
|
|
|
def test_repr_omits_content_bytes(self):
|
|
"""repr of Manifest, Fixture, AssetMapping does not include raw prompt/asset bytes."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
p = _write_tmp_manifest(tmp_dir, _make_minimal_manifest_dict())
|
|
m = _load_tmp_manifest(p)
|
|
r_manifest = repr(m)
|
|
r_fixture = repr(m.fixture)
|
|
r_asset = repr(m.fixture.assets[0])
|
|
self.assertNotIn(_SENTINEL_PROMPT, r_manifest)
|
|
self.assertNotIn(_SENTINEL_PROMPT, r_fixture)
|
|
self.assertNotIn(_SENTINEL_PROMPT, r_asset)
|
|
self.assertNotIn("content=", r_asset)
|
|
|
|
def test_digest_signatures_exact(self):
|
|
"""digest helpers reject legacy override arguments."""
|
|
sig_ws = inspect.signature(digest_workspace_inputs)
|
|
self.assertEqual(list(sig_ws.parameters.keys()), ["assets"])
|
|
sig_manifest = inspect.signature(digest_manifest_and_resolved_inputs)
|
|
self.assertEqual(list(sig_manifest.parameters.keys()), ["manifest"])
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
p = _write_tmp_manifest(tmp_dir, _make_minimal_manifest_dict())
|
|
m = _load_tmp_manifest(p)
|
|
with self.assertRaises(TypeError):
|
|
digest_workspace_inputs(m.fixture.assets, read_content=True) # type: ignore
|
|
with self.assertRaises(TypeError):
|
|
digest_manifest_and_resolved_inputs(m, prompt_content=b"test") # type: ignore
|
|
|
|
def test_asset_input_order_equivalence_and_canonicalization(self):
|
|
"""Assets passed in different order produce identical sorted assets, checksum, and digest."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d_order1 = _make_minimal_manifest_dict()
|
|
d_order1["fixture"]["assets"] = [
|
|
{"source": "scripts/fixtures/agent-comparison-benchmark/reference.txt", "workspace_path": "workspace/z_ref.txt"},
|
|
{"source": "scripts/fixtures/agent-comparison-benchmark/prompt.md", "workspace_path": "workspace/a_prompt.md"},
|
|
]
|
|
|
|
d_order2 = _make_minimal_manifest_dict()
|
|
d_order2["fixture"]["assets"] = [
|
|
{"source": "scripts/fixtures/agent-comparison-benchmark/prompt.md", "workspace_path": "workspace/a_prompt.md"},
|
|
{"source": "scripts/fixtures/agent-comparison-benchmark/reference.txt", "workspace_path": "workspace/z_ref.txt"},
|
|
]
|
|
|
|
p1 = _write_tmp_manifest(tmp_dir, d_order1, "order1.json")
|
|
p2 = _write_tmp_manifest(tmp_dir, d_order2, "order2.json")
|
|
m1 = _load_tmp_manifest(p1)
|
|
m2 = _load_tmp_manifest(p2)
|
|
|
|
self.assertEqual(m1.fixture.assets[0].workspace_path, "workspace/a_prompt.md")
|
|
self.assertEqual(m1.fixture.assets[1].workspace_path, "workspace/z_ref.txt")
|
|
self.assertEqual(m1.fixture.assets, m2.fixture.assets)
|
|
self.assertEqual(m1.fixture.checksum, m2.fixture.checksum)
|
|
self.assertEqual(m1.digest, m2.digest)
|
|
|
|
def test_input_drift_changes_digest(self):
|
|
"""Altering manifest, prompt content, asset path, or asset content changes m.digest."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d1 = _make_minimal_manifest_dict()
|
|
p1 = _write_tmp_manifest(tmp_dir, d1, "m1.json")
|
|
m1 = _load_tmp_manifest(p1)
|
|
|
|
# A valid evaluator binding drift remains loadable and changes the
|
|
# digest even though the rubric revision itself is closed.
|
|
d2 = _make_minimal_manifest_dict()
|
|
d2["evaluator"]["iop"]["route_id"] = "gpt-5.6-luna-alt"
|
|
p2 = _write_tmp_manifest(tmp_dir, d2, "m2.json")
|
|
m2 = _load_tmp_manifest(p2)
|
|
self.assertNotEqual(m1.digest, m2.digest)
|
|
|
|
# Prompt content drift
|
|
prompt_changed = replace(
|
|
m1,
|
|
fixture=replace(m1.fixture, prompt_content=m1.fixture.prompt_content + b" changed"),
|
|
)
|
|
self.assertNotEqual(digest_manifest_and_resolved_inputs(prompt_changed), m1.digest)
|
|
|
|
# Asset source drift
|
|
asset0 = m1.fixture.assets[0]
|
|
source_changed = replace(
|
|
m1,
|
|
fixture=replace(
|
|
m1.fixture,
|
|
assets=(replace(asset0, source="changed/source.txt"),) + m1.fixture.assets[1:],
|
|
),
|
|
)
|
|
self.assertNotEqual(digest_manifest_and_resolved_inputs(source_changed), m1.digest)
|
|
|
|
# Asset workspace_path drift
|
|
d3 = _make_minimal_manifest_dict()
|
|
d3["fixture"]["assets"][0]["workspace_path"] = "workspace/other_ref.txt"
|
|
p3 = _write_tmp_manifest(tmp_dir, d3, "m3.json")
|
|
m3 = _load_tmp_manifest(p3)
|
|
self.assertNotEqual(m1.fixture.checksum, m3.fixture.checksum)
|
|
self.assertNotEqual(m1.digest, m3.digest)
|
|
|
|
dest_changed = replace(
|
|
m1,
|
|
fixture=replace(
|
|
m1.fixture,
|
|
assets=(replace(asset0, workspace_path="workspace/other_ref.txt"),) + m1.fixture.assets[1:],
|
|
),
|
|
)
|
|
self.assertNotEqual(digest_manifest_and_resolved_inputs(dest_changed), m1.digest)
|
|
|
|
# Asset content drift
|
|
content_changed = replace(
|
|
m1,
|
|
fixture=replace(
|
|
m1.fixture,
|
|
assets=(replace(asset0, content=b"changed asset bytes"),) + m1.fixture.assets[1:],
|
|
),
|
|
)
|
|
self.assertNotEqual(digest_manifest_and_resolved_inputs(content_changed), m1.digest)
|
|
|
|
asset_a = AssetMapping(source="src.txt", workspace_path="w.txt", content=b"content A")
|
|
asset_b = AssetMapping(source="src.txt", workspace_path="w.txt", content=b"content B")
|
|
self.assertNotEqual(digest_workspace_inputs([asset_a]), digest_workspace_inputs([asset_b]))
|
|
|
|
|
|
class TestCLI(unittest.TestCase):
|
|
"""Public CLI tests."""
|
|
|
|
def _run_cli(self, *args: str) -> subprocess.CompletedProcess:
|
|
cli = str(_REPO_ROOT / "scripts" / "agent_comparison_benchmark.py")
|
|
return subprocess.run(
|
|
[sys.executable, cli, *args],
|
|
capture_output=True,
|
|
text=True,
|
|
cwd=str(_REPO_ROOT),
|
|
)
|
|
|
|
def test_cli_validate_valid_manifest(self):
|
|
"""Valid manifest exits 0 with sanitized single success line."""
|
|
example = str(
|
|
_REPO_ROOT
|
|
/ "scripts"
|
|
/ "fixtures"
|
|
/ "agent-comparison-benchmark-manifest.example.json"
|
|
)
|
|
if not Path(example).exists():
|
|
self.skipTest("example manifest not found")
|
|
result = self._run_cli("validate", "--manifest", example)
|
|
self.assertEqual(result.returncode, 0)
|
|
stdout_lines = [line for line in result.stdout.splitlines() if line.strip()]
|
|
stderr_lines = [line for line in result.stderr.splitlines() if line.strip()]
|
|
self.assertEqual(len(stdout_lines), 1)
|
|
self.assertEqual(stdout_lines[0], "ok: manifest is valid")
|
|
self.assertEqual(len(stderr_lines), 0)
|
|
|
|
def test_cli_validate_missing_file(self):
|
|
"""Missing manifest file exits 69 with single sanitized line."""
|
|
result = self._run_cli("validate", "--manifest", "/nonexistent/path.json")
|
|
self.assertEqual(result.returncode, 69)
|
|
stdout_lines = [line for line in result.stdout.splitlines() if line.strip()]
|
|
stderr_lines = [line for line in result.stderr.splitlines() if line.strip()]
|
|
self.assertEqual(len(stdout_lines), 0)
|
|
self.assertEqual(len(stderr_lines), 1)
|
|
self.assertIn("error:", stderr_lines[0])
|
|
self.assertNotIn("Traceback", result.stderr)
|
|
|
|
def test_cli_validate_malformed_json(self):
|
|
"""Malformed JSON exits 69 with single sanitized line."""
|
|
with tempfile.NamedTemporaryFile(
|
|
mode="w", suffix=".json", delete=False
|
|
) as f:
|
|
f.write("{invalid json")
|
|
tmp_path = f.name
|
|
try:
|
|
result = self._run_cli("validate", "--manifest", tmp_path)
|
|
self.assertEqual(result.returncode, 69)
|
|
stdout_lines = [line for line in result.stdout.splitlines() if line.strip()]
|
|
stderr_lines = [line for line in result.stderr.splitlines() if line.strip()]
|
|
self.assertEqual(len(stdout_lines), 0)
|
|
self.assertEqual(len(stderr_lines), 1)
|
|
self.assertIn("error:", stderr_lines[0])
|
|
self.assertNotIn("Traceback", result.stderr)
|
|
finally:
|
|
os.unlink(tmp_path)
|
|
|
|
def test_cli_validate_secret_manifest(self):
|
|
"""Manifest with secret values exits 69 without echoing secrets."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict()
|
|
d["pipeline_version"] = _SENTINEL_SECRET
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
result = self._run_cli("validate", "--manifest", str(path))
|
|
self.assertEqual(result.returncode, 69)
|
|
self.assertNotIn(_SENTINEL_SECRET, result.stdout)
|
|
self.assertNotIn(_SENTINEL_SECRET, result.stderr)
|
|
self.assertNotIn("Traceback", result.stderr)
|
|
|
|
def test_cli_usage_error(self):
|
|
"""Missing subcommand exits 64 with single sanitized line."""
|
|
result = self._run_cli()
|
|
self.assertEqual(result.returncode, 64)
|
|
stdout_lines = [line for line in result.stdout.splitlines() if line.strip()]
|
|
stderr_lines = [line for line in result.stderr.splitlines() if line.strip()]
|
|
self.assertEqual(len(stdout_lines), 0)
|
|
self.assertEqual(len(stderr_lines), 1)
|
|
self.assertEqual(stderr_lines[0], "error: invalid usage")
|
|
|
|
def test_cli_validate_no_manifest_flag(self):
|
|
"""Missing --manifest flag exits 64 with single sanitized line."""
|
|
result = self._run_cli("validate")
|
|
self.assertEqual(result.returncode, 64)
|
|
stdout_lines = [line for line in result.stdout.splitlines() if line.strip()]
|
|
stderr_lines = [line for line in result.stderr.splitlines() if line.strip()]
|
|
self.assertEqual(len(stdout_lines), 0)
|
|
self.assertEqual(len(stderr_lines), 1)
|
|
self.assertEqual(stderr_lines[0], "error: invalid usage")
|
|
|
|
def test_cli_validate_invalid_utf8(self):
|
|
"""Invalid UTF-8 manifest file exits 69 with single sanitized error line."""
|
|
with tempfile.NamedTemporaryFile(mode="wb", suffix=".json", delete=False) as f:
|
|
f.write(b"\x80\xff\xfe")
|
|
tmp_path = f.name
|
|
try:
|
|
result = self._run_cli("validate", "--manifest", tmp_path)
|
|
self.assertEqual(result.returncode, 69)
|
|
stdout_lines = [line for line in result.stdout.splitlines() if line.strip()]
|
|
stderr_lines = [line for line in result.stderr.splitlines() if line.strip()]
|
|
self.assertEqual(len(stdout_lines), 0)
|
|
self.assertEqual(len(stderr_lines), 1)
|
|
self.assertEqual(stderr_lines[0], "error: invalid JSON format")
|
|
self.assertNotIn("Traceback", result.stderr)
|
|
finally:
|
|
os.unlink(tmp_path)
|
|
|
|
def test_cli_validate_checksum_mismatch(self):
|
|
"""Manifest with checksum mismatch exits 69 with single sanitized error line."""
|
|
with tempfile.TemporaryDirectory(dir=_REPO_ROOT) as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict()
|
|
d["fixture"]["checksum"] = "sha256:" + "0" * 64
|
|
p = tmp_dir / "manifest.json"
|
|
p.write_text(json.dumps(d, indent=2), encoding="utf-8")
|
|
result = self._run_cli("validate", "--manifest", str(p))
|
|
self.assertEqual(result.returncode, 69)
|
|
stdout_lines = [line for line in result.stdout.splitlines() if line.strip()]
|
|
stderr_lines = [line for line in result.stderr.splitlines() if line.strip()]
|
|
self.assertEqual(len(stdout_lines), 0)
|
|
self.assertEqual(len(stderr_lines), 1)
|
|
self.assertEqual(stderr_lines[0], "error: fixture.checksum mismatch")
|
|
self.assertNotIn("Traceback", result.stderr)
|
|
|
|
def test_cli_validate_secret_missing_path(self):
|
|
"""Secret in missing path exits 69 with single sanitized error line without echoing secret."""
|
|
with tempfile.TemporaryDirectory(dir=_REPO_ROOT) as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict()
|
|
d["fixture"]["prompt"] = f"nonexistent/{_SENTINEL_SECRET}/prompt.md"
|
|
p = tmp_dir / "manifest.json"
|
|
p.write_text(json.dumps(d, indent=2), encoding="utf-8")
|
|
result = self._run_cli("validate", "--manifest", str(p))
|
|
self.assertEqual(result.returncode, 69)
|
|
stdout_lines = [line for line in result.stdout.splitlines() if line.strip()]
|
|
stderr_lines = [line for line in result.stderr.splitlines() if line.strip()]
|
|
self.assertEqual(stdout_lines, [])
|
|
self.assertEqual(len(stderr_lines), 1)
|
|
self.assertNotIn(_SENTINEL_SECRET, result.stdout)
|
|
self.assertNotIn(_SENTINEL_SECRET, result.stderr)
|
|
self.assertNotIn("Traceback", result.stderr)
|
|
|
|
def test_cli_validate_secret_unknown_argument(self):
|
|
"""Secret in unknown CLI flag exits 64 without echoing secret."""
|
|
example = str(
|
|
_REPO_ROOT
|
|
/ "scripts"
|
|
/ "fixtures"
|
|
/ "agent-comparison-benchmark-manifest.example.json"
|
|
)
|
|
result = self._run_cli("validate", "--manifest", example, f"--secret={_SENTINEL_SECRET}")
|
|
self.assertEqual(result.returncode, 64)
|
|
stdout_lines = [line for line in result.stdout.splitlines() if line.strip()]
|
|
stderr_lines = [line for line in result.stderr.splitlines() if line.strip()]
|
|
self.assertEqual(len(stdout_lines), 0)
|
|
self.assertEqual(len(stderr_lines), 1)
|
|
self.assertEqual(stderr_lines[0], "error: invalid usage")
|
|
self.assertNotIn(_SENTINEL_SECRET, result.stdout)
|
|
self.assertNotIn(_SENTINEL_SECRET, result.stderr)
|
|
self.assertNotIn("Traceback", result.stderr)
|
|
|
|
|
|
class TestValidateManifestBytes(unittest.TestCase):
|
|
"""validate_manifest_bytes tests."""
|
|
|
|
def test_validate_bytes_valid(self):
|
|
"""Valid bytes validate without disk write."""
|
|
d = _make_minimal_manifest_dict()
|
|
# Compute correct checksum for the fixture
|
|
d["fixture"]["checksum"] = _compute_fixture_checksum(_REPO_ROOT, d["fixture"])
|
|
data = json.dumps(d).encode("utf-8")
|
|
m = validate_manifest_bytes(data, repo_root=_REPO_ROOT)
|
|
self.assertEqual(m.pipeline_version, "2")
|
|
|
|
def test_validate_bytes_invalid(self):
|
|
"""Invalid bytes raise error."""
|
|
data = b"{invalid"
|
|
with self.assertRaises(ManifestValidationError):
|
|
validate_manifest_bytes(data)
|
|
|
|
|
|
class TestFrozenReturnTypes(unittest.TestCase):
|
|
"""Return type immutability tests."""
|
|
|
|
def test_manifest_is_frozen(self):
|
|
"""Manifest is a frozen dataclass."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
path = _write_tmp_manifest(tmp_dir, _make_minimal_manifest_dict())
|
|
m = _load_tmp_manifest(path)
|
|
self.assertTrue(hasattr(m, "__dataclass_fields__"))
|
|
with self.assertRaises(AttributeError):
|
|
m.repetitions = 99
|
|
|
|
def test_timeout_is_frozen(self):
|
|
"""Timeout is frozen."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
path = _write_tmp_manifest(tmp_dir, _make_minimal_manifest_dict())
|
|
m = _load_tmp_manifest(path)
|
|
with self.assertRaises(AttributeError):
|
|
m.timeout.run_seconds = 999
|
|
|
|
def test_viewport_is_frozen(self):
|
|
"""Viewport is frozen."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
path = _write_tmp_manifest(tmp_dir, _make_minimal_manifest_dict())
|
|
m = _load_tmp_manifest(path)
|
|
with self.assertRaises(AttributeError):
|
|
m.viewports[0].width = 9999
|
|
|
|
def test_cell_is_frozen(self):
|
|
"""MatrixCell is frozen."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
path = _write_tmp_manifest(tmp_dir, _make_minimal_manifest_dict())
|
|
m = _load_tmp_manifest(path)
|
|
with self.assertRaises(AttributeError):
|
|
m.matrix[0].id = "changed"
|
|
|
|
def test_tuple_fields_are_tuples(self):
|
|
"""tuple fields are actual tuples, not lists."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
path = _write_tmp_manifest(tmp_dir, _make_minimal_manifest_dict())
|
|
m = _load_tmp_manifest(path)
|
|
self.assertIsInstance(m.viewports, tuple)
|
|
self.assertIsInstance(m.matrix, tuple)
|
|
self.assertIsInstance(m.fixture.assets, tuple)
|
|
self.assertIsInstance(m.matrix[0].iop.expected_bindings, tuple)
|
|
|
|
|
|
class TestEdgeCases(unittest.TestCase):
|
|
"""Additional edge cases."""
|
|
|
|
def test_non_object_top_level_rejected(self):
|
|
"""Top-level JSON array is rejected."""
|
|
with tempfile.NamedTemporaryFile(
|
|
mode="w", suffix=".json", delete=False
|
|
) as f:
|
|
f.write("[]")
|
|
tmp_path = f.name
|
|
try:
|
|
with self.assertRaises(ManifestValidationError):
|
|
_load_tmp_manifest(tmp_path)
|
|
finally:
|
|
os.unlink(tmp_path)
|
|
|
|
def test_missing_required_field_rejected(self):
|
|
"""Missing required top-level field is rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict()
|
|
del d["timeout"]
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestValidationError):
|
|
_load_tmp_manifest(path)
|
|
|
|
def test_fixture_missing_required_field_rejected(self):
|
|
"""Missing fixture.version is rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict()
|
|
del d["fixture"]["version"]
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestValidationError):
|
|
_load_tmp_manifest(path)
|
|
|
|
def test_fixture_missing_prompt_file_rejected(self):
|
|
"""Prompt file that does not exist is rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict()
|
|
d["fixture"]["prompt"] = "nonexistent_prompt.md"
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestPathError):
|
|
_load_tmp_manifest(path)
|
|
|
|
def test_fixture_missing_asset_file_rejected(self):
|
|
"""Asset source file that does not exist is rejected."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict()
|
|
d["fixture"]["assets"] = [
|
|
{
|
|
"source": "nonexistent_source.txt",
|
|
"workspace_path": "workspace/dst.txt",
|
|
}
|
|
]
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
with self.assertRaises(ManifestPathError):
|
|
_load_tmp_manifest(path)
|
|
|
|
def test_multiple_assets_loaded(self):
|
|
"""Manifest with multiple assets loads correctly."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict()
|
|
d["fixture"]["assets"] = [
|
|
{
|
|
"source": "scripts/fixtures/agent-comparison-benchmark/prompt.md",
|
|
"workspace_path": "workspace/prompt.md",
|
|
},
|
|
{
|
|
"source": "scripts/fixtures/agent-comparison-benchmark/reference.txt",
|
|
"workspace_path": "workspace/reference.txt",
|
|
},
|
|
]
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
m = _load_tmp_manifest(path)
|
|
self.assertEqual(len(m.fixture.assets), 2)
|
|
|
|
def test_file_not_found(self):
|
|
"""Non-existent manifest file raises ManifestValidationError."""
|
|
with self.assertRaises(ManifestValidationError):
|
|
load_manifest("/nonexistent/manifest.json")
|
|
|
|
def test_caller_request_vs_evidence_separation(self):
|
|
"""request_model/requested_effort are separate from route/binding evidence."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_dir = Path(tmp)
|
|
d = _make_minimal_manifest_dict(
|
|
matrix=[
|
|
{
|
|
"id": "separation-test",
|
|
"caller": "claude",
|
|
"iop": {
|
|
"request_model": "claude-sonnet-4-20250514",
|
|
"requested_effort": "high",
|
|
"route_kind": "direct",
|
|
"route_id": "claude-direct",
|
|
"expected_bindings": [
|
|
{
|
|
"stage": "request",
|
|
"model": "claude-sonnet-4-20250514",
|
|
"effort": "high",
|
|
}
|
|
],
|
|
},
|
|
}
|
|
]
|
|
)
|
|
path = _write_tmp_manifest(tmp_dir, d)
|
|
m = _load_tmp_manifest(path)
|
|
cell = m.matrix[0]
|
|
# request_model and requested_effort are on iop, not in bindings
|
|
self.assertEqual(cell.iop.request_model, "claude-sonnet-4-20250514")
|
|
self.assertEqual(cell.iop.requested_effort, "high")
|
|
# route_kind/route_id are preflight evidence, not adapter inputs
|
|
self.assertEqual(cell.iop.route_kind, "direct")
|
|
self.assertEqual(cell.iop.route_id, "claude-direct")
|
|
# bindings contain the expected evidence
|
|
binding = cell.iop.expected_bindings[0]
|
|
self.assertEqual(binding.stage, "request")
|
|
self.assertEqual(binding.model, "claude-sonnet-4-20250514")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|