2485 lines
104 KiB
Python
2485 lines
104 KiB
Python
"""Network-free integration tests for public benchmark preflight."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import contextlib
|
|
import datetime
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
import re
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import unittest
|
|
from dataclasses import replace
|
|
from pathlib import Path
|
|
from unittest import mock
|
|
from urllib.error import HTTPError
|
|
|
|
from scripts import agent_comparison_benchmark as benchmark_cli
|
|
from scripts.agent_benchmark import live_iop
|
|
from scripts.agent_benchmark.browser_cdp import RenderObservation, ViewportObservation
|
|
from scripts.agent_benchmark.attempts import (
|
|
CapabilityUnavailable,
|
|
PreflightObservation,
|
|
RunBusyError,
|
|
RunStore,
|
|
Slot,
|
|
collect_preflight_observations,
|
|
preflight_manifest,
|
|
)
|
|
from scripts.agent_benchmark.connectivity import (
|
|
ISSUE_RESUME_CODES,
|
|
CallerCapability,
|
|
ConnectivityIssue,
|
|
EffectiveBinding,
|
|
RequestedEffectiveBinding,
|
|
canonical_evidence_bytes,
|
|
make_result,
|
|
)
|
|
from scripts.agent_benchmark.codex_iop import CodexInvocationResult
|
|
from scripts.agent_benchmark.manifest import (
|
|
AssetMapping,
|
|
ExpectedBinding,
|
|
MatrixCell,
|
|
digest_workspace_inputs,
|
|
load_manifest,
|
|
)
|
|
from scripts.agent_benchmark.lifecycle import (
|
|
CLOCK_HARNESS_MONOTONIC,
|
|
COMPLETION_EXIT_AFTER_IDLE,
|
|
METRIC_NAMES,
|
|
SOURCE_HARNESS,
|
|
SOURCE_WORKSPACE_POLL,
|
|
SUBMISSION_STDIN_ONCE,
|
|
UNIT_NANOSECONDS,
|
|
CaptureStream,
|
|
InvocationSpec,
|
|
InvocationResult,
|
|
LifecycleRecoveryError,
|
|
ParsedMetric,
|
|
env_pairs,
|
|
recover_invocation,
|
|
run_invocation,
|
|
spec_digest,
|
|
)
|
|
from scripts.agent_benchmark.measurement import (
|
|
AttemptMeasurement,
|
|
REASON_NOT_OBSERVED,
|
|
REASON_NOT_REPORTED,
|
|
WorkspaceWriteObservation,
|
|
load_measurement,
|
|
observed,
|
|
path_digest,
|
|
publish_measurement,
|
|
unavailable,
|
|
)
|
|
from scripts.agent_benchmark.scoring import BlindWorkspace, ScoringSummary, score_run
|
|
from scripts.agent_benchmark.web_validation import (
|
|
WEB_GATES,
|
|
build_web_validation,
|
|
load_web_validation,
|
|
publish_web_validation,
|
|
)
|
|
|
|
|
|
def _cell(cell_id: str, caller: str, model: str, effort: str) -> dict:
|
|
return {
|
|
"id": cell_id,
|
|
"caller": caller,
|
|
"iop": {
|
|
"request_model": model,
|
|
"requested_effort": effort,
|
|
"route_kind": "direct",
|
|
"route_id": cell_id,
|
|
"expected_bindings": [
|
|
{"stage": "request", "model": model, "effort": effort}
|
|
],
|
|
},
|
|
}
|
|
|
|
|
|
def _preset(cell_id: str, caller: str, model: str, effort: str) -> dict:
|
|
return {
|
|
"id": cell_id,
|
|
"caller": caller,
|
|
"iop": {
|
|
"request_model": model,
|
|
"requested_effort": effort,
|
|
"route_kind": "execution_preset",
|
|
"route_id": cell_id,
|
|
"expected_bindings": [
|
|
{"stage": stage, "model": model}
|
|
for stage in ("selector", "plan", "work", "review")
|
|
],
|
|
},
|
|
}
|
|
|
|
|
|
_SENTINEL_CLASSES = ("task", "secret", "endpoint", "config", "provider")
|
|
_LIVE_BRANCHES = ("claude", "agy", "codex")
|
|
|
|
|
|
def _branch_sentinels() -> dict[str, dict[str, str]]:
|
|
"""One distinct sentinel value per caller branch and leak class."""
|
|
return {
|
|
caller: {kind: f"branch-{kind}-sentinel-{caller}" for kind in _SENTINEL_CLASSES}
|
|
for caller in _LIVE_BRANCHES
|
|
}
|
|
|
|
|
|
# One production-shaped caller executable. It reads only its own argv, its own
|
|
# environment, the harness-submitted stdin task and its private caller config,
|
|
# then emits that caller's real stream shape. Every value it can observe is
|
|
# echoed back through the exact fields a real caller uses for content, so the
|
|
# published evidence proves the production redactors - not the test - removed
|
|
# them. The route is taken from the attempt's ``../prepared.json`` because this
|
|
# suite's matrix binds ``route_id`` to the cell id.
|
|
_CALLER_FIXTURE_BODY = r'''"""Production-shaped benchmark caller fixture."""
|
|
import json
|
|
import os
|
|
import sys
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
name = Path(__file__).name
|
|
config = json.loads(
|
|
Path(__file__).with_name(name + ".config.json").read_text(encoding="utf-8")
|
|
)
|
|
argv = sys.argv[1:]
|
|
task = sys.stdin.buffer.read().decode("utf-8", "replace")
|
|
prepared = json.loads((Path.cwd().parent / "prepared.json").read_text(encoding="utf-8"))
|
|
route_id = prepared["identity"]["cell_id"]
|
|
with open(Path(__file__).with_name(name + ".invocations"), "a", encoding="utf-8") as log:
|
|
log.write(prepared["identity"]["run_id"] + " " + route_id + "\n")
|
|
|
|
|
|
def option(flag):
|
|
return argv[argv.index(flag) + 1] if flag in argv else ""
|
|
|
|
|
|
def override(prefix):
|
|
for item in argv:
|
|
if item.startswith(prefix):
|
|
return json.loads(item[len(prefix):])
|
|
return ""
|
|
|
|
|
|
leak = json.dumps(
|
|
{"argv": argv, "env": dict(os.environ), "task": task, "config": config},
|
|
sort_keys=True,
|
|
)
|
|
sys.stderr.write(name + ": diagnostic " + leak + "\n")
|
|
sys.stderr.flush()
|
|
|
|
# Production-shaped benchmark output: exactly the three generated root files,
|
|
# using both fixture images. ``script.js`` is written last so the workspace
|
|
# observer still has a real caller-produced first-write observation.
|
|
Path("index.html").write_text("""<!doctype html><html><head><meta charset='utf-8'><meta name='viewport' content='width=device-width, initial-scale=1'><title>Orbit</title><link rel='stylesheet' href='styles.css'></head><body><header><nav aria-label='Primary'><a href='#main'>Home</a></nav></header><main id='main'><h1>Orbit landing page</h1><p>Accessible responsive fixture.</p><img src='assets/aurora-grid.svg' alt='Aurora grid artwork'><img src='assets/orbit-rings.svg' alt='Orbit rings artwork'><button type='button'>Start</button></main><script src='script.js'></script></body></html>""", encoding="utf-8")
|
|
Path("styles.css").write_text("""*{box-sizing:border-box}body{margin:0;background:#fff;color:#111;font:18px sans-serif}main{max-width:960px;margin:auto;padding:24px}img{display:block;max-width:100%;width:320px;height:auto;margin:16px 0}a:focus,button:focus{outline:3px solid #05f}@media(max-width:600px){main{padding:16px}img{width:100%}}""", encoding="utf-8")
|
|
Path("script.js").write_text("document.querySelector('button').addEventListener('click', () => {});", encoding="utf-8")
|
|
|
|
if name == "claude":
|
|
model = option("--model")
|
|
session = str(uuid.uuid4())
|
|
events = [
|
|
{"type": "system", "subtype": "init", "model": model,
|
|
"session_id": session, "tools": []},
|
|
{"type": "assistant", "session_id": session,
|
|
"message": {"model": model, "stop_reason": "end_turn",
|
|
"content": [{"type": "text", "text": leak}]}},
|
|
{"type": "result", "subtype": "success", "session_id": session,
|
|
"is_error": False, "duration_ms": 1234, "duration_api_ms": 1000,
|
|
"usage": {"input_tokens": 11, "output_tokens": 22,
|
|
"cache_read_input_tokens": 5},
|
|
"result": leak},
|
|
]
|
|
elif name == "agy":
|
|
binding = {"route_kind": "direct", "route_id": route_id,
|
|
"model": option("--model"), "effort": option("--effort")}
|
|
events = [
|
|
{"type": "metric", "subtype": "duration_ms", "value": 12.5},
|
|
dict(binding, type="iop", subtype="effective_binding",
|
|
stages=[{"stage": "request", "model": binding["model"],
|
|
"effort": binding["effort"]}]),
|
|
dict(binding, type="result", subtype="success", text=leak),
|
|
dict(binding, type="system", subtype="idle"),
|
|
]
|
|
else:
|
|
events = [
|
|
{"type": "item.completed", "item": {"type": "agent_message", "text": leak}},
|
|
{"type": "item.completed",
|
|
"item": {"id": "call-1", "type": "command_execution",
|
|
"duration_ms": 7.25, "output": leak}},
|
|
{"type": "turn.completed", "status": "completed",
|
|
"usage": {"input_tokens": 31, "cached_input_tokens": 8, "output_tokens": 12},
|
|
"iop_effective_binding": {
|
|
"route_kind": "direct", "route_id": route_id, "model": option("-m"),
|
|
"effort": override("model_reasoning_effort=")}},
|
|
]
|
|
|
|
for event in events:
|
|
sys.stdout.write(json.dumps(event) + "\n")
|
|
sys.stdout.flush()
|
|
'''
|
|
|
|
|
|
def _write_manifest(
|
|
root: Path, matrix: list[dict], output_id: str = "integration", *,
|
|
output_root: str | None = None, prompt: str = "public prompt fixture",
|
|
):
|
|
fixture_root = root / "scripts/fixtures"
|
|
fixture_root.mkdir(parents=True, exist_ok=True)
|
|
(fixture_root / "prompt.md").write_text(prompt, encoding="utf-8")
|
|
(fixture_root / "reference.txt").write_text("public reference", encoding="utf-8")
|
|
for image in ("aurora.svg", "orbit.svg"):
|
|
(fixture_root / image).write_text(
|
|
"<svg xmlns='http://www.w3.org/2000/svg' width='80' height='60'></svg>",
|
|
encoding="utf-8",
|
|
)
|
|
assets = (
|
|
AssetMapping(
|
|
"scripts/fixtures/reference.txt",
|
|
"brief/reference.txt",
|
|
b"public reference",
|
|
),
|
|
AssetMapping("scripts/fixtures/aurora.svg", "assets/aurora-grid.svg", (fixture_root / "aurora.svg").read_bytes()),
|
|
AssetMapping("scripts/fixtures/orbit.svg", "assets/orbit-rings.svg", (fixture_root / "orbit.svg").read_bytes()),
|
|
)
|
|
payload = {
|
|
"pipeline_version": "2",
|
|
"environment": "dev",
|
|
"testbed": "../iop-s2",
|
|
"repetitions": 1,
|
|
"session_policy": "fresh",
|
|
"setup_cache_policy": "isolated",
|
|
"timeout": {
|
|
"run_seconds": 5,
|
|
"idle_seconds": 1,
|
|
"quiet_seconds": 1,
|
|
"cleanup_grace_seconds": 1,
|
|
},
|
|
"viewports": [{"id": "desktop", "width": 900, "height": 700}, {"id": "mobile", "width": 375, "height": 700}],
|
|
"rubric_version": "landing-quality-v1",
|
|
"evaluator": {"caller": "codex", "iop": {"request_model": "judge", "requested_effort": "high", "route_kind": "direct", "route_id": "judge", "expected_bindings": [{"stage": "request", "model": "judge", "effort": "high"}]}},
|
|
"output_root": output_root or f"agent-test/runs/{output_id}",
|
|
"fixture": {
|
|
"version": "v1",
|
|
"prompt": "scripts/fixtures/prompt.md",
|
|
"assets": [
|
|
{
|
|
"source": "scripts/fixtures/reference.txt",
|
|
"workspace_path": "brief/reference.txt",
|
|
}
|
|
,{"source": "scripts/fixtures/aurora.svg", "workspace_path": "assets/aurora-grid.svg"}
|
|
,{"source": "scripts/fixtures/orbit.svg", "workspace_path": "assets/orbit-rings.svg"}
|
|
],
|
|
"checksum": digest_workspace_inputs(assets),
|
|
},
|
|
"matrix": matrix,
|
|
}
|
|
path = root / "manifest.json"
|
|
raw = json.dumps(payload, sort_keys=True).encode("utf-8")
|
|
path.write_bytes(raw)
|
|
return load_manifest(path, repo_root=root), raw, path
|
|
|
|
|
|
class FakeAdapter:
|
|
def __init__(
|
|
self,
|
|
caller: str,
|
|
efforts: tuple[str, ...],
|
|
issues_by_cell: dict[str, tuple[str, ...]] | None = None,
|
|
*,
|
|
sentinel: str = "",
|
|
) -> None:
|
|
self.capability = CallerCapability(
|
|
caller, ("direct", "execution_preset"), efforts
|
|
)
|
|
self.issues_by_cell = issues_by_cell or {}
|
|
self.sentinel = sentinel
|
|
self.calls: list[str] = []
|
|
self.invocations: list[tuple[str, str, str, bytes]] = []
|
|
self.fail_invocation = False
|
|
|
|
def preflight(self, cell: MatrixCell) -> PreflightObservation:
|
|
self.calls.append(cell.id)
|
|
issue_codes = self.issues_by_cell.get(cell.id, ())
|
|
if issue_codes:
|
|
binding = RequestedEffectiveBinding(
|
|
cell.id,
|
|
cell.caller,
|
|
cell.iop.route_kind,
|
|
cell.iop.route_id,
|
|
cell.iop.request_model,
|
|
cell.iop.requested_effort,
|
|
)
|
|
else:
|
|
binding = RequestedEffectiveBinding(
|
|
cell.id,
|
|
cell.caller,
|
|
cell.iop.route_kind,
|
|
cell.iop.route_id,
|
|
cell.iop.request_model,
|
|
cell.iop.requested_effort,
|
|
cell.iop.route_kind,
|
|
cell.iop.route_id,
|
|
cell.iop.request_model,
|
|
cell.iop.requested_effort,
|
|
tuple(
|
|
EffectiveBinding(item.stage, item.model, item.effort)
|
|
for item in cell.iop.expected_bindings
|
|
),
|
|
)
|
|
issues = tuple(
|
|
ConnectivityIssue(code, ISSUE_RESUME_CODES[code])
|
|
for code in issue_codes
|
|
)
|
|
return PreflightObservation(
|
|
make_result(cell, self.capability, binding, issues),
|
|
"sha256:" + "a" * 64,
|
|
"sha256:" + "b" * 64,
|
|
)
|
|
|
|
def invoke(
|
|
self,
|
|
cell,
|
|
prepared,
|
|
attempt,
|
|
control_dir,
|
|
task_payload,
|
|
timeout,
|
|
on_started,
|
|
):
|
|
run_root = Path(attempt.root).parents[3]
|
|
if not (run_root / "preflight/preflight-000001.json").is_file():
|
|
raise AssertionError("attempt allocated before preflight publication")
|
|
if cell.id != attempt.identity.cell_id or prepared.identity != attempt.identity:
|
|
raise AssertionError("execution identity drift")
|
|
self.invocations.append(
|
|
(cell.id, prepared.workspace_dir, prepared.session_id, task_payload)
|
|
)
|
|
|
|
if Path(control_dir).resolve(strict=False) != Path(attempt.root).resolve() / "control":
|
|
raise AssertionError("controller control binding drift")
|
|
source = (
|
|
"import sys; sys.stdin.buffer.read(); print('FAILED'); sys.exit(3)"
|
|
if self.fail_invocation
|
|
else "import sys; sys.stdin.buffer.read(); print('FINISH'); print('IDLE')"
|
|
)
|
|
spec = InvocationSpec(
|
|
argv=(sys.executable, "-u", "-c", source),
|
|
cwd=prepared.workspace_dir,
|
|
env=env_pairs({"PATH": os.environ.get("PATH", "/usr/bin:/bin")}),
|
|
submission_mode=SUBMISSION_STDIN_ONCE,
|
|
completion_mode=COMPLETION_EXIT_AFTER_IDLE,
|
|
timeout=timeout,
|
|
evidence_dir=attempt.root,
|
|
task_payload=task_payload,
|
|
control_dir=control_dir,
|
|
)
|
|
return run_invocation(
|
|
spec,
|
|
parse_event=lambda _stream, line: {
|
|
"FINISH": "finish",
|
|
"IDLE": "idle",
|
|
}.get(line.strip()),
|
|
on_started=lambda locator: on_started(locator, spec_digest(spec)),
|
|
)
|
|
|
|
def cleanup(self) -> None:
|
|
pass
|
|
|
|
|
|
class ConnectivityIntegrationTest(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.temp = tempfile.TemporaryDirectory(dir="/tmp", prefix="benchmark-preflight-")
|
|
self.root = Path(self.temp.name) / "repo"
|
|
self.root.mkdir()
|
|
self.matrix = [
|
|
_cell("claude-sonnet-direct", "claude", "claude-sonnet-5", "max"),
|
|
_cell("claude-gemini-direct", "claude", "gemini-3.6-flash", "high"),
|
|
_cell("claude-gpt-direct", "claude", "gpt-5.6-luna", "xhigh"),
|
|
_cell("agy-gemini-direct", "agy", "gemini-3.6-flash", "high"),
|
|
_cell("codex-gpt-direct", "codex", "gpt-5.6-luna", "xhigh"),
|
|
]
|
|
self.manifest, self.raw, self.path = _write_manifest(self.root, self.matrix)
|
|
self.store = RunStore(
|
|
self.root,
|
|
clock=lambda: datetime.datetime(
|
|
2026, 8, 10, 1, 2, 3, tzinfo=datetime.timezone.utc
|
|
),
|
|
token_hex=lambda _: "123456abcdef",
|
|
)
|
|
|
|
def tearDown(self) -> None:
|
|
self.temp.cleanup()
|
|
|
|
def _init_testbed(self) -> None:
|
|
testbed = self.root.parent / "iop-s2"
|
|
testbed.mkdir()
|
|
(testbed / "README.md").write_text("testbed", encoding="utf-8")
|
|
for command in (
|
|
("git", "init"),
|
|
("git", "config", "user.name", "test"),
|
|
("git", "config", "user.email", "test@example.invalid"),
|
|
("git", "add", "."),
|
|
("git", "commit", "-m", "testbed"),
|
|
):
|
|
subprocess.run(command, cwd=testbed, check=True, capture_output=True)
|
|
|
|
def _live_environment(self, *, token: str = "live-token-must-not-persist", manifest=None) -> dict[str, str]:
|
|
manifest = self.manifest if manifest is None else manifest
|
|
routes = []
|
|
seen: set[tuple[str, str]] = set()
|
|
evaluator = MatrixCell("evaluator", manifest.evaluator.caller, manifest.evaluator.iop)
|
|
for cell in (*manifest.matrix, evaluator):
|
|
key = (cell.iop.route_kind, cell.iop.route_id)
|
|
if key not in seen:
|
|
seen.add(key)
|
|
routes.append(
|
|
{
|
|
"route_kind": key[0],
|
|
"route_id": key[1],
|
|
"model": cell.iop.request_model,
|
|
"bindings": [
|
|
{
|
|
"stage": binding.stage,
|
|
"model": binding.model,
|
|
"effort": binding.effort,
|
|
}
|
|
for binding in cell.iop.expected_bindings
|
|
],
|
|
}
|
|
)
|
|
environment = {
|
|
"IOP_BENCH_CONFIG_OBSERVATION_ENV": "BENCH_CONFIG",
|
|
"BENCH_CONFIG": json.dumps({"schema_version": "1", "routes": routes}, sort_keys=True),
|
|
"BENCH_TOKEN": token,
|
|
}
|
|
for caller in ("CLAUDE", "AGY", "CODEX"):
|
|
environment[f"IOP_BENCH_{caller}_BASE_URL"] = "http://127.0.0.1:18083/v1"
|
|
environment[f"IOP_BENCH_{caller}_SECRET_ENV"] = "BENCH_TOKEN"
|
|
return environment
|
|
|
|
@staticmethod
|
|
def _score_measurement(attempt, caller: str) -> AttemptMeasurement:
|
|
timeline = {
|
|
"submitted_at": unavailable(REASON_NOT_OBSERVED, SOURCE_HARNESS),
|
|
"first_output_at": unavailable(REASON_NOT_OBSERVED, SOURCE_HARNESS),
|
|
"first_write_observed_at": unavailable(
|
|
REASON_NOT_OBSERVED, SOURCE_WORKSPACE_POLL
|
|
),
|
|
"first_write_mtime": unavailable(
|
|
REASON_NOT_OBSERVED, SOURCE_WORKSPACE_POLL
|
|
),
|
|
"total_duration": observed(
|
|
1, UNIT_NANOSECONDS, CLOCK_HARNESS_MONOTONIC, SOURCE_HARNESS
|
|
),
|
|
}
|
|
usage = {
|
|
name: unavailable(REASON_NOT_REPORTED, SOURCE_HARNESS)
|
|
for name in METRIC_NAMES
|
|
}
|
|
return AttemptMeasurement(
|
|
attempt.identity.run_id,
|
|
attempt.identity.cell_id,
|
|
attempt.identity.repetition,
|
|
attempt.identity.attempt,
|
|
caller,
|
|
"sha256:" + "3" * 64,
|
|
"success",
|
|
timeline,
|
|
usage,
|
|
WorkspaceWriteObservation(
|
|
False, None, None, "", 1, 0, REASON_NOT_OBSERVED
|
|
),
|
|
(),
|
|
)
|
|
|
|
@staticmethod
|
|
def _score_view(
|
|
attempt_root: Path,
|
|
ident: str,
|
|
width: int,
|
|
height: int,
|
|
image_paths: tuple[str, str],
|
|
) -> ViewportObservation:
|
|
screenshot = f"screenshot-{ident}.png"
|
|
png = b"\x89PNG\r\n\x1a\n" + ident.encode("ascii")
|
|
(attempt_root / screenshot).write_bytes(png)
|
|
images = tuple(
|
|
{
|
|
"src": path,
|
|
"alt": path,
|
|
"complete": True,
|
|
"natural_width": 80,
|
|
"natural_height": 60,
|
|
"visible": True,
|
|
"rect": {
|
|
"x": 0,
|
|
"y": 0,
|
|
"width": 80,
|
|
"height": 60,
|
|
"right": 80,
|
|
"bottom": 60,
|
|
},
|
|
}
|
|
for path in image_paths
|
|
)
|
|
return ViewportObservation(
|
|
ident,
|
|
width,
|
|
height,
|
|
screenshot,
|
|
"sha256:" + hashlib.sha256(png).hexdigest(),
|
|
len(png),
|
|
images,
|
|
{
|
|
"scroll_width": width,
|
|
"client_width": width,
|
|
"clipped": 0,
|
|
"overlaps": 0,
|
|
},
|
|
{
|
|
"h1_count": 1,
|
|
"headings": [1],
|
|
"heading_progression": True,
|
|
"main_count": 1,
|
|
"landmarks": 1,
|
|
"controls": [
|
|
{
|
|
"name": True,
|
|
"tab_index": 0,
|
|
"focused": True,
|
|
"focus_visible": True,
|
|
"contrast": 7.0,
|
|
}
|
|
],
|
|
"ax": {"nodes": 4, "non_ignored": 3, "named": 2},
|
|
},
|
|
)
|
|
|
|
def _successful_score_attempt(self, manifest, run):
|
|
cell = manifest.matrix[0]
|
|
with self.store.writer(run):
|
|
attempt = self.store.allocate(run, Slot(cell.id, 1))
|
|
workspace = Path(attempt.root) / "workspace"
|
|
for asset in manifest.fixture.assets:
|
|
target = workspace / asset.workspace_path
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
target.write_bytes(asset.content)
|
|
image_paths = tuple(
|
|
asset.workspace_path
|
|
for asset in manifest.fixture.assets
|
|
if Path(asset.workspace_path).suffix.lower() in (".png", ".svg")
|
|
)
|
|
if len(image_paths) != 2:
|
|
raise AssertionError("scoring fixture requires two images")
|
|
(workspace / "index.html").write_text(
|
|
"<main><h1>Ready</h1>"
|
|
+ "".join(
|
|
f"<img src='{path}' alt='{path}'>" for path in image_paths
|
|
)
|
|
+ "<button type='button'>Go</button>"
|
|
"<script src='script.js'></script></main>",
|
|
encoding="utf-8",
|
|
)
|
|
(workspace / "styles.css").write_text(
|
|
"body{color:#111;background:#fff}img{width:80px}"
|
|
"button:focus{outline:2px solid #05f}",
|
|
encoding="utf-8",
|
|
)
|
|
(workspace / "script.js").write_text(
|
|
"document.body.dataset.ready='1';", encoding="utf-8"
|
|
)
|
|
measurement = self._score_measurement(attempt, cell.caller)
|
|
publish_measurement(attempt.root, measurement)
|
|
render = RenderObservation(
|
|
"Chromium/Test",
|
|
"http://127.0.0.1:12345",
|
|
tuple(
|
|
{"kind": "local", "path": "/" + path, "allowed": True, "status": 200}
|
|
for path in ("index.html", *image_paths)
|
|
),
|
|
(),
|
|
tuple(
|
|
self._score_view(
|
|
Path(attempt.root),
|
|
viewport.id,
|
|
viewport.width,
|
|
viewport.height,
|
|
image_paths,
|
|
)
|
|
for viewport in manifest.viewports
|
|
),
|
|
)
|
|
publish_web_validation(
|
|
attempt.root,
|
|
build_web_validation(manifest, workspace, measurement, render),
|
|
)
|
|
return self.store.publish_terminal(
|
|
attempt, "success", result={"terminal_reason": "success"}
|
|
)
|
|
|
|
def _run_live_scoring_mutation(self, case, mutate):
|
|
secret = f"live-{case}-secret-exact-value"
|
|
raw = json.loads(self.raw)
|
|
raw["evaluator"]["iop"]["requested_effort"] = "xhigh"
|
|
raw["evaluator"]["iop"]["expected_bindings"] = [
|
|
{"stage": "request", "model": "judge", "effort": "xhigh"}
|
|
]
|
|
raw["output_root"] = f"agent-test/runs/{case}"
|
|
manifest_raw = json.dumps(raw, sort_keys=True).encode("utf-8")
|
|
manifest_path = self.root / f"{case}.json"
|
|
manifest_path.write_bytes(manifest_raw)
|
|
manifest = load_manifest(manifest_path, repo_root=self.root)
|
|
evaluator = MatrixCell(
|
|
"evaluator", manifest.evaluator.caller, manifest.evaluator.iop
|
|
)
|
|
run = self.store.create(manifest, manifest_raw)
|
|
attempt = self._successful_score_attempt(manifest, run)
|
|
environment = self._live_environment(token=secret, manifest=manifest)
|
|
base_url = environment["IOP_BENCH_CODEX_BASE_URL"]
|
|
|
|
def invoke(invocation, _on_started):
|
|
mutate(invocation, secret, base_url)
|
|
stream = CaptureStream("stdout", "", 0, 0, False)
|
|
lifecycle = InvocationResult(
|
|
True, "success", 0, None, True, True, True, False,
|
|
(), stream, replace(stream, stream="stderr"), "", "", None,
|
|
"sha256:" + "a" * 64,
|
|
"2026-08-11T00:00:00+00:00",
|
|
"2026-08-11T00:00:01+00:00", 1, (),
|
|
)
|
|
binding = (
|
|
evaluator.iop.route_kind,
|
|
evaluator.iop.route_id,
|
|
evaluator.iop.request_model,
|
|
evaluator.iop.requested_effort,
|
|
)
|
|
return CodexInvocationResult(lifecycle, binding)
|
|
|
|
adapter = live_iop.build_live_scoring_adapter(
|
|
environment,
|
|
observer=lambda _runtime: live_iop._Observation(
|
|
(evaluator.iop.request_model,), "sha256:" + "b" * 64, True
|
|
),
|
|
invoker=invoke,
|
|
)
|
|
summary = score_run(self.store, run, manifest, adapter=adapter)
|
|
score_root = Path(attempt.root) / "scoring" / "score-000001"
|
|
result = json.loads((score_root / "result.json").read_text())
|
|
allocation = json.loads((score_root / "allocation.json").read_text())
|
|
blind_root = Path(run.root) / allocation["blind_path"]
|
|
return summary, result, blind_root, secret, base_url
|
|
|
|
def _sentinel_live_environment(
|
|
self, manifest, sentinels: dict[str, dict[str, str]]
|
|
) -> dict[str, str]:
|
|
"""Give every branch its own endpoint and credential sentinel value."""
|
|
environment = self._live_environment(manifest=manifest)
|
|
del environment["BENCH_TOKEN"]
|
|
for caller, branch in sentinels.items():
|
|
prefix = f"IOP_BENCH_{caller.upper()}_"
|
|
secret_env = f"BENCH_TOKEN_{caller.upper()}"
|
|
environment[secret_env] = branch["secret"]
|
|
environment[prefix + "SECRET_ENV"] = secret_env
|
|
environment[prefix + "BASE_URL"] = f"http://{branch['endpoint']}.invalid:18083/v1"
|
|
return environment
|
|
|
|
@staticmethod
|
|
def _registry(issues: dict[str, tuple[str, ...]] | None = None, sentinel: str = ""):
|
|
issues = issues or {}
|
|
return {
|
|
"claude": FakeAdapter(
|
|
"claude", ("high", "max", "xhigh"), issues, sentinel=sentinel
|
|
),
|
|
"agy": FakeAdapter("agy", ("high",), issues, sentinel=sentinel),
|
|
"codex": FakeAdapter("codex", ("xhigh",), issues, sentinel=sentinel),
|
|
}
|
|
|
|
def test_all_three_callers_append_exact_ready_results_without_attempts(self) -> None:
|
|
registry = self._registry()
|
|
run, record = preflight_manifest(
|
|
self.store, self.manifest, self.raw, adapters=registry
|
|
)
|
|
self.assertEqual(record["status"], "ready")
|
|
self.assertEqual(
|
|
[result["cell"]["id"] for result in record["results"]],
|
|
[cell.id for cell in self.manifest.matrix],
|
|
)
|
|
self.assertEqual(len(self.store.preflights(run, self.manifest)), 1)
|
|
self.assertFalse((Path(run.root) / "cells").exists())
|
|
self.assertEqual(
|
|
{caller: adapter.calls for caller, adapter in registry.items()},
|
|
{
|
|
"claude": [
|
|
"claude-gemini-direct",
|
|
"claude-gpt-direct",
|
|
"claude-sonnet-direct",
|
|
],
|
|
"agy": ["agy-gemini-direct"],
|
|
"codex": ["codex-gpt-direct"],
|
|
},
|
|
)
|
|
|
|
def test_registration_and_implementation_blockers_are_distinct_and_no_attempt_allocates(self) -> None:
|
|
registry = self._registry(
|
|
{
|
|
"claude-sonnet-direct": ("credential_missing",),
|
|
"agy-gemini-direct": ("stream_incompatible",),
|
|
}
|
|
)
|
|
run, record = preflight_manifest(
|
|
self.store, self.manifest, self.raw, adapters=registry
|
|
)
|
|
statuses = [result["status"] for result in record["results"]]
|
|
self.assertEqual(statuses.count("registration_required"), 1)
|
|
self.assertEqual(statuses.count("implementation_gap"), 1)
|
|
self.assertEqual(record["status"], "implementation_gap")
|
|
self.assertFalse((Path(run.root) / "cells").exists())
|
|
self.assertEqual(self.store.status(run, self.manifest)["attempts"]["running"], 0)
|
|
|
|
@contextlib.contextmanager
|
|
def _production_shaped_callers(self, sentinels: dict[str, dict[str, str]]):
|
|
"""Publish one executable per caller before any invocation is built."""
|
|
# This checkout mounts /tmp with noexec. Keep the fixtures temporary
|
|
# and test-owned, but place their executable directory on the current
|
|
# executable test filesystem so the real caller adapters can launch
|
|
# them through their normal subprocess path.
|
|
bin_dir = tempfile.TemporaryDirectory(dir=Path.cwd(), prefix=".bc-")
|
|
self.addCleanup(bin_dir.cleanup)
|
|
fixture_bin = Path(bin_dir.name)
|
|
for caller, branch in sentinels.items():
|
|
executable = fixture_bin / caller
|
|
executable.write_text(
|
|
f"#!{sys.executable}\n" + _CALLER_FIXTURE_BODY, encoding="utf-8"
|
|
)
|
|
executable.chmod(0o700)
|
|
# The caller's private configuration is the only source of its
|
|
# config/provider sentinels, exactly as a real client config file.
|
|
(fixture_bin / f"{caller}.config.json").write_text(
|
|
json.dumps(
|
|
{
|
|
"caller": caller,
|
|
"config_identity": branch["config"],
|
|
"provider_id": branch["provider"],
|
|
},
|
|
sort_keys=True,
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
yield fixture_bin
|
|
|
|
@staticmethod
|
|
def _snapshot_run_root(run_root: Path) -> dict[str, bytes]:
|
|
return {
|
|
str(item.relative_to(run_root)): item.read_bytes()
|
|
for item in sorted(run_root.rglob("*"))
|
|
if item.is_file()
|
|
}
|
|
|
|
def _assert_one_published_spec_digest(self, attempt_root: Path) -> None:
|
|
"""Prove the admitted, journalled and published digests are one value."""
|
|
state = json.loads((attempt_root / "attempt.json").read_text(encoding="utf-8"))
|
|
result = json.loads(
|
|
(attempt_root / "lifecycle-result.json").read_text(encoding="utf-8")
|
|
)
|
|
header = json.loads(
|
|
(attempt_root / "lifecycle-journal.jsonl")
|
|
.read_text(encoding="utf-8")
|
|
.splitlines()[0]
|
|
)
|
|
self.assertIn("locator", state)
|
|
digests = {state["spec_digest"], result["spec_digest"], header["spec_digest"]}
|
|
self.assertEqual(len(digests), 1, attempt_root)
|
|
self.assertRegex(digests.pop(), r"^sha256:[0-9a-f]{64}$")
|
|
self.assertEqual(state["state"], "success")
|
|
self.assertIs(result["success"], True)
|
|
self.assertEqual(result["terminal_reason"], "success")
|
|
self.assertIs(result["finish_then_idle_then_quiet"], True)
|
|
self.assertIs(result["cleanup_complete"], True)
|
|
self.assertIs(result["process_group_alive"], False)
|
|
control_dir = Path(state["locator"]["control_dir"])
|
|
alias = control_dir.parent
|
|
self.assertEqual(control_dir.name, "control")
|
|
self.assertEqual(
|
|
state["locator"]["socket_path"], str(control_dir / "control.sock")
|
|
)
|
|
self.assertTrue(alias.name.startswith("iop-bench-attempt-"))
|
|
self.assertFalse(os.path.lexists(alias))
|
|
self.assertTrue((attempt_root / "control/locator.json").is_file())
|
|
self.assertTrue((attempt_root / "control/cleanup-receipt.json").is_file())
|
|
|
|
# The exact whole-total categories each caller reports through its own
|
|
# allowlist. Everything else must remain explicitly unavailable.
|
|
_EXPECTED_TOTALS = {
|
|
"claude": {
|
|
"total_duration", "model_duration", "model_calls", "input_tokens",
|
|
"output_tokens", "cached_input_tokens",
|
|
},
|
|
"agy": {"total_duration"},
|
|
"codex": {
|
|
"model_calls", "tool_calls", "input_tokens", "output_tokens",
|
|
"cached_input_tokens",
|
|
},
|
|
}
|
|
|
|
def _assert_measurement_evidence(
|
|
self, attempt_roots: list[Path], sentinels: dict[str, dict[str, str]]
|
|
) -> None:
|
|
"""Every attempt publishes one strict, source-aware, digested sidecar."""
|
|
callers = set()
|
|
for attempt_root in attempt_roots:
|
|
measurement = load_measurement(attempt_root)
|
|
caller = measurement.caller
|
|
callers.add(caller)
|
|
# This matrix binds the cell id to the caller name.
|
|
self.assertEqual(measurement.cell_id, caller)
|
|
self.assertEqual(measurement.terminal_reason, "success")
|
|
observed = {
|
|
name for name, item in measurement.usage.items()
|
|
if item.status == "observed"
|
|
}
|
|
self.assertEqual(observed, self._EXPECTED_TOTALS[caller], attempt_root)
|
|
# No caller reports a provider total, and none is reconstructed.
|
|
self.assertEqual(measurement.usage["total_tokens"].status, "unavailable")
|
|
self.assertIsNone(measurement.usage["total_tokens"].value)
|
|
self.assertEqual(measurement.timeline["first_output_at"].status, "observed")
|
|
self.assertEqual(
|
|
measurement.timeline["first_write_mtime"].clock, "filesystem_mtime"
|
|
)
|
|
self.assertTrue(measurement.observer.observed)
|
|
self.assertEqual(
|
|
measurement.observer.path_digest,
|
|
path_digest("index.html"),
|
|
)
|
|
web = load_web_validation(attempt_root)
|
|
self.assertEqual(web.status, "passed", web.record)
|
|
self.assertEqual(
|
|
[item["id"] for item in web.record["gates"]], list(WEB_GATES)
|
|
)
|
|
self.assertTrue(all(item["passed"] for item in web.record["gates"]))
|
|
expected_images = {
|
|
item["path"]
|
|
for item in web.record["workspace"]["inputs"]
|
|
if item["path"].startswith("assets/")
|
|
}
|
|
self.assertEqual(len(expected_images), 2)
|
|
self.assertEqual(
|
|
[item["id"] for item in web.record["viewports"]],
|
|
["desktop", "mobile"],
|
|
)
|
|
self.assertEqual(len(web.record["screenshots"]), 2)
|
|
for viewport, screenshot in zip(
|
|
web.record["viewports"], web.record["screenshots"]
|
|
):
|
|
self.assertEqual(
|
|
{item["src"] for item in viewport["images"]}, expected_images
|
|
)
|
|
self.assertEqual(
|
|
screenshot, {"id": viewport["id"], **viewport["screenshot"]}
|
|
)
|
|
data = (attempt_root / screenshot["file"]).read_bytes()
|
|
self.assertEqual(screenshot["size"], len(data))
|
|
self.assertEqual(
|
|
screenshot["digest"],
|
|
"sha256:" + hashlib.sha256(data).hexdigest(),
|
|
)
|
|
self.assertEqual(callers, set(_LIVE_BRANCHES))
|
|
|
|
def _assert_sentinels_absent(
|
|
self, published: dict[str, bytes], sentinels: dict[str, dict[str, str]]
|
|
) -> None:
|
|
for caller, branch in sentinels.items():
|
|
for kind, value in branch.items():
|
|
encoded = value.encode("ascii")
|
|
for relative, data in published.items():
|
|
self.assertNotIn(encoded, data, f"{caller}/{kind} in {relative}")
|
|
|
|
def test_cli_live_run_invokes_each_direct_cell_once(self) -> None:
|
|
short = tempfile.TemporaryDirectory(dir="/tmp", prefix="b")
|
|
self.addCleanup(short.cleanup)
|
|
root = Path(short.name) / "r"
|
|
root.mkdir()
|
|
matrix = [
|
|
_cell("claude", "claude", "sonnet", "max"),
|
|
_cell("agy", "agy", "gemini", "high"),
|
|
_cell("codex", "codex", "gpt", "xhigh"),
|
|
]
|
|
sentinels = _branch_sentinels()
|
|
manifest, _raw, path = _write_manifest(
|
|
root, matrix, output_root="agent-test/runs/r",
|
|
prompt="public prompt fixture\n"
|
|
+ "\n".join(sentinels[caller]["task"] for caller in _LIVE_BRANCHES),
|
|
)
|
|
testbed = root.parent / "iop-s2"
|
|
testbed.mkdir()
|
|
(testbed / "README.md").write_text("testbed", encoding="utf-8")
|
|
for command in (("git", "init"), ("git", "config", "user.name", "test"), ("git", "config", "user.email", "test@example.invalid"), ("git", "add", "."), ("git", "commit", "-m", "testbed")):
|
|
subprocess.run(command, cwd=testbed, check=True, capture_output=True)
|
|
environment = self._sentinel_live_environment(manifest, sentinels)
|
|
|
|
def observed(_runtime):
|
|
return live_iop._Observation(
|
|
tuple(sorted(cell.iop.request_model for cell in manifest.matrix)),
|
|
"sha256:" + "f" * 64,
|
|
True,
|
|
"agy 1.1.11",
|
|
"--print --output-format --sandbox --model --effort AGY_PROVIDER AGY_OPENAI_BASE_URL AGY_OPENAI_API_KEY stream-json",
|
|
)
|
|
|
|
stdout = io.StringIO()
|
|
stderr = io.StringIO()
|
|
with self._production_shaped_callers(sentinels) as fixture_bin:
|
|
with (
|
|
mock.patch.dict(
|
|
os.environ, {"PATH": f"{fixture_bin}:{os.environ['PATH']}"}
|
|
),
|
|
mock.patch.object(benchmark_cli, "_REPO_ROOT", root),
|
|
contextlib.redirect_stdout(stdout),
|
|
contextlib.redirect_stderr(stderr),
|
|
):
|
|
registry = live_iop.build_live_adapter_registry(
|
|
environment,
|
|
observer=observed,
|
|
binary_resolver=lambda name: str(fixture_bin / name),
|
|
)
|
|
with mock.patch.object(
|
|
benchmark_cli, "build_adapter_registry", return_value=registry
|
|
):
|
|
exit_code = benchmark_cli.main(["run", "--manifest", str(path)])
|
|
|
|
self.assertEqual(exit_code, 0, stderr.getvalue())
|
|
self.assertIn("ok: run run_id=", stdout.getvalue())
|
|
self.assertEqual(stderr.getvalue(), "")
|
|
calls = {
|
|
caller: (fixture_bin / f"{caller}.invocations")
|
|
.read_text(encoding="utf-8")
|
|
.splitlines()
|
|
for caller in _LIVE_BRANCHES
|
|
}
|
|
self.assertEqual({caller: len(item) for caller, item in calls.items()},
|
|
{"claude": 1, "agy": 1, "codex": 1})
|
|
|
|
run_roots = list((root / manifest.output_root).glob("run-*"))
|
|
self.assertEqual(len(run_roots), 1)
|
|
run_root = run_roots[0]
|
|
self.assertTrue((run_root / "preflight/preflight-000001.json").is_file())
|
|
published = self._snapshot_run_root(run_root)
|
|
attempt_roots = sorted(run_root.glob("cells/*/repetition-*/attempt-*"))
|
|
self.assertEqual(len(attempt_roots), len(manifest.matrix))
|
|
for attempt_root in attempt_roots:
|
|
self._assert_one_published_spec_digest(attempt_root)
|
|
self._assert_measurement_evidence(attempt_roots, sentinels)
|
|
self._assert_sentinels_absent(published, sentinels)
|
|
self.assertEqual(published, self._snapshot_run_root(run_root))
|
|
|
|
def test_cli_run_blocker_persists_preflight_and_allocates_zero_attempts(self) -> None:
|
|
registry = self._registry(
|
|
{"claude-sonnet-direct": ("credential_missing",)}
|
|
)
|
|
stdout = io.StringIO()
|
|
stderr = io.StringIO()
|
|
with (
|
|
mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root),
|
|
mock.patch.object(
|
|
benchmark_cli, "build_adapter_registry", return_value=registry
|
|
),
|
|
contextlib.redirect_stdout(stdout),
|
|
contextlib.redirect_stderr(stderr),
|
|
):
|
|
exit_code = benchmark_cli.main(["run", "--manifest", str(self.path)])
|
|
|
|
self.assertEqual(exit_code, 69)
|
|
self.assertEqual(stdout.getvalue(), "")
|
|
self.assertIn("error: preflight blocked", stderr.getvalue())
|
|
run_roots = list((self.root / self.manifest.output_root).glob("run-*"))
|
|
self.assertEqual(len(run_roots), 1)
|
|
self.assertTrue((run_roots[0] / "preflight/preflight-000001.json").is_file())
|
|
self.assertFalse((run_roots[0] / "cells").exists())
|
|
self.assertTrue(all(adapter.invocations == [] for adapter in registry.values()))
|
|
|
|
def test_cli_mixed_manifest_never_invokes_unobserved_preset_cells(self) -> None:
|
|
manifest, _, path = _write_manifest(
|
|
self.root,
|
|
[
|
|
_cell("direct-ready", "claude", "claude-sonnet-5", "max"),
|
|
_preset("preset-unobserved", "claude", "claude-sonnet-5", "max"),
|
|
],
|
|
output_id="mixed-unobserved",
|
|
)
|
|
registry = self._registry()
|
|
stdout = io.StringIO()
|
|
stderr = io.StringIO()
|
|
with (
|
|
mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root),
|
|
mock.patch.object(
|
|
benchmark_cli, "build_adapter_registry", return_value=registry
|
|
),
|
|
contextlib.redirect_stdout(stdout),
|
|
contextlib.redirect_stderr(stderr),
|
|
):
|
|
exit_code = benchmark_cli.main(["run", "--manifest", str(path)])
|
|
|
|
self.assertEqual(exit_code, 69)
|
|
self.assertEqual(stdout.getvalue(), "")
|
|
self.assertIn("error: benchmark execution failed", stderr.getvalue())
|
|
self.assertIn("completed=0 unresolved=2", stderr.getvalue())
|
|
run_roots = list((self.root / manifest.output_root).glob("run-*"))
|
|
self.assertEqual(len(run_roots), 1)
|
|
preflight = json.loads(
|
|
(run_roots[0] / "preflight/preflight-000001.json").read_text(
|
|
encoding="ascii"
|
|
)
|
|
)
|
|
self.assertEqual(preflight["status"], "ready")
|
|
self.assertEqual(
|
|
[result["cell"]["id"] for result in preflight["results"]],
|
|
["direct-ready"],
|
|
)
|
|
self.assertFalse((run_roots[0] / "cells").exists())
|
|
self.assertEqual(registry["claude"].calls, ["direct-ready"])
|
|
self.assertTrue(all(adapter.invocations == [] for adapter in registry.values()))
|
|
|
|
def test_cli_resume_retries_append_only_and_status_is_read_only(self) -> None:
|
|
self._init_testbed()
|
|
manifest, _, path = _write_manifest(
|
|
self.root,
|
|
[_cell("claude-only", "claude", "claude-sonnet-5", "max")],
|
|
output_id="retry",
|
|
)
|
|
failed = FakeAdapter("claude", ("max",))
|
|
failed.fail_invocation = True
|
|
self.addCleanup(failed.cleanup)
|
|
first_stdout = io.StringIO()
|
|
first_stderr = io.StringIO()
|
|
with (
|
|
mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root),
|
|
mock.patch.object(
|
|
benchmark_cli,
|
|
"build_adapter_registry",
|
|
return_value={"claude": failed},
|
|
),
|
|
contextlib.redirect_stdout(first_stdout),
|
|
contextlib.redirect_stderr(first_stderr),
|
|
):
|
|
first_exit = benchmark_cli.main(
|
|
["run", "--manifest", str(path)]
|
|
)
|
|
self.assertEqual(first_exit, 69)
|
|
matched = re.search(r"run_id=(run-[0-9A-Za-z-]+)", first_stderr.getvalue())
|
|
self.assertIsNotNone(matched)
|
|
run_id = matched.group(1) # type: ignore[union-attr]
|
|
run_root = self.root / manifest.output_root / run_id
|
|
first_attempt = next(run_root.glob("cells/*/repetition-*/attempt-000001"))
|
|
old_bytes = {
|
|
item.relative_to(first_attempt): item.read_bytes()
|
|
for item in first_attempt.rglob("*")
|
|
if item.is_file()
|
|
}
|
|
|
|
ready = FakeAdapter("claude", ("max",))
|
|
self.addCleanup(ready.cleanup)
|
|
resume_stdout = io.StringIO()
|
|
resume_stderr = io.StringIO()
|
|
with (
|
|
mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root),
|
|
mock.patch.object(
|
|
benchmark_cli,
|
|
"build_adapter_registry",
|
|
return_value={"claude": ready},
|
|
),
|
|
contextlib.redirect_stdout(resume_stdout),
|
|
contextlib.redirect_stderr(resume_stderr),
|
|
):
|
|
resume_exit = benchmark_cli.main(
|
|
[
|
|
"resume",
|
|
"--manifest",
|
|
str(path),
|
|
"--run-id",
|
|
run_id,
|
|
"--retry-failed",
|
|
]
|
|
)
|
|
|
|
self.assertEqual(resume_exit, 0, resume_stderr.getvalue())
|
|
self.assertIn("ok: resume", resume_stdout.getvalue())
|
|
self.assertEqual(
|
|
old_bytes,
|
|
{
|
|
relative: (first_attempt / relative).read_bytes()
|
|
for relative in old_bytes
|
|
},
|
|
)
|
|
self.assertTrue(
|
|
next(run_root.glob("cells/*/repetition-*/attempt-000002/attempt.json"))
|
|
.read_text(encoding="utf-8")
|
|
.find('"state":"success"')
|
|
>= 0
|
|
)
|
|
self.assertEqual(len(list((run_root / "preflight").glob("*.json"))), 2)
|
|
|
|
before_status = {
|
|
item.relative_to(run_root): item.read_bytes()
|
|
for item in run_root.rglob("*")
|
|
if item.is_file()
|
|
}
|
|
status_stdout = io.StringIO()
|
|
status_stderr = io.StringIO()
|
|
with (
|
|
mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root),
|
|
contextlib.redirect_stdout(status_stdout),
|
|
contextlib.redirect_stderr(status_stderr),
|
|
):
|
|
status_exit = benchmark_cli.main(
|
|
["status", "--manifest", str(path), "--run-id", run_id]
|
|
)
|
|
self.assertEqual(status_exit, 0, status_stderr.getvalue())
|
|
self.assertIn("'success': 1", status_stdout.getvalue())
|
|
self.assertEqual(
|
|
before_status,
|
|
{
|
|
item.relative_to(run_root): item.read_bytes()
|
|
for item in run_root.rglob("*")
|
|
if item.is_file()
|
|
},
|
|
)
|
|
|
|
def test_missing_adapter_is_rejected_before_output_root_mutation(self) -> None:
|
|
registry = self._registry()
|
|
del registry["codex"]
|
|
output_root = self.root / self.manifest.output_root
|
|
with self.assertRaises(CapabilityUnavailable):
|
|
preflight_manifest(
|
|
self.store, self.manifest, self.raw, adapters=registry
|
|
)
|
|
self.assertFalse(output_root.exists())
|
|
|
|
def test_generic_preset_cells_are_local_contract_only(self) -> None:
|
|
generic, _, _ = _write_manifest(
|
|
self.root,
|
|
[
|
|
_preset("claude-generic", "claude", "claude-sonnet-5", "high"),
|
|
_preset("agy-generic", "agy", "gemini-3.6-flash", "high"),
|
|
_preset("codex-generic", "codex", "gpt-5.6-luna", "xhigh"),
|
|
],
|
|
output_id="generic",
|
|
)
|
|
registry = self._registry()
|
|
observations = collect_preflight_observations(generic, registry)
|
|
self.assertEqual(observations, {})
|
|
self.assertTrue(all(adapter.calls == [] for adapter in registry.values()))
|
|
|
|
def test_generic_preset_only_public_preflight_fails_closed_without_run_state(self) -> None:
|
|
generic, raw, path = _write_manifest(
|
|
self.root,
|
|
[
|
|
_preset("claude-generic", "claude", "claude-sonnet-5", "high"),
|
|
_preset("agy-generic", "agy", "gemini-3.6-flash", "high"),
|
|
_preset("codex-generic", "codex", "gpt-5.6-luna", "xhigh"),
|
|
],
|
|
output_id="generic",
|
|
)
|
|
registry = self._registry()
|
|
output_root = self.root / generic.output_root
|
|
with self.assertRaises(Exception) as ctx:
|
|
preflight_manifest(
|
|
self.store, generic, raw, adapters=registry
|
|
)
|
|
self.assertIn("preflight requires a direct cell", str(ctx.exception))
|
|
self.assertFalse(output_root.exists())
|
|
self.assertTrue(all(adapter.calls == [] for adapter in registry.values()))
|
|
|
|
sentinel = "private_endpoint_and_token_must_not_appear"
|
|
stdout = io.StringIO()
|
|
stderr = io.StringIO()
|
|
with (
|
|
mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root),
|
|
mock.patch.object(benchmark_cli, "build_adapter_registry", return_value=registry),
|
|
contextlib.redirect_stdout(stdout),
|
|
contextlib.redirect_stderr(stderr),
|
|
):
|
|
exit_code = benchmark_cli.main(["preflight", "--manifest", str(path)])
|
|
self.assertEqual(exit_code, 69)
|
|
self.assertEqual(stdout.getvalue(), "")
|
|
self.assertNotIn(sentinel, stderr.getvalue())
|
|
self.assertFalse(output_root.exists())
|
|
|
|
def test_live_registry_dereferences_secret_names_without_persisting_values(self) -> None:
|
|
sentinel = "live-token-must-not-persist"
|
|
environment = self._live_environment(token=sentinel)
|
|
environment["ANTHROPIC_BASE_URL"] = "must-not-be-read"
|
|
|
|
def observed(_runtime):
|
|
return live_iop._Observation(
|
|
tuple(sorted(cell.iop.request_model for cell in self.manifest.matrix)),
|
|
"sha256:" + "c" * 64,
|
|
True,
|
|
"agy 1.1.11",
|
|
"--print --output-format --sandbox --model --effort AGY_PROVIDER AGY_OPENAI_BASE_URL AGY_OPENAI_API_KEY stream-json",
|
|
)
|
|
|
|
registry = live_iop.build_live_adapter_registry(
|
|
environment, observer=observed, binary_resolver=lambda _name: "/bin/true"
|
|
)
|
|
self.assertEqual(tuple(registry), ("claude", "agy", "codex"))
|
|
observations = collect_preflight_observations(self.manifest, registry)
|
|
self.assertEqual(set(observations), {cell.id for cell in self.manifest.matrix})
|
|
for observation in observations.values():
|
|
self.assertEqual(observation.result.status, "ready")
|
|
self.assertEqual(observation.result.binding.effective_model, observation.result.binding.requested_model)
|
|
durable = json.dumps(
|
|
{cell_id: item.result.status for cell_id, item in observations.items()},
|
|
sort_keys=True,
|
|
)
|
|
self.assertNotIn(sentinel, durable)
|
|
self.assertNotIn("must-not-be-read", durable)
|
|
|
|
def test_live_registry_ready_binding_requires_catalog_observation(self) -> None:
|
|
environment = self._live_environment(token="private-token")
|
|
registry = live_iop.build_live_adapter_registry(
|
|
environment,
|
|
observer=lambda _runtime: live_iop._Observation(
|
|
(), "sha256:" + "d" * 64, True
|
|
),
|
|
)
|
|
claude_cell = next(cell for cell in self.manifest.matrix if cell.caller == "claude")
|
|
observation = registry["claude"].preflight(claude_cell)
|
|
self.assertEqual(observation.result.status, "registration_required")
|
|
self.assertEqual([issue.code for issue in observation.result.issues], ["model_missing"])
|
|
self.assertIsNone(observation.result.binding.effective_model)
|
|
|
|
def test_live_scoring_adapter_preserves_manifest_route_and_ephemeral_secret(self) -> None:
|
|
secret = "evaluator-secret-must-not-persist"
|
|
source_identities = (
|
|
"source-cell-sentinel",
|
|
"source-route-sentinel",
|
|
"source-model-sentinel",
|
|
"source-effort-sentinel",
|
|
)
|
|
|
|
for route_kind in ("direct", "execution_preset"):
|
|
with self.subTest(route_kind=route_kind):
|
|
expected_bindings = (
|
|
(ExpectedBinding("request", "judge", "xhigh"),)
|
|
if route_kind == "direct"
|
|
else tuple(
|
|
ExpectedBinding(stage, "judge")
|
|
for stage in ("selector", "plan", "work", "review")
|
|
)
|
|
)
|
|
evaluator = MatrixCell(
|
|
"evaluator", self.manifest.evaluator.caller,
|
|
replace(
|
|
self.manifest.evaluator.iop,
|
|
requested_effort="xhigh",
|
|
route_kind=route_kind,
|
|
expected_bindings=expected_bindings,
|
|
),
|
|
)
|
|
scoring_manifest = replace(
|
|
self.manifest,
|
|
evaluator=replace(self.manifest.evaluator, iop=evaluator.iop),
|
|
)
|
|
environment = self._live_environment(
|
|
token=secret, manifest=scoring_manifest
|
|
)
|
|
captured = {}
|
|
|
|
def invoke(invocation, _on_started):
|
|
captured["spec"] = invocation.spec
|
|
stream = CaptureStream("stdout", "", 0, 0, False)
|
|
lifecycle = InvocationResult(
|
|
True, "success", 0, None, True, True, True, False,
|
|
(), stream, replace(stream, stream="stderr"), "", "", None,
|
|
"sha256:" + "a" * 64,
|
|
"2026-08-11T00:00:00+00:00",
|
|
"2026-08-11T00:00:01+00:00", 1, (),
|
|
)
|
|
binding = (
|
|
evaluator.iop.route_kind,
|
|
evaluator.iop.route_id,
|
|
evaluator.iop.request_model,
|
|
evaluator.iop.requested_effort,
|
|
)
|
|
return CodexInvocationResult(lifecycle, binding)
|
|
|
|
adapter = live_iop.build_live_scoring_adapter(
|
|
environment,
|
|
observer=lambda _runtime: live_iop._Observation(
|
|
(evaluator.iop.request_model,),
|
|
"sha256:" + "b" * 64,
|
|
True,
|
|
),
|
|
invoker=invoke,
|
|
)
|
|
observation = adapter.preflight(evaluator)
|
|
self.assertEqual(observation.result.status, "ready")
|
|
self.assertEqual(
|
|
observation.result.binding.effective_bindings,
|
|
tuple(
|
|
EffectiveBinding(item.stage, item.model, item.effort)
|
|
for item in evaluator.iop.expected_bindings
|
|
),
|
|
)
|
|
|
|
blind_root = self.root / "runs" / f"blind-{route_kind}"
|
|
for name in ("input", "session", "output"):
|
|
(blind_root / name).mkdir(parents=True, exist_ok=True)
|
|
blind = BlindWorkspace(
|
|
f"blind-{route_kind}", str(blind_root),
|
|
str(blind_root / "input"), str(blind_root / "session"),
|
|
str(blind_root / "output"), "sha256:" + "c" * 64,
|
|
"sha256:" + "d" * 64,
|
|
)
|
|
prompt = b"Evaluate only anonymous files under input/."
|
|
result = adapter.invoke(
|
|
evaluator,
|
|
blind,
|
|
prompt,
|
|
self.manifest.timeout,
|
|
lambda *_args: None,
|
|
)
|
|
self.assertTrue(result.success)
|
|
self.assertEqual(result.effective_binding[0], route_kind)
|
|
finalized = adapter.finalize_evidence(blind)
|
|
self.assertTrue(finalized.safe)
|
|
|
|
spec = captured["spec"]
|
|
visible = "\n".join(
|
|
(*spec.argv, spec.cwd, *(value for pair in spec.env for value in pair))
|
|
).casefold()
|
|
for identity in source_identities:
|
|
self.assertNotIn(identity, visible)
|
|
durable = b"".join(
|
|
path.read_bytes()
|
|
for path in blind_root.rglob("*")
|
|
if path.is_file()
|
|
)
|
|
self.assertNotIn(secret.encode("ascii"), durable)
|
|
evidence = canonical_evidence_bytes(
|
|
evaluator,
|
|
observation.result,
|
|
observation.endpoint_identity,
|
|
observation.config_identity,
|
|
)
|
|
self.assertNotIn(secret.encode("ascii"), evidence)
|
|
|
|
def test_live_scoring_alias_is_control_only(self) -> None:
|
|
evaluator = MatrixCell(
|
|
"evaluator",
|
|
self.manifest.evaluator.caller,
|
|
replace(
|
|
self.manifest.evaluator.iop,
|
|
requested_effort="xhigh",
|
|
expected_bindings=(ExpectedBinding("request", "judge", "xhigh"),),
|
|
),
|
|
)
|
|
scoring_manifest = replace(
|
|
self.manifest,
|
|
evaluator=replace(self.manifest.evaluator, iop=evaluator.iop),
|
|
)
|
|
captured: dict[str, object] = {}
|
|
|
|
def invoke(invocation, _on_started):
|
|
captured["spec"] = invocation.spec
|
|
evidence = Path(invocation.spec.evidence_dir)
|
|
(evidence / "lifecycle-journal.jsonl").write_text(
|
|
'{"record":"header"}\n{"record":"terminal"}\n',
|
|
encoding="utf-8",
|
|
)
|
|
(evidence / "lifecycle-result.json").write_text(
|
|
'{"record":"result"}\n', encoding="utf-8"
|
|
)
|
|
stream = CaptureStream("stdout", "", 0, 0, False)
|
|
lifecycle = InvocationResult(
|
|
True, "success", 0, None, True, True, True, False,
|
|
(), stream, replace(stream, stream="stderr"), "", "", None,
|
|
"sha256:" + "a" * 64,
|
|
"2026-08-11T00:00:00+00:00",
|
|
"2026-08-11T00:00:01+00:00", 1, (),
|
|
)
|
|
binding = (
|
|
evaluator.iop.route_kind,
|
|
evaluator.iop.route_id,
|
|
evaluator.iop.request_model,
|
|
evaluator.iop.requested_effort,
|
|
)
|
|
return CodexInvocationResult(lifecycle, binding)
|
|
|
|
adapter = live_iop.build_live_scoring_adapter(
|
|
self._live_environment(manifest=scoring_manifest),
|
|
observer=lambda _runtime: live_iop._Observation(
|
|
(evaluator.iop.request_model,), "sha256:" + "b" * 64, True
|
|
),
|
|
invoker=invoke,
|
|
)
|
|
self.assertEqual(adapter.preflight(evaluator).result.status, "ready")
|
|
blind_root = self.root / "alias-run" / "blind" / "blind-alias"
|
|
for name in ("input", "session", "output"):
|
|
(blind_root / name).mkdir(parents=True, exist_ok=True)
|
|
blind = BlindWorkspace(
|
|
"blind-alias",
|
|
str(blind_root),
|
|
str(blind_root / "input"),
|
|
str(blind_root / "session"),
|
|
str(blind_root / "output"),
|
|
"sha256:" + "c" * 64,
|
|
"sha256:" + "d" * 64,
|
|
)
|
|
result = adapter.invoke(
|
|
evaluator,
|
|
blind,
|
|
b"Evaluate anonymous output.",
|
|
self.manifest.timeout,
|
|
lambda *_args: None,
|
|
)
|
|
self.assertTrue(result.success)
|
|
spec = captured["spec"]
|
|
self.assertIsInstance(spec, InvocationSpec)
|
|
assert isinstance(spec, InvocationSpec)
|
|
self.assertEqual(Path(spec.evidence_dir), blind_root / "output")
|
|
alias = Path(spec.control_dir).parent
|
|
self.assertTrue(alias.is_symlink())
|
|
self.assertEqual(alias.resolve(strict=True), blind_root / "output")
|
|
sidecars = {
|
|
path.name: path.read_bytes()
|
|
for path in (
|
|
blind_root / "output" / "lifecycle-journal.jsonl",
|
|
blind_root / "output" / "lifecycle-result.json",
|
|
)
|
|
}
|
|
self.assertTrue(adapter.finalize_evidence(blind).safe)
|
|
self.assertFalse(alias.exists() or alias.is_symlink())
|
|
for name, data in sidecars.items():
|
|
self.assertEqual((blind_root / "output" / name).read_bytes(), data)
|
|
|
|
def test_live_scoring_scrubs_evaluator_secret_output(self) -> None:
|
|
secret = "live-evaluator-secret-exact-value"
|
|
base_url = "http://127.0.0.1:18083/v1"
|
|
evaluator = MatrixCell(
|
|
"evaluator",
|
|
self.manifest.evaluator.caller,
|
|
replace(
|
|
self.manifest.evaluator.iop,
|
|
requested_effort="xhigh",
|
|
expected_bindings=(
|
|
ExpectedBinding("request", "judge", "xhigh"),
|
|
),
|
|
),
|
|
)
|
|
scoring_manifest = replace(
|
|
self.manifest,
|
|
evaluator=replace(self.manifest.evaluator, iop=evaluator.iop),
|
|
)
|
|
environment = self._live_environment(
|
|
token=secret, manifest=scoring_manifest
|
|
)
|
|
|
|
def invoke(invocation, _on_started):
|
|
output = Path(invocation.spec.evidence_dir)
|
|
worksheet = _worksheet_payload = {
|
|
"rubric_version": "landing-quality-v1",
|
|
"categories": [
|
|
{
|
|
"id": ident,
|
|
"max_score": maximum,
|
|
"score": maximum,
|
|
"evidence": secret if index == 0 else "safe evidence",
|
|
}
|
|
for index, (ident, maximum) in enumerate(
|
|
(
|
|
("task_fidelity", 25),
|
|
("visual_hierarchy", 25),
|
|
("responsive_composition", 20),
|
|
("typography_readability", 15),
|
|
("polish_consistency", 15),
|
|
)
|
|
)
|
|
],
|
|
"total": 100,
|
|
}
|
|
(output / "worksheet.json").write_text(
|
|
json.dumps(worksheet), encoding="utf-8"
|
|
)
|
|
(output / "diagnostic.txt").write_text(
|
|
f"{base_url}\n{secret}\n", encoding="utf-8"
|
|
)
|
|
stream = CaptureStream("stdout", "", 0, 0, False)
|
|
lifecycle = InvocationResult(
|
|
True, "success", 0, None, True, True, True, False,
|
|
(), stream, replace(stream, stream="stderr"), "", "", None,
|
|
"sha256:" + "a" * 64,
|
|
"2026-08-11T00:00:00+00:00",
|
|
"2026-08-11T00:00:01+00:00", 1, (),
|
|
)
|
|
binding = (
|
|
evaluator.iop.route_kind,
|
|
evaluator.iop.route_id,
|
|
evaluator.iop.request_model,
|
|
evaluator.iop.requested_effort,
|
|
)
|
|
return CodexInvocationResult(lifecycle, binding)
|
|
|
|
adapter = live_iop.build_live_scoring_adapter(
|
|
environment,
|
|
observer=lambda _runtime: live_iop._Observation(
|
|
(evaluator.iop.request_model,),
|
|
"sha256:" + "b" * 64,
|
|
True,
|
|
),
|
|
invoker=invoke,
|
|
)
|
|
self.assertEqual(adapter.preflight(evaluator).result.status, "ready")
|
|
blind_root = self.root / "secret-run" / "blind" / "blind-secret"
|
|
for name in ("input", "session", "output"):
|
|
(blind_root / name).mkdir(parents=True, exist_ok=True)
|
|
blind = BlindWorkspace(
|
|
"blind-secret",
|
|
str(blind_root),
|
|
str(blind_root / "input"),
|
|
str(blind_root / "session"),
|
|
str(blind_root / "output"),
|
|
"sha256:" + "c" * 64,
|
|
"sha256:" + "d" * 64,
|
|
)
|
|
result = adapter.invoke(
|
|
evaluator,
|
|
blind,
|
|
b"Evaluate anonymous output.",
|
|
self.manifest.timeout,
|
|
lambda *_args: None,
|
|
)
|
|
self.assertTrue(result.success)
|
|
finalized = adapter.finalize_evidence(blind)
|
|
self.assertEqual(
|
|
(finalized.safe, finalized.reason),
|
|
(False, "runtime_secret_leak"),
|
|
)
|
|
durable = b"".join(
|
|
path.read_bytes()
|
|
for path in (self.root / "secret-run").rglob("*")
|
|
if path.is_file()
|
|
)
|
|
self.assertNotIn(secret.encode("utf-8"), durable)
|
|
self.assertNotIn(base_url.encode("utf-8"), durable)
|
|
self.assertFalse((blind_root / "output" / "worksheet.json").exists())
|
|
|
|
def test_live_scoring_scrubs_permission_denied_secret_paths(self) -> None:
|
|
retained_modes: dict[str, int] = {}
|
|
removed_paths: list[Path] = []
|
|
|
|
def mutate(invocation, secret, base_url):
|
|
blind_root = Path(invocation.spec.cwd)
|
|
input_root = blind_root / "input"
|
|
output_root = Path(invocation.spec.evidence_dir)
|
|
os.chmod(input_root, 0o700, follow_symlinks=False)
|
|
locked = input_root / ("locked-" + secret)
|
|
locked.mkdir()
|
|
payload = locked / "payload.bin"
|
|
payload.write_bytes(secret.encode("utf-8"))
|
|
os.chmod(payload, 0o000, follow_symlinks=False)
|
|
os.chmod(locked, 0o000, follow_symlinks=False)
|
|
removed_paths.append(locked)
|
|
|
|
denied = output_root / ("denied-" + secret + ".bin")
|
|
denied.write_bytes(base_url.encode("utf-8"))
|
|
os.chmod(denied, 0o000, follow_symlinks=False)
|
|
removed_paths.append(denied)
|
|
safe_dir = output_root / "safe-retained"
|
|
safe_dir.mkdir()
|
|
safe_file = output_root / "safe-retained.txt"
|
|
safe_file.write_text("safe evidence", encoding="utf-8")
|
|
os.chmod(safe_file, 0o000, follow_symlinks=False)
|
|
os.chmod(safe_dir, 0o000, follow_symlinks=False)
|
|
retained_modes["directory"] = 0o000
|
|
retained_modes["file"] = 0o000
|
|
|
|
summary, result, blind_root, secret, base_url = (
|
|
self._run_live_scoring_mutation("permission-denied-secret", mutate)
|
|
)
|
|
self.assertEqual((summary.scored, summary.scoring_failed), (0, 1))
|
|
self.assertEqual(result["reason"], "runtime_secret_leak")
|
|
self.assertNotIn("worksheet", result)
|
|
for path in removed_paths:
|
|
self.assertFalse(path.exists() or path.is_symlink())
|
|
safe_dir = blind_root / "output" / "safe-retained"
|
|
safe_file = blind_root / "output" / "safe-retained.txt"
|
|
self.assertEqual(stat.S_IMODE(os.lstat(safe_dir).st_mode), retained_modes["directory"])
|
|
self.assertEqual(stat.S_IMODE(os.lstat(safe_file).st_mode), retained_modes["file"])
|
|
self.assertNotIn(secret, result["reason"])
|
|
self.assertNotIn(base_url, result["reason"])
|
|
|
|
def test_live_scoring_classifies_safe_invalid_links_without_secret_claim(self) -> None:
|
|
cases = (
|
|
("input", "input_mutated"),
|
|
("output", "evaluator_output_leak"),
|
|
)
|
|
for root_kind, expected_reason in cases:
|
|
with self.subTest(root_kind=root_kind):
|
|
link_path: list[Path] = []
|
|
target_path: list[Path] = []
|
|
|
|
def mutate(invocation, _secret, _base_url):
|
|
blind_root = Path(invocation.spec.cwd)
|
|
selected = (
|
|
blind_root / "input"
|
|
if root_kind == "input"
|
|
else Path(invocation.spec.evidence_dir)
|
|
)
|
|
if root_kind == "input":
|
|
os.chmod(selected, 0o700, follow_symlinks=False)
|
|
target = selected / "index.html"
|
|
else:
|
|
target = selected / "safe-target.txt"
|
|
target.write_text("safe evidence", encoding="utf-8")
|
|
link = selected / "safe-invalid-link"
|
|
link.symlink_to(target.name)
|
|
link_path.append(link)
|
|
target_path.append(target)
|
|
|
|
summary, result, _blind_root, _secret, _base_url = (
|
|
self._run_live_scoring_mutation(
|
|
f"safe-link-{root_kind}", mutate
|
|
)
|
|
)
|
|
self.assertEqual(
|
|
(summary.scored, summary.scoring_failed), (0, 1)
|
|
)
|
|
self.assertEqual(result["reason"], expected_reason)
|
|
self.assertNotEqual(result["reason"], "runtime_secret_leak")
|
|
self.assertFalse(link_path[0].exists() or link_path[0].is_symlink())
|
|
self.assertTrue(target_path[0].is_file())
|
|
self.assertNotIn("worksheet", result)
|
|
|
|
def test_live_scoring_scrubs_secret_from_mutated_input_before_failure(self) -> None:
|
|
secret = "live-mutated-input-secret-exact-value"
|
|
evaluator = MatrixCell(
|
|
"evaluator",
|
|
self.manifest.evaluator.caller,
|
|
replace(
|
|
self.manifest.evaluator.iop,
|
|
requested_effort="xhigh",
|
|
expected_bindings=(ExpectedBinding("request", "judge", "xhigh"),),
|
|
),
|
|
)
|
|
manifest_payload = {
|
|
**json.loads(self.raw),
|
|
"evaluator": {
|
|
"caller": evaluator.caller,
|
|
"iop": {
|
|
"request_model": evaluator.iop.request_model,
|
|
"requested_effort": evaluator.iop.requested_effort,
|
|
"route_kind": evaluator.iop.route_kind,
|
|
"route_id": evaluator.iop.route_id,
|
|
"expected_bindings": [
|
|
{
|
|
"stage": item.stage,
|
|
"model": item.model,
|
|
"effort": item.effort,
|
|
}
|
|
for item in evaluator.iop.expected_bindings
|
|
],
|
|
},
|
|
},
|
|
"output_root": "agent-test/runs/mutated-input-secret",
|
|
}
|
|
manifest_raw = json.dumps(manifest_payload, sort_keys=True).encode("utf-8")
|
|
manifest_path = self.root / "mutated-input-secret.json"
|
|
manifest_path.write_bytes(manifest_raw)
|
|
scoring_manifest = load_manifest(manifest_path, repo_root=self.root)
|
|
evaluator = MatrixCell(
|
|
"evaluator",
|
|
scoring_manifest.evaluator.caller,
|
|
scoring_manifest.evaluator.iop,
|
|
)
|
|
run = self.store.create(scoring_manifest, manifest_raw)
|
|
attempt = self._successful_score_attempt(scoring_manifest, run)
|
|
environment = self._live_environment(
|
|
token=secret, manifest=scoring_manifest
|
|
)
|
|
base_url = environment["IOP_BENCH_CODEX_BASE_URL"]
|
|
invocation_count = 0
|
|
|
|
def invoke(invocation, _on_started):
|
|
nonlocal invocation_count
|
|
invocation_count += 1
|
|
if invocation_count == 1:
|
|
input_root = Path(invocation.spec.cwd) / "input"
|
|
os.chmod(input_root, 0o700, follow_symlinks=False)
|
|
target = input_root / "index.html"
|
|
os.chmod(target, 0o600, follow_symlinks=False)
|
|
target.write_bytes(target.read_bytes() + secret.encode("utf-8"))
|
|
sensitive_dir = input_root / ("copied-" + secret)
|
|
sensitive_dir.mkdir()
|
|
(sensitive_dir / "runtime.txt").write_text(
|
|
secret + "\n" + base_url, encoding="utf-8"
|
|
)
|
|
stream = CaptureStream("stdout", "", 0, 0, False)
|
|
lifecycle = InvocationResult(
|
|
True, "success", 0, None, True, True, True, False,
|
|
(), stream, replace(stream, stream="stderr"), "", "", None,
|
|
"sha256:" + "a" * 64,
|
|
"2026-08-11T00:00:00+00:00",
|
|
"2026-08-11T00:00:01+00:00", 1, (),
|
|
)
|
|
binding = (
|
|
evaluator.iop.route_kind,
|
|
evaluator.iop.route_id,
|
|
evaluator.iop.request_model,
|
|
evaluator.iop.requested_effort,
|
|
)
|
|
return CodexInvocationResult(lifecycle, binding)
|
|
|
|
adapter = live_iop.build_live_scoring_adapter(
|
|
environment,
|
|
observer=lambda _runtime: live_iop._Observation(
|
|
(evaluator.iop.request_model,), "sha256:" + "b" * 64, True
|
|
),
|
|
invoker=invoke,
|
|
)
|
|
first = score_run(
|
|
self.store, run, scoring_manifest, adapter=adapter
|
|
)
|
|
self.assertEqual((first.scored, first.scoring_failed), (0, 1))
|
|
first_score = Path(attempt.root) / "scoring" / "score-000001"
|
|
first_result = json.loads((first_score / "result.json").read_text())
|
|
self.assertEqual(first_result["reason"], "runtime_secret_leak")
|
|
self.assertNotIn("worksheet", first_result)
|
|
first_allocation = json.loads((first_score / "allocation.json").read_text())
|
|
first_blind = Path(run.root) / first_allocation["blind_path"]
|
|
self.assertFalse((first_blind / "output" / "worksheet.json").exists())
|
|
for path in (first_blind / "input", *(first_blind / "input").rglob("*")):
|
|
self.assertEqual(os.lstat(path).st_mode & 0o222, 0)
|
|
|
|
prior = {
|
|
path: path.read_bytes()
|
|
for path in Path(run.root).rglob("*")
|
|
if path.is_file()
|
|
}
|
|
retry = score_run(
|
|
self.store,
|
|
run,
|
|
scoring_manifest,
|
|
adapter=adapter,
|
|
retry_scoring_failed=True,
|
|
)
|
|
self.assertEqual((retry.scored, retry.scoring_failed), (0, 1))
|
|
self.assertEqual(invocation_count, 2)
|
|
for path, data in prior.items():
|
|
self.assertEqual(path.read_bytes(), data)
|
|
second_score = Path(attempt.root) / "scoring" / "score-000002"
|
|
second_allocation = json.loads(
|
|
(second_score / "allocation.json").read_text()
|
|
)
|
|
self.assertNotEqual(
|
|
first_allocation["blind_id"], second_allocation["blind_id"]
|
|
)
|
|
self.assertNotEqual(
|
|
first_allocation["session_identity"],
|
|
second_allocation["session_identity"],
|
|
)
|
|
for path in Path(run.root).rglob("*"):
|
|
relative = path.relative_to(run.root).as_posix().encode("utf-8")
|
|
self.assertNotIn(secret.encode("utf-8"), relative)
|
|
if path.is_file():
|
|
data = path.read_bytes()
|
|
self.assertNotIn(secret.encode("utf-8"), data)
|
|
self.assertNotIn(base_url.encode("utf-8"), data)
|
|
|
|
def test_live_scoring_survivor_cleanup_precedes_retry(self) -> None:
|
|
evaluator = MatrixCell(
|
|
"evaluator",
|
|
self.manifest.evaluator.caller,
|
|
replace(
|
|
self.manifest.evaluator.iop,
|
|
requested_effort="xhigh",
|
|
expected_bindings=(
|
|
ExpectedBinding("request", "judge", "xhigh"),
|
|
),
|
|
),
|
|
)
|
|
scoring_manifest = replace(
|
|
self.manifest,
|
|
evaluator=replace(self.manifest.evaluator, iop=evaluator.iop),
|
|
)
|
|
environment = self._live_environment(manifest=scoring_manifest)
|
|
locator_ready = threading.Event()
|
|
locators = []
|
|
worker_results = []
|
|
workers = []
|
|
first = True
|
|
|
|
def invoker(invocation, on_started):
|
|
nonlocal first
|
|
if not first:
|
|
self.assertTrue(workers)
|
|
self.assertFalse(workers[0].is_alive())
|
|
stream = CaptureStream("stdout", "", 0, 0, False)
|
|
lifecycle = InvocationResult(
|
|
True, "success", 0, None, True, True, True, False,
|
|
(), stream, replace(stream, stream="stderr"), "", "", None,
|
|
"sha256:" + "a" * 64,
|
|
"2026-08-11T00:00:00+00:00",
|
|
"2026-08-11T00:00:01+00:00", 1, (),
|
|
)
|
|
binding = (
|
|
evaluator.iop.route_kind,
|
|
evaluator.iop.route_id,
|
|
evaluator.iop.request_model,
|
|
evaluator.iop.requested_effort,
|
|
)
|
|
return CodexInvocationResult(lifecycle, binding)
|
|
first = False
|
|
spec = replace(
|
|
invocation.spec,
|
|
argv=(
|
|
sys.executable,
|
|
"-u",
|
|
"-c",
|
|
"import sys,time; sys.stdin.buffer.read(); "
|
|
"print('START', flush=True); time.sleep(30)",
|
|
),
|
|
env=env_pairs(
|
|
{"PATH": os.environ.get("PATH", "/usr/bin:/bin")}
|
|
),
|
|
task_payload=b"evaluate",
|
|
)
|
|
|
|
def run():
|
|
worker_results.append(
|
|
run_invocation(
|
|
spec,
|
|
parse_event=lambda _stream, _line: None,
|
|
on_started=lambda locator: (
|
|
on_started(locator),
|
|
locators.append(locator),
|
|
locator_ready.set(),
|
|
),
|
|
)
|
|
)
|
|
|
|
worker = threading.Thread(target=run)
|
|
workers.append(worker)
|
|
worker.start()
|
|
if not locator_ready.wait(5):
|
|
self.fail("live evaluator locator was not published")
|
|
deadline = time.monotonic() + 5
|
|
while not recover_invocation(locators[0], stop=False).caller_launched:
|
|
if time.monotonic() >= deadline:
|
|
self.fail("live evaluator did not launch")
|
|
time.sleep(0.01)
|
|
raise KeyboardInterrupt("simulated live scoring controller loss")
|
|
|
|
adapter = live_iop.build_live_scoring_adapter(
|
|
environment,
|
|
observer=lambda _runtime: live_iop._Observation(
|
|
(evaluator.iop.request_model,),
|
|
"sha256:" + "b" * 64,
|
|
True,
|
|
),
|
|
invoker=invoker,
|
|
)
|
|
self.assertEqual(adapter.preflight(evaluator).result.status, "ready")
|
|
blind_root = self.root / "recovery-run" / "blind" / "blind-recovery"
|
|
for name in ("input", "session", "output"):
|
|
(blind_root / name).mkdir(parents=True, exist_ok=True)
|
|
blind = BlindWorkspace(
|
|
"blind-recovery",
|
|
str(blind_root),
|
|
str(blind_root / "input"),
|
|
str(blind_root / "session"),
|
|
str(blind_root / "output"),
|
|
"sha256:" + "c" * 64,
|
|
"sha256:" + "d" * 64,
|
|
)
|
|
|
|
def cleanup():
|
|
if workers and workers[0].is_alive() and locators:
|
|
try:
|
|
recover_invocation(locators[0], stop=True)
|
|
except Exception:
|
|
pass
|
|
workers[0].join(5)
|
|
try:
|
|
adapter.finalize_evidence(blind)
|
|
except Exception:
|
|
pass
|
|
|
|
self.addCleanup(cleanup)
|
|
with self.assertRaises(KeyboardInterrupt):
|
|
adapter.invoke(
|
|
evaluator,
|
|
blind,
|
|
b"Evaluate anonymous output.",
|
|
self.manifest.timeout,
|
|
lambda locator, digest: self.assertRegex(
|
|
digest, r"^sha256:[0-9a-f]{64}$"
|
|
),
|
|
)
|
|
try:
|
|
stopped = recover_invocation(locators[0], stop=True)
|
|
except LifecycleRecoveryError:
|
|
stopped = None
|
|
if stopped is not None:
|
|
self.assertTrue(stopped.cleanup_complete)
|
|
self.assertFalse(stopped.process_group_alive)
|
|
workers[0].join(5)
|
|
self.assertFalse(workers[0].is_alive())
|
|
self.assertEqual(len(worker_results), 1)
|
|
self.assertTrue(worker_results[0].cleanup_complete)
|
|
self.assertFalse(worker_results[0].process_group_alive)
|
|
receipt = json.loads(
|
|
(
|
|
blind_root / "output" / "codex-control" / "cleanup-receipt.json"
|
|
).read_text()
|
|
)
|
|
self.assertTrue(receipt["cleanup_complete"])
|
|
self.assertFalse(receipt["process_group_alive"])
|
|
self.assertTrue(adapter.finalize_evidence(blind).safe)
|
|
|
|
retry = adapter.invoke(
|
|
evaluator,
|
|
blind,
|
|
b"Evaluate anonymous output.",
|
|
self.manifest.timeout,
|
|
lambda *_args: None,
|
|
)
|
|
self.assertTrue(retry.success)
|
|
self.assertTrue(adapter.finalize_evidence(blind).safe)
|
|
|
|
def test_catalog_only_never_creates_ready_binding(self) -> None:
|
|
environment = self._live_environment()
|
|
del environment["IOP_BENCH_CONFIG_OBSERVATION_ENV"]
|
|
registry = live_iop.build_live_adapter_registry(
|
|
environment,
|
|
observer=lambda _runtime: live_iop._Observation(
|
|
tuple(sorted(cell.iop.request_model for cell in self.manifest.matrix)),
|
|
"sha256:" + "d" * 64,
|
|
True,
|
|
),
|
|
)
|
|
cell = next(cell for cell in self.manifest.matrix if cell.caller == "claude")
|
|
result = registry["claude"].preflight(cell).result
|
|
self.assertEqual(result.status, "registration_required")
|
|
self.assertEqual([item.code for item in result.issues], ["route_missing"])
|
|
self.assertIsNone(result.binding.effective_model)
|
|
|
|
def test_config_owner_binding_is_passed_without_manifest_synthesis(self) -> None:
|
|
environment = self._live_environment()
|
|
cell = next(cell for cell in self.manifest.matrix if cell.id == "claude-sonnet-direct")
|
|
registry = live_iop.build_live_adapter_registry(
|
|
environment,
|
|
observer=lambda _runtime: live_iop._Observation(
|
|
tuple(sorted(item.iop.request_model for item in self.manifest.matrix)),
|
|
"sha256:" + "e" * 64,
|
|
True,
|
|
),
|
|
)
|
|
result = registry["claude"].preflight(cell).result
|
|
self.assertEqual(result.status, "ready")
|
|
self.assertEqual(result.binding.effective_route_id, cell.iop.route_id)
|
|
self.assertEqual(result.binding.effective_model, cell.iop.request_model)
|
|
self.assertEqual(
|
|
result.binding.effective_bindings,
|
|
(EffectiveBinding("request", cell.iop.request_model, cell.iop.requested_effort),),
|
|
)
|
|
|
|
def test_live_scoring_preset_requires_observed_stage_bindings(self) -> None:
|
|
raw = json.loads(self.path.read_text())
|
|
raw["evaluator"]["iop"] = {
|
|
"request_model": "judge",
|
|
"requested_effort": "xhigh",
|
|
"route_kind": "execution_preset",
|
|
"route_id": "judge-preset",
|
|
"expected_bindings": [
|
|
{"stage": stage, "model": "judge"}
|
|
for stage in ("selector", "plan", "work", "review")
|
|
],
|
|
}
|
|
raw["output_root"] = "agent-test/runs/preset-observation"
|
|
path = self.root / "preset-observation.json"
|
|
path.write_text(json.dumps(raw), encoding="utf-8")
|
|
manifest = load_manifest(path, repo_root=self.root)
|
|
evaluator = MatrixCell(
|
|
"evaluator", manifest.evaluator.caller, manifest.evaluator.iop
|
|
)
|
|
environment = self._live_environment(manifest=manifest)
|
|
observed = lambda _runtime: live_iop._Observation(
|
|
("judge",), "sha256:" + "8" * 64, True
|
|
)
|
|
|
|
ready = live_iop.build_live_scoring_adapter(
|
|
environment, observer=observed
|
|
).preflight(evaluator)
|
|
self.assertEqual(ready.result.status, "ready")
|
|
self.assertEqual(
|
|
ready.result.binding.effective_bindings,
|
|
tuple(
|
|
EffectiveBinding(item.stage, item.model, item.effort)
|
|
for item in evaluator.iop.expected_bindings
|
|
),
|
|
)
|
|
|
|
routes = json.loads(environment["BENCH_CONFIG"])["routes"]
|
|
index = next(
|
|
i for i, item in enumerate(routes) if item["route_id"] == "judge-preset"
|
|
)
|
|
cases = {
|
|
"missing": routes[index]["bindings"][:-1],
|
|
"reordered": list(reversed(routes[index]["bindings"])),
|
|
"substituted": [
|
|
(
|
|
{**binding, "model": "other-model"}
|
|
if binding["stage"] == "work"
|
|
else dict(binding)
|
|
)
|
|
for binding in routes[index]["bindings"]
|
|
],
|
|
}
|
|
for name, bindings in cases.items():
|
|
with self.subTest(case=name):
|
|
changed = [dict(item) for item in routes]
|
|
changed[index] = {**changed[index], "bindings": bindings}
|
|
candidate = {
|
|
**environment,
|
|
"BENCH_CONFIG": json.dumps(
|
|
{"schema_version": "1", "routes": changed},
|
|
sort_keys=True,
|
|
),
|
|
}
|
|
result = live_iop.build_live_scoring_adapter(
|
|
candidate, observer=observed
|
|
).preflight(evaluator).result
|
|
self.assertEqual(result.status, "implementation_gap")
|
|
self.assertEqual(
|
|
[issue.code for issue in result.issues],
|
|
["protocol_incompatible"],
|
|
)
|
|
self.assertIsNone(result.binding.effective_model)
|
|
|
|
def test_live_scoring_metrics_match_any_admitted_stage_model(self) -> None:
|
|
direct_evaluator = MatrixCell(
|
|
"evaluator",
|
|
self.manifest.evaluator.caller,
|
|
replace(
|
|
self.manifest.evaluator.iop,
|
|
requested_effort="xhigh",
|
|
expected_bindings=(ExpectedBinding("request", "judge", "xhigh"),),
|
|
),
|
|
)
|
|
direct_manifest = replace(
|
|
self.manifest,
|
|
evaluator=replace(self.manifest.evaluator, iop=direct_evaluator.iop),
|
|
)
|
|
|
|
raw = json.loads(self.path.read_text())
|
|
raw["evaluator"]["iop"] = {
|
|
"request_model": "judge-selector",
|
|
"requested_effort": "xhigh",
|
|
"route_kind": "execution_preset",
|
|
"route_id": "judge-heterogeneous",
|
|
"expected_bindings": [
|
|
{"stage": "selector", "model": "judge-selector"},
|
|
{"stage": "plan", "model": "judge-plan"},
|
|
{"stage": "work", "model": "judge-work"},
|
|
{"stage": "review", "model": "judge-review"},
|
|
],
|
|
}
|
|
raw["output_root"] = "agent-test/runs/heterogeneous-metrics"
|
|
path = self.root / "heterogeneous-metrics.json"
|
|
path.write_text(json.dumps(raw), encoding="utf-8")
|
|
preset_manifest = load_manifest(path, repo_root=self.root)
|
|
|
|
def exercise(
|
|
manifest, metric_model: str, suffix: str, metric_stage: str | None = None
|
|
):
|
|
evaluator = MatrixCell(
|
|
"evaluator", manifest.evaluator.caller, manifest.evaluator.iop
|
|
)
|
|
|
|
def invoke(_invocation, _on_started):
|
|
stream = CaptureStream("stdout", "", 0, 0, False)
|
|
metric = ParsedMetric(
|
|
"model_duration",
|
|
1,
|
|
UNIT_NANOSECONDS,
|
|
CLOCK_HARNESS_MONOTONIC,
|
|
SOURCE_HARNESS,
|
|
(
|
|
metric_stage
|
|
if metric_stage is not None
|
|
else (
|
|
"work"
|
|
if evaluator.iop.route_kind == "execution_preset"
|
|
else "request"
|
|
)
|
|
),
|
|
metric_model,
|
|
"call-1",
|
|
)
|
|
lifecycle = InvocationResult(
|
|
True, "success", 0, None, True, True, True, False,
|
|
(), stream, replace(stream, stream="stderr"), "", "", None,
|
|
"sha256:" + "a" * 64,
|
|
"2026-08-11T00:00:00+00:00",
|
|
"2026-08-11T00:00:01+00:00", 1, (metric,),
|
|
)
|
|
binding = (
|
|
evaluator.iop.route_kind,
|
|
evaluator.iop.route_id,
|
|
evaluator.iop.request_model,
|
|
evaluator.iop.requested_effort,
|
|
)
|
|
return CodexInvocationResult(lifecycle, binding)
|
|
|
|
adapter = live_iop.build_live_scoring_adapter(
|
|
self._live_environment(manifest=manifest),
|
|
observer=lambda _runtime: live_iop._Observation(
|
|
(evaluator.iop.request_model,), "sha256:" + "b" * 64, True
|
|
),
|
|
invoker=invoke,
|
|
)
|
|
self.assertEqual(adapter.preflight(evaluator).result.status, "ready")
|
|
blind_root = self.root / "metric-run" / suffix
|
|
for name in ("input", "session", "output"):
|
|
(blind_root / name).mkdir(parents=True, exist_ok=True)
|
|
blind = BlindWorkspace(
|
|
"blind-" + suffix,
|
|
str(blind_root),
|
|
str(blind_root / "input"),
|
|
str(blind_root / "session"),
|
|
str(blind_root / "output"),
|
|
"sha256:" + "c" * 64,
|
|
"sha256:" + "d" * 64,
|
|
)
|
|
try:
|
|
return adapter.invoke(
|
|
evaluator,
|
|
blind,
|
|
b"Evaluate anonymous output.",
|
|
manifest.timeout,
|
|
lambda *_args: None,
|
|
)
|
|
finally:
|
|
adapter.finalize_evidence(blind)
|
|
|
|
self.assertTrue(exercise(direct_manifest, "judge", "direct").success)
|
|
self.assertTrue(
|
|
exercise(preset_manifest, "judge-work", "preset-work").success
|
|
)
|
|
self.assertTrue(
|
|
exercise(
|
|
preset_manifest, "judge-plan", "preset-unqualified", metric_stage=""
|
|
).success
|
|
)
|
|
for stage, model in (("plan", "judge-work"), ("work", "judge-plan")):
|
|
with self.subTest(stage=stage, model=model):
|
|
with self.assertRaises(live_iop.LiveIopError) as raised:
|
|
exercise(
|
|
preset_manifest,
|
|
model,
|
|
f"preset-cross-{stage}",
|
|
metric_stage=stage,
|
|
)
|
|
self.assertEqual(
|
|
raised.exception.issue_code, "stream_incompatible"
|
|
)
|
|
with self.assertRaises(live_iop.LiveIopError) as raised:
|
|
exercise(preset_manifest, "unadmitted-model", "preset-unknown")
|
|
self.assertEqual(raised.exception.issue_code, "stream_incompatible")
|
|
|
|
def test_catalog_accepts_edge_routing_ids_and_rejects_malformed_records(self) -> None:
|
|
environment = self._live_environment()
|
|
runtime = live_iop._runtime_from_environment("claude", environment).runtime
|
|
self.assertIsNotNone(runtime)
|
|
|
|
class Response:
|
|
status = 200
|
|
|
|
def __init__(self, records) -> None:
|
|
self.body = json.dumps({"object": "list", "data": records}).encode()
|
|
|
|
def __enter__(self): return self
|
|
def __exit__(self, *_args): return False
|
|
def read(self): return self.body
|
|
|
|
model_ids = (
|
|
"claude-sonnet-5",
|
|
"gemini-3.6-flash",
|
|
"gpt-5.6-luna",
|
|
"qwen3.6:35b",
|
|
"ornith:35b",
|
|
"laguna-s:2.1",
|
|
)
|
|
with mock.patch.object(
|
|
live_iop,
|
|
"urlopen",
|
|
return_value=Response([{"id": model_id} for model_id in model_ids]),
|
|
):
|
|
models, _identity = live_iop._catalog(runtime) # type: ignore[arg-type]
|
|
self.assertEqual(models, tuple(sorted(model_ids)))
|
|
|
|
malformed = {
|
|
"numeric": [{"id": 1}],
|
|
"empty": [{"id": ""}],
|
|
"whitespace": [{"id": " \t"}],
|
|
"duplicate": [{"id": "claude-sonnet-5"}, {"id": "claude-sonnet-5"}],
|
|
}
|
|
for name, records in malformed.items():
|
|
with self.subTest(name=name):
|
|
with mock.patch.object(live_iop, "urlopen", return_value=Response(records)):
|
|
with self.assertRaises(live_iop.LiveIopError) as raised:
|
|
live_iop._catalog(runtime) # type: ignore[arg-type]
|
|
self.assertEqual(raised.exception.issue_code, "protocol_incompatible")
|
|
|
|
def test_live_failure_taxonomy_is_exact(self) -> None:
|
|
"""Every live setup/catalog boundary returns one closed issue/resume pair."""
|
|
environment = self._live_environment(token="secret-must-not-appear")
|
|
runtime_cases = (
|
|
("missing-base", {"IOP_BENCH_CLAUDE_BASE_URL": ""}, "endpoint_incompatible"),
|
|
("invalid-base", {"IOP_BENCH_CLAUDE_BASE_URL": "not-a-url"}, "endpoint_incompatible"),
|
|
("invalid-secret-ref", {"IOP_BENCH_CLAUDE_SECRET_ENV": "1BAD"}, "credential_missing"),
|
|
("missing-secret", {"BENCH_TOKEN": ""}, "credential_missing"),
|
|
)
|
|
for _name, updates, expected in runtime_cases:
|
|
candidate = {**environment, **updates}
|
|
resolution = live_iop._runtime_from_environment("claude", candidate)
|
|
self.assertIsNone(resolution.runtime)
|
|
self.assertEqual(resolution.issue_code, expected)
|
|
|
|
runtime = live_iop._runtime_from_environment("claude", environment).runtime
|
|
self.assertIsNotNone(runtime)
|
|
|
|
class Response:
|
|
def __init__(self, status, body): self.status, self.body = status, body
|
|
def __enter__(self): return self
|
|
def __exit__(self, *_args): return False
|
|
def read(self): return self.body
|
|
|
|
catalog_cases = (
|
|
(HTTPError("http://invalid", 401, "", None, None), "auth_incompatible"),
|
|
(HTTPError("http://invalid", 403, "", None, None), "auth_incompatible"),
|
|
(OSError("unreachable"), "endpoint_incompatible"),
|
|
(Response(502, b"{}"), "endpoint_incompatible"),
|
|
(Response(200, b"not-json"), "protocol_incompatible"),
|
|
(Response(200, b'{"data":[{"id":1}]}'), "protocol_incompatible"),
|
|
)
|
|
for outcome, expected in catalog_cases:
|
|
patch_kwargs = {"side_effect": outcome} if isinstance(outcome, BaseException) else {"return_value": outcome}
|
|
with mock.patch.object(live_iop, "urlopen", **patch_kwargs):
|
|
with self.assertRaises(live_iop.LiveIopError) as raised:
|
|
live_iop._catalog(runtime) # type: ignore[arg-type]
|
|
self.assertEqual(raised.exception.issue_code, expected)
|
|
|
|
claude = next(cell for cell in self.manifest.matrix if cell.caller == "claude")
|
|
routes = json.loads(environment["BENCH_CONFIG"])["routes"]
|
|
no_route = {**environment, "BENCH_CONFIG": json.dumps({"schema_version": "1", "routes": [item for item in routes if item["route_id"] != claude.iop.route_id]})}
|
|
no_model_routes = [
|
|
{**item, "model": "other-model"} if item["route_id"] == claude.iop.route_id else dict(item)
|
|
for item in routes
|
|
]
|
|
no_model = {**environment, "BENCH_CONFIG": json.dumps({"schema_version": "1", "routes": no_model_routes})}
|
|
unsupported = replace(next(cell for cell in self.manifest.matrix if cell.caller == "agy"), iop=replace(next(cell for cell in self.manifest.matrix if cell.caller == "agy").iop, requested_effort="max"))
|
|
observed = lambda _runtime: live_iop._Observation(tuple(sorted(cell.iop.request_model for cell in self.manifest.matrix)), "sha256:" + "1" * 64, True, "agy 1.1.11", "--print --output-format --sandbox --model --effort AGY_PROVIDER AGY_OPENAI_BASE_URL AGY_OPENAI_API_KEY stream-json")
|
|
checks = (
|
|
(live_iop.build_live_adapter_registry(no_route, observer=observed)["claude"].preflight(claude).result, "route_missing"),
|
|
(live_iop.build_live_adapter_registry(no_model, observer=observed)["claude"].preflight(claude).result, "model_missing"),
|
|
(live_iop.build_live_adapter_registry(environment, observer=lambda _runtime: (_ for _ in ()).throw(live_iop.LiveIopError("stream_incompatible")))["claude"].preflight(claude).result, "stream_incompatible"),
|
|
)
|
|
for result, issue_code in checks:
|
|
self.assertEqual(
|
|
(result.status, tuple((item.code, item.resume_code) for item in result.issues)),
|
|
("registration_required" if issue_code in ISSUE_RESUME_CODES and issue_code in {"route_missing", "model_missing", "effort_unsupported"} else "implementation_gap", ((issue_code, ISSUE_RESUME_CODES[issue_code]),)),
|
|
)
|
|
_, unsupported_issues = live_iop._binding_from_config(
|
|
unsupported,
|
|
CallerCapability("agy", ("direct", "execution_preset"), ("high", "low", "medium")),
|
|
live_iop._runtime_from_environment("agy", environment).runtime.config, # type: ignore[union-attr]
|
|
)
|
|
self.assertEqual(
|
|
tuple((item.code, item.resume_code) for item in unsupported_issues),
|
|
(("effort_unsupported", ISSUE_RESUME_CODES["effort_unsupported"]),),
|
|
)
|
|
|
|
def test_live_invocation_rejects_missing_or_mismatched_caller_binding(self) -> None:
|
|
environment = self._live_environment()
|
|
observed = lambda _runtime: live_iop._Observation(
|
|
tuple(sorted(cell.iop.request_model for cell in self.manifest.matrix)),
|
|
"sha256:" + "2" * 64, True, "agy 1.1.11",
|
|
"--print --output-format --sandbox --model --effort AGY_PROVIDER AGY_OPENAI_BASE_URL AGY_OPENAI_API_KEY stream-json",
|
|
)
|
|
registry = live_iop.build_live_adapter_registry(
|
|
environment, observer=observed, binary_resolver=lambda _name: "/bin/true"
|
|
)
|
|
agy = next(cell for cell in self.manifest.matrix if cell.caller == "agy")
|
|
codex = next(cell for cell in self.manifest.matrix if cell.caller == "codex")
|
|
self.assertEqual(registry["agy"].preflight(agy).result.status, "ready")
|
|
self.assertEqual(registry["codex"].preflight(codex).result.status, "ready")
|
|
|
|
agy_adapter = registry["agy"]
|
|
codex_adapter = registry["codex"]
|
|
agy_adapter._invokers = live_iop._InvokerSeams( # type: ignore[attr-defined]
|
|
live_iop._DEFAULT_INVOKERS.claude,
|
|
lambda *_args: None,
|
|
live_iop._DEFAULT_INVOKERS.codex,
|
|
)
|
|
codex_adapter._invokers = live_iop._InvokerSeams( # type: ignore[attr-defined]
|
|
live_iop._DEFAULT_INVOKERS.claude,
|
|
live_iop._DEFAULT_INVOKERS.agy,
|
|
lambda *_args: type("Mismatch", (), {"effective_binding": None, "lifecycle": None})(),
|
|
)
|
|
with mock.patch.object(live_iop, "build_agy_invocation", return_value=object()):
|
|
with self.assertRaises(live_iop.LiveIopError) as raised:
|
|
agy_adapter.invoke(
|
|
agy, object(), object(), "/tmp/control", b"task",
|
|
self.manifest.timeout, lambda *_args: None,
|
|
)
|
|
self.assertEqual(raised.exception.issue_code, "stream_incompatible")
|
|
with mock.patch.object(live_iop, "build_codex_invocation", return_value=type("Invocation", (), {"spec": object()})()):
|
|
with self.assertRaises(live_iop.LiveIopError) as raised:
|
|
codex_adapter.invoke(
|
|
codex, object(), object(), "/tmp/control", b"task",
|
|
self.manifest.timeout, lambda *_args: None,
|
|
)
|
|
self.assertEqual(raised.exception.issue_code, "stream_incompatible")
|
|
|
|
def test_cli_missing_live_input_fails_closed_without_secret_or_attempt(self) -> None:
|
|
sentinel = "missing-input-token"
|
|
stdout = io.StringIO()
|
|
stderr = io.StringIO()
|
|
with (
|
|
mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root),
|
|
contextlib.redirect_stdout(stdout),
|
|
contextlib.redirect_stderr(stderr),
|
|
):
|
|
exit_code = benchmark_cli.main(["run", "--manifest", str(self.path)])
|
|
self.assertEqual(exit_code, 69)
|
|
self.assertEqual(stdout.getvalue(), "")
|
|
self.assertIn("error: preflight blocked", stderr.getvalue())
|
|
self.assertNotIn(sentinel, stderr.getvalue())
|
|
run_roots = list((self.root / self.manifest.output_root).glob("run-*"))
|
|
self.assertEqual(len(run_roots), 1)
|
|
self.assertFalse((run_roots[0] / "cells").exists())
|
|
|
|
def test_concurrent_writer_fails_fast_without_partial_record(self) -> None:
|
|
registry = self._registry()
|
|
observations = collect_preflight_observations(self.manifest, registry)
|
|
run = self.store.create(self.manifest, self.raw)
|
|
result: list[BaseException] = []
|
|
|
|
def append() -> None:
|
|
try:
|
|
self.store.record_preflight(run, self.manifest, observations)
|
|
except BaseException as exc:
|
|
result.append(exc)
|
|
|
|
with self.store.writer(run):
|
|
worker = threading.Thread(target=append)
|
|
worker.start()
|
|
worker.join(5)
|
|
self.assertFalse(worker.is_alive())
|
|
self.assertEqual(len(result), 1)
|
|
self.assertIsInstance(result[0], RunBusyError)
|
|
self.assertEqual(self.store.preflights(run, self.manifest), ())
|
|
|
|
def test_cli_fake_registry_reports_only_closed_summary(self) -> None:
|
|
sentinel = "private_endpoint_and_token_must_not_appear"
|
|
registry = self._registry(
|
|
{"codex-gpt-direct": ("credential_missing",)}, sentinel=sentinel
|
|
)
|
|
stdout = io.StringIO()
|
|
stderr = io.StringIO()
|
|
with (
|
|
mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root),
|
|
mock.patch.object(benchmark_cli, "build_adapter_registry", return_value=registry),
|
|
contextlib.redirect_stdout(stdout),
|
|
contextlib.redirect_stderr(stderr),
|
|
):
|
|
exit_code = benchmark_cli.main(
|
|
["preflight", "--manifest", str(self.path)]
|
|
)
|
|
self.assertEqual(exit_code, 69)
|
|
self.assertEqual(stdout.getvalue(), "")
|
|
self.assertIn("status=registration_required", stderr.getvalue())
|
|
self.assertIn("registration_required=1", stderr.getvalue())
|
|
self.assertNotIn(sentinel, stderr.getvalue())
|
|
run_roots = list((self.root / self.manifest.output_root).glob("run-*"))
|
|
self.assertEqual(len(run_roots), 1)
|
|
durable = b"".join(
|
|
path.read_bytes() for path in run_roots[0].rglob("*") if path.is_file()
|
|
)
|
|
self.assertNotIn(sentinel.encode("ascii"), durable)
|
|
self.assertFalse((run_roots[0] / "cells").exists())
|
|
|
|
def test_cli_score_missing_run_fails_closed_with_run_id(self) -> None:
|
|
run_id = "run-20260811T010203Z-000000000000"
|
|
stdout = io.StringIO()
|
|
stderr = io.StringIO()
|
|
with (
|
|
mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root),
|
|
contextlib.redirect_stdout(stdout),
|
|
contextlib.redirect_stderr(stderr),
|
|
):
|
|
exit_code = benchmark_cli.main(
|
|
["score", "--manifest", str(self.path), "--run-id", run_id]
|
|
)
|
|
self.assertEqual(exit_code, 69)
|
|
self.assertEqual(stdout.getvalue(), "")
|
|
self.assertEqual(
|
|
stderr.getvalue(),
|
|
f"error: benchmark scoring is unavailable run_id={run_id}\n",
|
|
)
|
|
|
|
def test_cli_score_prints_only_closed_counts_and_forwards_retry(self) -> None:
|
|
run = self.store.create(self.manifest, self.raw)
|
|
cases = (
|
|
(ScoringSummary(run.run_id, 2, 1, 0, 0), 0, "ok: score "),
|
|
(ScoringSummary(run.run_id, 0, 1, 1, 2), 69, "error: benchmark scoring failed "),
|
|
)
|
|
for summary, expected_exit, prefix in cases:
|
|
with self.subTest(summary=summary):
|
|
stdout = io.StringIO()
|
|
stderr = io.StringIO()
|
|
with (
|
|
mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root),
|
|
mock.patch.object(
|
|
benchmark_cli, "build_live_scoring_adapter",
|
|
return_value=object(),
|
|
),
|
|
mock.patch.object(
|
|
benchmark_cli, "score_run", return_value=summary
|
|
) as score,
|
|
contextlib.redirect_stdout(stdout),
|
|
contextlib.redirect_stderr(stderr),
|
|
):
|
|
exit_code = benchmark_cli.main(
|
|
[
|
|
"score", "--manifest", str(self.path),
|
|
"--run-id", run.run_id, "--retry-scoring-failed",
|
|
]
|
|
)
|
|
self.assertEqual(exit_code, expected_exit)
|
|
rendered = stdout.getvalue() or stderr.getvalue()
|
|
self.assertEqual(
|
|
rendered,
|
|
prefix
|
|
+ f"run_id={run.run_id} scored={summary.scored} "
|
|
+ f"unscored={summary.unscored} "
|
|
+ f"scoring_failed={summary.scoring_failed} "
|
|
+ f"blocked={summary.blocked}\n",
|
|
)
|
|
score.assert_called_once()
|
|
self.assertTrue(score.call_args.kwargs["retry_scoring_failed"])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|