toki
0be1a3dcce
Epic 3 준비 전에 caller별 IOP direct preflight와 attempt recovery의 검증된 완료 상태를 원격 checkpoint로 보존한다.
1156 lines
49 KiB
Python
1156 lines
49 KiB
Python
"""Network-free integration tests for public benchmark preflight."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import contextlib
|
|
import datetime
|
|
import io
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
import unittest
|
|
from dataclasses import replace
|
|
from pathlib import Path
|
|
from unittest import mock
|
|
from urllib.error import HTTPError
|
|
|
|
from scripts import agent_comparison_benchmark as benchmark_cli
|
|
from scripts.agent_benchmark import live_iop
|
|
from scripts.agent_benchmark.attempts import (
|
|
CapabilityUnavailable,
|
|
PreflightObservation,
|
|
RunBusyError,
|
|
RunStore,
|
|
collect_preflight_observations,
|
|
preflight_manifest,
|
|
)
|
|
from scripts.agent_benchmark.connectivity import (
|
|
ISSUE_RESUME_CODES,
|
|
CallerCapability,
|
|
ConnectivityIssue,
|
|
EffectiveBinding,
|
|
RequestedEffectiveBinding,
|
|
make_result,
|
|
)
|
|
from scripts.agent_benchmark.manifest import (
|
|
AssetMapping,
|
|
MatrixCell,
|
|
digest_workspace_inputs,
|
|
load_manifest,
|
|
)
|
|
from scripts.agent_benchmark.lifecycle import (
|
|
COMPLETION_EXIT_AFTER_IDLE,
|
|
SUBMISSION_STDIN_ONCE,
|
|
InvocationSpec,
|
|
env_pairs,
|
|
run_invocation,
|
|
spec_digest,
|
|
)
|
|
|
|
|
|
def _cell(cell_id: str, caller: str, model: str, effort: str) -> dict:
|
|
return {
|
|
"id": cell_id,
|
|
"caller": caller,
|
|
"iop": {
|
|
"request_model": model,
|
|
"requested_effort": effort,
|
|
"route_kind": "direct",
|
|
"route_id": cell_id,
|
|
"expected_bindings": [
|
|
{"stage": "request", "model": model, "effort": effort}
|
|
],
|
|
},
|
|
}
|
|
|
|
|
|
def _preset(cell_id: str, caller: str, model: str, effort: str) -> dict:
|
|
return {
|
|
"id": cell_id,
|
|
"caller": caller,
|
|
"iop": {
|
|
"request_model": model,
|
|
"requested_effort": effort,
|
|
"route_kind": "execution_preset",
|
|
"route_id": cell_id,
|
|
"expected_bindings": [
|
|
{"stage": stage, "model": model}
|
|
for stage in ("selector", "plan", "work", "review")
|
|
],
|
|
},
|
|
}
|
|
|
|
|
|
_SENTINEL_CLASSES = ("task", "secret", "endpoint", "config", "provider")
|
|
_LIVE_BRANCHES = ("claude", "agy", "codex")
|
|
|
|
|
|
def _branch_sentinels() -> dict[str, dict[str, str]]:
|
|
"""One distinct sentinel value per caller branch and leak class."""
|
|
return {
|
|
caller: {kind: f"branch-{kind}-sentinel-{caller}" for kind in _SENTINEL_CLASSES}
|
|
for caller in _LIVE_BRANCHES
|
|
}
|
|
|
|
|
|
# One production-shaped caller executable. It reads only its own argv, its own
|
|
# environment, the harness-submitted stdin task and its private caller config,
|
|
# then emits that caller's real stream shape. Every value it can observe is
|
|
# echoed back through the exact fields a real caller uses for content, so the
|
|
# published evidence proves the production redactors - not the test - removed
|
|
# them. The route is taken from the attempt's ``../prepared.json`` because this
|
|
# suite's matrix binds ``route_id`` to the cell id.
|
|
_CALLER_FIXTURE_BODY = r'''"""Production-shaped benchmark caller fixture."""
|
|
import json
|
|
import os
|
|
import sys
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
name = Path(__file__).name
|
|
config = json.loads(
|
|
Path(__file__).with_name(name + ".config.json").read_text(encoding="utf-8")
|
|
)
|
|
argv = sys.argv[1:]
|
|
task = sys.stdin.buffer.read().decode("utf-8", "replace")
|
|
prepared = json.loads((Path.cwd().parent / "prepared.json").read_text(encoding="utf-8"))
|
|
route_id = prepared["identity"]["cell_id"]
|
|
with open(Path(__file__).with_name(name + ".invocations"), "a", encoding="utf-8") as log:
|
|
log.write(prepared["identity"]["run_id"] + " " + route_id + "\n")
|
|
|
|
|
|
def option(flag):
|
|
return argv[argv.index(flag) + 1] if flag in argv else ""
|
|
|
|
|
|
def override(prefix):
|
|
for item in argv:
|
|
if item.startswith(prefix):
|
|
return json.loads(item[len(prefix):])
|
|
return ""
|
|
|
|
|
|
leak = json.dumps(
|
|
{"argv": argv, "env": dict(os.environ), "task": task, "config": config},
|
|
sort_keys=True,
|
|
)
|
|
sys.stderr.write(name + ": diagnostic " + leak + "\n")
|
|
sys.stderr.flush()
|
|
|
|
if name == "claude":
|
|
model = option("--model")
|
|
session = str(uuid.uuid4())
|
|
events = [
|
|
{"type": "system", "subtype": "init", "model": model,
|
|
"session_id": session, "tools": []},
|
|
{"type": "assistant", "session_id": session,
|
|
"message": {"model": model, "stop_reason": "end_turn",
|
|
"content": [{"type": "text", "text": leak}]}},
|
|
{"type": "result", "subtype": "success", "session_id": session,
|
|
"is_error": False, "result": leak},
|
|
]
|
|
elif name == "agy":
|
|
binding = {"route_kind": "direct", "route_id": route_id,
|
|
"model": option("--model"), "effort": option("--effort")}
|
|
events = [
|
|
dict(binding, type="iop", subtype="effective_binding",
|
|
stages=[{"stage": "request", "model": binding["model"],
|
|
"effort": binding["effort"]}]),
|
|
dict(binding, type="result", subtype="success", text=leak),
|
|
dict(binding, type="system", subtype="idle"),
|
|
]
|
|
else:
|
|
events = [
|
|
{"type": "item.completed", "item": {"type": "agent_message", "text": leak}},
|
|
{"type": "turn.completed", "status": "completed",
|
|
"iop_effective_binding": {
|
|
"route_kind": "direct", "route_id": route_id, "model": option("-m"),
|
|
"effort": override("model_reasoning_effort=")}},
|
|
]
|
|
|
|
for event in events:
|
|
sys.stdout.write(json.dumps(event) + "\n")
|
|
sys.stdout.flush()
|
|
'''
|
|
|
|
|
|
def _write_manifest(
|
|
root: Path, matrix: list[dict], output_id: str = "integration", *,
|
|
output_root: str | None = None, prompt: str = "public prompt fixture",
|
|
):
|
|
fixture_root = root / "scripts/fixtures"
|
|
fixture_root.mkdir(parents=True, exist_ok=True)
|
|
(fixture_root / "prompt.md").write_text(prompt, encoding="utf-8")
|
|
(fixture_root / "reference.txt").write_text("public reference", encoding="utf-8")
|
|
assets = (
|
|
AssetMapping(
|
|
"scripts/fixtures/reference.txt",
|
|
"workspace/reference.txt",
|
|
b"public reference",
|
|
),
|
|
)
|
|
payload = {
|
|
"pipeline_version": "1",
|
|
"environment": "dev",
|
|
"testbed": "../iop-s2",
|
|
"repetitions": 1,
|
|
"session_policy": "fresh",
|
|
"setup_cache_policy": "isolated",
|
|
"timeout": {
|
|
"run_seconds": 5,
|
|
"idle_seconds": 1,
|
|
"quiet_seconds": 1,
|
|
"cleanup_grace_seconds": 1,
|
|
},
|
|
"viewports": [{"id": "desktop", "width": 1, "height": 1}],
|
|
"rubric_version": "v1",
|
|
"output_root": output_root or f"agent-test/runs/{output_id}",
|
|
"fixture": {
|
|
"version": "v1",
|
|
"prompt": "scripts/fixtures/prompt.md",
|
|
"assets": [
|
|
{
|
|
"source": "scripts/fixtures/reference.txt",
|
|
"workspace_path": "workspace/reference.txt",
|
|
}
|
|
],
|
|
"checksum": digest_workspace_inputs(assets),
|
|
},
|
|
"matrix": matrix,
|
|
}
|
|
path = root / "manifest.json"
|
|
raw = json.dumps(payload, sort_keys=True).encode("utf-8")
|
|
path.write_bytes(raw)
|
|
return load_manifest(path, repo_root=root), raw, path
|
|
|
|
|
|
class FakeAdapter:
|
|
def __init__(
|
|
self,
|
|
caller: str,
|
|
efforts: tuple[str, ...],
|
|
issues_by_cell: dict[str, tuple[str, ...]] | None = None,
|
|
*,
|
|
sentinel: str = "",
|
|
) -> None:
|
|
self.capability = CallerCapability(
|
|
caller, ("direct", "execution_preset"), efforts
|
|
)
|
|
self.issues_by_cell = issues_by_cell or {}
|
|
self.sentinel = sentinel
|
|
self.calls: list[str] = []
|
|
self.invocations: list[tuple[str, str, str, bytes]] = []
|
|
self.fail_invocation = False
|
|
|
|
def preflight(self, cell: MatrixCell) -> PreflightObservation:
|
|
self.calls.append(cell.id)
|
|
issue_codes = self.issues_by_cell.get(cell.id, ())
|
|
if issue_codes:
|
|
binding = RequestedEffectiveBinding(
|
|
cell.id,
|
|
cell.caller,
|
|
cell.iop.route_kind,
|
|
cell.iop.route_id,
|
|
cell.iop.request_model,
|
|
cell.iop.requested_effort,
|
|
)
|
|
else:
|
|
binding = RequestedEffectiveBinding(
|
|
cell.id,
|
|
cell.caller,
|
|
cell.iop.route_kind,
|
|
cell.iop.route_id,
|
|
cell.iop.request_model,
|
|
cell.iop.requested_effort,
|
|
cell.iop.route_kind,
|
|
cell.iop.route_id,
|
|
cell.iop.request_model,
|
|
cell.iop.requested_effort,
|
|
tuple(
|
|
EffectiveBinding(item.stage, item.model, item.effort)
|
|
for item in cell.iop.expected_bindings
|
|
),
|
|
)
|
|
issues = tuple(
|
|
ConnectivityIssue(code, ISSUE_RESUME_CODES[code])
|
|
for code in issue_codes
|
|
)
|
|
return PreflightObservation(
|
|
make_result(cell, self.capability, binding, issues),
|
|
"sha256:" + "a" * 64,
|
|
"sha256:" + "b" * 64,
|
|
)
|
|
|
|
def invoke(
|
|
self,
|
|
cell,
|
|
prepared,
|
|
attempt,
|
|
control_dir,
|
|
task_payload,
|
|
timeout,
|
|
on_started,
|
|
):
|
|
run_root = Path(attempt.root).parents[3]
|
|
if not (run_root / "preflight/preflight-000001.json").is_file():
|
|
raise AssertionError("attempt allocated before preflight publication")
|
|
if cell.id != attempt.identity.cell_id or prepared.identity != attempt.identity:
|
|
raise AssertionError("execution identity drift")
|
|
self.invocations.append(
|
|
(cell.id, prepared.workspace_dir, prepared.session_id, task_payload)
|
|
)
|
|
|
|
if Path(control_dir).resolve(strict=False) != Path(attempt.root).resolve() / "control":
|
|
raise AssertionError("controller control binding drift")
|
|
source = (
|
|
"import sys; sys.stdin.buffer.read(); print('FAILED'); sys.exit(3)"
|
|
if self.fail_invocation
|
|
else "import sys; sys.stdin.buffer.read(); print('FINISH'); print('IDLE')"
|
|
)
|
|
spec = InvocationSpec(
|
|
argv=(sys.executable, "-u", "-c", source),
|
|
cwd=prepared.workspace_dir,
|
|
env=env_pairs({"PATH": os.environ.get("PATH", "/usr/bin:/bin")}),
|
|
submission_mode=SUBMISSION_STDIN_ONCE,
|
|
completion_mode=COMPLETION_EXIT_AFTER_IDLE,
|
|
timeout=timeout,
|
|
evidence_dir=attempt.root,
|
|
task_payload=task_payload,
|
|
control_dir=control_dir,
|
|
)
|
|
return run_invocation(
|
|
spec,
|
|
parse_event=lambda _stream, line: {
|
|
"FINISH": "finish",
|
|
"IDLE": "idle",
|
|
}.get(line.strip()),
|
|
on_started=lambda locator: on_started(locator, spec_digest(spec)),
|
|
)
|
|
|
|
def cleanup(self) -> None:
|
|
pass
|
|
|
|
|
|
class ConnectivityIntegrationTest(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.temp = tempfile.TemporaryDirectory(dir="/tmp", prefix="benchmark-preflight-")
|
|
self.root = Path(self.temp.name) / "repo"
|
|
self.root.mkdir()
|
|
self.matrix = [
|
|
_cell("claude-sonnet-direct", "claude", "claude-sonnet-5", "max"),
|
|
_cell("claude-gemini-direct", "claude", "gemini-3.6-flash", "high"),
|
|
_cell("claude-gpt-direct", "claude", "gpt-5.6-luna", "xhigh"),
|
|
_cell("agy-gemini-direct", "agy", "gemini-3.6-flash", "high"),
|
|
_cell("codex-gpt-direct", "codex", "gpt-5.6-luna", "xhigh"),
|
|
]
|
|
self.manifest, self.raw, self.path = _write_manifest(self.root, self.matrix)
|
|
self.store = RunStore(
|
|
self.root,
|
|
clock=lambda: datetime.datetime(
|
|
2026, 8, 10, 1, 2, 3, tzinfo=datetime.timezone.utc
|
|
),
|
|
token_hex=lambda _: "123456abcdef",
|
|
)
|
|
|
|
def tearDown(self) -> None:
|
|
self.temp.cleanup()
|
|
|
|
def _init_testbed(self) -> None:
|
|
testbed = self.root.parent / "iop-s2"
|
|
testbed.mkdir()
|
|
(testbed / "README.md").write_text("testbed", encoding="utf-8")
|
|
for command in (
|
|
("git", "init"),
|
|
("git", "config", "user.name", "test"),
|
|
("git", "config", "user.email", "test@example.invalid"),
|
|
("git", "add", "."),
|
|
("git", "commit", "-m", "testbed"),
|
|
):
|
|
subprocess.run(command, cwd=testbed, check=True, capture_output=True)
|
|
|
|
def _live_environment(self, *, token: str = "live-token-must-not-persist", manifest=None) -> dict[str, str]:
|
|
manifest = self.manifest if manifest is None else manifest
|
|
routes = []
|
|
seen: set[tuple[str, str]] = set()
|
|
for cell in manifest.matrix:
|
|
key = (cell.iop.route_kind, cell.iop.route_id)
|
|
if key not in seen:
|
|
seen.add(key)
|
|
routes.append({"route_kind": key[0], "route_id": key[1], "model": cell.iop.request_model})
|
|
environment = {
|
|
"IOP_BENCH_CONFIG_OBSERVATION_ENV": "BENCH_CONFIG",
|
|
"BENCH_CONFIG": json.dumps({"schema_version": "1", "routes": routes}, sort_keys=True),
|
|
"BENCH_TOKEN": token,
|
|
}
|
|
for caller in ("CLAUDE", "AGY", "CODEX"):
|
|
environment[f"IOP_BENCH_{caller}_BASE_URL"] = "http://127.0.0.1:18083/v1"
|
|
environment[f"IOP_BENCH_{caller}_SECRET_ENV"] = "BENCH_TOKEN"
|
|
return environment
|
|
|
|
def _sentinel_live_environment(
|
|
self, manifest, sentinels: dict[str, dict[str, str]]
|
|
) -> dict[str, str]:
|
|
"""Give every branch its own endpoint and credential sentinel value."""
|
|
environment = self._live_environment(manifest=manifest)
|
|
del environment["BENCH_TOKEN"]
|
|
for caller, branch in sentinels.items():
|
|
prefix = f"IOP_BENCH_{caller.upper()}_"
|
|
secret_env = f"BENCH_TOKEN_{caller.upper()}"
|
|
environment[secret_env] = branch["secret"]
|
|
environment[prefix + "SECRET_ENV"] = secret_env
|
|
environment[prefix + "BASE_URL"] = f"http://{branch['endpoint']}.invalid:18083/v1"
|
|
return environment
|
|
|
|
@staticmethod
|
|
def _registry(issues: dict[str, tuple[str, ...]] | None = None, sentinel: str = ""):
|
|
issues = issues or {}
|
|
return {
|
|
"claude": FakeAdapter(
|
|
"claude", ("high", "max", "xhigh"), issues, sentinel=sentinel
|
|
),
|
|
"agy": FakeAdapter("agy", ("high",), issues, sentinel=sentinel),
|
|
"codex": FakeAdapter("codex", ("xhigh",), issues, sentinel=sentinel),
|
|
}
|
|
|
|
def test_all_three_callers_append_exact_ready_results_without_attempts(self) -> None:
|
|
registry = self._registry()
|
|
run, record = preflight_manifest(
|
|
self.store, self.manifest, self.raw, adapters=registry
|
|
)
|
|
self.assertEqual(record["status"], "ready")
|
|
self.assertEqual(
|
|
[result["cell"]["id"] for result in record["results"]],
|
|
[cell.id for cell in self.manifest.matrix],
|
|
)
|
|
self.assertEqual(len(self.store.preflights(run, self.manifest)), 1)
|
|
self.assertFalse((Path(run.root) / "cells").exists())
|
|
self.assertEqual(
|
|
{caller: adapter.calls for caller, adapter in registry.items()},
|
|
{
|
|
"claude": [
|
|
"claude-gemini-direct",
|
|
"claude-gpt-direct",
|
|
"claude-sonnet-direct",
|
|
],
|
|
"agy": ["agy-gemini-direct"],
|
|
"codex": ["codex-gpt-direct"],
|
|
},
|
|
)
|
|
|
|
def test_registration_and_implementation_blockers_are_distinct_and_no_attempt_allocates(self) -> None:
|
|
registry = self._registry(
|
|
{
|
|
"claude-sonnet-direct": ("credential_missing",),
|
|
"agy-gemini-direct": ("stream_incompatible",),
|
|
}
|
|
)
|
|
run, record = preflight_manifest(
|
|
self.store, self.manifest, self.raw, adapters=registry
|
|
)
|
|
statuses = [result["status"] for result in record["results"]]
|
|
self.assertEqual(statuses.count("registration_required"), 1)
|
|
self.assertEqual(statuses.count("implementation_gap"), 1)
|
|
self.assertEqual(record["status"], "implementation_gap")
|
|
self.assertFalse((Path(run.root) / "cells").exists())
|
|
self.assertEqual(self.store.status(run, self.manifest)["attempts"]["running"], 0)
|
|
|
|
@contextlib.contextmanager
|
|
def _production_shaped_callers(self, sentinels: dict[str, dict[str, str]]):
|
|
"""Publish one executable per caller before any invocation is built."""
|
|
# This checkout mounts /tmp with noexec. Keep the fixtures temporary
|
|
# and test-owned, but place their executable directory on the current
|
|
# executable test filesystem so the real caller adapters can launch
|
|
# them through their normal subprocess path.
|
|
bin_dir = tempfile.TemporaryDirectory(dir=Path.cwd(), prefix=".bc-")
|
|
self.addCleanup(bin_dir.cleanup)
|
|
fixture_bin = Path(bin_dir.name)
|
|
for caller, branch in sentinels.items():
|
|
executable = fixture_bin / caller
|
|
executable.write_text(
|
|
f"#!{sys.executable}\n" + _CALLER_FIXTURE_BODY, encoding="utf-8"
|
|
)
|
|
executable.chmod(0o700)
|
|
# The caller's private configuration is the only source of its
|
|
# config/provider sentinels, exactly as a real client config file.
|
|
(fixture_bin / f"{caller}.config.json").write_text(
|
|
json.dumps(
|
|
{
|
|
"caller": caller,
|
|
"config_identity": branch["config"],
|
|
"provider_id": branch["provider"],
|
|
},
|
|
sort_keys=True,
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
yield fixture_bin
|
|
|
|
@staticmethod
|
|
def _snapshot_run_root(run_root: Path) -> dict[str, bytes]:
|
|
return {
|
|
str(item.relative_to(run_root)): item.read_bytes()
|
|
for item in sorted(run_root.rglob("*"))
|
|
if item.is_file()
|
|
}
|
|
|
|
def _assert_one_published_spec_digest(self, attempt_root: Path) -> None:
|
|
"""Prove the admitted, journalled and published digests are one value."""
|
|
state = json.loads((attempt_root / "attempt.json").read_text(encoding="utf-8"))
|
|
result = json.loads(
|
|
(attempt_root / "lifecycle-result.json").read_text(encoding="utf-8")
|
|
)
|
|
header = json.loads(
|
|
(attempt_root / "lifecycle-journal.jsonl")
|
|
.read_text(encoding="utf-8")
|
|
.splitlines()[0]
|
|
)
|
|
self.assertIn("locator", state)
|
|
digests = {state["spec_digest"], result["spec_digest"], header["spec_digest"]}
|
|
self.assertEqual(len(digests), 1, attempt_root)
|
|
self.assertRegex(digests.pop(), r"^sha256:[0-9a-f]{64}$")
|
|
self.assertEqual(state["state"], "success")
|
|
self.assertIs(result["success"], True)
|
|
self.assertEqual(result["terminal_reason"], "success")
|
|
self.assertIs(result["finish_then_idle_then_quiet"], True)
|
|
self.assertIs(result["cleanup_complete"], True)
|
|
self.assertIs(result["process_group_alive"], False)
|
|
control_dir = Path(state["locator"]["control_dir"])
|
|
alias = control_dir.parent
|
|
self.assertEqual(control_dir.name, "control")
|
|
self.assertEqual(
|
|
state["locator"]["socket_path"], str(control_dir / "control.sock")
|
|
)
|
|
self.assertTrue(alias.name.startswith("iop-bench-attempt-"))
|
|
self.assertFalse(os.path.lexists(alias))
|
|
self.assertTrue((attempt_root / "control/locator.json").is_file())
|
|
self.assertTrue((attempt_root / "control/cleanup-receipt.json").is_file())
|
|
|
|
def _assert_sentinels_absent(
|
|
self, published: dict[str, bytes], sentinels: dict[str, dict[str, str]]
|
|
) -> None:
|
|
for caller, branch in sentinels.items():
|
|
for kind, value in branch.items():
|
|
encoded = value.encode("ascii")
|
|
for relative, data in published.items():
|
|
self.assertNotIn(encoded, data, f"{caller}/{kind} in {relative}")
|
|
|
|
def test_cli_live_run_invokes_each_direct_cell_once(self) -> None:
|
|
short = tempfile.TemporaryDirectory(dir="/tmp", prefix="b")
|
|
self.addCleanup(short.cleanup)
|
|
root = Path(short.name) / "r"
|
|
root.mkdir()
|
|
matrix = [
|
|
_cell("claude", "claude", "sonnet", "max"),
|
|
_cell("agy", "agy", "gemini", "high"),
|
|
_cell("codex", "codex", "gpt", "xhigh"),
|
|
]
|
|
sentinels = _branch_sentinels()
|
|
manifest, _raw, path = _write_manifest(
|
|
root, matrix, output_root="agent-test/runs/r",
|
|
prompt="public prompt fixture\n"
|
|
+ "\n".join(sentinels[caller]["task"] for caller in _LIVE_BRANCHES),
|
|
)
|
|
testbed = root.parent / "iop-s2"
|
|
testbed.mkdir()
|
|
(testbed / "README.md").write_text("testbed", encoding="utf-8")
|
|
for command in (("git", "init"), ("git", "config", "user.name", "test"), ("git", "config", "user.email", "test@example.invalid"), ("git", "add", "."), ("git", "commit", "-m", "testbed")):
|
|
subprocess.run(command, cwd=testbed, check=True, capture_output=True)
|
|
environment = self._sentinel_live_environment(manifest, sentinels)
|
|
|
|
def observed(_runtime):
|
|
return live_iop._Observation(
|
|
tuple(sorted(cell.iop.request_model for cell in manifest.matrix)),
|
|
"sha256:" + "f" * 64,
|
|
True,
|
|
"agy 1.1.11",
|
|
"--print --output-format --sandbox --model --effort AGY_PROVIDER AGY_OPENAI_BASE_URL AGY_OPENAI_API_KEY stream-json",
|
|
)
|
|
|
|
stdout = io.StringIO()
|
|
stderr = io.StringIO()
|
|
with self._production_shaped_callers(sentinels) as fixture_bin:
|
|
with (
|
|
mock.patch.dict(
|
|
os.environ, {"PATH": f"{fixture_bin}:{os.environ['PATH']}"}
|
|
),
|
|
mock.patch.object(benchmark_cli, "_REPO_ROOT", root),
|
|
contextlib.redirect_stdout(stdout),
|
|
contextlib.redirect_stderr(stderr),
|
|
):
|
|
registry = live_iop.build_live_adapter_registry(
|
|
environment,
|
|
observer=observed,
|
|
binary_resolver=lambda name: str(fixture_bin / name),
|
|
)
|
|
with mock.patch.object(
|
|
benchmark_cli, "build_adapter_registry", return_value=registry
|
|
):
|
|
exit_code = benchmark_cli.main(["run", "--manifest", str(path)])
|
|
|
|
self.assertEqual(exit_code, 0, stderr.getvalue())
|
|
self.assertIn("ok: run run_id=", stdout.getvalue())
|
|
self.assertEqual(stderr.getvalue(), "")
|
|
calls = {
|
|
caller: (fixture_bin / f"{caller}.invocations")
|
|
.read_text(encoding="utf-8")
|
|
.splitlines()
|
|
for caller in _LIVE_BRANCHES
|
|
}
|
|
self.assertEqual({caller: len(item) for caller, item in calls.items()},
|
|
{"claude": 1, "agy": 1, "codex": 1})
|
|
|
|
run_roots = list((root / manifest.output_root).glob("run-*"))
|
|
self.assertEqual(len(run_roots), 1)
|
|
run_root = run_roots[0]
|
|
self.assertTrue((run_root / "preflight/preflight-000001.json").is_file())
|
|
published = self._snapshot_run_root(run_root)
|
|
attempt_roots = sorted(run_root.glob("cells/*/repetition-*/attempt-*"))
|
|
self.assertEqual(len(attempt_roots), len(manifest.matrix))
|
|
for attempt_root in attempt_roots:
|
|
self._assert_one_published_spec_digest(attempt_root)
|
|
self._assert_sentinels_absent(published, sentinels)
|
|
self.assertEqual(published, self._snapshot_run_root(run_root))
|
|
|
|
def test_cli_run_blocker_persists_preflight_and_allocates_zero_attempts(self) -> None:
|
|
registry = self._registry(
|
|
{"claude-sonnet-direct": ("credential_missing",)}
|
|
)
|
|
stdout = io.StringIO()
|
|
stderr = io.StringIO()
|
|
with (
|
|
mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root),
|
|
mock.patch.object(
|
|
benchmark_cli, "build_adapter_registry", return_value=registry
|
|
),
|
|
contextlib.redirect_stdout(stdout),
|
|
contextlib.redirect_stderr(stderr),
|
|
):
|
|
exit_code = benchmark_cli.main(["run", "--manifest", str(self.path)])
|
|
|
|
self.assertEqual(exit_code, 69)
|
|
self.assertEqual(stdout.getvalue(), "")
|
|
self.assertIn("error: preflight blocked", stderr.getvalue())
|
|
run_roots = list((self.root / self.manifest.output_root).glob("run-*"))
|
|
self.assertEqual(len(run_roots), 1)
|
|
self.assertTrue((run_roots[0] / "preflight/preflight-000001.json").is_file())
|
|
self.assertFalse((run_roots[0] / "cells").exists())
|
|
self.assertTrue(all(adapter.invocations == [] for adapter in registry.values()))
|
|
|
|
def test_cli_mixed_manifest_never_invokes_unobserved_preset_cells(self) -> None:
|
|
manifest, _, path = _write_manifest(
|
|
self.root,
|
|
[
|
|
_cell("direct-ready", "claude", "claude-sonnet-5", "max"),
|
|
_preset("preset-unobserved", "claude", "claude-sonnet-5", "max"),
|
|
],
|
|
output_id="mixed-unobserved",
|
|
)
|
|
registry = self._registry()
|
|
stdout = io.StringIO()
|
|
stderr = io.StringIO()
|
|
with (
|
|
mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root),
|
|
mock.patch.object(
|
|
benchmark_cli, "build_adapter_registry", return_value=registry
|
|
),
|
|
contextlib.redirect_stdout(stdout),
|
|
contextlib.redirect_stderr(stderr),
|
|
):
|
|
exit_code = benchmark_cli.main(["run", "--manifest", str(path)])
|
|
|
|
self.assertEqual(exit_code, 69)
|
|
self.assertEqual(stdout.getvalue(), "")
|
|
self.assertIn("error: benchmark execution failed", stderr.getvalue())
|
|
self.assertIn("completed=0 unresolved=2", stderr.getvalue())
|
|
run_roots = list((self.root / manifest.output_root).glob("run-*"))
|
|
self.assertEqual(len(run_roots), 1)
|
|
preflight = json.loads(
|
|
(run_roots[0] / "preflight/preflight-000001.json").read_text(
|
|
encoding="ascii"
|
|
)
|
|
)
|
|
self.assertEqual(preflight["status"], "ready")
|
|
self.assertEqual(
|
|
[result["cell"]["id"] for result in preflight["results"]],
|
|
["direct-ready"],
|
|
)
|
|
self.assertFalse((run_roots[0] / "cells").exists())
|
|
self.assertEqual(registry["claude"].calls, ["direct-ready"])
|
|
self.assertTrue(all(adapter.invocations == [] for adapter in registry.values()))
|
|
|
|
def test_cli_resume_retries_append_only_and_status_is_read_only(self) -> None:
|
|
self._init_testbed()
|
|
manifest, _, path = _write_manifest(
|
|
self.root,
|
|
[_cell("claude-only", "claude", "claude-sonnet-5", "max")],
|
|
output_id="retry",
|
|
)
|
|
failed = FakeAdapter("claude", ("max",))
|
|
failed.fail_invocation = True
|
|
self.addCleanup(failed.cleanup)
|
|
first_stdout = io.StringIO()
|
|
first_stderr = io.StringIO()
|
|
with (
|
|
mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root),
|
|
mock.patch.object(
|
|
benchmark_cli,
|
|
"build_adapter_registry",
|
|
return_value={"claude": failed},
|
|
),
|
|
contextlib.redirect_stdout(first_stdout),
|
|
contextlib.redirect_stderr(first_stderr),
|
|
):
|
|
first_exit = benchmark_cli.main(
|
|
["run", "--manifest", str(path)]
|
|
)
|
|
self.assertEqual(first_exit, 69)
|
|
matched = re.search(r"run_id=(run-[0-9A-Za-z-]+)", first_stderr.getvalue())
|
|
self.assertIsNotNone(matched)
|
|
run_id = matched.group(1) # type: ignore[union-attr]
|
|
run_root = self.root / manifest.output_root / run_id
|
|
first_attempt = next(run_root.glob("cells/*/repetition-*/attempt-000001"))
|
|
old_bytes = {
|
|
item.relative_to(first_attempt): item.read_bytes()
|
|
for item in first_attempt.rglob("*")
|
|
if item.is_file()
|
|
}
|
|
|
|
ready = FakeAdapter("claude", ("max",))
|
|
self.addCleanup(ready.cleanup)
|
|
resume_stdout = io.StringIO()
|
|
resume_stderr = io.StringIO()
|
|
with (
|
|
mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root),
|
|
mock.patch.object(
|
|
benchmark_cli,
|
|
"build_adapter_registry",
|
|
return_value={"claude": ready},
|
|
),
|
|
contextlib.redirect_stdout(resume_stdout),
|
|
contextlib.redirect_stderr(resume_stderr),
|
|
):
|
|
resume_exit = benchmark_cli.main(
|
|
[
|
|
"resume",
|
|
"--manifest",
|
|
str(path),
|
|
"--run-id",
|
|
run_id,
|
|
"--retry-failed",
|
|
]
|
|
)
|
|
|
|
self.assertEqual(resume_exit, 0, resume_stderr.getvalue())
|
|
self.assertIn("ok: resume", resume_stdout.getvalue())
|
|
self.assertEqual(
|
|
old_bytes,
|
|
{
|
|
relative: (first_attempt / relative).read_bytes()
|
|
for relative in old_bytes
|
|
},
|
|
)
|
|
self.assertTrue(
|
|
next(run_root.glob("cells/*/repetition-*/attempt-000002/attempt.json"))
|
|
.read_text(encoding="utf-8")
|
|
.find('"state":"success"')
|
|
>= 0
|
|
)
|
|
self.assertEqual(len(list((run_root / "preflight").glob("*.json"))), 2)
|
|
|
|
before_status = {
|
|
item.relative_to(run_root): item.read_bytes()
|
|
for item in run_root.rglob("*")
|
|
if item.is_file()
|
|
}
|
|
status_stdout = io.StringIO()
|
|
status_stderr = io.StringIO()
|
|
with (
|
|
mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root),
|
|
contextlib.redirect_stdout(status_stdout),
|
|
contextlib.redirect_stderr(status_stderr),
|
|
):
|
|
status_exit = benchmark_cli.main(
|
|
["status", "--manifest", str(path), "--run-id", run_id]
|
|
)
|
|
self.assertEqual(status_exit, 0, status_stderr.getvalue())
|
|
self.assertIn("'success': 1", status_stdout.getvalue())
|
|
self.assertEqual(
|
|
before_status,
|
|
{
|
|
item.relative_to(run_root): item.read_bytes()
|
|
for item in run_root.rglob("*")
|
|
if item.is_file()
|
|
},
|
|
)
|
|
|
|
def test_missing_adapter_is_rejected_before_output_root_mutation(self) -> None:
|
|
registry = self._registry()
|
|
del registry["codex"]
|
|
output_root = self.root / self.manifest.output_root
|
|
with self.assertRaises(CapabilityUnavailable):
|
|
preflight_manifest(
|
|
self.store, self.manifest, self.raw, adapters=registry
|
|
)
|
|
self.assertFalse(output_root.exists())
|
|
|
|
def test_generic_preset_cells_are_local_contract_only(self) -> None:
|
|
generic, _, _ = _write_manifest(
|
|
self.root,
|
|
[
|
|
_preset("claude-generic", "claude", "claude-sonnet-5", "high"),
|
|
_preset("agy-generic", "agy", "gemini-3.6-flash", "high"),
|
|
_preset("codex-generic", "codex", "gpt-5.6-luna", "xhigh"),
|
|
],
|
|
output_id="generic",
|
|
)
|
|
registry = self._registry()
|
|
observations = collect_preflight_observations(generic, registry)
|
|
self.assertEqual(observations, {})
|
|
self.assertTrue(all(adapter.calls == [] for adapter in registry.values()))
|
|
|
|
def test_generic_preset_only_public_preflight_fails_closed_without_run_state(self) -> None:
|
|
generic, raw, path = _write_manifest(
|
|
self.root,
|
|
[
|
|
_preset("claude-generic", "claude", "claude-sonnet-5", "high"),
|
|
_preset("agy-generic", "agy", "gemini-3.6-flash", "high"),
|
|
_preset("codex-generic", "codex", "gpt-5.6-luna", "xhigh"),
|
|
],
|
|
output_id="generic",
|
|
)
|
|
registry = self._registry()
|
|
output_root = self.root / generic.output_root
|
|
with self.assertRaises(Exception) as ctx:
|
|
preflight_manifest(
|
|
self.store, generic, raw, adapters=registry
|
|
)
|
|
self.assertIn("preflight requires a direct cell", str(ctx.exception))
|
|
self.assertFalse(output_root.exists())
|
|
self.assertTrue(all(adapter.calls == [] for adapter in registry.values()))
|
|
|
|
sentinel = "private_endpoint_and_token_must_not_appear"
|
|
stdout = io.StringIO()
|
|
stderr = io.StringIO()
|
|
with (
|
|
mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root),
|
|
mock.patch.object(benchmark_cli, "build_adapter_registry", return_value=registry),
|
|
contextlib.redirect_stdout(stdout),
|
|
contextlib.redirect_stderr(stderr),
|
|
):
|
|
exit_code = benchmark_cli.main(["preflight", "--manifest", str(path)])
|
|
self.assertEqual(exit_code, 69)
|
|
self.assertEqual(stdout.getvalue(), "")
|
|
self.assertNotIn(sentinel, stderr.getvalue())
|
|
self.assertFalse(output_root.exists())
|
|
|
|
def test_live_registry_dereferences_secret_names_without_persisting_values(self) -> None:
|
|
sentinel = "live-token-must-not-persist"
|
|
environment = self._live_environment(token=sentinel)
|
|
environment["ANTHROPIC_BASE_URL"] = "must-not-be-read"
|
|
|
|
def observed(_runtime):
|
|
return live_iop._Observation(
|
|
tuple(sorted(cell.iop.request_model for cell in self.manifest.matrix)),
|
|
"sha256:" + "c" * 64,
|
|
True,
|
|
"agy 1.1.11",
|
|
"--print --output-format --sandbox --model --effort AGY_PROVIDER AGY_OPENAI_BASE_URL AGY_OPENAI_API_KEY stream-json",
|
|
)
|
|
|
|
registry = live_iop.build_live_adapter_registry(
|
|
environment, observer=observed, binary_resolver=lambda _name: "/bin/true"
|
|
)
|
|
self.assertEqual(tuple(registry), ("claude", "agy", "codex"))
|
|
observations = collect_preflight_observations(self.manifest, registry)
|
|
self.assertEqual(set(observations), {cell.id for cell in self.manifest.matrix})
|
|
for observation in observations.values():
|
|
self.assertEqual(observation.result.status, "ready")
|
|
self.assertEqual(observation.result.binding.effective_model, observation.result.binding.requested_model)
|
|
durable = json.dumps(
|
|
{cell_id: item.result.status for cell_id, item in observations.items()},
|
|
sort_keys=True,
|
|
)
|
|
self.assertNotIn(sentinel, durable)
|
|
self.assertNotIn("must-not-be-read", durable)
|
|
|
|
def test_live_registry_ready_binding_requires_catalog_observation(self) -> None:
|
|
environment = self._live_environment(token="private-token")
|
|
registry = live_iop.build_live_adapter_registry(
|
|
environment,
|
|
observer=lambda _runtime: live_iop._Observation(
|
|
(), "sha256:" + "d" * 64, True
|
|
),
|
|
)
|
|
claude_cell = next(cell for cell in self.manifest.matrix if cell.caller == "claude")
|
|
observation = registry["claude"].preflight(claude_cell)
|
|
self.assertEqual(observation.result.status, "registration_required")
|
|
self.assertEqual([issue.code for issue in observation.result.issues], ["model_missing"])
|
|
self.assertIsNone(observation.result.binding.effective_model)
|
|
|
|
def test_catalog_only_never_creates_ready_binding(self) -> None:
|
|
environment = self._live_environment()
|
|
del environment["IOP_BENCH_CONFIG_OBSERVATION_ENV"]
|
|
registry = live_iop.build_live_adapter_registry(
|
|
environment,
|
|
observer=lambda _runtime: live_iop._Observation(
|
|
tuple(sorted(cell.iop.request_model for cell in self.manifest.matrix)),
|
|
"sha256:" + "d" * 64,
|
|
True,
|
|
),
|
|
)
|
|
cell = next(cell for cell in self.manifest.matrix if cell.caller == "claude")
|
|
result = registry["claude"].preflight(cell).result
|
|
self.assertEqual(result.status, "registration_required")
|
|
self.assertEqual([item.code for item in result.issues], ["route_missing"])
|
|
self.assertIsNone(result.binding.effective_model)
|
|
|
|
def test_config_owner_binding_is_passed_without_manifest_synthesis(self) -> None:
|
|
environment = self._live_environment()
|
|
cell = next(cell for cell in self.manifest.matrix if cell.id == "claude-sonnet-direct")
|
|
registry = live_iop.build_live_adapter_registry(
|
|
environment,
|
|
observer=lambda _runtime: live_iop._Observation(
|
|
tuple(sorted(item.iop.request_model for item in self.manifest.matrix)),
|
|
"sha256:" + "e" * 64,
|
|
True,
|
|
),
|
|
)
|
|
result = registry["claude"].preflight(cell).result
|
|
self.assertEqual(result.status, "ready")
|
|
self.assertEqual(result.binding.effective_route_id, cell.iop.route_id)
|
|
self.assertEqual(result.binding.effective_model, cell.iop.request_model)
|
|
self.assertEqual(
|
|
result.binding.effective_bindings,
|
|
(EffectiveBinding("request", cell.iop.request_model, cell.iop.requested_effort),),
|
|
)
|
|
|
|
def test_catalog_accepts_edge_routing_ids_and_rejects_malformed_records(self) -> None:
|
|
environment = self._live_environment()
|
|
runtime = live_iop._runtime_from_environment("claude", environment).runtime
|
|
self.assertIsNotNone(runtime)
|
|
|
|
class Response:
|
|
status = 200
|
|
|
|
def __init__(self, records) -> None:
|
|
self.body = json.dumps({"object": "list", "data": records}).encode()
|
|
|
|
def __enter__(self): return self
|
|
def __exit__(self, *_args): return False
|
|
def read(self): return self.body
|
|
|
|
model_ids = (
|
|
"claude-sonnet-5",
|
|
"gemini-3.6-flash",
|
|
"gpt-5.6-luna",
|
|
"qwen3.6:35b",
|
|
"ornith:35b",
|
|
"laguna-s:2.1",
|
|
)
|
|
with mock.patch.object(
|
|
live_iop,
|
|
"urlopen",
|
|
return_value=Response([{"id": model_id} for model_id in model_ids]),
|
|
):
|
|
models, _identity = live_iop._catalog(runtime) # type: ignore[arg-type]
|
|
self.assertEqual(models, tuple(sorted(model_ids)))
|
|
|
|
malformed = {
|
|
"numeric": [{"id": 1}],
|
|
"empty": [{"id": ""}],
|
|
"whitespace": [{"id": " \t"}],
|
|
"duplicate": [{"id": "claude-sonnet-5"}, {"id": "claude-sonnet-5"}],
|
|
}
|
|
for name, records in malformed.items():
|
|
with self.subTest(name=name):
|
|
with mock.patch.object(live_iop, "urlopen", return_value=Response(records)):
|
|
with self.assertRaises(live_iop.LiveIopError) as raised:
|
|
live_iop._catalog(runtime) # type: ignore[arg-type]
|
|
self.assertEqual(raised.exception.issue_code, "protocol_incompatible")
|
|
|
|
def test_live_failure_taxonomy_is_exact(self) -> None:
|
|
"""Every live setup/catalog boundary returns one closed issue/resume pair."""
|
|
environment = self._live_environment(token="secret-must-not-appear")
|
|
runtime_cases = (
|
|
("missing-base", {"IOP_BENCH_CLAUDE_BASE_URL": ""}, "endpoint_incompatible"),
|
|
("invalid-base", {"IOP_BENCH_CLAUDE_BASE_URL": "not-a-url"}, "endpoint_incompatible"),
|
|
("invalid-secret-ref", {"IOP_BENCH_CLAUDE_SECRET_ENV": "1BAD"}, "credential_missing"),
|
|
("missing-secret", {"BENCH_TOKEN": ""}, "credential_missing"),
|
|
)
|
|
for _name, updates, expected in runtime_cases:
|
|
candidate = {**environment, **updates}
|
|
resolution = live_iop._runtime_from_environment("claude", candidate)
|
|
self.assertIsNone(resolution.runtime)
|
|
self.assertEqual(resolution.issue_code, expected)
|
|
|
|
runtime = live_iop._runtime_from_environment("claude", environment).runtime
|
|
self.assertIsNotNone(runtime)
|
|
|
|
class Response:
|
|
def __init__(self, status, body): self.status, self.body = status, body
|
|
def __enter__(self): return self
|
|
def __exit__(self, *_args): return False
|
|
def read(self): return self.body
|
|
|
|
catalog_cases = (
|
|
(HTTPError("http://invalid", 401, "", None, None), "auth_incompatible"),
|
|
(HTTPError("http://invalid", 403, "", None, None), "auth_incompatible"),
|
|
(OSError("unreachable"), "endpoint_incompatible"),
|
|
(Response(502, b"{}"), "endpoint_incompatible"),
|
|
(Response(200, b"not-json"), "protocol_incompatible"),
|
|
(Response(200, b'{"data":[{"id":1}]}'), "protocol_incompatible"),
|
|
)
|
|
for outcome, expected in catalog_cases:
|
|
patch_kwargs = {"side_effect": outcome} if isinstance(outcome, BaseException) else {"return_value": outcome}
|
|
with mock.patch.object(live_iop, "urlopen", **patch_kwargs):
|
|
with self.assertRaises(live_iop.LiveIopError) as raised:
|
|
live_iop._catalog(runtime) # type: ignore[arg-type]
|
|
self.assertEqual(raised.exception.issue_code, expected)
|
|
|
|
claude = next(cell for cell in self.manifest.matrix if cell.caller == "claude")
|
|
routes = json.loads(environment["BENCH_CONFIG"])["routes"]
|
|
no_route = {**environment, "BENCH_CONFIG": json.dumps({"schema_version": "1", "routes": [item for item in routes if item["route_id"] != claude.iop.route_id]})}
|
|
no_model_routes = [
|
|
{**item, "model": "other-model"} if item["route_id"] == claude.iop.route_id else dict(item)
|
|
for item in routes
|
|
]
|
|
no_model = {**environment, "BENCH_CONFIG": json.dumps({"schema_version": "1", "routes": no_model_routes})}
|
|
unsupported = replace(next(cell for cell in self.manifest.matrix if cell.caller == "agy"), iop=replace(next(cell for cell in self.manifest.matrix if cell.caller == "agy").iop, requested_effort="max"))
|
|
observed = lambda _runtime: live_iop._Observation(tuple(sorted(cell.iop.request_model for cell in self.manifest.matrix)), "sha256:" + "1" * 64, True, "agy 1.1.11", "--print --output-format --sandbox --model --effort AGY_PROVIDER AGY_OPENAI_BASE_URL AGY_OPENAI_API_KEY stream-json")
|
|
checks = (
|
|
(live_iop.build_live_adapter_registry(no_route, observer=observed)["claude"].preflight(claude).result, "route_missing"),
|
|
(live_iop.build_live_adapter_registry(no_model, observer=observed)["claude"].preflight(claude).result, "model_missing"),
|
|
(live_iop.build_live_adapter_registry(environment, observer=lambda _runtime: (_ for _ in ()).throw(live_iop.LiveIopError("stream_incompatible")))["claude"].preflight(claude).result, "stream_incompatible"),
|
|
)
|
|
for result, issue_code in checks:
|
|
self.assertEqual(
|
|
(result.status, tuple((item.code, item.resume_code) for item in result.issues)),
|
|
("registration_required" if issue_code in ISSUE_RESUME_CODES and issue_code in {"route_missing", "model_missing", "effort_unsupported"} else "implementation_gap", ((issue_code, ISSUE_RESUME_CODES[issue_code]),)),
|
|
)
|
|
_, unsupported_issues = live_iop._binding_from_config(
|
|
unsupported,
|
|
CallerCapability("agy", ("direct", "execution_preset"), ("high", "low", "medium")),
|
|
live_iop._runtime_from_environment("agy", environment).runtime.config, # type: ignore[union-attr]
|
|
)
|
|
self.assertEqual(
|
|
tuple((item.code, item.resume_code) for item in unsupported_issues),
|
|
(("effort_unsupported", ISSUE_RESUME_CODES["effort_unsupported"]),),
|
|
)
|
|
|
|
def test_live_invocation_rejects_missing_or_mismatched_caller_binding(self) -> None:
|
|
environment = self._live_environment()
|
|
observed = lambda _runtime: live_iop._Observation(
|
|
tuple(sorted(cell.iop.request_model for cell in self.manifest.matrix)),
|
|
"sha256:" + "2" * 64, True, "agy 1.1.11",
|
|
"--print --output-format --sandbox --model --effort AGY_PROVIDER AGY_OPENAI_BASE_URL AGY_OPENAI_API_KEY stream-json",
|
|
)
|
|
registry = live_iop.build_live_adapter_registry(
|
|
environment, observer=observed, binary_resolver=lambda _name: "/bin/true"
|
|
)
|
|
agy = next(cell for cell in self.manifest.matrix if cell.caller == "agy")
|
|
codex = next(cell for cell in self.manifest.matrix if cell.caller == "codex")
|
|
self.assertEqual(registry["agy"].preflight(agy).result.status, "ready")
|
|
self.assertEqual(registry["codex"].preflight(codex).result.status, "ready")
|
|
|
|
agy_adapter = registry["agy"]
|
|
codex_adapter = registry["codex"]
|
|
agy_adapter._invokers = live_iop._InvokerSeams( # type: ignore[attr-defined]
|
|
live_iop._DEFAULT_INVOKERS.claude,
|
|
lambda *_args: None,
|
|
live_iop._DEFAULT_INVOKERS.codex,
|
|
)
|
|
codex_adapter._invokers = live_iop._InvokerSeams( # type: ignore[attr-defined]
|
|
live_iop._DEFAULT_INVOKERS.claude,
|
|
live_iop._DEFAULT_INVOKERS.agy,
|
|
lambda *_args: type("Mismatch", (), {"effective_binding": None, "lifecycle": None})(),
|
|
)
|
|
with mock.patch.object(live_iop, "build_agy_invocation", return_value=object()):
|
|
with self.assertRaises(live_iop.LiveIopError) as raised:
|
|
agy_adapter.invoke(
|
|
agy, object(), object(), "/tmp/control", b"task",
|
|
self.manifest.timeout, lambda *_args: None,
|
|
)
|
|
self.assertEqual(raised.exception.issue_code, "stream_incompatible")
|
|
with mock.patch.object(live_iop, "build_codex_invocation", return_value=type("Invocation", (), {"spec": object()})()):
|
|
with self.assertRaises(live_iop.LiveIopError) as raised:
|
|
codex_adapter.invoke(
|
|
codex, object(), object(), "/tmp/control", b"task",
|
|
self.manifest.timeout, lambda *_args: None,
|
|
)
|
|
self.assertEqual(raised.exception.issue_code, "stream_incompatible")
|
|
|
|
def test_cli_missing_live_input_fails_closed_without_secret_or_attempt(self) -> None:
|
|
sentinel = "missing-input-token"
|
|
stdout = io.StringIO()
|
|
stderr = io.StringIO()
|
|
with (
|
|
mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root),
|
|
contextlib.redirect_stdout(stdout),
|
|
contextlib.redirect_stderr(stderr),
|
|
):
|
|
exit_code = benchmark_cli.main(["run", "--manifest", str(self.path)])
|
|
self.assertEqual(exit_code, 69)
|
|
self.assertEqual(stdout.getvalue(), "")
|
|
self.assertIn("error: preflight blocked", stderr.getvalue())
|
|
self.assertNotIn(sentinel, stderr.getvalue())
|
|
run_roots = list((self.root / self.manifest.output_root).glob("run-*"))
|
|
self.assertEqual(len(run_roots), 1)
|
|
self.assertFalse((run_roots[0] / "cells").exists())
|
|
|
|
def test_concurrent_writer_fails_fast_without_partial_record(self) -> None:
|
|
registry = self._registry()
|
|
observations = collect_preflight_observations(self.manifest, registry)
|
|
run = self.store.create(self.manifest, self.raw)
|
|
result: list[BaseException] = []
|
|
|
|
def append() -> None:
|
|
try:
|
|
self.store.record_preflight(run, self.manifest, observations)
|
|
except BaseException as exc:
|
|
result.append(exc)
|
|
|
|
with self.store.writer(run):
|
|
worker = threading.Thread(target=append)
|
|
worker.start()
|
|
worker.join(5)
|
|
self.assertFalse(worker.is_alive())
|
|
self.assertEqual(len(result), 1)
|
|
self.assertIsInstance(result[0], RunBusyError)
|
|
self.assertEqual(self.store.preflights(run, self.manifest), ())
|
|
|
|
def test_cli_fake_registry_reports_only_closed_summary(self) -> None:
|
|
sentinel = "private_endpoint_and_token_must_not_appear"
|
|
registry = self._registry(
|
|
{"codex-gpt-direct": ("credential_missing",)}, sentinel=sentinel
|
|
)
|
|
stdout = io.StringIO()
|
|
stderr = io.StringIO()
|
|
with (
|
|
mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root),
|
|
mock.patch.object(benchmark_cli, "build_adapter_registry", return_value=registry),
|
|
contextlib.redirect_stdout(stdout),
|
|
contextlib.redirect_stderr(stderr),
|
|
):
|
|
exit_code = benchmark_cli.main(
|
|
["preflight", "--manifest", str(self.path)]
|
|
)
|
|
self.assertEqual(exit_code, 69)
|
|
self.assertEqual(stdout.getvalue(), "")
|
|
self.assertIn("status=registration_required", stderr.getvalue())
|
|
self.assertIn("registration_required=1", stderr.getvalue())
|
|
self.assertNotIn(sentinel, stderr.getvalue())
|
|
run_roots = list((self.root / self.manifest.output_root).glob("run-*"))
|
|
self.assertEqual(len(run_roots), 1)
|
|
durable = b"".join(
|
|
path.read_bytes() for path in run_roots[0].rglob("*") if path.is_file()
|
|
)
|
|
self.assertNotIn(sentinel.encode("ascii"), durable)
|
|
self.assertFalse((run_roots[0] / "cells").exists())
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|