toki
de4d8f4ff8
세 Agent의 direct route를 동일한 fail-closed preflight와 격리 실행 경계에서 비교하고, 관측되지 않은 preset 셀이 실행되는 것을 막기 위해 연결 계약과 증거 수집 흐름을 고정한다.
677 lines
25 KiB
Python
677 lines
25 KiB
Python
"""Network-free integration tests for public benchmark preflight."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import contextlib
|
|
import datetime
|
|
import io
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
import unittest
|
|
from pathlib import Path
|
|
from unittest import mock
|
|
|
|
from scripts import agent_comparison_benchmark as benchmark_cli
|
|
from scripts.agent_benchmark.attempts import (
|
|
CapabilityUnavailable,
|
|
PreflightObservation,
|
|
RunBusyError,
|
|
RunStore,
|
|
collect_preflight_observations,
|
|
preflight_manifest,
|
|
)
|
|
from scripts.agent_benchmark.connectivity import (
|
|
ISSUE_RESUME_CODES,
|
|
CallerCapability,
|
|
ConnectivityIssue,
|
|
EffectiveBinding,
|
|
RequestedEffectiveBinding,
|
|
make_result,
|
|
)
|
|
from scripts.agent_benchmark.manifest import (
|
|
AssetMapping,
|
|
MatrixCell,
|
|
digest_workspace_inputs,
|
|
load_manifest,
|
|
)
|
|
from scripts.agent_benchmark.lifecycle import (
|
|
COMPLETION_EXIT_AFTER_IDLE,
|
|
SUBMISSION_STDIN_ONCE,
|
|
InvocationSpec,
|
|
env_pairs,
|
|
run_invocation,
|
|
spec_digest,
|
|
)
|
|
|
|
|
|
def _cell(cell_id: str, caller: str, model: str, effort: str) -> dict:
|
|
return {
|
|
"id": cell_id,
|
|
"caller": caller,
|
|
"iop": {
|
|
"request_model": model,
|
|
"requested_effort": effort,
|
|
"route_kind": "direct",
|
|
"route_id": cell_id,
|
|
"expected_bindings": [
|
|
{"stage": "request", "model": model, "effort": effort}
|
|
],
|
|
},
|
|
}
|
|
|
|
|
|
def _preset(cell_id: str, caller: str, model: str, effort: str) -> dict:
|
|
return {
|
|
"id": cell_id,
|
|
"caller": caller,
|
|
"iop": {
|
|
"request_model": model,
|
|
"requested_effort": effort,
|
|
"route_kind": "execution_preset",
|
|
"route_id": cell_id,
|
|
"expected_bindings": [
|
|
{"stage": stage, "model": model}
|
|
for stage in ("selector", "plan", "work", "review")
|
|
],
|
|
},
|
|
}
|
|
|
|
|
|
def _write_manifest(root: Path, matrix: list[dict], output_id: str = "integration"):
|
|
fixture_root = root / "scripts/fixtures"
|
|
fixture_root.mkdir(parents=True, exist_ok=True)
|
|
(fixture_root / "prompt.md").write_text("public prompt fixture", encoding="utf-8")
|
|
(fixture_root / "reference.txt").write_text("public reference", encoding="utf-8")
|
|
assets = (
|
|
AssetMapping(
|
|
"scripts/fixtures/reference.txt",
|
|
"workspace/reference.txt",
|
|
b"public reference",
|
|
),
|
|
)
|
|
payload = {
|
|
"pipeline_version": "1",
|
|
"environment": "dev",
|
|
"testbed": "../iop-s2",
|
|
"repetitions": 1,
|
|
"session_policy": "fresh",
|
|
"setup_cache_policy": "isolated",
|
|
"timeout": {
|
|
"run_seconds": 5,
|
|
"idle_seconds": 1,
|
|
"quiet_seconds": 1,
|
|
"cleanup_grace_seconds": 1,
|
|
},
|
|
"viewports": [{"id": "desktop", "width": 1, "height": 1}],
|
|
"rubric_version": "v1",
|
|
"output_root": f"agent-test/runs/{output_id}",
|
|
"fixture": {
|
|
"version": "v1",
|
|
"prompt": "scripts/fixtures/prompt.md",
|
|
"assets": [
|
|
{
|
|
"source": "scripts/fixtures/reference.txt",
|
|
"workspace_path": "workspace/reference.txt",
|
|
}
|
|
],
|
|
"checksum": digest_workspace_inputs(assets),
|
|
},
|
|
"matrix": matrix,
|
|
}
|
|
path = root / "manifest.json"
|
|
raw = json.dumps(payload, sort_keys=True).encode("utf-8")
|
|
path.write_bytes(raw)
|
|
return load_manifest(path, repo_root=root), raw, path
|
|
|
|
|
|
class FakeAdapter:
|
|
def __init__(
|
|
self,
|
|
caller: str,
|
|
efforts: tuple[str, ...],
|
|
issues_by_cell: dict[str, tuple[str, ...]] | None = None,
|
|
*,
|
|
sentinel: str = "",
|
|
) -> None:
|
|
self.capability = CallerCapability(
|
|
caller, ("direct", "execution_preset"), efforts
|
|
)
|
|
self.issues_by_cell = issues_by_cell or {}
|
|
self.sentinel = sentinel
|
|
self.calls: list[str] = []
|
|
self.invocations: list[tuple[str, str, str, bytes]] = []
|
|
self.fail_invocation = False
|
|
self._control_aliases: list[Path] = []
|
|
|
|
def preflight(self, cell: MatrixCell) -> PreflightObservation:
|
|
self.calls.append(cell.id)
|
|
issue_codes = self.issues_by_cell.get(cell.id, ())
|
|
if issue_codes:
|
|
binding = RequestedEffectiveBinding(
|
|
cell.id,
|
|
cell.caller,
|
|
cell.iop.route_kind,
|
|
cell.iop.route_id,
|
|
cell.iop.request_model,
|
|
cell.iop.requested_effort,
|
|
)
|
|
else:
|
|
binding = RequestedEffectiveBinding(
|
|
cell.id,
|
|
cell.caller,
|
|
cell.iop.route_kind,
|
|
cell.iop.route_id,
|
|
cell.iop.request_model,
|
|
cell.iop.requested_effort,
|
|
cell.iop.route_kind,
|
|
cell.iop.route_id,
|
|
cell.iop.request_model,
|
|
cell.iop.requested_effort,
|
|
tuple(
|
|
EffectiveBinding(item.stage, item.model, item.effort)
|
|
for item in cell.iop.expected_bindings
|
|
),
|
|
)
|
|
issues = tuple(
|
|
ConnectivityIssue(code, ISSUE_RESUME_CODES[code])
|
|
for code in issue_codes
|
|
)
|
|
return PreflightObservation(
|
|
make_result(cell, self.capability, binding, issues),
|
|
"sha256:" + "a" * 64,
|
|
"sha256:" + "b" * 64,
|
|
)
|
|
|
|
def invoke(
|
|
self,
|
|
cell,
|
|
prepared,
|
|
attempt,
|
|
task_payload,
|
|
timeout,
|
|
on_started,
|
|
):
|
|
run_root = Path(attempt.root).parents[3]
|
|
if not (run_root / "preflight/preflight-000001.json").is_file():
|
|
raise AssertionError("attempt allocated before preflight publication")
|
|
if cell.id != attempt.identity.cell_id or prepared.identity != attempt.identity:
|
|
raise AssertionError("execution identity drift")
|
|
self.invocations.append(
|
|
(cell.id, prepared.workspace_dir, prepared.session_id, task_payload)
|
|
)
|
|
|
|
alias = Path(tempfile.mkdtemp(dir="/tmp", prefix="bi"))
|
|
alias.rmdir()
|
|
alias.symlink_to(Path(attempt.root), target_is_directory=True)
|
|
self._control_aliases.append(alias)
|
|
source = (
|
|
"import sys; sys.stdin.buffer.read(); print('FAILED'); sys.exit(3)"
|
|
if self.fail_invocation
|
|
else "import sys; sys.stdin.buffer.read(); print('FINISH'); print('IDLE')"
|
|
)
|
|
spec = InvocationSpec(
|
|
argv=(sys.executable, "-u", "-c", source),
|
|
cwd=prepared.workspace_dir,
|
|
env=env_pairs({"PATH": os.environ.get("PATH", "/usr/bin:/bin")}),
|
|
submission_mode=SUBMISSION_STDIN_ONCE,
|
|
completion_mode=COMPLETION_EXIT_AFTER_IDLE,
|
|
timeout=timeout,
|
|
evidence_dir=attempt.root,
|
|
task_payload=task_payload,
|
|
control_dir=str(alias / "control"),
|
|
)
|
|
return run_invocation(
|
|
spec,
|
|
parse_event=lambda _stream, line: {
|
|
"FINISH": "finish",
|
|
"IDLE": "idle",
|
|
}.get(line.strip()),
|
|
on_started=lambda locator: on_started(locator, spec_digest(spec)),
|
|
)
|
|
|
|
def cleanup(self) -> None:
|
|
for alias in self._control_aliases:
|
|
alias.unlink(missing_ok=True)
|
|
|
|
|
|
class ConnectivityIntegrationTest(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.temp = tempfile.TemporaryDirectory(dir="/tmp", prefix="benchmark-preflight-")
|
|
self.root = Path(self.temp.name) / "repo"
|
|
self.root.mkdir()
|
|
self.matrix = [
|
|
_cell("claude-sonnet-direct", "claude", "claude-sonnet-5", "max"),
|
|
_cell("claude-gemini-direct", "claude", "gemini-3.6-flash", "high"),
|
|
_cell("claude-gpt-direct", "claude", "gpt-5.6-luna", "xhigh"),
|
|
_cell("agy-gemini-direct", "agy", "gemini-3.6-flash", "high"),
|
|
_cell("codex-gpt-direct", "codex", "gpt-5.6-luna", "xhigh"),
|
|
]
|
|
self.manifest, self.raw, self.path = _write_manifest(self.root, self.matrix)
|
|
self.store = RunStore(
|
|
self.root,
|
|
clock=lambda: datetime.datetime(
|
|
2026, 8, 10, 1, 2, 3, tzinfo=datetime.timezone.utc
|
|
),
|
|
token_hex=lambda _: "123456abcdef",
|
|
)
|
|
|
|
def tearDown(self) -> None:
|
|
self.temp.cleanup()
|
|
|
|
def _init_testbed(self) -> None:
|
|
testbed = self.root.parent / "iop-s2"
|
|
testbed.mkdir()
|
|
(testbed / "README.md").write_text("testbed", encoding="utf-8")
|
|
for command in (
|
|
("git", "init"),
|
|
("git", "config", "user.name", "test"),
|
|
("git", "config", "user.email", "test@example.invalid"),
|
|
("git", "add", "."),
|
|
("git", "commit", "-m", "testbed"),
|
|
):
|
|
subprocess.run(command, cwd=testbed, check=True, capture_output=True)
|
|
|
|
@staticmethod
|
|
def _registry(issues: dict[str, tuple[str, ...]] | None = None, sentinel: str = ""):
|
|
issues = issues or {}
|
|
return {
|
|
"claude": FakeAdapter(
|
|
"claude", ("high", "max", "xhigh"), issues, sentinel=sentinel
|
|
),
|
|
"agy": FakeAdapter("agy", ("high",), issues, sentinel=sentinel),
|
|
"codex": FakeAdapter("codex", ("xhigh",), issues, sentinel=sentinel),
|
|
}
|
|
|
|
def test_all_three_callers_append_exact_ready_results_without_attempts(self) -> None:
|
|
registry = self._registry()
|
|
run, record = preflight_manifest(
|
|
self.store, self.manifest, self.raw, adapters=registry
|
|
)
|
|
self.assertEqual(record["status"], "ready")
|
|
self.assertEqual(
|
|
[result["cell"]["id"] for result in record["results"]],
|
|
[cell.id for cell in self.manifest.matrix],
|
|
)
|
|
self.assertEqual(len(self.store.preflights(run, self.manifest)), 1)
|
|
self.assertFalse((Path(run.root) / "cells").exists())
|
|
self.assertEqual(
|
|
{caller: adapter.calls for caller, adapter in registry.items()},
|
|
{
|
|
"claude": [
|
|
"claude-gemini-direct",
|
|
"claude-gpt-direct",
|
|
"claude-sonnet-direct",
|
|
],
|
|
"agy": ["agy-gemini-direct"],
|
|
"codex": ["codex-gpt-direct"],
|
|
},
|
|
)
|
|
|
|
def test_registration_and_implementation_blockers_are_distinct_and_no_attempt_allocates(self) -> None:
|
|
registry = self._registry(
|
|
{
|
|
"claude-sonnet-direct": ("credential_missing",),
|
|
"agy-gemini-direct": ("stream_incompatible",),
|
|
}
|
|
)
|
|
run, record = preflight_manifest(
|
|
self.store, self.manifest, self.raw, adapters=registry
|
|
)
|
|
statuses = [result["status"] for result in record["results"]]
|
|
self.assertEqual(statuses.count("registration_required"), 1)
|
|
self.assertEqual(statuses.count("implementation_gap"), 1)
|
|
self.assertEqual(record["status"], "implementation_gap")
|
|
self.assertFalse((Path(run.root) / "cells").exists())
|
|
self.assertEqual(self.store.status(run, self.manifest)["attempts"]["running"], 0)
|
|
|
|
def test_cli_run_ready_submits_each_cell_once_in_fresh_workspace(self) -> None:
|
|
self._init_testbed()
|
|
registry = self._registry()
|
|
for adapter in registry.values():
|
|
self.addCleanup(adapter.cleanup)
|
|
stdout = io.StringIO()
|
|
stderr = io.StringIO()
|
|
with (
|
|
mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root),
|
|
mock.patch.object(
|
|
benchmark_cli, "build_adapter_registry", return_value=registry
|
|
),
|
|
contextlib.redirect_stdout(stdout),
|
|
contextlib.redirect_stderr(stderr),
|
|
):
|
|
exit_code = benchmark_cli.main(
|
|
["run", "--manifest", str(self.path)]
|
|
)
|
|
|
|
self.assertEqual(exit_code, 0, stderr.getvalue())
|
|
self.assertIn("ok: run run_id=", stdout.getvalue())
|
|
self.assertEqual(stderr.getvalue(), "")
|
|
invocations = [
|
|
invocation
|
|
for adapter in registry.values()
|
|
for invocation in adapter.invocations
|
|
]
|
|
self.assertEqual(len(invocations), len(self.manifest.matrix))
|
|
self.assertEqual(
|
|
sorted(item[0] for item in invocations),
|
|
sorted(cell.id for cell in self.manifest.matrix),
|
|
)
|
|
self.assertEqual(
|
|
{item[3] for item in invocations},
|
|
{self.manifest.fixture.prompt_content},
|
|
)
|
|
self.assertEqual(len({item[1] for item in invocations}), len(invocations))
|
|
self.assertEqual(len({item[2] for item in invocations}), len(invocations))
|
|
|
|
run_roots = list((self.root / self.manifest.output_root).glob("run-*"))
|
|
self.assertEqual(len(run_roots), 1)
|
|
self.assertTrue((run_roots[0] / "preflight/preflight-000001.json").is_file())
|
|
self.assertEqual(
|
|
len(list(run_roots[0].glob("cells/*/repetition-*/attempt-*"))),
|
|
len(self.manifest.matrix),
|
|
)
|
|
|
|
def test_cli_run_blocker_persists_preflight_and_allocates_zero_attempts(self) -> None:
|
|
registry = self._registry(
|
|
{"claude-sonnet-direct": ("credential_missing",)}
|
|
)
|
|
stdout = io.StringIO()
|
|
stderr = io.StringIO()
|
|
with (
|
|
mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root),
|
|
mock.patch.object(
|
|
benchmark_cli, "build_adapter_registry", return_value=registry
|
|
),
|
|
contextlib.redirect_stdout(stdout),
|
|
contextlib.redirect_stderr(stderr),
|
|
):
|
|
exit_code = benchmark_cli.main(["run", "--manifest", str(self.path)])
|
|
|
|
self.assertEqual(exit_code, 69)
|
|
self.assertEqual(stdout.getvalue(), "")
|
|
self.assertIn("error: preflight blocked", stderr.getvalue())
|
|
run_roots = list((self.root / self.manifest.output_root).glob("run-*"))
|
|
self.assertEqual(len(run_roots), 1)
|
|
self.assertTrue((run_roots[0] / "preflight/preflight-000001.json").is_file())
|
|
self.assertFalse((run_roots[0] / "cells").exists())
|
|
self.assertTrue(all(adapter.invocations == [] for adapter in registry.values()))
|
|
|
|
def test_cli_mixed_manifest_never_invokes_unobserved_preset_cells(self) -> None:
|
|
manifest, _, path = _write_manifest(
|
|
self.root,
|
|
[
|
|
_cell("direct-ready", "claude", "claude-sonnet-5", "max"),
|
|
_preset("preset-unobserved", "claude", "claude-sonnet-5", "max"),
|
|
],
|
|
output_id="mixed-unobserved",
|
|
)
|
|
registry = self._registry()
|
|
stdout = io.StringIO()
|
|
stderr = io.StringIO()
|
|
with (
|
|
mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root),
|
|
mock.patch.object(
|
|
benchmark_cli, "build_adapter_registry", return_value=registry
|
|
),
|
|
contextlib.redirect_stdout(stdout),
|
|
contextlib.redirect_stderr(stderr),
|
|
):
|
|
exit_code = benchmark_cli.main(["run", "--manifest", str(path)])
|
|
|
|
self.assertEqual(exit_code, 69)
|
|
self.assertEqual(stdout.getvalue(), "")
|
|
self.assertIn("error: benchmark execution failed", stderr.getvalue())
|
|
self.assertIn("completed=0 unresolved=2", stderr.getvalue())
|
|
run_roots = list((self.root / manifest.output_root).glob("run-*"))
|
|
self.assertEqual(len(run_roots), 1)
|
|
preflight = json.loads(
|
|
(run_roots[0] / "preflight/preflight-000001.json").read_text(
|
|
encoding="ascii"
|
|
)
|
|
)
|
|
self.assertEqual(preflight["status"], "ready")
|
|
self.assertEqual(
|
|
[result["cell"]["id"] for result in preflight["results"]],
|
|
["direct-ready"],
|
|
)
|
|
self.assertFalse((run_roots[0] / "cells").exists())
|
|
self.assertEqual(registry["claude"].calls, ["direct-ready"])
|
|
self.assertTrue(all(adapter.invocations == [] for adapter in registry.values()))
|
|
|
|
def test_cli_resume_retries_append_only_and_status_is_read_only(self) -> None:
|
|
self._init_testbed()
|
|
manifest, _, path = _write_manifest(
|
|
self.root,
|
|
[_cell("claude-only", "claude", "claude-sonnet-5", "max")],
|
|
output_id="retry",
|
|
)
|
|
failed = FakeAdapter("claude", ("max",))
|
|
failed.fail_invocation = True
|
|
self.addCleanup(failed.cleanup)
|
|
first_stdout = io.StringIO()
|
|
first_stderr = io.StringIO()
|
|
with (
|
|
mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root),
|
|
mock.patch.object(
|
|
benchmark_cli,
|
|
"build_adapter_registry",
|
|
return_value={"claude": failed},
|
|
),
|
|
contextlib.redirect_stdout(first_stdout),
|
|
contextlib.redirect_stderr(first_stderr),
|
|
):
|
|
first_exit = benchmark_cli.main(
|
|
["run", "--manifest", str(path)]
|
|
)
|
|
self.assertEqual(first_exit, 69)
|
|
matched = re.search(r"run_id=(run-[0-9A-Za-z-]+)", first_stderr.getvalue())
|
|
self.assertIsNotNone(matched)
|
|
run_id = matched.group(1) # type: ignore[union-attr]
|
|
run_root = self.root / manifest.output_root / run_id
|
|
first_attempt = next(run_root.glob("cells/*/repetition-*/attempt-000001"))
|
|
old_bytes = {
|
|
item.relative_to(first_attempt): item.read_bytes()
|
|
for item in first_attempt.rglob("*")
|
|
if item.is_file()
|
|
}
|
|
|
|
ready = FakeAdapter("claude", ("max",))
|
|
self.addCleanup(ready.cleanup)
|
|
resume_stdout = io.StringIO()
|
|
resume_stderr = io.StringIO()
|
|
with (
|
|
mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root),
|
|
mock.patch.object(
|
|
benchmark_cli,
|
|
"build_adapter_registry",
|
|
return_value={"claude": ready},
|
|
),
|
|
contextlib.redirect_stdout(resume_stdout),
|
|
contextlib.redirect_stderr(resume_stderr),
|
|
):
|
|
resume_exit = benchmark_cli.main(
|
|
[
|
|
"resume",
|
|
"--manifest",
|
|
str(path),
|
|
"--run-id",
|
|
run_id,
|
|
"--retry-failed",
|
|
]
|
|
)
|
|
|
|
self.assertEqual(resume_exit, 0, resume_stderr.getvalue())
|
|
self.assertIn("ok: resume", resume_stdout.getvalue())
|
|
self.assertEqual(
|
|
old_bytes,
|
|
{
|
|
relative: (first_attempt / relative).read_bytes()
|
|
for relative in old_bytes
|
|
},
|
|
)
|
|
self.assertTrue(
|
|
next(run_root.glob("cells/*/repetition-*/attempt-000002/attempt.json"))
|
|
.read_text(encoding="utf-8")
|
|
.find('"state":"success"')
|
|
>= 0
|
|
)
|
|
self.assertEqual(len(list((run_root / "preflight").glob("*.json"))), 2)
|
|
|
|
before_status = {
|
|
item.relative_to(run_root): item.read_bytes()
|
|
for item in run_root.rglob("*")
|
|
if item.is_file()
|
|
}
|
|
status_stdout = io.StringIO()
|
|
status_stderr = io.StringIO()
|
|
with (
|
|
mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root),
|
|
contextlib.redirect_stdout(status_stdout),
|
|
contextlib.redirect_stderr(status_stderr),
|
|
):
|
|
status_exit = benchmark_cli.main(
|
|
["status", "--manifest", str(path), "--run-id", run_id]
|
|
)
|
|
self.assertEqual(status_exit, 0, status_stderr.getvalue())
|
|
self.assertIn("'success': 1", status_stdout.getvalue())
|
|
self.assertEqual(
|
|
before_status,
|
|
{
|
|
item.relative_to(run_root): item.read_bytes()
|
|
for item in run_root.rglob("*")
|
|
if item.is_file()
|
|
},
|
|
)
|
|
|
|
def test_missing_adapter_is_rejected_before_output_root_mutation(self) -> None:
|
|
registry = self._registry()
|
|
del registry["codex"]
|
|
output_root = self.root / self.manifest.output_root
|
|
with self.assertRaises(CapabilityUnavailable):
|
|
preflight_manifest(
|
|
self.store, self.manifest, self.raw, adapters=registry
|
|
)
|
|
self.assertFalse(output_root.exists())
|
|
|
|
def test_generic_preset_cells_are_local_contract_only(self) -> None:
|
|
generic, _, _ = _write_manifest(
|
|
self.root,
|
|
[
|
|
_preset("claude-generic", "claude", "claude-sonnet-5", "high"),
|
|
_preset("agy-generic", "agy", "gemini-3.6-flash", "high"),
|
|
_preset("codex-generic", "codex", "gpt-5.6-luna", "xhigh"),
|
|
],
|
|
output_id="generic",
|
|
)
|
|
registry = self._registry()
|
|
observations = collect_preflight_observations(generic, registry)
|
|
self.assertEqual(observations, {})
|
|
self.assertTrue(all(adapter.calls == [] for adapter in registry.values()))
|
|
|
|
def test_generic_preset_only_public_preflight_fails_closed_without_run_state(self) -> None:
|
|
generic, raw, path = _write_manifest(
|
|
self.root,
|
|
[
|
|
_preset("claude-generic", "claude", "claude-sonnet-5", "high"),
|
|
_preset("agy-generic", "agy", "gemini-3.6-flash", "high"),
|
|
_preset("codex-generic", "codex", "gpt-5.6-luna", "xhigh"),
|
|
],
|
|
output_id="generic",
|
|
)
|
|
registry = self._registry()
|
|
output_root = self.root / generic.output_root
|
|
with self.assertRaises(Exception) as ctx:
|
|
preflight_manifest(
|
|
self.store, generic, raw, adapters=registry
|
|
)
|
|
self.assertIn("preflight requires a direct cell", str(ctx.exception))
|
|
self.assertFalse(output_root.exists())
|
|
self.assertTrue(all(adapter.calls == [] for adapter in registry.values()))
|
|
|
|
sentinel = "private_endpoint_and_token_must_not_appear"
|
|
stdout = io.StringIO()
|
|
stderr = io.StringIO()
|
|
with (
|
|
mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root),
|
|
mock.patch.object(benchmark_cli, "build_adapter_registry", return_value=registry),
|
|
contextlib.redirect_stdout(stdout),
|
|
contextlib.redirect_stderr(stderr),
|
|
):
|
|
exit_code = benchmark_cli.main(["preflight", "--manifest", str(path)])
|
|
self.assertEqual(exit_code, 69)
|
|
self.assertEqual(stdout.getvalue(), "")
|
|
self.assertNotIn(sentinel, stderr.getvalue())
|
|
self.assertFalse(output_root.exists())
|
|
|
|
def test_public_registry_is_exact_and_network_free_fail_closed(self) -> None:
|
|
registry = benchmark_cli.build_adapter_registry()
|
|
self.assertEqual(tuple(registry), ("claude", "agy", "codex"))
|
|
|
|
observations = collect_preflight_observations(self.manifest, registry)
|
|
self.assertEqual(set(observations), {cell.id for cell in self.manifest.matrix})
|
|
for observation in observations.values():
|
|
self.assertEqual(observation.result.status, "implementation_gap")
|
|
self.assertEqual(
|
|
[issue.code for issue in observation.result.issues],
|
|
["stream_incompatible"],
|
|
)
|
|
self.assertIsNone(observation.result.binding.effective_route_kind)
|
|
|
|
def test_concurrent_writer_fails_fast_without_partial_record(self) -> None:
|
|
registry = self._registry()
|
|
observations = collect_preflight_observations(self.manifest, registry)
|
|
run = self.store.create(self.manifest, self.raw)
|
|
result: list[BaseException] = []
|
|
|
|
def append() -> None:
|
|
try:
|
|
self.store.record_preflight(run, self.manifest, observations)
|
|
except BaseException as exc:
|
|
result.append(exc)
|
|
|
|
with self.store.writer(run):
|
|
worker = threading.Thread(target=append)
|
|
worker.start()
|
|
worker.join(5)
|
|
self.assertFalse(worker.is_alive())
|
|
self.assertEqual(len(result), 1)
|
|
self.assertIsInstance(result[0], RunBusyError)
|
|
self.assertEqual(self.store.preflights(run, self.manifest), ())
|
|
|
|
def test_cli_fake_registry_reports_only_closed_summary(self) -> None:
|
|
sentinel = "private_endpoint_and_token_must_not_appear"
|
|
registry = self._registry(
|
|
{"codex-gpt-direct": ("credential_missing",)}, sentinel=sentinel
|
|
)
|
|
stdout = io.StringIO()
|
|
stderr = io.StringIO()
|
|
with (
|
|
mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root),
|
|
mock.patch.object(benchmark_cli, "build_adapter_registry", return_value=registry),
|
|
contextlib.redirect_stdout(stdout),
|
|
contextlib.redirect_stderr(stderr),
|
|
):
|
|
exit_code = benchmark_cli.main(
|
|
["preflight", "--manifest", str(self.path)]
|
|
)
|
|
self.assertEqual(exit_code, 69)
|
|
self.assertEqual(stdout.getvalue(), "")
|
|
self.assertIn("status=registration_required", stderr.getvalue())
|
|
self.assertIn("registration_required=1", stderr.getvalue())
|
|
self.assertNotIn(sentinel, stderr.getvalue())
|
|
run_roots = list((self.root / self.manifest.output_root).glob("run-*"))
|
|
self.assertEqual(len(run_roots), 1)
|
|
durable = b"".join(
|
|
path.read_bytes() for path in run_roots[0].rglob("*") if path.is_file()
|
|
)
|
|
self.assertNotIn(sentinel.encode("ascii"), durable)
|
|
self.assertFalse((run_roots[0] / "cells").exists())
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|