252 lines
12 KiB
Python
252 lines
12 KiB
Python
"""Credential-free tests for the isolated Codex IOP benchmark adapter."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
from scripts.agent_benchmark.codex_iop import (
|
|
BASE_URL_ENV_KEY,
|
|
SECRET_ENV_KEY,
|
|
CodexJSONLError,
|
|
CodexJSONLParser,
|
|
CodexRuntimeError,
|
|
build_codex_invocation,
|
|
build_codex_spec,
|
|
redact_codex_jsonl,
|
|
run_codex_invocation,
|
|
runtime_from_environment,
|
|
)
|
|
from scripts.agent_benchmark.lifecycle import (
|
|
REASON_DUPLICATE_EVENT,
|
|
REASON_NONZERO_EXIT,
|
|
REASON_PARSER_ERROR,
|
|
)
|
|
from scripts.agent_benchmark.manifest import ExpectedBinding, IopCell, MatrixCell, Timeout
|
|
from scripts.agent_benchmark.workspace import AttemptIdentity, PreparedWorkspace, TestbedProvenance
|
|
|
|
|
|
_ENDPOINT = "https://iop.private.example/v1"
|
|
_SECRET = "iop_test_secret_123456789"
|
|
_PROMPT = b"private benchmark prompt must not persist"
|
|
|
|
|
|
def _cell(*, effort: str = "xhigh") -> MatrixCell:
|
|
return MatrixCell(
|
|
"codex-gpt", "codex",
|
|
IopCell("gpt-5.6-luna", effort, "direct", "iop-gpt", (
|
|
ExpectedBinding("request", "gpt-5.6-luna", effort),
|
|
)),
|
|
)
|
|
|
|
|
|
class CodexIOPTest(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.tmp = tempfile.TemporaryDirectory()
|
|
self.root = Path(self.tmp.name)
|
|
self.workspace = self.root / "workspace"
|
|
self.workspace.mkdir()
|
|
self.session = self.root / "session"
|
|
self.session.mkdir()
|
|
self.evidence = self.root / "attempt"
|
|
self.evidence.mkdir()
|
|
|
|
def tearDown(self) -> None:
|
|
self.tmp.cleanup()
|
|
|
|
def _runtime(self):
|
|
return runtime_from_environment({
|
|
BASE_URL_ENV_KEY: _ENDPOINT,
|
|
SECRET_ENV_KEY: _SECRET,
|
|
"PATH": os.environ.get("PATH", "/usr/bin:/bin"),
|
|
})
|
|
|
|
def _prepared(self) -> PreparedWorkspace:
|
|
return PreparedWorkspace(
|
|
identity=AttemptIdentity("run-0001", "codex-gpt", 1, 1),
|
|
attempt_root=str(self.evidence), workspace_dir=str(self.workspace),
|
|
session_dir=str(self.session), session_id="fresh-session", session_is_fresh=True,
|
|
workspace_checksum="sha256:" + "0" * 64, setup_cache_policy="isolated",
|
|
testbed_provenance=TestbedProvenance("../iop-s2", "main", "0" * 40, "0" * 64, True),
|
|
prepared_at="2026-08-10T00:00:00+00:00",
|
|
)
|
|
|
|
def _timeout(self) -> Timeout:
|
|
return Timeout(5, 2, 1, 1)
|
|
|
|
def _fake_codex(self, records: list[object], exit_code: int = 0) -> str:
|
|
path = self.root / f"fake-codex-{len(list(self.root.glob('fake-codex-*')))}.py"
|
|
lines = [
|
|
"#!/usr/bin/env python3", "import json, sys", "task = sys.stdin.read()",
|
|
]
|
|
for record in records:
|
|
if record == "TASK":
|
|
lines.append("print(json.dumps({'type': 'turn.completed', 'status': 'completed', 'content': task}), flush=True)")
|
|
elif isinstance(record, str):
|
|
lines.append(f"print({record!r}, flush=True)")
|
|
else:
|
|
lines.append(f"print(json.dumps({record!r}), flush=True)")
|
|
lines.append(f"raise SystemExit({exit_code})")
|
|
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
|
return str(path)
|
|
|
|
def test_exact_isolated_responses_spec_uses_one_stdin_submission(self) -> None:
|
|
spec = build_codex_spec(_cell(), self._prepared(), self._runtime(), _PROMPT, self._timeout())
|
|
argv = list(spec.argv)
|
|
codex = argv[argv.index("--") + 1:]
|
|
self.assertFalse((self.workspace / ".git").exists())
|
|
self.assertEqual(codex.count("--skip-git-repo-check"), 1)
|
|
self.assertEqual(codex[:12], [
|
|
"codex", "exec", "--json", "--ephemeral", "--ignore-user-config",
|
|
"--strict-config", "--skip-git-repo-check", "-C", str(self.workspace),
|
|
"-m", "gpt-5.6-luna", "-c",
|
|
])
|
|
self.assertEqual(codex[-1], "-")
|
|
overrides = [codex[index + 1] for index, item in enumerate(codex[:-1]) if item == "-c"]
|
|
self.assertEqual(overrides, [
|
|
'model_provider="iop_benchmark"',
|
|
'model_providers.iop_benchmark.name="IOP Benchmark"',
|
|
f'model_providers.iop_benchmark.base_url="{_ENDPOINT}"',
|
|
'model_providers.iop_benchmark.env_key="IOP_BENCHMARK_API_KEY"',
|
|
'model_providers.iop_benchmark.wire_api="responses"',
|
|
'model_reasoning_effort="xhigh"',
|
|
])
|
|
self.assertEqual(spec.submission_mode, "stdin_once")
|
|
self.assertEqual(spec.completion_mode, "exit_after_idle")
|
|
self.assertEqual(dict(spec.env)[SECRET_ENV_KEY], _SECRET)
|
|
self.assertNotIn(BASE_URL_ENV_KEY, dict(spec.env))
|
|
self.assertNotIn("OPENAI_API_KEY", dict(spec.env))
|
|
|
|
def test_runtime_and_effort_are_closed(self) -> None:
|
|
with self.assertRaises(CodexRuntimeError):
|
|
runtime_from_environment({BASE_URL_ENV_KEY: _ENDPOINT, SECRET_ENV_KEY: _SECRET})
|
|
with self.assertRaises(CodexRuntimeError):
|
|
runtime_from_environment({BASE_URL_ENV_KEY: _ENDPOINT, SECRET_ENV_KEY: _SECRET, "PATH": "/bin", "EXTRA": "x"})
|
|
with self.assertRaises(CodexRuntimeError):
|
|
build_codex_spec(_cell(effort="high"), self._prepared(), self._runtime(), _PROMPT, self._timeout())
|
|
|
|
def test_fixture_finish_then_verified_idle_and_structural_redaction(self) -> None:
|
|
parser = CodexJSONLParser(_cell(), "fixture-nonce-0001")
|
|
fixture = Path("scripts/fixtures/agent-comparison-benchmark/codex-iop-stream.jsonl")
|
|
events = [parser.parse("stdout", line) for line in fixture.read_text(encoding="utf-8").splitlines()]
|
|
self.assertEqual([events[0], events[-1]], [None, "idle"])
|
|
tool = events[1]
|
|
self.assertEqual((tool.name, tool.value, tool.call_id), ("tool_duration", 7_250_000, "call-1"))
|
|
# A tool interval is reported inside the turn, so it is published as an
|
|
# overlapping interval instead of a subtractable slice.
|
|
self.assertTrue(tool.overlap)
|
|
self.assertEqual(events[2][-1], "finish")
|
|
turn = {metric.name: metric.value for metric in events[2][:-1]}
|
|
self.assertEqual(turn, {
|
|
"cached_input_tokens": 8, "input_tokens": 31, "output_tokens": 12,
|
|
"reasoning_tokens": 4, "model_calls": 1, "tool_calls": 1,
|
|
})
|
|
# The fixture omits the provider total, so it is never reconstructed
|
|
# from the reported categories.
|
|
self.assertNotIn("total_tokens", turn)
|
|
redacted = redact_codex_jsonl(
|
|
json.dumps({"type": "turn.completed", "content": _PROMPT.decode(), "endpoint": _ENDPOINT, "token": _SECRET}),
|
|
(_ENDPOINT, _SECRET, _PROMPT.decode()),
|
|
)
|
|
self.assertNotIn(_PROMPT.decode(), redacted)
|
|
self.assertNotIn(_ENDPOINT, redacted)
|
|
self.assertNotIn(_SECRET, redacted)
|
|
self.assertIn("[redacted]", redacted)
|
|
|
|
def test_bridge_proves_finish_then_idle_after_child_exit(self) -> None:
|
|
fake = self._fake_codex(["TASK"])
|
|
invocation = build_codex_invocation(_cell(), self._prepared(), self._runtime(), _PROMPT, self._timeout(), codex_executable=(sys.executable, fake))
|
|
result = run_codex_invocation(invocation, lambda _: None)
|
|
self.assertTrue(result.lifecycle.success)
|
|
self.assertEqual([event.kind for event in result.lifecycle.events], [
|
|
"submitted", "first_output", "metric:model_calls", "metric:tool_calls",
|
|
"finish", "idle", "exited", "quiet",
|
|
])
|
|
capture = result.lifecycle.stdout.text
|
|
self.assertNotIn(_PROMPT.decode(), capture)
|
|
self.assertNotIn(_ENDPOINT, capture)
|
|
self.assertNotIn(_SECRET, capture)
|
|
self.assertIsNone(result.effective_binding)
|
|
|
|
def test_child_failure_never_synthesizes_idle(self) -> None:
|
|
fake = self._fake_codex([{"type": "turn.completed", "status": "completed"}], exit_code=7)
|
|
result = run_codex_invocation(build_codex_invocation(_cell(), self._prepared(), self._runtime(), _PROMPT, self._timeout(), codex_executable=(sys.executable, fake)), lambda _: None)
|
|
self.assertFalse(result.lifecycle.success)
|
|
self.assertEqual(result.lifecycle.terminal_reason, REASON_NONZERO_EXIT)
|
|
self.assertNotIn("idle", [event.kind for event in result.lifecycle.events])
|
|
|
|
def test_duplicate_malformed_and_unverified_idle_fail_closed(self) -> None:
|
|
cases = (
|
|
([{"type": "turn.completed", "status": "completed"}, {"type": "turn.completed", "status": "completed"}], REASON_DUPLICATE_EVENT),
|
|
(["not-json"], REASON_PARSER_ERROR),
|
|
([{"type": "adapter.idle", "adapter": "codex_iop", "nonce": "not-the-bridge-nonce", "child_exit": 0}], REASON_PARSER_ERROR),
|
|
)
|
|
for index, (records, reason) in enumerate(cases):
|
|
with self.subTest(reason=reason):
|
|
evidence = self.root / f"attempt-{reason}-{index}"
|
|
evidence.mkdir()
|
|
prepared = self._prepared().__class__(**{**self._prepared().__dict__, "attempt_root": str(evidence)})
|
|
fake = self._fake_codex(records)
|
|
result = run_codex_invocation(build_codex_invocation(_cell(), prepared, self._runtime(), _PROMPT, self._timeout(), codex_executable=(sys.executable, fake)), lambda _: None)
|
|
self.assertFalse(result.lifecycle.success)
|
|
self.assertEqual(result.lifecycle.terminal_reason, reason)
|
|
|
|
def test_tool_intervals_require_one_explicit_unique_pairing(self) -> None:
|
|
parser = CodexJSONLParser(_cell(), "0123456789abcdef")
|
|
# An item without a reported duration carries no interval to pair.
|
|
self.assertIsNone(parser.parse("stdout", json.dumps(
|
|
{"type": "item.completed", "item": {"type": "agent_message", "text": "x"}}
|
|
)))
|
|
paired = json.dumps({"type": "item.completed", "item": {"id": "call-1", "duration_ms": 3}})
|
|
self.assertEqual(parser.parse("stdout", paired).call_id, "call-1")
|
|
for name, item in {
|
|
"duplicate-call": {"id": "call-1", "duration_ms": 4},
|
|
"unpaired-duration": {"duration_ms": 4},
|
|
"unsafe-call-id": {"id": "call 1", "duration_ms": 4},
|
|
"string-duration": {"id": "call-2", "duration_ms": "4"},
|
|
"negative-duration": {"id": "call-3", "duration_ms": -4},
|
|
}.items():
|
|
with self.subTest(name=name):
|
|
with self.assertRaises(CodexJSONLError):
|
|
parser.parse("stdout", json.dumps({"type": "item.completed", "item": item}))
|
|
turn = parser.parse("stdout", json.dumps({"type": "turn.completed"}))
|
|
counts = {metric.name: metric.value for metric in turn[:-1]}
|
|
self.assertEqual(counts, {"model_calls": 1, "tool_calls": 1})
|
|
|
|
def test_unknown_or_fractional_turn_usage_fails_closed(self) -> None:
|
|
for usage in (
|
|
{"unknown_tokens": 1},
|
|
{"input_tokens": 1.5},
|
|
{"input_tokens": True},
|
|
{"input_tokens": -1},
|
|
):
|
|
with self.subTest(usage=usage):
|
|
parser = CodexJSONLParser(_cell(), "0123456789abcdef")
|
|
with self.assertRaises(CodexJSONLError):
|
|
parser.parse("stdout", json.dumps({
|
|
"type": "turn.completed", "status": "completed", "usage": usage,
|
|
}))
|
|
|
|
def test_effective_binding_is_optional_but_any_observation_is_exact(self) -> None:
|
|
parser = CodexJSONLParser(_cell(), "0123456789abcdef")
|
|
self.assertIsNone(parser.effective_binding)
|
|
self.assertEqual(parser.connectivity_result().status, "implementation_gap")
|
|
parser.parse("stdout", json.dumps({"type": "thread.started", "iop_effective_binding": {
|
|
"route_kind": "direct", "route_id": "iop-gpt", "model": "gpt-5.6-luna", "effort": "xhigh",
|
|
}}))
|
|
self.assertEqual(parser.effective_binding, ("direct", "iop-gpt", "gpt-5.6-luna", "xhigh"))
|
|
with self.assertRaises(CodexJSONLError):
|
|
parser.connectivity_result()
|
|
mismatch = CodexJSONLParser(_cell(), "0123456789abcdef")
|
|
with self.assertRaises(CodexJSONLError):
|
|
mismatch.parse("stdout", json.dumps({"type": "thread.started", "iop_effective_binding": {
|
|
"route_kind": "direct", "route_id": "iop-gpt", "model": "gpt-alias", "effort": "xhigh",
|
|
}}))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|