"""Credential-free tests for the isolated Codex IOP benchmark adapter.""" from __future__ import annotations import json import os import sys import tempfile import unittest from pathlib import Path from scripts.agent_benchmark.codex_iop import ( BASE_URL_ENV_KEY, SECRET_ENV_KEY, CodexJSONLError, CodexJSONLParser, CodexRuntimeError, build_codex_invocation, build_codex_spec, redact_codex_jsonl, run_codex_invocation, runtime_from_environment, ) from scripts.agent_benchmark.lifecycle import ( REASON_DUPLICATE_EVENT, REASON_NONZERO_EXIT, REASON_PARSER_ERROR, ) from scripts.agent_benchmark.manifest import ExpectedBinding, IopCell, MatrixCell, Timeout from scripts.agent_benchmark.workspace import AttemptIdentity, PreparedWorkspace, TestbedProvenance _ENDPOINT = "https://iop.private.example/v1" _SECRET = "iop_test_secret_123456789" _PROMPT = b"private benchmark prompt must not persist" def _cell(*, effort: str = "xhigh") -> MatrixCell: return MatrixCell( "codex-gpt", "codex", IopCell("gpt-5.6-luna", effort, "direct", "iop-gpt", ( ExpectedBinding("request", "gpt-5.6-luna", effort), )), ) class CodexIOPTest(unittest.TestCase): def setUp(self) -> None: self.tmp = tempfile.TemporaryDirectory() self.root = Path(self.tmp.name) self.workspace = self.root / "workspace" self.workspace.mkdir() self.session = self.root / "session" self.session.mkdir() self.evidence = self.root / "attempt" self.evidence.mkdir() def tearDown(self) -> None: self.tmp.cleanup() def _runtime(self): return runtime_from_environment({ BASE_URL_ENV_KEY: _ENDPOINT, SECRET_ENV_KEY: _SECRET, "PATH": os.environ.get("PATH", "/usr/bin:/bin"), }) def _prepared(self) -> PreparedWorkspace: return PreparedWorkspace( identity=AttemptIdentity("run-0001", "codex-gpt", 1, 1), attempt_root=str(self.evidence), workspace_dir=str(self.workspace), session_dir=str(self.session), session_id="fresh-session", session_is_fresh=True, workspace_checksum="sha256:" + "0" * 64, setup_cache_policy="isolated", testbed_provenance=TestbedProvenance("../iop-s2", "main", "0" * 40, "0" * 64, True), prepared_at="2026-08-10T00:00:00+00:00", ) def _timeout(self) -> Timeout: return Timeout(5, 2, 1, 1) def _fake_codex(self, records: list[object], exit_code: int = 0) -> str: path = self.root / f"fake-codex-{len(list(self.root.glob('fake-codex-*')))}.py" lines = [ "#!/usr/bin/env python3", "import json, sys", "task = sys.stdin.read()", ] for record in records: if record == "TASK": lines.append("print(json.dumps({'type': 'turn.completed', 'status': 'completed', 'content': task}), flush=True)") elif isinstance(record, str): lines.append(f"print({record!r}, flush=True)") else: lines.append(f"print(json.dumps({record!r}), flush=True)") lines.append(f"raise SystemExit({exit_code})") path.write_text("\n".join(lines) + "\n", encoding="utf-8") return str(path) def test_exact_isolated_responses_spec_uses_one_stdin_submission(self) -> None: spec = build_codex_spec(_cell(), self._prepared(), self._runtime(), _PROMPT, self._timeout()) argv = list(spec.argv) codex = argv[argv.index("--") + 1:] self.assertFalse((self.workspace / ".git").exists()) self.assertEqual(codex.count("--skip-git-repo-check"), 1) self.assertEqual(codex[:12], [ "codex", "exec", "--json", "--ephemeral", "--ignore-user-config", "--strict-config", "--skip-git-repo-check", "-C", str(self.workspace), "-m", "gpt-5.6-luna", "-c", ]) self.assertEqual(codex[-1], "-") overrides = [codex[index + 1] for index, item in enumerate(codex[:-1]) if item == "-c"] self.assertEqual(overrides, [ 'model_provider="iop_benchmark"', 'model_providers.iop_benchmark.name="IOP Benchmark"', f'model_providers.iop_benchmark.base_url="{_ENDPOINT}"', 'model_providers.iop_benchmark.env_key="IOP_BENCHMARK_API_KEY"', 'model_providers.iop_benchmark.wire_api="responses"', 'model_reasoning_effort="xhigh"', ]) self.assertEqual(spec.submission_mode, "stdin_once") self.assertEqual(spec.completion_mode, "exit_after_idle") self.assertEqual(dict(spec.env)[SECRET_ENV_KEY], _SECRET) self.assertNotIn(BASE_URL_ENV_KEY, dict(spec.env)) self.assertNotIn("OPENAI_API_KEY", dict(spec.env)) def test_runtime_and_effort_are_closed(self) -> None: with self.assertRaises(CodexRuntimeError): runtime_from_environment({BASE_URL_ENV_KEY: _ENDPOINT, SECRET_ENV_KEY: _SECRET}) with self.assertRaises(CodexRuntimeError): runtime_from_environment({BASE_URL_ENV_KEY: _ENDPOINT, SECRET_ENV_KEY: _SECRET, "PATH": "/bin", "EXTRA": "x"}) with self.assertRaises(CodexRuntimeError): build_codex_spec(_cell(effort="high"), self._prepared(), self._runtime(), _PROMPT, self._timeout()) def test_fixture_finish_then_verified_idle_and_structural_redaction(self) -> None: parser = CodexJSONLParser(_cell(), "fixture-nonce-0001") fixture = Path("scripts/fixtures/agent-comparison-benchmark/codex-iop-stream.jsonl") events = [parser.parse("stdout", line) for line in fixture.read_text(encoding="utf-8").splitlines()] self.assertEqual(events, [None, "finish", "idle"]) redacted = redact_codex_jsonl( json.dumps({"type": "turn.completed", "content": _PROMPT.decode(), "endpoint": _ENDPOINT, "token": _SECRET}), (_ENDPOINT, _SECRET, _PROMPT.decode()), ) self.assertNotIn(_PROMPT.decode(), redacted) self.assertNotIn(_ENDPOINT, redacted) self.assertNotIn(_SECRET, redacted) self.assertIn("[redacted]", redacted) def test_bridge_proves_finish_then_idle_after_child_exit(self) -> None: fake = self._fake_codex(["TASK"]) invocation = build_codex_invocation(_cell(), self._prepared(), self._runtime(), _PROMPT, self._timeout(), codex_executable=(sys.executable, fake)) result = run_codex_invocation(invocation, lambda _: None) self.assertTrue(result.lifecycle.success) self.assertEqual([event.kind for event in result.lifecycle.events], ["submitted", "finish", "idle", "exited", "quiet"]) capture = result.lifecycle.stdout.text self.assertNotIn(_PROMPT.decode(), capture) self.assertNotIn(_ENDPOINT, capture) self.assertNotIn(_SECRET, capture) self.assertIsNone(result.effective_binding) def test_child_failure_never_synthesizes_idle(self) -> None: fake = self._fake_codex([{"type": "turn.completed", "status": "completed"}], exit_code=7) result = run_codex_invocation(build_codex_invocation(_cell(), self._prepared(), self._runtime(), _PROMPT, self._timeout(), codex_executable=(sys.executable, fake)), lambda _: None) self.assertFalse(result.lifecycle.success) self.assertEqual(result.lifecycle.terminal_reason, REASON_NONZERO_EXIT) self.assertNotIn("idle", [event.kind for event in result.lifecycle.events]) def test_duplicate_malformed_and_unverified_idle_fail_closed(self) -> None: cases = ( ([{"type": "turn.completed", "status": "completed"}, {"type": "turn.completed", "status": "completed"}], REASON_DUPLICATE_EVENT), (["not-json"], REASON_PARSER_ERROR), ([{"type": "adapter.idle", "adapter": "codex_iop", "nonce": "not-the-bridge-nonce", "child_exit": 0}], REASON_PARSER_ERROR), ) for index, (records, reason) in enumerate(cases): with self.subTest(reason=reason): evidence = self.root / f"attempt-{reason}-{index}" evidence.mkdir() prepared = self._prepared().__class__(**{**self._prepared().__dict__, "attempt_root": str(evidence)}) fake = self._fake_codex(records) result = run_codex_invocation(build_codex_invocation(_cell(), prepared, self._runtime(), _PROMPT, self._timeout(), codex_executable=(sys.executable, fake)), lambda _: None) self.assertFalse(result.lifecycle.success) self.assertEqual(result.lifecycle.terminal_reason, reason) def test_effective_binding_is_optional_but_any_observation_is_exact(self) -> None: parser = CodexJSONLParser(_cell(), "0123456789abcdef") self.assertIsNone(parser.effective_binding) self.assertEqual(parser.connectivity_result().status, "implementation_gap") parser.parse("stdout", json.dumps({"type": "thread.started", "iop_effective_binding": { "route_kind": "direct", "route_id": "iop-gpt", "model": "gpt-5.6-luna", "effort": "xhigh", }})) self.assertEqual(parser.effective_binding, ("direct", "iop-gpt", "gpt-5.6-luna", "xhigh")) with self.assertRaises(CodexJSONLError): parser.connectivity_result() mismatch = CodexJSONLParser(_cell(), "0123456789abcdef") with self.assertRaises(CodexJSONLError): mismatch.parse("stdout", json.dumps({"type": "thread.started", "iop_effective_binding": { "route_kind": "direct", "route_id": "iop-gpt", "model": "gpt-alias", "effort": "xhigh", }})) if __name__ == "__main__": unittest.main()