toki
de4d8f4ff8
세 Agent의 direct route를 동일한 fail-closed preflight와 격리 실행 경계에서 비교하고, 관측되지 않은 preset 셀이 실행되는 것을 막기 위해 연결 계약과 증거 수집 흐름을 고정한다.
199 lines
9.5 KiB
Python
199 lines
9.5 KiB
Python
"""Credential-free tests for the isolated Codex IOP benchmark adapter."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
from scripts.agent_benchmark.codex_iop import (
|
|
BASE_URL_ENV_KEY,
|
|
SECRET_ENV_KEY,
|
|
CodexJSONLError,
|
|
CodexJSONLParser,
|
|
CodexRuntimeError,
|
|
build_codex_invocation,
|
|
build_codex_spec,
|
|
redact_codex_jsonl,
|
|
run_codex_invocation,
|
|
runtime_from_environment,
|
|
)
|
|
from scripts.agent_benchmark.lifecycle import (
|
|
REASON_DUPLICATE_EVENT,
|
|
REASON_NONZERO_EXIT,
|
|
REASON_PARSER_ERROR,
|
|
)
|
|
from scripts.agent_benchmark.manifest import ExpectedBinding, IopCell, MatrixCell, Timeout
|
|
from scripts.agent_benchmark.workspace import AttemptIdentity, PreparedWorkspace, TestbedProvenance
|
|
|
|
|
|
_ENDPOINT = "https://iop.private.example/v1"
|
|
_SECRET = "iop_test_secret_123456789"
|
|
_PROMPT = b"private benchmark prompt must not persist"
|
|
|
|
|
|
def _cell(*, effort: str = "xhigh") -> MatrixCell:
|
|
return MatrixCell(
|
|
"codex-gpt", "codex",
|
|
IopCell("gpt-5.6-luna", effort, "direct", "iop-gpt", (
|
|
ExpectedBinding("request", "gpt-5.6-luna", effort),
|
|
)),
|
|
)
|
|
|
|
|
|
class CodexIOPTest(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.tmp = tempfile.TemporaryDirectory()
|
|
self.root = Path(self.tmp.name)
|
|
self.workspace = self.root / "workspace"
|
|
self.workspace.mkdir()
|
|
self.session = self.root / "session"
|
|
self.session.mkdir()
|
|
self.evidence = self.root / "attempt"
|
|
self.evidence.mkdir()
|
|
|
|
def tearDown(self) -> None:
|
|
self.tmp.cleanup()
|
|
|
|
def _runtime(self):
|
|
return runtime_from_environment({
|
|
BASE_URL_ENV_KEY: _ENDPOINT,
|
|
SECRET_ENV_KEY: _SECRET,
|
|
"PATH": os.environ.get("PATH", "/usr/bin:/bin"),
|
|
})
|
|
|
|
def _prepared(self) -> PreparedWorkspace:
|
|
return PreparedWorkspace(
|
|
identity=AttemptIdentity("run-0001", "codex-gpt", 1, 1),
|
|
attempt_root=str(self.evidence), workspace_dir=str(self.workspace),
|
|
session_dir=str(self.session), session_id="fresh-session", session_is_fresh=True,
|
|
workspace_checksum="sha256:" + "0" * 64, setup_cache_policy="isolated",
|
|
testbed_provenance=TestbedProvenance("../iop-s2", "main", "0" * 40, "0" * 64, True),
|
|
prepared_at="2026-08-10T00:00:00+00:00",
|
|
)
|
|
|
|
def _timeout(self) -> Timeout:
|
|
return Timeout(5, 2, 1, 1)
|
|
|
|
def _fake_codex(self, records: list[object], exit_code: int = 0) -> str:
|
|
path = self.root / f"fake-codex-{len(list(self.root.glob('fake-codex-*')))}.py"
|
|
lines = [
|
|
"#!/usr/bin/env python3", "import json, sys", "task = sys.stdin.read()",
|
|
]
|
|
for record in records:
|
|
if record == "TASK":
|
|
lines.append("print(json.dumps({'type': 'turn.completed', 'status': 'completed', 'content': task}), flush=True)")
|
|
elif isinstance(record, str):
|
|
lines.append(f"print({record!r}, flush=True)")
|
|
else:
|
|
lines.append(f"print(json.dumps({record!r}), flush=True)")
|
|
lines.append(f"raise SystemExit({exit_code})")
|
|
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
|
return str(path)
|
|
|
|
def test_exact_isolated_responses_spec_uses_one_stdin_submission(self) -> None:
|
|
spec = build_codex_spec(_cell(), self._prepared(), self._runtime(), _PROMPT, self._timeout())
|
|
argv = list(spec.argv)
|
|
codex = argv[argv.index("--") + 1:]
|
|
self.assertFalse((self.workspace / ".git").exists())
|
|
self.assertEqual(codex.count("--skip-git-repo-check"), 1)
|
|
self.assertEqual(codex[:12], [
|
|
"codex", "exec", "--json", "--ephemeral", "--ignore-user-config",
|
|
"--strict-config", "--skip-git-repo-check", "-C", str(self.workspace),
|
|
"-m", "gpt-5.6-luna", "-c",
|
|
])
|
|
self.assertEqual(codex[-1], "-")
|
|
overrides = [codex[index + 1] for index, item in enumerate(codex[:-1]) if item == "-c"]
|
|
self.assertEqual(overrides, [
|
|
'model_provider="iop_benchmark"',
|
|
'model_providers.iop_benchmark.name="IOP Benchmark"',
|
|
f'model_providers.iop_benchmark.base_url="{_ENDPOINT}"',
|
|
'model_providers.iop_benchmark.env_key="IOP_BENCHMARK_API_KEY"',
|
|
'model_providers.iop_benchmark.wire_api="responses"',
|
|
'model_reasoning_effort="xhigh"',
|
|
])
|
|
self.assertEqual(spec.submission_mode, "stdin_once")
|
|
self.assertEqual(spec.completion_mode, "exit_after_idle")
|
|
self.assertEqual(dict(spec.env)[SECRET_ENV_KEY], _SECRET)
|
|
self.assertNotIn(BASE_URL_ENV_KEY, dict(spec.env))
|
|
self.assertNotIn("OPENAI_API_KEY", dict(spec.env))
|
|
|
|
def test_runtime_and_effort_are_closed(self) -> None:
|
|
with self.assertRaises(CodexRuntimeError):
|
|
runtime_from_environment({BASE_URL_ENV_KEY: _ENDPOINT, SECRET_ENV_KEY: _SECRET})
|
|
with self.assertRaises(CodexRuntimeError):
|
|
runtime_from_environment({BASE_URL_ENV_KEY: _ENDPOINT, SECRET_ENV_KEY: _SECRET, "PATH": "/bin", "EXTRA": "x"})
|
|
with self.assertRaises(CodexRuntimeError):
|
|
build_codex_spec(_cell(effort="high"), self._prepared(), self._runtime(), _PROMPT, self._timeout())
|
|
|
|
def test_fixture_finish_then_verified_idle_and_structural_redaction(self) -> None:
|
|
parser = CodexJSONLParser(_cell(), "fixture-nonce-0001")
|
|
fixture = Path("scripts/fixtures/agent-comparison-benchmark/codex-iop-stream.jsonl")
|
|
events = [parser.parse("stdout", line) for line in fixture.read_text(encoding="utf-8").splitlines()]
|
|
self.assertEqual(events, [None, "finish", "idle"])
|
|
redacted = redact_codex_jsonl(
|
|
json.dumps({"type": "turn.completed", "content": _PROMPT.decode(), "endpoint": _ENDPOINT, "token": _SECRET}),
|
|
(_ENDPOINT, _SECRET, _PROMPT.decode()),
|
|
)
|
|
self.assertNotIn(_PROMPT.decode(), redacted)
|
|
self.assertNotIn(_ENDPOINT, redacted)
|
|
self.assertNotIn(_SECRET, redacted)
|
|
self.assertIn("[redacted]", redacted)
|
|
|
|
def test_bridge_proves_finish_then_idle_after_child_exit(self) -> None:
|
|
fake = self._fake_codex(["TASK"])
|
|
invocation = build_codex_invocation(_cell(), self._prepared(), self._runtime(), _PROMPT, self._timeout(), codex_executable=(sys.executable, fake))
|
|
result = run_codex_invocation(invocation, lambda _: None)
|
|
self.assertTrue(result.lifecycle.success)
|
|
self.assertEqual([event.kind for event in result.lifecycle.events], ["submitted", "finish", "idle", "exited", "quiet"])
|
|
capture = result.lifecycle.stdout.text
|
|
self.assertNotIn(_PROMPT.decode(), capture)
|
|
self.assertNotIn(_ENDPOINT, capture)
|
|
self.assertNotIn(_SECRET, capture)
|
|
self.assertIsNone(result.effective_binding)
|
|
|
|
def test_child_failure_never_synthesizes_idle(self) -> None:
|
|
fake = self._fake_codex([{"type": "turn.completed", "status": "completed"}], exit_code=7)
|
|
result = run_codex_invocation(build_codex_invocation(_cell(), self._prepared(), self._runtime(), _PROMPT, self._timeout(), codex_executable=(sys.executable, fake)), lambda _: None)
|
|
self.assertFalse(result.lifecycle.success)
|
|
self.assertEqual(result.lifecycle.terminal_reason, REASON_NONZERO_EXIT)
|
|
self.assertNotIn("idle", [event.kind for event in result.lifecycle.events])
|
|
|
|
def test_duplicate_malformed_and_unverified_idle_fail_closed(self) -> None:
|
|
cases = (
|
|
([{"type": "turn.completed", "status": "completed"}, {"type": "turn.completed", "status": "completed"}], REASON_DUPLICATE_EVENT),
|
|
(["not-json"], REASON_PARSER_ERROR),
|
|
([{"type": "adapter.idle", "adapter": "codex_iop", "nonce": "not-the-bridge-nonce", "child_exit": 0}], REASON_PARSER_ERROR),
|
|
)
|
|
for index, (records, reason) in enumerate(cases):
|
|
with self.subTest(reason=reason):
|
|
evidence = self.root / f"attempt-{reason}-{index}"
|
|
evidence.mkdir()
|
|
prepared = self._prepared().__class__(**{**self._prepared().__dict__, "attempt_root": str(evidence)})
|
|
fake = self._fake_codex(records)
|
|
result = run_codex_invocation(build_codex_invocation(_cell(), prepared, self._runtime(), _PROMPT, self._timeout(), codex_executable=(sys.executable, fake)), lambda _: None)
|
|
self.assertFalse(result.lifecycle.success)
|
|
self.assertEqual(result.lifecycle.terminal_reason, reason)
|
|
|
|
def test_effective_binding_is_optional_but_any_observation_is_exact(self) -> None:
|
|
parser = CodexJSONLParser(_cell(), "0123456789abcdef")
|
|
self.assertIsNone(parser.effective_binding)
|
|
self.assertEqual(parser.connectivity_result().status, "implementation_gap")
|
|
parser.parse("stdout", json.dumps({"type": "thread.started", "iop_effective_binding": {
|
|
"route_kind": "direct", "route_id": "iop-gpt", "model": "gpt-5.6-luna", "effort": "xhigh",
|
|
}}))
|
|
self.assertEqual(parser.effective_binding, ("direct", "iop-gpt", "gpt-5.6-luna", "xhigh"))
|
|
with self.assertRaises(CodexJSONLError):
|
|
parser.connectivity_result()
|
|
mismatch = CodexJSONLParser(_cell(), "0123456789abcdef")
|
|
with self.assertRaises(CodexJSONLError):
|
|
mismatch.parse("stdout", json.dumps({"type": "thread.started", "iop_effective_binding": {
|
|
"route_kind": "direct", "route_id": "iop-gpt", "model": "gpt-alias", "effort": "xhigh",
|
|
}}))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|