"""Isolated, secret-safe Codex ``exec`` adapter for IOP benchmark cells. The adapter deliberately owns no public runner command. It converts a frozen manifest cell and a caller-supplied runtime into one bounded lifecycle invocation. The only executable it starts is a small bridge in the lifecycle owned process group; that bridge starts ``codex exec``, forwards its JSONL, and emits an idle marker only after the child has exited and both output streams reached EOF. """ from __future__ import annotations import json import os import re import secrets import subprocess import sys import threading from dataclasses import dataclass from pathlib import Path from typing import Any, Callable, Mapping from urllib.parse import urlsplit # The bridge is intentionally invoked by absolute script path from an isolated # workspace. Make the repository package importable without inheriting an # ambient PYTHONPATH. if __package__ in (None, ""): sys.path.insert(0, str(Path(__file__).resolve().parents[2])) from scripts.agent_benchmark.lifecycle import ( COMPLETION_EXIT_AFTER_IDLE, SUBMISSION_STDIN_ONCE, InvocationResult, InvocationSpec, LifecycleMetricError, LifecycleValidationError, ParsedMetric, SupervisorLocator, count_metric, duration_metric, env_pairs, exact_value_redactor, is_reported_number, run_invocation, ) from scripts.agent_benchmark.connectivity import ( ISSUE_RESUME_CODES, CallerCapability, ConnectivityIssue, ConnectivityResult, RequestedEffectiveBinding, make_result, ) from scripts.agent_benchmark.manifest import MatrixCell, Timeout, TOKEN_RE from scripts.agent_benchmark.workspace import PreparedWorkspace PROVIDER_ID = "iop_benchmark" SECRET_ENV_KEY = "IOP_BENCHMARK_API_KEY" BASE_URL_ENV_KEY = "IOP_BENCHMARK_BASE_URL" SUPPORTED_EFFORTS = ("xhigh",) _BRIDGE_IDLE_TYPE = "adapter.idle" _BRIDGE_ID = "codex_iop" _SENSITIVE_KEYS = frozenset({ "api_key", "authorization", "base_url", "command", "content", "endpoint", "input", "instructions", "message", "output", "prompt", "secret", "text", "token", "tool_input", "tool_output", "url", }) _SAFE_STRING_KEYS = frozenset({ "adapter", "effort", "model", "nonce", "reasoning_effort", "route_id", "route_kind", "stage", "status", "type", }) # The exact reported Codex usage keys this adapter observes. A tool interval is # consumed only when the completed item explicitly pairs one bounded duration # with one call identifier; nothing is derived from event arrival order. _CODEX_USAGE_FIELDS = { "input_tokens": "input_tokens", "cached_input_tokens": "cached_input_tokens", "output_tokens": "output_tokens", "reasoning_output_tokens": "reasoning_tokens", "total_tokens": "total_tokens", } _CODEX_ITEM_COMPLETED = "item.completed" _CODEX_CALL_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.:-]{0,63}$") class CodexIOPError(Exception): """Base class for closed Codex adapter validation failures.""" class CodexRuntimeError(CodexIOPError): """The caller-supplied runtime is absent, malformed, or unsafe.""" class CodexJSONLError(CodexIOPError): """A Codex JSONL event cannot safely satisfy the adapter protocol.""" @dataclass(frozen=True) class CodexRuntime: """The two runtime-only values needed to reach the IOP Responses surface.""" base_url: str api_key: str path: str @dataclass(frozen=True) class CodexInvocation: """One immutable spec plus its parser and capture redactor.""" spec: InvocationSpec parser: "CodexJSONLParser" redact: Callable[[str], str] @dataclass(frozen=True) class CodexInvocationResult: """Lifecycle result and independently observed effective binding, if any.""" lifecycle: InvocationResult effective_binding: tuple[str, str, str, str] | None def runtime_from_environment(environment: Mapping[str, str]) -> CodexRuntime: """Read exactly the named IOP runtime values without consulting ambient env.""" if not isinstance(environment, Mapping): raise CodexRuntimeError("invalid runtime environment") required = {BASE_URL_ENV_KEY, SECRET_ENV_KEY, "PATH"} if set(environment) != required or not all(isinstance(key, str) for key in environment): raise CodexRuntimeError("runtime environment has unsupported keys") values = {key: environment[key] for key in required} if not all(isinstance(value, str) and value for value in values.values()): raise CodexRuntimeError("runtime environment is incomplete") parsed = urlsplit(values[BASE_URL_ENV_KEY]) if parsed.scheme not in ("http", "https") or not parsed.netloc or parsed.query or parsed.fragment: raise CodexRuntimeError("invalid IOP base URL") return CodexRuntime(values[BASE_URL_ENV_KEY], values[SECRET_ENV_KEY], values["PATH"]) def _require_cell(cell: MatrixCell) -> None: if not isinstance(cell, MatrixCell) or cell.caller != "codex": raise CodexRuntimeError("Codex adapter requires a Codex matrix cell") if cell.iop.requested_effort not in SUPPORTED_EFFORTS: raise CodexRuntimeError("unsupported Codex reasoning effort") if not TOKEN_RE.fullmatch(cell.iop.request_model): raise CodexRuntimeError("invalid requested model") def codex_capability() -> CallerCapability: """The closed local capability claimed by this adapter implementation.""" return CallerCapability("codex", ("direct", "execution_preset"), SUPPORTED_EFFORTS) def _require_prepared(prepared: PreparedWorkspace) -> None: if not isinstance(prepared, PreparedWorkspace) or not prepared.session_is_fresh: raise CodexRuntimeError("Codex invocation requires a fresh prepared workspace") if not Path(prepared.workspace_dir).is_dir() or not Path(prepared.session_dir).is_dir(): raise CodexRuntimeError("prepared workspace is unavailable") if not Path(prepared.attempt_root).is_dir(): raise CodexRuntimeError("prepared attempt root is unavailable") def _toml_string(value: str) -> str: if not isinstance(value, str) or any(ord(char) < 0x20 for char in value): raise CodexRuntimeError("invalid provider configuration value") # JSON strings are valid TOML basic strings and avoid hand-built quoting. return json.dumps(value, ensure_ascii=True) def _provider_overrides(cell: MatrixCell, runtime: CodexRuntime) -> tuple[str, ...]: return ( f"model_provider={_toml_string(PROVIDER_ID)}", f"model_providers.{PROVIDER_ID}.name={_toml_string('IOP Benchmark')}", f"model_providers.{PROVIDER_ID}.base_url={_toml_string(runtime.base_url)}", f"model_providers.{PROVIDER_ID}.env_key={_toml_string(SECRET_ENV_KEY)}", f"model_providers.{PROVIDER_ID}.wire_api={_toml_string('responses')}", f"model_reasoning_effort={_toml_string(cell.iop.requested_effort)}", ) def build_codex_spec( cell: MatrixCell, prepared: PreparedWorkspace, runtime: CodexRuntime, task_payload: bytes, timeout: Timeout, *, codex_executable: str | tuple[str, ...] = "codex", ) -> InvocationSpec: """Build one isolated Codex invocation without reading user configuration. The secret remains only in the child environment. The base URL is an ephemeral process argument required by Codex's provider override; neither is serialized into fixture, capture, lifecycle result, or durable config. """ _require_cell(cell) _require_prepared(prepared) if not isinstance(runtime, CodexRuntime): raise CodexRuntimeError("invalid Codex runtime") if not isinstance(task_payload, bytes) or not task_payload: raise CodexRuntimeError("Codex task payload is required") try: task_payload.decode("utf-8") except UnicodeDecodeError as exc: raise CodexRuntimeError("Codex task payload must be UTF-8") from exc if not isinstance(timeout, Timeout): raise CodexRuntimeError("invalid timeout") if isinstance(codex_executable, str): executable_argv = (codex_executable,) elif isinstance(codex_executable, tuple) and codex_executable and all( isinstance(item, str) and item for item in codex_executable ): # Test-only seam for a non-executable fixture script; normal callers # always receive the exact single-token ``codex`` command above. executable_argv = codex_executable else: raise CodexRuntimeError("invalid Codex executable") codex_argv: list[str] = [ *executable_argv, "exec", "--json", "--ephemeral", "--ignore-user-config", "--strict-config", "--skip-git-repo-check", "-C", prepared.workspace_dir, "-m", cell.iop.request_model, ] for override in _provider_overrides(cell, runtime): codex_argv.extend(("-c", override)) codex_argv.append("-") return InvocationSpec( # The lifecycle changes cwd to the isolated workspace, so use this # module's absolute script path rather than relying on repository # import resolution in the child bridge. argv=(sys.executable, str(Path(__file__).resolve()), "--bridge", "--", *codex_argv), cwd=prepared.workspace_dir, env=env_pairs({"PATH": runtime.path, "HOME": prepared.session_dir, SECRET_ENV_KEY: runtime.api_key}), env_allowlist=(SECRET_ENV_KEY,), submission_mode=SUBMISSION_STDIN_ONCE, completion_mode=COMPLETION_EXIT_AFTER_IDLE, timeout=timeout, evidence_dir=prepared.attempt_root, task_payload=task_payload, control_dir=str(Path(prepared.attempt_root) / "codex-control"), ) def redact_codex_jsonl(text: str, secrets: tuple[str, ...]) -> str: """Remove JSON content and runtime values before any line becomes evidence.""" exact = exact_value_redactor(secrets) try: value = json.loads(text) except (TypeError, json.JSONDecodeError): return "[redacted]" def redact(value: Any, key: str | None = None) -> Any: if key is not None and key.lower() in _SENSITIVE_KEYS: return "[redacted]" if isinstance(value, dict): return {str(item_key): redact(item_value, str(item_key)) for item_key, item_value in value.items()} if isinstance(value, list): return [redact(item) for item in value] if isinstance(value, str): return exact(value) if key in _SAFE_STRING_KEYS else "[redacted]" return value return json.dumps(redact(value), ensure_ascii=True, separators=(",", ":")) class CodexJSONLParser: """Parse only terminal evidence and optional explicit IOP effective binding.""" def __init__(self, cell: MatrixCell, idle_nonce: str) -> None: _require_cell(cell) if not isinstance(idle_nonce, str) or len(idle_nonce) < 16: raise CodexRuntimeError("invalid bridge idle nonce") self._cell = cell self._idle_nonce = idle_nonce self._effective_binding: tuple[str, str, str, str] | None = None self._turns = 0 self._tool_calls: set[str] = set() @property def effective_binding(self) -> tuple[str, str, str, str] | None: return self._effective_binding def connectivity_result(self) -> ConnectivityResult: """Classify absent evidence without synthesizing unreported stage bindings.""" iop = self._cell.iop if self._effective_binding is not None: # Codex's optional scalar observation proves neither the complete # stage list nor its exact order. Never manufacture that missing # contract evidence from the manifest just to produce ``ready``. raise CodexJSONLError("effective stage binding is unavailable") binding = RequestedEffectiveBinding( self._cell.id, self._cell.caller, iop.route_kind, iop.route_id, iop.request_model, iop.requested_effort, None, None, None, None, (), ) issues: tuple[ConnectivityIssue, ...] = ( ConnectivityIssue("stream_incompatible", ISSUE_RESUME_CODES["stream_incompatible"]), ) return make_result(self._cell, codex_capability(), binding, issues) def parse(self, stream: str, line: str) -> str | ParsedMetric | tuple[Any, ...] | None: if stream != "stdout": return None try: record = json.loads(line) except (TypeError, json.JSONDecodeError) as exc: raise CodexJSONLError("malformed Codex JSONL") from exc if not isinstance(record, dict) or not isinstance(record.get("type"), str): raise CodexJSONLError("malformed Codex JSONL") self._observe_effective_binding(record) if record["type"] == "turn.completed": status = record.get("status") if status is not None and status not in ("completed", "success"): raise CodexJSONLError("unsuccessful Codex terminal turn") self._turns += 1 return (*self._turn_observations(record), "finish") if record["type"] == _CODEX_ITEM_COMPLETED: return self._tool_interval(record) if record["type"] == _BRIDGE_IDLE_TYPE: if record != {"type": _BRIDGE_IDLE_TYPE, "adapter": _BRIDGE_ID, "nonce": self._idle_nonce, "child_exit": 0}: raise CodexJSONLError("unverified bridge idle marker") return "idle" return None def _turn_observations(self, record: dict[str, Any]) -> tuple[ParsedMetric, ...]: """Observe reported turn usage plus the turn and tool call counts.""" model = self._cell.iop.request_model usage = record.get("usage") if usage is not None and ( not isinstance(usage, dict) or not set(usage) <= set(_CODEX_USAGE_FIELDS) ): raise CodexJSONLError("invalid Codex usage observation") try: observations = [ count_metric(_CODEX_USAGE_FIELDS[field], value, model=model) for field, value in sorted((usage or {}).items()) ] observations.append(count_metric("model_calls", self._turns, model=model)) observations.append( count_metric("tool_calls", len(self._tool_calls), model=model) ) except LifecycleMetricError as exc: raise CodexJSONLError("invalid Codex usage observation") from exc return tuple(observations) def _tool_interval(self, record: dict[str, Any]) -> ParsedMetric | None: """Observe one explicitly paired tool interval; never infer a pairing.""" item = record.get("item") if not isinstance(item, dict) or "duration_ms" not in item: return None call_id = item.get("id") if not isinstance(call_id, str) or _CODEX_CALL_ID_RE.fullmatch(call_id) is None: raise CodexJSONLError("unpaired Codex tool interval") if not is_reported_number(item["duration_ms"]): raise CodexJSONLError("invalid Codex tool interval") if call_id in self._tool_calls: raise CodexJSONLError("duplicate Codex tool interval") try: observation = duration_metric( "tool_duration", item["duration_ms"], reported_unit="ms", model=self._cell.iop.request_model, call_id=call_id, overlap=True, ) except LifecycleMetricError as exc: raise CodexJSONLError("invalid Codex tool interval") from exc self._tool_calls.add(call_id) return observation def _observe_effective_binding(self, record: dict[str, Any]) -> None: observed = record.get("iop_effective_binding") if observed is None: return if not isinstance(observed, dict) or set(observed) != {"route_kind", "route_id", "model", "effort"}: raise CodexJSONLError("invalid effective binding observation") values = tuple(observed[key] for key in ("route_kind", "route_id", "model", "effort")) if not all(isinstance(value, str) and TOKEN_RE.fullmatch(value) for value in values): raise CodexJSONLError("invalid effective binding observation") expected = ( self._cell.iop.route_kind, self._cell.iop.route_id, self._cell.iop.request_model, self._cell.iop.requested_effort, ) if values != expected: raise CodexJSONLError("effective binding substitution") if self._effective_binding is not None: raise CodexJSONLError("duplicate effective binding observation") self._effective_binding = values # type: ignore[assignment] def build_codex_invocation( cell: MatrixCell, prepared: PreparedWorkspace, runtime: CodexRuntime, task_payload: bytes, timeout: Timeout, *, codex_executable: str | tuple[str, ...] = "codex", ) -> CodexInvocation: """Pair the closed invocation spec with its parser and structural redactor.""" nonce = secrets.token_hex(16) spec = build_codex_spec(cell, prepared, runtime, task_payload, timeout, codex_executable=codex_executable) parser = CodexJSONLParser(cell, nonce) # The nonce must reach the bridge but not Codex. Put it before the bridge # delimiter so the bridge removes it before starting the child. argv = (*spec.argv[:3], f"--idle-nonce={nonce}", *spec.argv[3:]) task_text = task_payload.decode("utf-8") return CodexInvocation(spec=InvocationSpec(**{**spec.__dict__, "argv": argv}), parser=parser, redact=lambda line: redact_codex_jsonl(line, (runtime.base_url, runtime.api_key, task_text))) def run_codex_invocation( invocation: CodexInvocation, on_started: Callable[[SupervisorLocator], None], ) -> CodexInvocationResult: """Run exactly one prepared Codex invocation through the generic lifecycle.""" if not isinstance(invocation, CodexInvocation) or not callable(on_started): raise CodexRuntimeError("invalid Codex invocation") result = run_invocation(invocation.spec, parse_event=invocation.parser.parse, on_started=on_started, redact=invocation.redact) return CodexInvocationResult(result, invocation.parser.effective_binding) def _forward_stream(stream: Any, destination: Any) -> None: try: for line in iter(stream.readline, b""): destination.buffer.write(line) destination.buffer.flush() finally: stream.close() def _bridge(argv: list[str]) -> int: """Own a child Codex process and emit idle only after exit plus both EOFs.""" if not argv or not argv[0].startswith("--idle-nonce="): return 64 nonce = argv.pop(0).partition("=")[2] if not nonce or not argv or argv.pop(0) != "--": return 64 try: child = subprocess.Popen(argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True) except OSError: return 127 assert child.stdin is not None and child.stdout is not None and child.stderr is not None payload = sys.stdin.buffer.read() try: child.stdin.write(payload) child.stdin.close() except OSError: child.terminate() stdout_thread = threading.Thread(target=_forward_stream, args=(child.stdout, sys.stdout), daemon=True) stderr_thread = threading.Thread(target=_forward_stream, args=(child.stderr, sys.stderr), daemon=True) stdout_thread.start() stderr_thread.start() exit_code = child.wait() stdout_thread.join() stderr_thread.join() if exit_code == 0: sys.stdout.write(json.dumps({"type": _BRIDGE_IDLE_TYPE, "adapter": _BRIDGE_ID, "nonce": nonce, "child_exit": 0}, separators=(",", ":")) + "\n") sys.stdout.flush() return exit_code def main(argv: list[str] | None = None) -> int: values = list(sys.argv[1:] if argv is None else argv) if not values or values.pop(0) != "--bridge": return 64 return _bridge(values) if __name__ == "__main__": raise SystemExit(main())