473 lines
20 KiB
Python
473 lines
20 KiB
Python
"""Isolated, secret-safe Codex ``exec`` adapter for IOP benchmark cells.
|
|
|
|
The adapter deliberately owns no public runner command. It converts a frozen
|
|
manifest cell and a caller-supplied runtime into one bounded lifecycle
|
|
invocation. The only executable it starts is a small bridge in the lifecycle
|
|
owned process group; that bridge starts ``codex exec``, forwards its JSONL, and
|
|
emits an idle marker only after the child has exited and both output streams
|
|
reached EOF.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import re
|
|
import secrets
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Any, Callable, Mapping
|
|
from urllib.parse import urlsplit
|
|
|
|
# The bridge is intentionally invoked by absolute script path from an isolated
|
|
# workspace. Make the repository package importable without inheriting an
|
|
# ambient PYTHONPATH.
|
|
if __package__ in (None, ""):
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
|
|
|
from scripts.agent_benchmark.lifecycle import (
|
|
COMPLETION_EXIT_AFTER_IDLE,
|
|
SUBMISSION_STDIN_ONCE,
|
|
InvocationResult,
|
|
InvocationSpec,
|
|
LifecycleMetricError,
|
|
LifecycleValidationError,
|
|
ParsedMetric,
|
|
SupervisorLocator,
|
|
count_metric,
|
|
duration_metric,
|
|
env_pairs,
|
|
exact_value_redactor,
|
|
is_reported_number,
|
|
run_invocation,
|
|
)
|
|
from scripts.agent_benchmark.connectivity import (
|
|
ISSUE_RESUME_CODES,
|
|
CallerCapability,
|
|
ConnectivityIssue,
|
|
ConnectivityResult,
|
|
RequestedEffectiveBinding,
|
|
make_result,
|
|
)
|
|
from scripts.agent_benchmark.manifest import MatrixCell, Timeout, TOKEN_RE
|
|
from scripts.agent_benchmark.workspace import PreparedWorkspace
|
|
|
|
|
|
PROVIDER_ID = "iop_benchmark"
|
|
SECRET_ENV_KEY = "IOP_BENCHMARK_API_KEY"
|
|
BASE_URL_ENV_KEY = "IOP_BENCHMARK_BASE_URL"
|
|
SUPPORTED_EFFORTS = ("xhigh",)
|
|
_BRIDGE_IDLE_TYPE = "adapter.idle"
|
|
_BRIDGE_ID = "codex_iop"
|
|
_SENSITIVE_KEYS = frozenset({
|
|
"api_key", "authorization", "base_url", "command", "content", "endpoint",
|
|
"input", "instructions", "message", "output", "prompt", "secret", "text",
|
|
"token", "tool_input", "tool_output", "url",
|
|
})
|
|
_SAFE_STRING_KEYS = frozenset({
|
|
"adapter", "effort", "model", "nonce", "reasoning_effort", "route_id",
|
|
"route_kind", "stage", "status", "type",
|
|
})
|
|
# The exact reported Codex usage keys this adapter observes. A tool interval is
|
|
# consumed only when the completed item explicitly pairs one bounded duration
|
|
# with one call identifier; nothing is derived from event arrival order.
|
|
_CODEX_USAGE_FIELDS = {
|
|
"input_tokens": "input_tokens",
|
|
"cached_input_tokens": "cached_input_tokens",
|
|
"output_tokens": "output_tokens",
|
|
"reasoning_output_tokens": "reasoning_tokens",
|
|
"total_tokens": "total_tokens",
|
|
}
|
|
_CODEX_ITEM_COMPLETED = "item.completed"
|
|
_CODEX_CALL_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.:-]{0,63}$")
|
|
|
|
|
|
class CodexIOPError(Exception):
|
|
"""Base class for closed Codex adapter validation failures."""
|
|
|
|
|
|
class CodexRuntimeError(CodexIOPError):
|
|
"""The caller-supplied runtime is absent, malformed, or unsafe."""
|
|
|
|
|
|
class CodexJSONLError(CodexIOPError):
|
|
"""A Codex JSONL event cannot safely satisfy the adapter protocol."""
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class CodexRuntime:
|
|
"""The two runtime-only values needed to reach the IOP Responses surface."""
|
|
|
|
base_url: str
|
|
api_key: str
|
|
path: str
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class CodexInvocation:
|
|
"""One immutable spec plus its parser and capture redactor."""
|
|
|
|
spec: InvocationSpec
|
|
parser: "CodexJSONLParser"
|
|
redact: Callable[[str], str]
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class CodexInvocationResult:
|
|
"""Lifecycle result and independently observed effective binding, if any."""
|
|
|
|
lifecycle: InvocationResult
|
|
effective_binding: tuple[str, str, str, str] | None
|
|
|
|
|
|
def runtime_from_environment(environment: Mapping[str, str]) -> CodexRuntime:
|
|
"""Read exactly the named IOP runtime values without consulting ambient env."""
|
|
if not isinstance(environment, Mapping):
|
|
raise CodexRuntimeError("invalid runtime environment")
|
|
required = {BASE_URL_ENV_KEY, SECRET_ENV_KEY, "PATH"}
|
|
if set(environment) != required or not all(isinstance(key, str) for key in environment):
|
|
raise CodexRuntimeError("runtime environment has unsupported keys")
|
|
values = {key: environment[key] for key in required}
|
|
if not all(isinstance(value, str) and value for value in values.values()):
|
|
raise CodexRuntimeError("runtime environment is incomplete")
|
|
parsed = urlsplit(values[BASE_URL_ENV_KEY])
|
|
if parsed.scheme not in ("http", "https") or not parsed.netloc or parsed.query or parsed.fragment:
|
|
raise CodexRuntimeError("invalid IOP base URL")
|
|
return CodexRuntime(values[BASE_URL_ENV_KEY], values[SECRET_ENV_KEY], values["PATH"])
|
|
|
|
|
|
def _require_cell(cell: MatrixCell) -> None:
|
|
if not isinstance(cell, MatrixCell) or cell.caller != "codex":
|
|
raise CodexRuntimeError("Codex adapter requires a Codex matrix cell")
|
|
if cell.iop.requested_effort not in SUPPORTED_EFFORTS:
|
|
raise CodexRuntimeError("unsupported Codex reasoning effort")
|
|
if not TOKEN_RE.fullmatch(cell.iop.request_model):
|
|
raise CodexRuntimeError("invalid requested model")
|
|
|
|
|
|
def codex_capability() -> CallerCapability:
|
|
"""The closed local capability claimed by this adapter implementation."""
|
|
return CallerCapability("codex", ("direct", "execution_preset"), SUPPORTED_EFFORTS)
|
|
|
|
|
|
def _require_prepared(prepared: PreparedWorkspace) -> None:
|
|
if not isinstance(prepared, PreparedWorkspace) or not prepared.session_is_fresh:
|
|
raise CodexRuntimeError("Codex invocation requires a fresh prepared workspace")
|
|
if not Path(prepared.workspace_dir).is_dir() or not Path(prepared.session_dir).is_dir():
|
|
raise CodexRuntimeError("prepared workspace is unavailable")
|
|
if not Path(prepared.attempt_root).is_dir():
|
|
raise CodexRuntimeError("prepared attempt root is unavailable")
|
|
|
|
|
|
def _toml_string(value: str) -> str:
|
|
if not isinstance(value, str) or any(ord(char) < 0x20 for char in value):
|
|
raise CodexRuntimeError("invalid provider configuration value")
|
|
# JSON strings are valid TOML basic strings and avoid hand-built quoting.
|
|
return json.dumps(value, ensure_ascii=True)
|
|
|
|
|
|
def _provider_overrides(cell: MatrixCell, runtime: CodexRuntime) -> tuple[str, ...]:
|
|
return (
|
|
f"model_provider={_toml_string(PROVIDER_ID)}",
|
|
f"model_providers.{PROVIDER_ID}.name={_toml_string('IOP Benchmark')}",
|
|
f"model_providers.{PROVIDER_ID}.base_url={_toml_string(runtime.base_url)}",
|
|
f"model_providers.{PROVIDER_ID}.env_key={_toml_string(SECRET_ENV_KEY)}",
|
|
f"model_providers.{PROVIDER_ID}.wire_api={_toml_string('responses')}",
|
|
f"model_reasoning_effort={_toml_string(cell.iop.requested_effort)}",
|
|
)
|
|
|
|
|
|
def build_codex_spec(
|
|
cell: MatrixCell,
|
|
prepared: PreparedWorkspace,
|
|
runtime: CodexRuntime,
|
|
task_payload: bytes,
|
|
timeout: Timeout,
|
|
*,
|
|
codex_executable: str | tuple[str, ...] = "codex",
|
|
) -> InvocationSpec:
|
|
"""Build one isolated Codex invocation without reading user configuration.
|
|
|
|
The secret remains only in the child environment. The base URL is an
|
|
ephemeral process argument required by Codex's provider override; neither
|
|
is serialized into fixture, capture, lifecycle result, or durable config.
|
|
"""
|
|
_require_cell(cell)
|
|
_require_prepared(prepared)
|
|
if not isinstance(runtime, CodexRuntime):
|
|
raise CodexRuntimeError("invalid Codex runtime")
|
|
if not isinstance(task_payload, bytes) or not task_payload:
|
|
raise CodexRuntimeError("Codex task payload is required")
|
|
try:
|
|
task_payload.decode("utf-8")
|
|
except UnicodeDecodeError as exc:
|
|
raise CodexRuntimeError("Codex task payload must be UTF-8") from exc
|
|
if not isinstance(timeout, Timeout):
|
|
raise CodexRuntimeError("invalid timeout")
|
|
if isinstance(codex_executable, str):
|
|
executable_argv = (codex_executable,)
|
|
elif isinstance(codex_executable, tuple) and codex_executable and all(
|
|
isinstance(item, str) and item for item in codex_executable
|
|
):
|
|
# Test-only seam for a non-executable fixture script; normal callers
|
|
# always receive the exact single-token ``codex`` command above.
|
|
executable_argv = codex_executable
|
|
else:
|
|
raise CodexRuntimeError("invalid Codex executable")
|
|
|
|
codex_argv: list[str] = [
|
|
*executable_argv, "exec", "--json", "--ephemeral", "--ignore-user-config",
|
|
"--strict-config", "--skip-git-repo-check", "-C", prepared.workspace_dir,
|
|
"-m", cell.iop.request_model,
|
|
]
|
|
for override in _provider_overrides(cell, runtime):
|
|
codex_argv.extend(("-c", override))
|
|
codex_argv.append("-")
|
|
return InvocationSpec(
|
|
# The lifecycle changes cwd to the isolated workspace, so use this
|
|
# module's absolute script path rather than relying on repository
|
|
# import resolution in the child bridge.
|
|
argv=(sys.executable, str(Path(__file__).resolve()), "--bridge", "--", *codex_argv),
|
|
cwd=prepared.workspace_dir,
|
|
env=env_pairs({"PATH": runtime.path, "HOME": prepared.session_dir, SECRET_ENV_KEY: runtime.api_key}),
|
|
env_allowlist=(SECRET_ENV_KEY,),
|
|
submission_mode=SUBMISSION_STDIN_ONCE,
|
|
completion_mode=COMPLETION_EXIT_AFTER_IDLE,
|
|
timeout=timeout,
|
|
evidence_dir=prepared.attempt_root,
|
|
task_payload=task_payload,
|
|
control_dir=str(Path(prepared.attempt_root) / "codex-control"),
|
|
)
|
|
|
|
|
|
def redact_codex_jsonl(text: str, secrets: tuple[str, ...]) -> str:
|
|
"""Remove JSON content and runtime values before any line becomes evidence."""
|
|
exact = exact_value_redactor(secrets)
|
|
try:
|
|
value = json.loads(text)
|
|
except (TypeError, json.JSONDecodeError):
|
|
return "[redacted]"
|
|
|
|
def redact(value: Any, key: str | None = None) -> Any:
|
|
if key is not None and key.lower() in _SENSITIVE_KEYS:
|
|
return "[redacted]"
|
|
if isinstance(value, dict):
|
|
return {str(item_key): redact(item_value, str(item_key)) for item_key, item_value in value.items()}
|
|
if isinstance(value, list):
|
|
return [redact(item) for item in value]
|
|
if isinstance(value, str):
|
|
return exact(value) if key in _SAFE_STRING_KEYS else "[redacted]"
|
|
return value
|
|
|
|
return json.dumps(redact(value), ensure_ascii=True, separators=(",", ":"))
|
|
|
|
|
|
class CodexJSONLParser:
|
|
"""Parse only terminal evidence and optional explicit IOP effective binding."""
|
|
|
|
def __init__(self, cell: MatrixCell, idle_nonce: str) -> None:
|
|
_require_cell(cell)
|
|
if not isinstance(idle_nonce, str) or len(idle_nonce) < 16:
|
|
raise CodexRuntimeError("invalid bridge idle nonce")
|
|
self._cell = cell
|
|
self._idle_nonce = idle_nonce
|
|
self._effective_binding: tuple[str, str, str, str] | None = None
|
|
self._turns = 0
|
|
self._tool_calls: set[str] = set()
|
|
|
|
@property
|
|
def effective_binding(self) -> tuple[str, str, str, str] | None:
|
|
return self._effective_binding
|
|
|
|
def connectivity_result(self) -> ConnectivityResult:
|
|
"""Classify absent evidence without synthesizing unreported stage bindings."""
|
|
iop = self._cell.iop
|
|
if self._effective_binding is not None:
|
|
# Codex's optional scalar observation proves neither the complete
|
|
# stage list nor its exact order. Never manufacture that missing
|
|
# contract evidence from the manifest just to produce ``ready``.
|
|
raise CodexJSONLError("effective stage binding is unavailable")
|
|
binding = RequestedEffectiveBinding(
|
|
self._cell.id, self._cell.caller, iop.route_kind, iop.route_id,
|
|
iop.request_model, iop.requested_effort,
|
|
None, None, None, None, (),
|
|
)
|
|
issues: tuple[ConnectivityIssue, ...] = (
|
|
ConnectivityIssue("stream_incompatible", ISSUE_RESUME_CODES["stream_incompatible"]),
|
|
)
|
|
return make_result(self._cell, codex_capability(), binding, issues)
|
|
|
|
def parse(self, stream: str, line: str) -> str | ParsedMetric | tuple[Any, ...] | None:
|
|
if stream != "stdout":
|
|
return None
|
|
try:
|
|
record = json.loads(line)
|
|
except (TypeError, json.JSONDecodeError) as exc:
|
|
raise CodexJSONLError("malformed Codex JSONL") from exc
|
|
if not isinstance(record, dict) or not isinstance(record.get("type"), str):
|
|
raise CodexJSONLError("malformed Codex JSONL")
|
|
self._observe_effective_binding(record)
|
|
if record["type"] == "turn.completed":
|
|
status = record.get("status")
|
|
if status is not None and status not in ("completed", "success"):
|
|
raise CodexJSONLError("unsuccessful Codex terminal turn")
|
|
self._turns += 1
|
|
return (*self._turn_observations(record), "finish")
|
|
if record["type"] == _CODEX_ITEM_COMPLETED:
|
|
return self._tool_interval(record)
|
|
if record["type"] == _BRIDGE_IDLE_TYPE:
|
|
if record != {"type": _BRIDGE_IDLE_TYPE, "adapter": _BRIDGE_ID, "nonce": self._idle_nonce, "child_exit": 0}:
|
|
raise CodexJSONLError("unverified bridge idle marker")
|
|
return "idle"
|
|
return None
|
|
|
|
def _turn_observations(self, record: dict[str, Any]) -> tuple[ParsedMetric, ...]:
|
|
"""Observe reported turn usage plus the turn and tool call counts."""
|
|
model = self._cell.iop.request_model
|
|
usage = record.get("usage")
|
|
if usage is not None and (
|
|
not isinstance(usage, dict) or not set(usage) <= set(_CODEX_USAGE_FIELDS)
|
|
):
|
|
raise CodexJSONLError("invalid Codex usage observation")
|
|
try:
|
|
observations = [
|
|
count_metric(_CODEX_USAGE_FIELDS[field], value, model=model)
|
|
for field, value in sorted((usage or {}).items())
|
|
]
|
|
observations.append(count_metric("model_calls", self._turns, model=model))
|
|
observations.append(
|
|
count_metric("tool_calls", len(self._tool_calls), model=model)
|
|
)
|
|
except LifecycleMetricError as exc:
|
|
raise CodexJSONLError("invalid Codex usage observation") from exc
|
|
return tuple(observations)
|
|
|
|
def _tool_interval(self, record: dict[str, Any]) -> ParsedMetric | None:
|
|
"""Observe one explicitly paired tool interval; never infer a pairing."""
|
|
item = record.get("item")
|
|
if not isinstance(item, dict) or "duration_ms" not in item:
|
|
return None
|
|
call_id = item.get("id")
|
|
if not isinstance(call_id, str) or _CODEX_CALL_ID_RE.fullmatch(call_id) is None:
|
|
raise CodexJSONLError("unpaired Codex tool interval")
|
|
if not is_reported_number(item["duration_ms"]):
|
|
raise CodexJSONLError("invalid Codex tool interval")
|
|
if call_id in self._tool_calls:
|
|
raise CodexJSONLError("duplicate Codex tool interval")
|
|
try:
|
|
observation = duration_metric(
|
|
"tool_duration", item["duration_ms"], reported_unit="ms",
|
|
model=self._cell.iop.request_model, call_id=call_id, overlap=True,
|
|
)
|
|
except LifecycleMetricError as exc:
|
|
raise CodexJSONLError("invalid Codex tool interval") from exc
|
|
self._tool_calls.add(call_id)
|
|
return observation
|
|
|
|
def _observe_effective_binding(self, record: dict[str, Any]) -> None:
|
|
observed = record.get("iop_effective_binding")
|
|
if observed is None:
|
|
return
|
|
if not isinstance(observed, dict) or set(observed) != {"route_kind", "route_id", "model", "effort"}:
|
|
raise CodexJSONLError("invalid effective binding observation")
|
|
values = tuple(observed[key] for key in ("route_kind", "route_id", "model", "effort"))
|
|
if not all(isinstance(value, str) and TOKEN_RE.fullmatch(value) for value in values):
|
|
raise CodexJSONLError("invalid effective binding observation")
|
|
expected = (
|
|
self._cell.iop.route_kind,
|
|
self._cell.iop.route_id,
|
|
self._cell.iop.request_model,
|
|
self._cell.iop.requested_effort,
|
|
)
|
|
if values != expected:
|
|
raise CodexJSONLError("effective binding substitution")
|
|
if self._effective_binding is not None:
|
|
raise CodexJSONLError("duplicate effective binding observation")
|
|
self._effective_binding = values # type: ignore[assignment]
|
|
|
|
|
|
def build_codex_invocation(
|
|
cell: MatrixCell,
|
|
prepared: PreparedWorkspace,
|
|
runtime: CodexRuntime,
|
|
task_payload: bytes,
|
|
timeout: Timeout,
|
|
*,
|
|
codex_executable: str | tuple[str, ...] = "codex",
|
|
) -> CodexInvocation:
|
|
"""Pair the closed invocation spec with its parser and structural redactor."""
|
|
nonce = secrets.token_hex(16)
|
|
spec = build_codex_spec(cell, prepared, runtime, task_payload, timeout, codex_executable=codex_executable)
|
|
parser = CodexJSONLParser(cell, nonce)
|
|
# The nonce must reach the bridge but not Codex. Put it before the bridge
|
|
# delimiter so the bridge removes it before starting the child.
|
|
argv = (*spec.argv[:3], f"--idle-nonce={nonce}", *spec.argv[3:])
|
|
task_text = task_payload.decode("utf-8")
|
|
return CodexInvocation(spec=InvocationSpec(**{**spec.__dict__, "argv": argv}), parser=parser,
|
|
redact=lambda line: redact_codex_jsonl(line, (runtime.base_url, runtime.api_key, task_text)))
|
|
|
|
|
|
def run_codex_invocation(
|
|
invocation: CodexInvocation,
|
|
on_started: Callable[[SupervisorLocator], None],
|
|
) -> CodexInvocationResult:
|
|
"""Run exactly one prepared Codex invocation through the generic lifecycle."""
|
|
if not isinstance(invocation, CodexInvocation) or not callable(on_started):
|
|
raise CodexRuntimeError("invalid Codex invocation")
|
|
result = run_invocation(invocation.spec, parse_event=invocation.parser.parse,
|
|
on_started=on_started, redact=invocation.redact)
|
|
return CodexInvocationResult(result, invocation.parser.effective_binding)
|
|
|
|
|
|
def _forward_stream(stream: Any, destination: Any) -> None:
|
|
try:
|
|
for line in iter(stream.readline, b""):
|
|
destination.buffer.write(line)
|
|
destination.buffer.flush()
|
|
finally:
|
|
stream.close()
|
|
|
|
|
|
def _bridge(argv: list[str]) -> int:
|
|
"""Own a child Codex process and emit idle only after exit plus both EOFs."""
|
|
if not argv or not argv[0].startswith("--idle-nonce="):
|
|
return 64
|
|
nonce = argv.pop(0).partition("=")[2]
|
|
if not nonce or not argv or argv.pop(0) != "--":
|
|
return 64
|
|
try:
|
|
child = subprocess.Popen(argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
|
|
except OSError:
|
|
return 127
|
|
assert child.stdin is not None and child.stdout is not None and child.stderr is not None
|
|
payload = sys.stdin.buffer.read()
|
|
try:
|
|
child.stdin.write(payload)
|
|
child.stdin.close()
|
|
except OSError:
|
|
child.terminate()
|
|
stdout_thread = threading.Thread(target=_forward_stream, args=(child.stdout, sys.stdout), daemon=True)
|
|
stderr_thread = threading.Thread(target=_forward_stream, args=(child.stderr, sys.stderr), daemon=True)
|
|
stdout_thread.start()
|
|
stderr_thread.start()
|
|
exit_code = child.wait()
|
|
stdout_thread.join()
|
|
stderr_thread.join()
|
|
if exit_code == 0:
|
|
sys.stdout.write(json.dumps({"type": _BRIDGE_IDLE_TYPE, "adapter": _BRIDGE_ID, "nonce": nonce, "child_exit": 0}, separators=(",", ":")) + "\n")
|
|
sys.stdout.flush()
|
|
return exit_code
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
values = list(sys.argv[1:] if argv is None else argv)
|
|
if not values or values.pop(0) != "--bridge":
|
|
return 64
|
|
return _bridge(values)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|