toki
de4d8f4ff8
세 Agent의 direct route를 동일한 fail-closed preflight와 격리 실행 경계에서 비교하고, 관측되지 않은 preset 셀이 실행되는 것을 막기 위해 연결 계약과 증거 수집 흐름을 고정한다.
407 lines
17 KiB
Python
407 lines
17 KiB
Python
"""Isolated, secret-safe Codex ``exec`` adapter for IOP benchmark cells.
|
|
|
|
The adapter deliberately owns no public runner command. It converts a frozen
|
|
manifest cell and a caller-supplied runtime into one bounded lifecycle
|
|
invocation. The only executable it starts is a small bridge in the lifecycle
|
|
owned process group; that bridge starts ``codex exec``, forwards its JSONL, and
|
|
emits an idle marker only after the child has exited and both output streams
|
|
reached EOF.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import secrets
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Any, Callable, Mapping
|
|
from urllib.parse import urlsplit
|
|
|
|
# The bridge is intentionally invoked by absolute script path from an isolated
|
|
# workspace. Make the repository package importable without inheriting an
|
|
# ambient PYTHONPATH.
|
|
if __package__ in (None, ""):
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
|
|
|
from scripts.agent_benchmark.lifecycle import (
|
|
COMPLETION_EXIT_AFTER_IDLE,
|
|
SUBMISSION_STDIN_ONCE,
|
|
InvocationResult,
|
|
InvocationSpec,
|
|
LifecycleValidationError,
|
|
SupervisorLocator,
|
|
env_pairs,
|
|
exact_value_redactor,
|
|
run_invocation,
|
|
)
|
|
from scripts.agent_benchmark.connectivity import (
|
|
ISSUE_RESUME_CODES,
|
|
CallerCapability,
|
|
ConnectivityIssue,
|
|
ConnectivityResult,
|
|
RequestedEffectiveBinding,
|
|
make_result,
|
|
)
|
|
from scripts.agent_benchmark.manifest import MatrixCell, Timeout, TOKEN_RE
|
|
from scripts.agent_benchmark.workspace import PreparedWorkspace
|
|
|
|
|
|
PROVIDER_ID = "iop_benchmark"
|
|
SECRET_ENV_KEY = "IOP_BENCHMARK_API_KEY"
|
|
BASE_URL_ENV_KEY = "IOP_BENCHMARK_BASE_URL"
|
|
SUPPORTED_EFFORTS = ("xhigh",)
|
|
_BRIDGE_IDLE_TYPE = "adapter.idle"
|
|
_BRIDGE_ID = "codex_iop"
|
|
_SENSITIVE_KEYS = frozenset({
|
|
"api_key", "authorization", "base_url", "command", "content", "endpoint",
|
|
"input", "instructions", "message", "output", "prompt", "secret", "text",
|
|
"token", "tool_input", "tool_output", "url",
|
|
})
|
|
_SAFE_STRING_KEYS = frozenset({
|
|
"adapter", "effort", "model", "nonce", "reasoning_effort", "route_id",
|
|
"route_kind", "stage", "status", "type",
|
|
})
|
|
|
|
|
|
class CodexIOPError(Exception):
|
|
"""Base class for closed Codex adapter validation failures."""
|
|
|
|
|
|
class CodexRuntimeError(CodexIOPError):
|
|
"""The caller-supplied runtime is absent, malformed, or unsafe."""
|
|
|
|
|
|
class CodexJSONLError(CodexIOPError):
|
|
"""A Codex JSONL event cannot safely satisfy the adapter protocol."""
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class CodexRuntime:
|
|
"""The two runtime-only values needed to reach the IOP Responses surface."""
|
|
|
|
base_url: str
|
|
api_key: str
|
|
path: str
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class CodexInvocation:
|
|
"""One immutable spec plus its parser and capture redactor."""
|
|
|
|
spec: InvocationSpec
|
|
parser: "CodexJSONLParser"
|
|
redact: Callable[[str], str]
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class CodexInvocationResult:
|
|
"""Lifecycle result and independently observed effective binding, if any."""
|
|
|
|
lifecycle: InvocationResult
|
|
effective_binding: tuple[str, str, str, str] | None
|
|
|
|
|
|
def runtime_from_environment(environment: Mapping[str, str]) -> CodexRuntime:
|
|
"""Read exactly the named IOP runtime values without consulting ambient env."""
|
|
if not isinstance(environment, Mapping):
|
|
raise CodexRuntimeError("invalid runtime environment")
|
|
required = {BASE_URL_ENV_KEY, SECRET_ENV_KEY, "PATH"}
|
|
if set(environment) != required or not all(isinstance(key, str) for key in environment):
|
|
raise CodexRuntimeError("runtime environment has unsupported keys")
|
|
values = {key: environment[key] for key in required}
|
|
if not all(isinstance(value, str) and value for value in values.values()):
|
|
raise CodexRuntimeError("runtime environment is incomplete")
|
|
parsed = urlsplit(values[BASE_URL_ENV_KEY])
|
|
if parsed.scheme not in ("http", "https") or not parsed.netloc or parsed.query or parsed.fragment:
|
|
raise CodexRuntimeError("invalid IOP base URL")
|
|
return CodexRuntime(values[BASE_URL_ENV_KEY], values[SECRET_ENV_KEY], values["PATH"])
|
|
|
|
|
|
def _require_cell(cell: MatrixCell) -> None:
|
|
if not isinstance(cell, MatrixCell) or cell.caller != "codex":
|
|
raise CodexRuntimeError("Codex adapter requires a Codex matrix cell")
|
|
if cell.iop.requested_effort not in SUPPORTED_EFFORTS:
|
|
raise CodexRuntimeError("unsupported Codex reasoning effort")
|
|
if not TOKEN_RE.fullmatch(cell.iop.request_model):
|
|
raise CodexRuntimeError("invalid requested model")
|
|
|
|
|
|
def codex_capability() -> CallerCapability:
|
|
"""The closed local capability claimed by this adapter implementation."""
|
|
return CallerCapability("codex", ("direct", "execution_preset"), SUPPORTED_EFFORTS)
|
|
|
|
|
|
def _require_prepared(prepared: PreparedWorkspace) -> None:
|
|
if not isinstance(prepared, PreparedWorkspace) or not prepared.session_is_fresh:
|
|
raise CodexRuntimeError("Codex invocation requires a fresh prepared workspace")
|
|
if not Path(prepared.workspace_dir).is_dir() or not Path(prepared.session_dir).is_dir():
|
|
raise CodexRuntimeError("prepared workspace is unavailable")
|
|
if not Path(prepared.attempt_root).is_dir():
|
|
raise CodexRuntimeError("prepared attempt root is unavailable")
|
|
|
|
|
|
def _toml_string(value: str) -> str:
|
|
if not isinstance(value, str) or any(ord(char) < 0x20 for char in value):
|
|
raise CodexRuntimeError("invalid provider configuration value")
|
|
# JSON strings are valid TOML basic strings and avoid hand-built quoting.
|
|
return json.dumps(value, ensure_ascii=True)
|
|
|
|
|
|
def _provider_overrides(cell: MatrixCell, runtime: CodexRuntime) -> tuple[str, ...]:
|
|
return (
|
|
f"model_provider={_toml_string(PROVIDER_ID)}",
|
|
f"model_providers.{PROVIDER_ID}.name={_toml_string('IOP Benchmark')}",
|
|
f"model_providers.{PROVIDER_ID}.base_url={_toml_string(runtime.base_url)}",
|
|
f"model_providers.{PROVIDER_ID}.env_key={_toml_string(SECRET_ENV_KEY)}",
|
|
f"model_providers.{PROVIDER_ID}.wire_api={_toml_string('responses')}",
|
|
f"model_reasoning_effort={_toml_string(cell.iop.requested_effort)}",
|
|
)
|
|
|
|
|
|
def build_codex_spec(
|
|
cell: MatrixCell,
|
|
prepared: PreparedWorkspace,
|
|
runtime: CodexRuntime,
|
|
task_payload: bytes,
|
|
timeout: Timeout,
|
|
*,
|
|
codex_executable: str | tuple[str, ...] = "codex",
|
|
) -> InvocationSpec:
|
|
"""Build one isolated Codex invocation without reading user configuration.
|
|
|
|
The secret remains only in the child environment. The base URL is an
|
|
ephemeral process argument required by Codex's provider override; neither
|
|
is serialized into fixture, capture, lifecycle result, or durable config.
|
|
"""
|
|
_require_cell(cell)
|
|
_require_prepared(prepared)
|
|
if not isinstance(runtime, CodexRuntime):
|
|
raise CodexRuntimeError("invalid Codex runtime")
|
|
if not isinstance(task_payload, bytes) or not task_payload:
|
|
raise CodexRuntimeError("Codex task payload is required")
|
|
try:
|
|
task_payload.decode("utf-8")
|
|
except UnicodeDecodeError as exc:
|
|
raise CodexRuntimeError("Codex task payload must be UTF-8") from exc
|
|
if not isinstance(timeout, Timeout):
|
|
raise CodexRuntimeError("invalid timeout")
|
|
if isinstance(codex_executable, str):
|
|
executable_argv = (codex_executable,)
|
|
elif isinstance(codex_executable, tuple) and codex_executable and all(
|
|
isinstance(item, str) and item for item in codex_executable
|
|
):
|
|
# Test-only seam for a non-executable fixture script; normal callers
|
|
# always receive the exact single-token ``codex`` command above.
|
|
executable_argv = codex_executable
|
|
else:
|
|
raise CodexRuntimeError("invalid Codex executable")
|
|
|
|
codex_argv: list[str] = [
|
|
*executable_argv, "exec", "--json", "--ephemeral", "--ignore-user-config",
|
|
"--strict-config", "--skip-git-repo-check", "-C", prepared.workspace_dir,
|
|
"-m", cell.iop.request_model,
|
|
]
|
|
for override in _provider_overrides(cell, runtime):
|
|
codex_argv.extend(("-c", override))
|
|
codex_argv.append("-")
|
|
return InvocationSpec(
|
|
# The lifecycle changes cwd to the isolated workspace, so use this
|
|
# module's absolute script path rather than relying on repository
|
|
# import resolution in the child bridge.
|
|
argv=(sys.executable, str(Path(__file__).resolve()), "--bridge", "--", *codex_argv),
|
|
cwd=prepared.workspace_dir,
|
|
env=env_pairs({"PATH": runtime.path, "HOME": prepared.session_dir, SECRET_ENV_KEY: runtime.api_key}),
|
|
env_allowlist=(SECRET_ENV_KEY,),
|
|
submission_mode=SUBMISSION_STDIN_ONCE,
|
|
completion_mode=COMPLETION_EXIT_AFTER_IDLE,
|
|
timeout=timeout,
|
|
evidence_dir=prepared.attempt_root,
|
|
task_payload=task_payload,
|
|
control_dir=str(Path(prepared.attempt_root) / "codex-control"),
|
|
)
|
|
|
|
|
|
def redact_codex_jsonl(text: str, secrets: tuple[str, ...]) -> str:
|
|
"""Remove JSON content and runtime values before any line becomes evidence."""
|
|
exact = exact_value_redactor(secrets)
|
|
try:
|
|
value = json.loads(text)
|
|
except (TypeError, json.JSONDecodeError):
|
|
return "[redacted]"
|
|
|
|
def redact(value: Any, key: str | None = None) -> Any:
|
|
if key is not None and key.lower() in _SENSITIVE_KEYS:
|
|
return "[redacted]"
|
|
if isinstance(value, dict):
|
|
return {str(item_key): redact(item_value, str(item_key)) for item_key, item_value in value.items()}
|
|
if isinstance(value, list):
|
|
return [redact(item) for item in value]
|
|
if isinstance(value, str):
|
|
return exact(value) if key in _SAFE_STRING_KEYS else "[redacted]"
|
|
return value
|
|
|
|
return json.dumps(redact(value), ensure_ascii=True, separators=(",", ":"))
|
|
|
|
|
|
class CodexJSONLParser:
|
|
"""Parse only terminal evidence and optional explicit IOP effective binding."""
|
|
|
|
def __init__(self, cell: MatrixCell, idle_nonce: str) -> None:
|
|
_require_cell(cell)
|
|
if not isinstance(idle_nonce, str) or len(idle_nonce) < 16:
|
|
raise CodexRuntimeError("invalid bridge idle nonce")
|
|
self._cell = cell
|
|
self._idle_nonce = idle_nonce
|
|
self._effective_binding: tuple[str, str, str, str] | None = None
|
|
|
|
@property
|
|
def effective_binding(self) -> tuple[str, str, str, str] | None:
|
|
return self._effective_binding
|
|
|
|
def connectivity_result(self) -> ConnectivityResult:
|
|
"""Classify absent evidence without synthesizing unreported stage bindings."""
|
|
iop = self._cell.iop
|
|
if self._effective_binding is not None:
|
|
# Codex's optional scalar observation proves neither the complete
|
|
# stage list nor its exact order. Never manufacture that missing
|
|
# contract evidence from the manifest just to produce ``ready``.
|
|
raise CodexJSONLError("effective stage binding is unavailable")
|
|
binding = RequestedEffectiveBinding(
|
|
self._cell.id, self._cell.caller, iop.route_kind, iop.route_id,
|
|
iop.request_model, iop.requested_effort,
|
|
None, None, None, None, (),
|
|
)
|
|
issues: tuple[ConnectivityIssue, ...] = (
|
|
ConnectivityIssue("stream_incompatible", ISSUE_RESUME_CODES["stream_incompatible"]),
|
|
)
|
|
return make_result(self._cell, codex_capability(), binding, issues)
|
|
|
|
def parse(self, stream: str, line: str) -> str | None:
|
|
if stream != "stdout":
|
|
return None
|
|
try:
|
|
record = json.loads(line)
|
|
except (TypeError, json.JSONDecodeError) as exc:
|
|
raise CodexJSONLError("malformed Codex JSONL") from exc
|
|
if not isinstance(record, dict) or not isinstance(record.get("type"), str):
|
|
raise CodexJSONLError("malformed Codex JSONL")
|
|
self._observe_effective_binding(record)
|
|
if record["type"] == "turn.completed":
|
|
status = record.get("status")
|
|
if status is not None and status not in ("completed", "success"):
|
|
raise CodexJSONLError("unsuccessful Codex terminal turn")
|
|
return "finish"
|
|
if record["type"] == _BRIDGE_IDLE_TYPE:
|
|
if record != {"type": _BRIDGE_IDLE_TYPE, "adapter": _BRIDGE_ID, "nonce": self._idle_nonce, "child_exit": 0}:
|
|
raise CodexJSONLError("unverified bridge idle marker")
|
|
return "idle"
|
|
return None
|
|
|
|
def _observe_effective_binding(self, record: dict[str, Any]) -> None:
|
|
observed = record.get("iop_effective_binding")
|
|
if observed is None:
|
|
return
|
|
if not isinstance(observed, dict) or set(observed) != {"route_kind", "route_id", "model", "effort"}:
|
|
raise CodexJSONLError("invalid effective binding observation")
|
|
values = tuple(observed[key] for key in ("route_kind", "route_id", "model", "effort"))
|
|
if not all(isinstance(value, str) and TOKEN_RE.fullmatch(value) for value in values):
|
|
raise CodexJSONLError("invalid effective binding observation")
|
|
expected = (
|
|
self._cell.iop.route_kind,
|
|
self._cell.iop.route_id,
|
|
self._cell.iop.request_model,
|
|
self._cell.iop.requested_effort,
|
|
)
|
|
if values != expected:
|
|
raise CodexJSONLError("effective binding substitution")
|
|
if self._effective_binding is not None:
|
|
raise CodexJSONLError("duplicate effective binding observation")
|
|
self._effective_binding = values # type: ignore[assignment]
|
|
|
|
|
|
def build_codex_invocation(
|
|
cell: MatrixCell,
|
|
prepared: PreparedWorkspace,
|
|
runtime: CodexRuntime,
|
|
task_payload: bytes,
|
|
timeout: Timeout,
|
|
*,
|
|
codex_executable: str | tuple[str, ...] = "codex",
|
|
) -> CodexInvocation:
|
|
"""Pair the closed invocation spec with its parser and structural redactor."""
|
|
nonce = secrets.token_hex(16)
|
|
spec = build_codex_spec(cell, prepared, runtime, task_payload, timeout, codex_executable=codex_executable)
|
|
parser = CodexJSONLParser(cell, nonce)
|
|
# The nonce must reach the bridge but not Codex. Put it before the bridge
|
|
# delimiter so the bridge removes it before starting the child.
|
|
argv = (*spec.argv[:3], f"--idle-nonce={nonce}", *spec.argv[3:])
|
|
task_text = task_payload.decode("utf-8")
|
|
return CodexInvocation(spec=InvocationSpec(**{**spec.__dict__, "argv": argv}), parser=parser,
|
|
redact=lambda line: redact_codex_jsonl(line, (runtime.base_url, runtime.api_key, task_text)))
|
|
|
|
|
|
def run_codex_invocation(
|
|
invocation: CodexInvocation,
|
|
on_started: Callable[[SupervisorLocator], None],
|
|
) -> CodexInvocationResult:
|
|
"""Run exactly one prepared Codex invocation through the generic lifecycle."""
|
|
if not isinstance(invocation, CodexInvocation) or not callable(on_started):
|
|
raise CodexRuntimeError("invalid Codex invocation")
|
|
result = run_invocation(invocation.spec, parse_event=invocation.parser.parse,
|
|
on_started=on_started, redact=invocation.redact)
|
|
return CodexInvocationResult(result, invocation.parser.effective_binding)
|
|
|
|
|
|
def _forward_stream(stream: Any, destination: Any) -> None:
|
|
try:
|
|
for line in iter(stream.readline, b""):
|
|
destination.buffer.write(line)
|
|
destination.buffer.flush()
|
|
finally:
|
|
stream.close()
|
|
|
|
|
|
def _bridge(argv: list[str]) -> int:
|
|
"""Own a child Codex process and emit idle only after exit plus both EOFs."""
|
|
if not argv or not argv[0].startswith("--idle-nonce="):
|
|
return 64
|
|
nonce = argv.pop(0).partition("=")[2]
|
|
if not nonce or not argv or argv.pop(0) != "--":
|
|
return 64
|
|
try:
|
|
child = subprocess.Popen(argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
|
|
except OSError:
|
|
return 127
|
|
assert child.stdin is not None and child.stdout is not None and child.stderr is not None
|
|
payload = sys.stdin.buffer.read()
|
|
try:
|
|
child.stdin.write(payload)
|
|
child.stdin.close()
|
|
except OSError:
|
|
child.terminate()
|
|
stdout_thread = threading.Thread(target=_forward_stream, args=(child.stdout, sys.stdout), daemon=True)
|
|
stderr_thread = threading.Thread(target=_forward_stream, args=(child.stderr, sys.stderr), daemon=True)
|
|
stdout_thread.start()
|
|
stderr_thread.start()
|
|
exit_code = child.wait()
|
|
stdout_thread.join()
|
|
stderr_thread.join()
|
|
if exit_code == 0:
|
|
sys.stdout.write(json.dumps({"type": _BRIDGE_IDLE_TYPE, "adapter": _BRIDGE_ID, "nonce": nonce, "child_exit": 0}, separators=(",", ":")) + "\n")
|
|
sys.stdout.flush()
|
|
return exit_code
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
values = list(sys.argv[1:] if argv is None else argv)
|
|
if not values or values.pop(0) != "--bridge":
|
|
return 64
|
|
return _bridge(values)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|