"""Explicit, secret-safe live IOP boundary for benchmark callers. Only the six ``IOP_BENCH__{BASE_URL,SECRET_ENV}`` values select this boundary. The referenced secret is retained in a private runtime object and is never included in connectivity evidence, command output, or run metadata. """ from __future__ import annotations import hashlib import json import os import re import shutil import stat import subprocess import tempfile from collections.abc import Callable, Mapping from contextlib import ExitStack, contextmanager from dataclasses import dataclass, replace from pathlib import Path from typing import Any, Iterator from urllib.parse import urlsplit, urlunsplit from urllib.error import HTTPError from urllib.request import Request, urlopen from scripts.agent_benchmark.agy_iop import ( AGY_CALLER, AgyEventParser, AgyRuntimeInputs, AgyRuntimeObservation, build_agy_invocation, inspect_agy_iop_capability, preflight_agy_iop, run_agy_invocation, _runtime_identity as _agy_runtime_identity, ) from scripts.agent_benchmark.attempts import Attempt, ExecutionAdapter, PreflightObservation from scripts.agent_benchmark.claude_iop import ClaudeIopAdapter, ClaudeIopRuntime, claude_capability from scripts.agent_benchmark.codex_iop import ( BASE_URL_ENV_KEY, SECRET_ENV_KEY, build_codex_invocation, codex_capability, run_codex_invocation, runtime_from_environment, ) from scripts.agent_benchmark.connectivity import ( ISSUE_CODE_ORDER, ISSUE_RESUME_CODES, CallerCapability, ConnectivityIssue, EffectiveBinding, RequestedEffectiveBinding, make_result, ) from scripts.agent_benchmark.lifecycle import InvocationResult, run_invocation, spec_digest from scripts.agent_benchmark.manifest import ( CALLER_ENUM, STAGE_ENUM, MatrixCell, TOKEN_RE, Timeout, ) from scripts.agent_benchmark.scoring import ( BlindWorkspace, ScoringAdapter, ScoringEvidenceFinalization, ScoringInvocationResult, ) from scripts.agent_benchmark.workspace import ( AttemptIdentity, PreparedWorkspace, TestbedProvenance, ) _ENV_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]{0,63}$") _CALLERS = ("claude", "agy", "codex") _TIMEOUT_SECONDS = 10 _SECRET_SCAN_MAX_ENTRIES = 100_000 _SECRET_SCAN_MAX_DEPTH = 64 _SECRET_SCAN_MAX_FILE_BYTES = 32 * 1024 * 1024 class LiveIopError(Exception): """Private boundary failure converted into one closed public issue.""" def __init__(self, issue_code: str) -> None: if issue_code not in ISSUE_RESUME_CODES: raise ValueError("invalid live IOP issue") self.issue_code = issue_code super().__init__(issue_code) @dataclass(frozen=True) class _Runtime: caller: str base_url: str secret: str endpoint_identity: str config: "_ConfigObservation" @dataclass(frozen=True) class _RouteObservation: route_kind: str route_id: str model: str bindings: tuple[EffectiveBinding, ...] @dataclass(frozen=True) class _ConfigObservation: routes: tuple[_RouteObservation, ...] identity: str @dataclass(frozen=True) class _Observation: catalog_models: tuple[str, ...] config_identity: str caller_ready: bool agy_version: str = "" agy_help: str = "" @dataclass(frozen=True) class _RuntimeResolution: runtime: _Runtime | None issue_code: str | None @dataclass(frozen=True) class _InvokerSeams: """Test seams below registry construction; production uses real callers.""" claude: Callable[..., InvocationResult] agy: Callable[..., InvocationResult] codex: Callable[..., Any] def _identity(label: str, value: str) -> str: return "sha256:" + hashlib.sha256( b"iop-benchmark-live-v1\0" + label.encode("ascii") + b"\0" + value.encode("utf-8") ).hexdigest() def _base_url(value: str) -> str: if not isinstance(value, str) or not value: raise LiveIopError("endpoint_incompatible") parsed = urlsplit(value) if parsed.scheme not in ("http", "https") or not parsed.netloc or parsed.query or parsed.fragment: raise LiveIopError("endpoint_incompatible") return value.rstrip("/") def _models_url(base_url: str) -> str: parsed = urlsplit(base_url) path = parsed.path.rstrip("/") if path.endswith("/v1"): path += "/models" else: path += "/v1/models" return urlunsplit((parsed.scheme, parsed.netloc, path, "", "")) def _command(argv: tuple[str, ...]) -> str: try: completed = subprocess.run( argv, check=False, capture_output=True, text=True, timeout=_TIMEOUT_SECONDS, env={"PATH": os.environ.get("PATH", "/usr/bin:/bin"), "LANG": "C.UTF-8", "LC_ALL": "C.UTF-8"}, ) except (OSError, subprocess.SubprocessError) as exc: raise LiveIopError("stream_incompatible") from exc if completed.returncode != 0: raise LiveIopError("stream_incompatible") return completed.stdout + completed.stderr def _caller_binary(name: str) -> str: path = shutil.which(name) if not path or not os.path.isfile(path) or not os.access(path, os.X_OK): raise LiveIopError("stream_incompatible") return os.path.realpath(path) def _catalog(runtime: _Runtime) -> tuple[tuple[str, ...], str]: request = Request(_models_url(runtime.base_url), headers={"Authorization": f"Bearer {runtime.secret}"}) try: with urlopen(request, timeout=_TIMEOUT_SECONDS) as response: if response.status in (401, 403): raise LiveIopError("auth_incompatible") if response.status != 200: raise LiveIopError("endpoint_incompatible") payload = json.loads(response.read().decode("utf-8")) except HTTPError as exc: if exc.code in (401, 403): raise LiveIopError("auth_incompatible") from exc raise LiveIopError("endpoint_incompatible") from exc except LiveIopError: raise except OSError as exc: # Connection refusal, DNS failure, and timeout all mean that this # boundary cannot reach a compatible endpoint. They are distinct # from a reachable endpoint with an invalid response schema. raise LiveIopError("endpoint_incompatible") from exc except ValueError as exc: raise LiveIopError("protocol_incompatible") from exc records = payload.get("data") if isinstance(payload, dict) else None if not isinstance(records, list): raise LiveIopError("protocol_incompatible") model_ids: list[str] = [] for item in records: model_id = item.get("id") if isinstance(item, dict) else None if not isinstance(model_id, str) or not model_id.strip(): raise LiveIopError("protocol_incompatible") model_ids.append(model_id) models = tuple(sorted(model_ids)) if not models or len(set(models)) != len(models): raise LiveIopError("protocol_incompatible") return models, _identity("catalog", "\n".join(models)) def _observe(runtime: _Runtime) -> _Observation: models, config_identity = _catalog(runtime) if runtime.caller == "claude": _command(("claude", "--version")) _command(("claude", "--help")) return _Observation(models, config_identity, True) if runtime.caller == AGY_CALLER: version = _command(("agy", "--version")) help_output = _command(("agy", "--help")) return _Observation(models, config_identity, True, version, help_output) if runtime.caller == "codex": _command(("codex", "--version")) _command(("codex", "exec", "--help")) return _Observation(models, config_identity, True) raise LiveIopError("protocol_incompatible") def _config_from_environment(environment: Mapping[str, str]) -> _ConfigObservation: reference = environment.get("IOP_BENCH_CONFIG_OBSERVATION_ENV") if not isinstance(reference, str) or not _ENV_NAME.fullmatch(reference): raise LiveIopError("route_missing") raw = environment.get(reference) if not isinstance(raw, str) or not raw: raise LiveIopError("route_missing") try: value = json.loads(raw) except (TypeError, json.JSONDecodeError) as exc: raise LiveIopError("protocol_incompatible") from exc if not isinstance(value, dict) or set(value) != {"schema_version", "routes"} or value.get("schema_version") != "1": raise LiveIopError("protocol_incompatible") raw_routes = value.get("routes") if not isinstance(raw_routes, list) or not raw_routes: raise LiveIopError("route_missing") routes: list[_RouteObservation] = [] for item in raw_routes: if not isinstance(item, dict) or set(item) != { "route_kind", "route_id", "model", "bindings" }: raise LiveIopError("protocol_incompatible") route_kind, route_id, model = (item.get(name) for name in ("route_kind", "route_id", "model")) if route_kind not in ("direct", "execution_preset") or not all(isinstance(field, str) and TOKEN_RE.fullmatch(field) for field in (route_id, model)): raise LiveIopError("protocol_incompatible") raw_bindings = item.get("bindings") if not isinstance(raw_bindings, list) or not raw_bindings: raise LiveIopError("protocol_incompatible") bindings: list[EffectiveBinding] = [] stages: set[str] = set() for raw_binding in raw_bindings: if not isinstance(raw_binding, dict) or set(raw_binding) != { "stage", "model", "effort" }: raise LiveIopError("protocol_incompatible") stage = raw_binding.get("stage") bound_model = raw_binding.get("model") effort = raw_binding.get("effort") if ( stage not in STAGE_ENUM or stage in stages or not isinstance(bound_model, str) or not TOKEN_RE.fullmatch(bound_model) or ( effort is not None and ( not isinstance(effort, str) or not TOKEN_RE.fullmatch(effort) ) ) ): raise LiveIopError("protocol_incompatible") stages.add(stage) bindings.append(EffectiveBinding(stage, bound_model, effort)) routes.append( _RouteObservation(route_kind, route_id, model, tuple(bindings)) ) if len({(item.route_kind, item.route_id) for item in routes}) != len(routes): raise LiveIopError("protocol_incompatible") routes.sort(key=lambda item: (item.route_kind, item.route_id, item.model)) canonical_routes = [ { "route_kind": item.route_kind, "route_id": item.route_id, "model": item.model, "bindings": [ { "stage": binding.stage, "model": binding.model, "effort": binding.effort, } for binding in item.bindings ], } for item in routes ] canonical = json.dumps( {"schema_version": "1", "routes": canonical_routes}, sort_keys=True, separators=(",", ":"), ensure_ascii=True, ) return _ConfigObservation(tuple(routes), _identity("config", canonical)) def _runtime_from_environment(caller: str, environment: Mapping[str, str]) -> _RuntimeResolution: prefix = f"IOP_BENCH_{caller.upper()}_" base_key = prefix + "BASE_URL" secret_ref_key = prefix + "SECRET_ENV" base_url = environment.get(base_key) secret_ref = environment.get(secret_ref_key) try: normalized = _base_url(base_url) except LiveIopError as exc: return _RuntimeResolution(None, exc.issue_code) if not isinstance(secret_ref, str) or not _ENV_NAME.fullmatch(secret_ref): return _RuntimeResolution(None, "credential_missing") secret = environment.get(secret_ref) if not isinstance(secret, str) or not secret: return _RuntimeResolution(None, "credential_missing") try: config = _config_from_environment(environment) except LiveIopError as exc: return _RuntimeResolution(None, exc.issue_code) return _RuntimeResolution(_Runtime(caller, normalized, secret, _identity("endpoint", normalized), config), None) def _requested(cell: MatrixCell) -> RequestedEffectiveBinding: return RequestedEffectiveBinding( cell.id, cell.caller, cell.iop.route_kind, cell.iop.route_id, cell.iop.request_model, cell.iop.requested_effort, ) def _issues(*codes: str) -> tuple[ConnectivityIssue, ...]: selected = set(codes) return tuple(ConnectivityIssue(code, ISSUE_RESUME_CODES[code]) for code in ISSUE_CODE_ORDER if code in selected) def _binding_from_config(cell: MatrixCell, capability: CallerCapability, config: _ConfigObservation) -> tuple[RequestedEffectiveBinding, tuple[ConnectivityIssue, ...]]: """Admit a manifest route from config ownership and caller capability only.""" requested = _requested(cell) if cell.iop.route_kind not in capability.route_kinds: return requested, _issues("protocol_incompatible") if cell.iop.requested_effort not in capability.efforts: return requested, _issues("effort_unsupported") route = next((item for item in config.routes if (item.route_kind, item.route_id) == (cell.iop.route_kind, cell.iop.route_id)), None) if route is None: return requested, _issues("route_missing") if route.model != cell.iop.request_model: return requested, _issues("model_missing") expected = tuple( EffectiveBinding(binding.stage, binding.model, binding.effort) for binding in cell.iop.expected_bindings ) if route.bindings != expected: return requested, _issues("protocol_incompatible") return RequestedEffectiveBinding( cell.id, cell.caller, cell.iop.route_kind, cell.iop.route_id, cell.iop.request_model, cell.iop.requested_effort, route.route_kind, route.route_id, route.model, cell.iop.requested_effort, route.bindings, ), () def _bound_observations( result: InvocationResult, admitted: RequestedEffectiveBinding ) -> InvocationResult: """Return the lifecycle result only when every observation stays bound. Typed observations ride on the lifecycle result, so this boundary keeps the result intact and refuses any observation whose model label names something other than the admitted binding. """ if not isinstance(result, InvocationResult): raise LiveIopError("stream_incompatible") admitted_models = { value for value in (admitted.effective_model, admitted.requested_model) if value } admitted_models.update( binding.model for binding in admitted.effective_bindings if binding.model ) admitted_by_stage = { binding.stage: binding.model for binding in admitted.effective_bindings if binding.stage and binding.model } for metric in result.metrics: if ( metric.stage and metric.model and admitted_by_stage.get(metric.stage) != metric.model ): raise LiveIopError("stream_incompatible") if metric.model and metric.model not in admitted_models: raise LiveIopError("stream_incompatible") return result def _default_claude_invoker(adapter: ClaudeIopAdapter, spec: Any, **kwargs: Any) -> InvocationResult: return run_invocation(spec, **kwargs) def _default_agy_invoker(spec: Any, parser: AgyEventParser, preflight: Any, on_started: Callable[..., None]) -> InvocationResult: return run_agy_invocation(spec, parser, preflight, on_started) def _default_codex_invoker(invocation: Any, on_started: Callable[..., None]) -> Any: return run_codex_invocation(invocation, on_started) _DEFAULT_INVOKERS = _InvokerSeams(_default_claude_invoker, _default_agy_invoker, _default_codex_invoker) @contextmanager def _temporary_owned_permissions( path: Path, required: int, *, expected: os.stat_result | None = None, ) -> Iterator[os.stat_result]: """Grant minimum owner access and restore a retained inode exactly.""" try: info = os.lstat(path) except OSError as exc: raise LiveIopError("stream_incompatible") from exc current_uid = getattr(os, "geteuid", lambda: info.st_uid)() if ( stat.S_ISLNK(info.st_mode) or info.st_uid != current_uid or ( expected is not None and (info.st_dev, info.st_ino) != (expected.st_dev, expected.st_ino) ) ): raise LiveIopError("stream_incompatible") original_mode = stat.S_IMODE(info.st_mode) temporary_mode = original_mode | required changed = temporary_mode != original_mode try: if changed: os.chmod(path, temporary_mode, follow_symlinks=False) current = os.lstat(path) if ( stat.S_ISLNK(current.st_mode) or current.st_uid != current_uid or (current.st_dev, current.st_ino) != (info.st_dev, info.st_ino) or stat.S_IMODE(current.st_mode) & required != required ): raise LiveIopError("stream_incompatible") yield current except OSError as exc: raise LiveIopError("stream_incompatible") from exc finally: if changed: try: current = os.lstat(path) if ( stat.S_ISLNK(current.st_mode) or current.st_uid != current_uid or (current.st_dev, current.st_ino) != (info.st_dev, info.st_ino) ): raise LiveIopError("stream_incompatible") os.chmod(path, original_mode, follow_symlinks=False) restored = os.lstat(path) if ( (restored.st_dev, restored.st_ino) != (info.st_dev, info.st_ino) or stat.S_IMODE(restored.st_mode) != original_mode ): raise LiveIopError("stream_incompatible") except OSError as exc: raise LiveIopError("stream_incompatible") from exc @contextmanager def _temporary_directory_chain( root: Path, parent: Path, *, writable_parent: bool = False ) -> Iterator[None]: try: relative = parent.relative_to(root) except ValueError as exc: raise LiveIopError("stream_incompatible") from exc chain = [root] current = root for part in relative.parts: current = current / part chain.append(current) with ExitStack() as stack: for index, directory in enumerate(chain): required = stat.S_IXUSR if writable_parent and index == len(chain) - 1: required |= stat.S_IWUSR stack.enter_context(_temporary_owned_permissions(directory, required)) yield def _owned_plain_directory(path: Path) -> Path: try: info = os.lstat(path) except OSError as exc: raise LiveIopError("stream_incompatible") from exc current_uid = getattr(os, "geteuid", lambda: info.st_uid)() if ( not stat.S_ISDIR(info.st_mode) or stat.S_ISLNK(info.st_mode) or info.st_uid != current_uid ): raise LiveIopError("stream_incompatible") with _temporary_owned_permissions(path, stat.S_IXUSR, expected=info): try: resolved = path.resolve(strict=True) except OSError as exc: raise LiveIopError("stream_incompatible") from exc return resolved def _blind_controlled_roots( blind: BlindWorkspace, ) -> tuple[Path, tuple[Path, Path, Path]]: blind_path = Path(blind.root) blind_root = _owned_plain_directory(blind_path) controlled = ( _owned_plain_directory(Path(blind.input_dir)), _owned_plain_directory(Path(blind.session_dir)), _owned_plain_directory(Path(blind.output_dir)), ) expected = tuple(blind_root / name for name in ("input", "session", "output")) if controlled != expected: raise LiveIopError("stream_incompatible") return blind_root, controlled def _walk_no_follow(root: Path) -> tuple[tuple[Path, os.stat_result], ...]: entries: list[tuple[Path, os.stat_result]] = [] def visit( directory: Path, depth: int, expected: os.stat_result | None = None ) -> None: if depth > _SECRET_SCAN_MAX_DEPTH: raise LiveIopError("stream_incompatible") with _temporary_owned_permissions( directory, stat.S_IRUSR | stat.S_IXUSR, expected=expected ): try: with os.scandir(directory) as iterator: children = sorted(iterator, key=lambda item: item.name) except OSError as exc: raise LiveIopError("stream_incompatible") from exc for entry in children: try: info = entry.stat(follow_symlinks=False) except OSError as exc: raise LiveIopError("stream_incompatible") from exc path = directory / entry.name entries.append((path, info)) if len(entries) > _SECRET_SCAN_MAX_ENTRIES: raise LiveIopError("stream_incompatible") if stat.S_ISDIR(info.st_mode) and not stat.S_ISLNK(info.st_mode): visit(path, depth + 1, info) visit(root, 0) return tuple(entries) def _read_bounded_regular( root: Path, path: Path, expected: os.stat_result ) -> bytes: if expected.st_size > _SECRET_SCAN_MAX_FILE_BYTES: raise LiveIopError("stream_incompatible") flags = os.O_RDONLY | os.O_CLOEXEC | os.O_NONBLOCK if hasattr(os, "O_NOFOLLOW"): flags |= os.O_NOFOLLOW with _temporary_directory_chain(root, path.parent): with _temporary_owned_permissions(path, stat.S_IRUSR, expected=expected): try: descriptor = os.open(path, flags) except OSError as exc: raise LiveIopError("stream_incompatible") from exc try: current = os.fstat(descriptor) if ( not stat.S_ISREG(current.st_mode) or current.st_uid != getattr(os, "geteuid", lambda: current.st_uid)() or current.st_dev != expected.st_dev or current.st_ino != expected.st_ino or current.st_size != expected.st_size or current.st_size > _SECRET_SCAN_MAX_FILE_BYTES ): raise LiveIopError("stream_incompatible") data = bytearray() while len(data) < current.st_size: chunk = os.read(descriptor, current.st_size - len(data)) if not chunk: raise LiveIopError("stream_incompatible") data.extend(chunk) if os.read(descriptor, 1): raise LiveIopError("stream_incompatible") final = os.fstat(descriptor) if ( final.st_size != current.st_size or final.st_mtime_ns != current.st_mtime_ns ): raise LiveIopError("stream_incompatible") return bytes(data) except OSError as exc: raise LiveIopError("stream_incompatible") from exc finally: os.close(descriptor) def _contains_sensitive(data: bytes, sensitive: tuple[bytes, ...]) -> bool: return any(value in data for value in sensitive) def _relative_bytes(root: Path, path: Path) -> bytes: try: return os.fsencode(path.relative_to(root).as_posix()) except ValueError as exc: raise LiveIopError("stream_incompatible") from exc def _unlink_owned_entry( root: Path, path: Path, expected: os.stat_result, *, directory: bool, ) -> None: with _temporary_directory_chain(root, path.parent, writable_parent=True): try: current = os.lstat(path) current_uid = getattr(os, "geteuid", lambda: current.st_uid)() if ( current.st_uid != current_uid or (current.st_dev, current.st_ino) != (expected.st_dev, expected.st_ino) or (directory and not stat.S_ISDIR(current.st_mode)) or (not directory and stat.S_ISDIR(current.st_mode)) ): raise LiveIopError("stream_incompatible") if directory: path.rmdir() else: path.unlink() except OSError as exc: raise LiveIopError("stream_incompatible") from exc @dataclass class _SanitizationOutcome: secret: bool = False input_invalid: bool = False output_invalid: bool = False def record(self, root_kind: str, *, secret: bool, invalid: bool) -> None: self.secret = self.secret or secret if invalid and root_kind == "input": self.input_invalid = True elif invalid: self.output_invalid = True def finalization(self) -> ScoringEvidenceFinalization: if self.secret: return ScoringEvidenceFinalization(False, "runtime_secret_leak") if self.input_invalid: return ScoringEvidenceFinalization(False, "input_mutated") if self.output_invalid: return ScoringEvidenceFinalization(False, "evaluator_output_leak") return ScoringEvidenceFinalization(True) def _remove_sensitive_blind_paths( roots: tuple[Path, Path, Path], sensitive: tuple[bytes, ...] ) -> _SanitizationOutcome: outcome = _SanitizationOutcome() for root_kind, root in zip(("input", "session", "output"), roots): entries = sorted( _walk_no_follow(root), key=lambda item: (len(item[0].parts), item[0].as_posix()), reverse=True, ) for path, info in entries: current_uid = getattr(os, "geteuid", lambda: info.st_uid)() if info.st_uid != current_uid: raise LiveIopError("stream_incompatible") path_leak = _contains_sensitive(_relative_bytes(root, path), sensitive) mode = info.st_mode if stat.S_ISDIR(mode) and not stat.S_ISLNK(mode): if path_leak: _unlink_owned_entry(root, path, info, directory=True) outcome.record(root_kind, secret=True, invalid=False) continue if stat.S_ISLNK(mode): with _temporary_directory_chain(root, path.parent): try: link_value = os.fsencode(os.readlink(path)) current = os.lstat(path) except OSError as exc: raise LiveIopError("stream_incompatible") from exc if (current.st_dev, current.st_ino) != (info.st_dev, info.st_ino): raise LiveIopError("stream_incompatible") link_leak = path_leak or _contains_sensitive(link_value, sensitive) _unlink_owned_entry(root, path, info, directory=False) outcome.record(root_kind, secret=link_leak, invalid=not link_leak) continue if stat.S_ISREG(mode): content_leak = False if not path_leak: content_leak = _contains_sensitive( _read_bounded_regular(root, path, info), sensitive ) if path_leak or content_leak: _unlink_owned_entry(root, path, info, directory=False) outcome.record(root_kind, secret=True, invalid=False) continue if stat.S_ISSOCK(mode): if path_leak: _unlink_owned_entry(root, path, info, directory=False) outcome.record(root_kind, secret=True, invalid=False) continue if path_leak: _unlink_owned_entry(root, path, info, directory=False) outcome.record(root_kind, secret=True, invalid=False) continue raise LiveIopError("stream_incompatible") return outcome def _freeze_sanitized_input(root: Path) -> None: entries = _walk_no_follow(root) directories = [root] current_uid = getattr(os, "geteuid", lambda: os.lstat(root).st_uid)() for path, info in entries: if info.st_uid != current_uid: raise LiveIopError("stream_incompatible") try: if stat.S_ISDIR(info.st_mode) and not stat.S_ISLNK(info.st_mode): directories.append(path) elif stat.S_ISREG(info.st_mode) and not stat.S_ISLNK(info.st_mode): os.chmod(path, 0o400, follow_symlinks=False) else: raise LiveIopError("stream_incompatible") except OSError as exc: raise LiveIopError("stream_incompatible") from exc for directory in sorted( directories, key=lambda item: len(item.parts), reverse=True ): try: os.chmod(directory, 0o500, follow_symlinks=False) except OSError as exc: raise LiveIopError("stream_incompatible") from exc def _verify_sensitive_absent(root: Path, sensitive: tuple[bytes, ...]) -> None: root = _owned_plain_directory(root) for path, info in _walk_no_follow(root): if _contains_sensitive(_relative_bytes(root, path), sensitive): raise LiveIopError("stream_incompatible") if stat.S_ISREG(info.st_mode): if _contains_sensitive( _read_bounded_regular(root, path, info), sensitive ): raise LiveIopError("stream_incompatible") elif stat.S_ISLNK(info.st_mode): with _temporary_directory_chain(root, path.parent): try: link_value = os.fsencode(os.readlink(path)) except OSError as exc: raise LiveIopError("stream_incompatible") from exc if _contains_sensitive(link_value, sensitive): raise LiveIopError("stream_incompatible") def _bind_live_spec( cell: MatrixCell, prepared: PreparedWorkspace, attempt: Attempt, control_dir: str, spec: Any, ) -> Any: """Immutably bind one caller spec to the controller-owned attempt paths.""" if ( not isinstance(prepared, PreparedWorkspace) or not isinstance(attempt, Attempt) or cell.id != attempt.identity.cell_id or prepared.identity != attempt.identity or not isinstance(control_dir, str) or not control_dir ): raise LiveIopError("stream_incompatible") try: attempt_root = Path(attempt.root).resolve(strict=True) prepared_root = Path(prepared.attempt_root).resolve(strict=True) workspace_root = Path(prepared.workspace_dir).resolve(strict=True) session_root = Path(prepared.session_dir).resolve(strict=True) evidence_root = Path(spec.evidence_dir).resolve(strict=True) control_path = Path(control_dir) alias = control_path.parent if ( not control_path.is_absolute() or control_path.name != "control" or control_path.exists() or control_path.is_symlink() or not alias.is_symlink() or alias.resolve(strict=True) != attempt_root or control_path.resolve(strict=False) != attempt_root / "control" ): raise LiveIopError("stream_incompatible") except (OSError, RuntimeError, TypeError, ValueError) as exc: raise LiveIopError("stream_incompatible") from exc if ( prepared_root != attempt_root or evidence_root != attempt_root or workspace_root != attempt_root / "workspace" or session_root != attempt_root / "session" ): raise LiveIopError("stream_incompatible") return replace(spec, control_dir=control_dir) class _LiveAdapter: """One caller's live observation and invocation boundary.""" def __init__( self, caller: str, capability: CallerCapability, runtime: _RuntimeResolution, observer: Callable[[_Runtime], _Observation] = _observe, binary_resolver: Callable[[str], str] = _caller_binary, invokers: _InvokerSeams = _DEFAULT_INVOKERS, ) -> None: self.caller = caller self.capability = capability self._runtime_resolution = runtime self._observer = observer self._binary_resolver = binary_resolver self._invokers = invokers self._agy_preflight: Any = None self._admitted_bindings: dict[str, RequestedEffectiveBinding] = {} def preflight(self, cell: MatrixCell) -> PreflightObservation: if cell.caller != self.caller: raise LiveIopError("protocol_incompatible") resolution = self._runtime_resolution runtime = resolution.runtime if runtime is None: result = make_result(cell, self.capability, _requested(cell), _issues(resolution.issue_code or "credential_missing")) return PreflightObservation(result, _identity("missing", self.caller), _identity("missing-config", self.caller)) try: observed = self._observer(runtime) except LiveIopError as exc: result = make_result(cell, self.capability, _requested(cell), _issues(exc.issue_code)) return PreflightObservation(result, runtime.endpoint_identity, _identity("unobserved-config", self.caller)) if not observed.caller_ready: result = make_result(cell, self.capability, _requested(cell), _issues("stream_incompatible")) return PreflightObservation(result, runtime.endpoint_identity, runtime.config.identity) if self.caller == AGY_CALLER: capability = inspect_agy_iop_capability(observed.agy_version, observed.agy_help) agy_observation = AgyRuntimeObservation( cell.id, cell.iop.route_kind, cell.iop.route_id, _agy_runtime_identity("endpoint", runtime.base_url), _agy_runtime_identity("credential", runtime.secret), runtime.config.identity, ) try: self._agy_preflight = preflight_agy_iop( cell, capability, AgyRuntimeInputs(self._binary_resolver("agy"), runtime.base_url, runtime.secret), agy_observation, ) except Exception: result = make_result(cell, self.capability, _requested(cell), _issues("stream_incompatible")) return PreflightObservation(result, runtime.endpoint_identity, runtime.config.identity) if self._agy_preflight.issues: result = make_result(cell, self.capability, _requested(cell), self._agy_preflight.issues) return PreflightObservation(result, runtime.endpoint_identity, runtime.config.identity) binding, issues = _binding_from_config(cell, self.capability, runtime.config) if not issues and binding.effective_model not in observed.catalog_models: binding, issues = _requested(cell), _issues("model_missing") result = make_result(cell, self.capability, binding, issues) if result.status == "ready": self._admitted_bindings[cell.id] = result.binding return PreflightObservation(result, runtime.endpoint_identity, runtime.config.identity) def invoke( self, cell: MatrixCell, prepared: PreparedWorkspace, attempt: Attempt, control_dir: str, task_payload: bytes, timeout: Timeout, on_started: Callable[..., None], ) -> InvocationResult: if cell.caller != self.caller: raise LiveIopError("protocol_incompatible") runtime = self._runtime_resolution.runtime if runtime is None: raise LiveIopError(self._runtime_resolution.issue_code or "protocol_incompatible") admitted = self._admitted_bindings.get(cell.id) if admitted is None: raise LiveIopError("stream_incompatible") if self.caller == "claude": adapter = ClaudeIopAdapter(cell, prepared, ClaudeIopRuntime("claude", runtime.base_url, runtime.secret)) spec = _bind_live_spec( cell, prepared, attempt, control_dir, adapter.invocation(task_payload.decode("utf-8"), attempt.root, timeout), ) return _bound_observations( self._invokers.claude(adapter, spec, parse_event=adapter.parser(), redact=adapter.redactor(task_payload.decode("utf-8")), on_started=lambda locator: on_started(locator, spec_digest(spec))), admitted, ) if self.caller == AGY_CALLER: if self._agy_preflight is None: raise LiveIopError("stream_incompatible") spec = _bind_live_spec( cell, prepared, attempt, control_dir, build_agy_invocation( cell, prepared, task_payload, timeout, self._agy_preflight ), ) parser = AgyEventParser(cell) result = self._invokers.agy(spec, parser, self._agy_preflight, lambda locator: on_started(locator, spec_digest(spec))) observed = parser.observed_result(self._agy_preflight.capability, result) if observed.status != "ready" or observed.binding != admitted: raise LiveIopError("stream_incompatible") return _bound_observations(result, admitted) if self.caller == "codex": invocation = build_codex_invocation(cell, prepared, runtime_from_environment({BASE_URL_ENV_KEY: runtime.base_url, SECRET_ENV_KEY: runtime.secret, "PATH": os.environ.get("PATH", "/usr/bin:/bin")}), task_payload, timeout) invocation = replace( invocation, spec=_bind_live_spec( cell, prepared, attempt, control_dir, invocation.spec, ), ) result = self._invokers.codex(invocation, lambda locator: on_started(locator, spec_digest(invocation.spec))) expected = (admitted.effective_route_kind, admitted.effective_route_id, admitted.effective_model, admitted.effective_effort) if result.effective_binding != expected: raise LiveIopError("stream_incompatible") return _bound_observations(result.lifecycle, admitted) raise LiveIopError("protocol_incompatible") def build_live_adapter_registry( environment: Mapping[str, str], *, observer: Callable[[_Runtime], _Observation] = _observe, binary_resolver: Callable[[str], str] = _caller_binary, invokers: _InvokerSeams = _DEFAULT_INVOKERS, ) -> dict[str, ExecutionAdapter]: """Build the fixed caller registry without reading ambient caller settings.""" if not isinstance(environment, Mapping): raise LiveIopError("protocol_incompatible") registry: dict[str, ExecutionAdapter] = { "claude": _LiveAdapter("claude", claude_capability(), _runtime_from_environment("claude", environment), observer, binary_resolver, invokers), "agy": _LiveAdapter(AGY_CALLER, CallerCapability(AGY_CALLER, ("direct", "execution_preset"), ("high", "low", "medium")), _runtime_from_environment("agy", environment), observer, binary_resolver, invokers), "codex": _LiveAdapter("codex", codex_capability(), _runtime_from_environment("codex", environment), observer, binary_resolver, invokers), } if tuple(registry) != CALLER_ENUM: raise LiveIopError("protocol_incompatible") return registry class _LiveScoringAdapter: """Codex-only scoring adapter sharing the live config and secret boundary.""" def __init__( self, live: _LiveAdapter, *, invoker: Callable[[Any, Callable[..., None]], Any] = _default_codex_invoker, ) -> None: self.capability = codex_capability() self._live = live self._invoker = invoker self._control_aliases: dict[str, Path] = {} def preflight(self, cell: MatrixCell) -> PreflightObservation: if cell.id != "evaluator" or cell.caller != "codex": raise LiveIopError("protocol_incompatible") return self._live.preflight(cell) def invoke( self, cell: MatrixCell, blind: BlindWorkspace, task_payload: bytes, timeout: Timeout, on_started: Callable[..., None], ) -> ScoringInvocationResult: if cell.id != "evaluator" or cell.caller != "codex": raise LiveIopError("protocol_incompatible") runtime = self._live._runtime_resolution.runtime admitted = self._live._admitted_bindings.get(cell.id) if runtime is None or admitted is None: raise LiveIopError("stream_incompatible") blind_root = Path(blind.root).resolve(strict=True) input_root = Path(blind.input_dir).resolve(strict=True) session_root = Path(blind.session_dir).resolve(strict=True) output_root = Path(blind.output_dir).resolve(strict=True) if ( input_root != blind_root / "input" or session_root != blind_root / "session" or output_root != blind_root / "output" or any(path.is_symlink() for path in (blind_root, input_root, session_root, output_root)) ): raise LiveIopError("stream_incompatible") prepared = PreparedWorkspace( identity=AttemptIdentity("run-blind", "blind", 1, 1), attempt_root=str(output_root), workspace_dir=str(blind_root), session_dir=str(session_root), session_id=blind.session_identity, session_is_fresh=True, workspace_checksum=blind.input_digest, setup_cache_policy="isolated", testbed_provenance=TestbedProvenance( path="opaque", branch="opaque", head="opaque", status_digest="sha256:" + "0" * 64, clean=True, ), prepared_at="opaque", ) invocation = build_codex_invocation( cell, prepared, runtime_from_environment( { BASE_URL_ENV_KEY: runtime.base_url, SECRET_ENV_KEY: runtime.secret, "PATH": os.environ.get("PATH", "/usr/bin:/bin"), } ), task_payload, timeout, ) alias = Path(tempfile.gettempdir()).resolve() / ( "iop-bench-score-" + hashlib.sha256(os.fsencode(str(output_root))).hexdigest()[:20] ) try: os.symlink(str(output_root), alias, target_is_directory=True) except FileExistsError: try: if not alias.is_symlink() or alias.resolve(strict=True) != output_root: raise LiveIopError("stream_incompatible") except OSError as exc: raise LiveIopError("stream_incompatible") from exc except OSError as exc: raise LiveIopError("stream_incompatible") from exc self._control_aliases[blind.blind_id] = alias invocation = replace( invocation, spec=replace( invocation.spec, evidence_dir=str(output_root), control_dir=str(alias / "codex-control"), ), ) result = self._invoker( invocation, lambda locator: on_started(locator, spec_digest(invocation.spec)), ) expected = ( admitted.effective_route_kind, admitted.effective_route_id, admitted.effective_model, admitted.effective_effort, ) if result.effective_binding != expected: return ScoringInvocationResult( False, "binding_mismatch", result.effective_binding ) lifecycle = _bound_observations(result.lifecycle, admitted) return ScoringInvocationResult( lifecycle.success, lifecycle.terminal_reason, result.effective_binding, ) def finalize_evidence( self, blind: BlindWorkspace ) -> ScoringEvidenceFinalization: """Remove exact evaluator runtime values before controller publication.""" runtime = self._live._runtime_resolution.runtime if runtime is None: raise LiveIopError("stream_incompatible") blind_root, controlled = _blind_controlled_roots(blind) sensitive = tuple( dict.fromkeys( value.encode("utf-8") for value in (runtime.secret, runtime.base_url) if value ) ) alias = self._control_aliases.pop(blind.blind_id, None) if alias is not None and (alias.exists() or alias.is_symlink()): try: if not alias.is_symlink() or alias.resolve(strict=True) != controlled[2]: raise LiveIopError("stream_incompatible") alias.unlink() except OSError as exc: raise LiveIopError("stream_incompatible") from exc outcome = _remove_sensitive_blind_paths(controlled, sensitive) _freeze_sanitized_input(controlled[0]) _verify_sensitive_absent(blind_root.parent.parent, sensitive) return outcome.finalization() def build_live_scoring_adapter( environment: Mapping[str, str], *, observer: Callable[[_Runtime], _Observation] = _observe, binary_resolver: Callable[[str], str] = _caller_binary, invoker: Callable[[Any, Callable[..., None]], Any] = _default_codex_invoker, ) -> ScoringAdapter: """Build the manifest-bound Codex evaluator without caller fallback.""" if not isinstance(environment, Mapping): raise LiveIopError("protocol_incompatible") live = _LiveAdapter( "codex", codex_capability(), _runtime_from_environment("codex", environment), observer, binary_resolver, _DEFAULT_INVOKERS, ) return _LiveScoringAdapter(live, invoker=invoker)