iop/scripts/agent_comparison_benchmark.py

#!/usr/bin/env python3
"""
Public CLI for the agent comparison benchmark manifest.

Usage:
    python3 scripts/agent_comparison_benchmark.py validate --manifest PATH
    python3 scripts/agent_comparison_benchmark.py preflight --manifest PATH
    python3 scripts/agent_comparison_benchmark.py run --manifest PATH
    python3 scripts/agent_comparison_benchmark.py resume --manifest PATH --run-id RUN_ID
    python3 scripts/agent_comparison_benchmark.py status --manifest PATH --run-id RUN_ID

Exits:
    0   - manifest is valid or every direct preflight cell is ready
    64  - usage error (missing args, bad flags)
    69  - validation/state failed or preflight is blocked
"""

from __future__ import annotations

import argparse
import hashlib
import sys
from pathlib import Path
from typing import Callable

# Ensure the repo root is on sys.path for imports.
_REPO_ROOT = Path(__file__).resolve().parent.parent
if str(_REPO_ROOT) not in sys.path:
    sys.path.insert(0, str(_REPO_ROOT))

from scripts.agent_benchmark.manifest import (
    CALLER_ENUM,
    MatrixCell,
    ManifestError,
    Timeout,
    load_manifest,
)
from scripts.agent_benchmark.attempts import (
    Attempt,
    CapabilityUnavailable,
    ExecutionAdapter,
    PreflightObservation,
    RunStore,
    preflight_manifest,
    run_slots,
)
from scripts.agent_benchmark.connectivity import (
    ISSUE_RESUME_CODES,
    CallerCapability,
    ConnectivityIssue,
    RequestedEffectiveBinding,
    make_result,
)
from scripts.agent_benchmark.claude_iop import claude_capability
from scripts.agent_benchmark.agy_iop import AGY_CALLER
from scripts.agent_benchmark.codex_iop import codex_capability
from scripts.agent_benchmark.lifecycle import InvocationResult, SupervisorLocator
from scripts.agent_benchmark.workspace import PreparedWorkspace, prepare_workspace

EXIT_VALID = 0
EXIT_USAGE = 64
EXIT_INVALID = 69


class _RegisteredExecutionAdapter:
    """Typed execution registration with an explicit live-observation gap.

    A later authorized-live adapter can replace these registrations without
    changing the CLI, evidence schema, or run writer. Until then a direct cell
    is never reported ready from requested values alone, so invoke is unreachable.
    """

    def __init__(self, capability: CallerCapability) -> None:
        self.capability = capability

    @staticmethod
    def _identity(caller: str, kind: str) -> str:
        raw = f"iop-benchmark-unobserved-v1:{caller}:{kind}".encode("ascii")
        return "sha256:" + hashlib.sha256(raw).hexdigest()

    def preflight(self, cell: MatrixCell) -> PreflightObservation:
        binding = RequestedEffectiveBinding(
            cell.id,
            cell.caller,
            cell.iop.route_kind,
            cell.iop.route_id,
            cell.iop.request_model,
            cell.iop.requested_effort,
        )
        issue = ConnectivityIssue(
            "stream_incompatible", ISSUE_RESUME_CODES["stream_incompatible"]
        )
        result = make_result(cell, self.capability, binding, (issue,))
        return PreflightObservation(
            result,
            self._identity(cell.caller, "endpoint"),
            self._identity(cell.caller, "config"),
        )

    def invoke(
        self,
        cell: MatrixCell,
        prepared: PreparedWorkspace,
        attempt: Attempt,
        task_payload: bytes,
        timeout: Timeout,
        on_started: Callable[[SupervisorLocator, str], None],
    ) -> InvocationResult:
        """Remain unreachable until a live observer replaces this registration."""
        raise CapabilityUnavailable("capability-unavailable: caller-adapter")


def build_adapter_registry() -> dict[str, ExecutionAdapter]:
    """Build the exact three-caller registry from completed adapter modules."""
    registry: dict[str, ExecutionAdapter] = {
        "claude": _RegisteredExecutionAdapter(claude_capability()),
        # agy's completed module exposes its documented caller constant while
        # the same closed capability tuple is enforced by its preflight parser.
        "agy": _RegisteredExecutionAdapter(
            CallerCapability(
                AGY_CALLER,
                ("direct", "execution_preset"),
                ("high", "low", "medium"),
            )
        ),
        "codex": _RegisteredExecutionAdapter(codex_capability()),
    }
    if tuple(registry) != CALLER_ENUM:
        raise RuntimeError("caller adapter registry is invalid")
    return registry


class _SanitizedArgumentParser(argparse.ArgumentParser):
    def error(self, message: str) -> None:
        print("error: invalid usage", file=sys.stderr)
        sys.exit(EXIT_USAGE)


def _build_parser() -> argparse.ArgumentParser:
    parser = _SanitizedArgumentParser(
        prog="agent_comparison_benchmark",
        description="Agent comparison benchmark manifest tools.",
    )
    sub = parser.add_subparsers(dest="command", required=True)
    p_validate = sub.add_parser(
        "validate",
        help="Validate a benchmark manifest JSON file.",
    )
    p_validate.add_argument(
        "--manifest",
        required=True,
        help="Path to the manifest JSON file.",
    )
    for command in ("preflight", "run", "resume", "status"):
        entry = sub.add_parser(command, help=f"Safely {command} benchmark state.")
        entry.add_argument("--manifest", required=True, help="Path to the manifest JSON file.")
        if command in {"resume", "status"}:
            entry.add_argument("--run-id", required=True, help="Harness-generated run id.")
        if command == "resume":
            entry.add_argument("--retry-failed", action="store_true")
    return parser


def _cmd_validate(args: argparse.Namespace) -> int:
    manifest_path = Path(args.manifest)
    if not manifest_path.is_file():
        print("error: manifest is unavailable", file=sys.stderr)
        return EXIT_INVALID

    try:
        load_manifest(manifest_path)
    except ManifestError as exc:
        print(f"error: {exc}", file=sys.stderr)
        return EXIT_INVALID
    except Exception:
        print("error: manifest validation failed", file=sys.stderr)
        return EXIT_INVALID

    print("ok: manifest is valid")
    return EXIT_VALID


def _cmd_state(args: argparse.Namespace) -> int:
    try:
        manifest_path = Path(args.manifest)
        manifest = load_manifest(manifest_path, repo_root=_REPO_ROOT)
        raw = manifest_path.read_bytes()
        store = RunStore(_REPO_ROOT)
        if args.command == "run":
            run = store.create(manifest, raw)
        else:
            run = store.open(manifest, args.run_id, raw)
        if args.command == "status":
            print("ok: " + str(store.status(run, manifest)["attempts"]))
            return EXIT_VALID

        completed = run_slots(
            store,
            run,
            manifest,
            adapters=build_adapter_registry(),
            prepare=lambda bound_manifest, attempt: prepare_workspace(
                bound_manifest,
                attempt.root,
                attempt.identity,
                repo_root=_REPO_ROOT,
            ),
            retry_failed=bool(getattr(args, "retry_failed", False)),
        )
        status = store.status(run, manifest)
        preflight = status["preflight"]
        preflight_summary = (
            f"run_id={run.run_id} status={preflight['latest_status']} "
            f"ready={preflight['ready']} "
            f"registration_required={preflight['registration_required']} "
            f"implementation_gap={preflight['implementation_gap']}"
        )
        if preflight["latest_status"] != "ready":
            print("error: preflight blocked " + preflight_summary, file=sys.stderr)
            return EXIT_INVALID

        attempts = status["attempts"]
        attempt_summary = " ".join(
            f"{state}={attempts[state]}"
            for state in (
                "success", "failed", "timed_out", "cancelled", "interrupted", "running"
            )
        )
        unresolved = 0
        for slot in store.slots(manifest):
            retained = store.attempts(run, slot)
            if not retained or retained[-1].state != "success":
                unresolved += 1
        summary = (
            f"run_id={run.run_id} completed={len(completed)} "
            f"unresolved={unresolved} {attempt_summary}"
        )
        if unresolved:
            print("error: benchmark execution failed " + summary, file=sys.stderr)
            return EXIT_INVALID
        print(f"ok: {args.command} " + summary)
        return EXIT_VALID
    except CapabilityUnavailable:
        print("error: capability unavailable", file=sys.stderr)
    except Exception:
        print("error: benchmark state is unavailable", file=sys.stderr)
    return EXIT_INVALID


def _cmd_preflight(args: argparse.Namespace) -> int:
    try:
        manifest_path = Path(args.manifest)
        manifest = load_manifest(manifest_path, repo_root=_REPO_ROOT)
        raw = manifest_path.read_bytes()
        run, record = preflight_manifest(
            RunStore(_REPO_ROOT),
            manifest,
            raw,
            adapters=build_adapter_registry(),
        )
        counts = {status: 0 for status in ("ready", "registration_required", "implementation_gap")}
        for result in record["results"]:
            counts[result["status"]] += 1
        summary = (
            f"run_id={run.run_id} status={record['status']} "
            f"ready={counts['ready']} "
            f"registration_required={counts['registration_required']} "
            f"implementation_gap={counts['implementation_gap']}"
        )
        if record["status"] == "ready":
            print("ok: preflight " + summary)
            return EXIT_VALID
        print("error: preflight blocked " + summary, file=sys.stderr)
    except CapabilityUnavailable:
        print("error: capability unavailable", file=sys.stderr)
    except Exception:
        print("error: benchmark preflight is unavailable", file=sys.stderr)
    return EXIT_INVALID


def main(argv: list[str] | None = None) -> int:
    parser = _build_parser()
    try:
        args = parser.parse_args(argv)
    except SystemExit as exc:
        return exc.code if isinstance(exc.code, int) else EXIT_USAGE

    if args.command == "validate":
        return _cmd_validate(args)
    if args.command == "preflight":
        return _cmd_preflight(args)
    if args.command in {"run", "resume", "status"}:
        return _cmd_state(args)

    return EXIT_USAGE


if __name__ == "__main__":
    sys.exit(main())