iop/scripts/agent_comparison_benchmark.py

#!/usr/bin/env python3
"""
Public CLI for the agent comparison benchmark manifest.

Usage:
    python3 scripts/agent_comparison_benchmark.py validate --manifest PATH
    python3 scripts/agent_comparison_benchmark.py preflight --manifest PATH
    python3 scripts/agent_comparison_benchmark.py run --manifest PATH
    python3 scripts/agent_comparison_benchmark.py resume --manifest PATH --run-id RUN_ID
    python3 scripts/agent_comparison_benchmark.py status --manifest PATH --run-id RUN_ID
    python3 scripts/agent_comparison_benchmark.py score --manifest PATH --run-id RUN_ID

Exits:
    0   - manifest is valid or every matrix preflight cell is ready
    64  - usage error (missing args, bad flags)
    69  - validation/state failed or preflight is blocked
"""

from __future__ import annotations

import argparse
import os
import sys
from pathlib import Path
from collections.abc import Mapping

# Ensure the repo root is on sys.path for imports.
_REPO_ROOT = Path(__file__).resolve().parent.parent
if str(_REPO_ROOT) not in sys.path:
    sys.path.insert(0, str(_REPO_ROOT))

from scripts.agent_benchmark.manifest import CALLER_ENUM, ManifestError, load_manifest
from scripts.agent_benchmark.attempts import (
    AttemptStateError,
    CapabilityUnavailable,
    ExecutionAdapter,
    RunStore,
    preflight_manifest,
    run_slots,
)
from scripts.agent_benchmark.live_iop import (
    build_live_adapter_registry,
    build_live_scoring_adapter,
)
from scripts.agent_benchmark.scoring import ScoringError, score_run
from scripts.agent_benchmark.reporting import ReportError, publish_report
from scripts.agent_benchmark.workspace import prepare_workspace

EXIT_VALID = 0
EXIT_USAGE = 64
EXIT_INVALID = 69


def build_adapter_registry(
    environment: Mapping[str, str] | None = None,
) -> dict[str, ExecutionAdapter]:
    """Build the exact three-caller registry from explicit live inputs only."""
    registry = build_live_adapter_registry(
        os.environ if environment is None else environment
    )
    if tuple(registry) != CALLER_ENUM:
        raise RuntimeError("caller adapter registry is invalid")
    return registry


class _SanitizedArgumentParser(argparse.ArgumentParser):
    def error(self, message: str) -> None:
        print("error: invalid usage", file=sys.stderr)
        sys.exit(EXIT_USAGE)


def _build_parser() -> argparse.ArgumentParser:
    parser = _SanitizedArgumentParser(
        prog="agent_comparison_benchmark",
        description="Agent comparison benchmark manifest tools.",
    )
    sub = parser.add_subparsers(dest="command", required=True)
    p_validate = sub.add_parser(
        "validate",
        help="Validate a benchmark manifest JSON file.",
    )
    p_validate.add_argument(
        "--manifest",
        required=True,
        help="Path to the manifest JSON file.",
    )
    for command in ("preflight", "run", "resume", "status", "score", "report"):
        entry = sub.add_parser(command, help=f"Safely {command} benchmark state.")
        entry.add_argument("--manifest", required=True, help="Path to the manifest JSON file.")
        if command in {"resume", "status", "score", "report"}:
            entry.add_argument("--run-id", required=True, help="Harness-generated run id.")
        if command == "resume":
            entry.add_argument("--retry-failed", action="store_true")
        if command == "score":
            entry.add_argument("--retry-scoring-failed", action="store_true")
    return parser


def _cmd_validate(args: argparse.Namespace) -> int:
    manifest_path = Path(args.manifest)
    if not manifest_path.is_file():
        print("error: manifest is unavailable", file=sys.stderr)
        return EXIT_INVALID

    try:
        load_manifest(manifest_path)
    except ManifestError as exc:
        print(f"error: {exc}", file=sys.stderr)
        return EXIT_INVALID
    except Exception:
        print("error: manifest validation failed", file=sys.stderr)
        return EXIT_INVALID

    print("ok: manifest is valid")
    return EXIT_VALID


def _cmd_state(args: argparse.Namespace) -> int:
    try:
        manifest_path = Path(args.manifest)
        manifest = load_manifest(manifest_path, repo_root=_REPO_ROOT)
        raw = manifest_path.read_bytes()
        store = RunStore(_REPO_ROOT)
        if args.command == "run":
            run = store.create(manifest, raw)
        else:
            run = store.open(manifest, args.run_id, raw)
        if args.command == "status":
            status = store.status(run, manifest)
            attempts = status["attempts"]
            outcomes = status["outcomes"]
            attempt_summary = " ".join(
                f"{state}={attempts[state]}"
                for state in (
                    "completed", "timed_out", "cancelled", "interrupted", "running"
                )
            )
            axes = " ".join(
                f"{axis}_{name}={outcomes[axis][name]}"
                for axis in ("product", "harness", "process", "artifact")
                for name in outcomes[axis]
            )
            print(
                f"ok: status run_id={run.run_id} "
                f"unresolved={outcomes['unresolved']} {attempt_summary} {axes}"
            )
            return EXIT_VALID

        completed = run_slots(
            store,
            run,
            manifest,
            adapters=build_adapter_registry(),
            prepare=lambda bound_manifest, attempt: prepare_workspace(
                bound_manifest,
                attempt.root,
                attempt.identity,
                repo_root=_REPO_ROOT,
            ),
            retry_failed=bool(getattr(args, "retry_failed", False)),
        )
        status = store.status(run, manifest)
        preflight = status["preflight"]
        preflight_summary = (
            f"run_id={run.run_id} status={preflight['latest_status']} "
            f"ready={preflight['ready']} "
            f"registration_required={preflight['registration_required']} "
            f"implementation_gap={preflight['implementation_gap']}"
        )
        if preflight["latest_status"] != "ready":
            print("error: preflight blocked " + preflight_summary, file=sys.stderr)
            return EXIT_INVALID

        attempts = status["attempts"]
        attempt_summary = " ".join(
            f"{state}={attempts[state]}"
            for state in (
                "completed", "timed_out", "cancelled", "interrupted", "running"
            )
        )
        outcomes = status["outcomes"]
        unresolved = outcomes["unresolved"]
        axes = " ".join(
            f"{prefix}_{name}={counts[name]}"
            for prefix, counts in (
                ("product", outcomes["product"]),
                ("harness", outcomes["harness"]),
                ("process", outcomes["process"]),
                ("artifact", outcomes["artifact"]),
            )
            for name in counts
        )
        summary = (
            f"run_id={run.run_id} executed={len(completed)} "
            f"unresolved={unresolved} {attempt_summary} {axes}"
        )
        if unresolved:
            print("error: benchmark execution failed " + summary, file=sys.stderr)
            return EXIT_INVALID
        print(f"ok: {args.command} " + summary)
        return EXIT_VALID
    except CapabilityUnavailable:
        print("error: capability unavailable", file=sys.stderr)
    except Exception:
        print("error: benchmark state is unavailable", file=sys.stderr)
    return EXIT_INVALID


def _cmd_preflight(args: argparse.Namespace) -> int:
    try:
        manifest_path = Path(args.manifest)
        manifest = load_manifest(manifest_path, repo_root=_REPO_ROOT)
        raw = manifest_path.read_bytes()
        run, record = preflight_manifest(
            RunStore(_REPO_ROOT),
            manifest,
            raw,
            adapters=build_adapter_registry(),
        )
        counts = {status: 0 for status in ("ready", "registration_required", "implementation_gap")}
        for result in record["results"]:
            counts[result["status"]] += 1
        summary = (
            f"run_id={run.run_id} status={record['status']} "
            f"ready={counts['ready']} "
            f"registration_required={counts['registration_required']} "
            f"implementation_gap={counts['implementation_gap']}"
        )
        if record["status"] == "ready":
            print("ok: preflight " + summary)
            return EXIT_VALID
        print("error: preflight blocked " + summary, file=sys.stderr)
    except CapabilityUnavailable:
        print("error: capability unavailable", file=sys.stderr)
    except Exception:
        print("error: benchmark preflight is unavailable", file=sys.stderr)
    return EXIT_INVALID


def _cmd_score(args: argparse.Namespace) -> int:
    run_id = str(args.run_id)
    try:
        manifest_path = Path(args.manifest)
        manifest = load_manifest(manifest_path, repo_root=_REPO_ROOT)
        raw = manifest_path.read_bytes()
        store = RunStore(_REPO_ROOT)
        run = store.open(manifest, run_id, raw)
        summary = score_run(
            store,
            run,
            manifest,
            adapter=build_live_scoring_adapter(os.environ),
            retry_scoring_failed=bool(args.retry_scoring_failed),
        )
        counts = (
            f"run_id={summary.run_id} scored={summary.scored} "
            f"unscored={summary.unscored} "
            f"scoring_failed={summary.scoring_failed} blocked={summary.blocked}"
        )
        if summary.scoring_failed or summary.blocked:
            print("error: benchmark scoring failed " + counts, file=sys.stderr)
            return EXIT_INVALID
        print("ok: score " + counts)
        return EXIT_VALID
    except (ManifestError, ScoringError, AttemptStateError, OSError):
        print(
            f"error: benchmark scoring is unavailable run_id={run_id}",
            file=sys.stderr,
        )
    except Exception:
        print(
            f"error: benchmark scoring is unavailable run_id={run_id}",
            file=sys.stderr,
        )
    return EXIT_INVALID


def _cmd_report(args: argparse.Namespace) -> int:
    run_id = str(args.run_id)
    try:
        manifest_path = Path(args.manifest)
        manifest = load_manifest(manifest_path, repo_root=_REPO_ROOT)
        raw = manifest_path.read_bytes()
        store = RunStore(_REPO_ROOT)
        run = store.open(manifest, run_id, raw)
        path = publish_report(store, run, manifest)
        rel = str(path.relative_to(_REPO_ROOT))
        print(f"ok: report run_id={run_id} path={rel}")
        return EXIT_VALID
    except (ManifestError, ReportError, AttemptStateError, OSError):
        print("error: benchmark report is unavailable", file=sys.stderr)
    except Exception:
        print("error: benchmark report is unavailable", file=sys.stderr)
    return EXIT_INVALID


def main(argv: list[str] | None = None) -> int:
    parser = _build_parser()
    try:
        args = parser.parse_args(argv)
    except SystemExit as exc:
        return exc.code if isinstance(exc.code, int) else EXIT_USAGE

    if args.command == "validate":
        return _cmd_validate(args)
    if args.command == "preflight":
        return _cmd_preflight(args)
    if args.command == "score":
        return _cmd_score(args)
    if args.command == "report":
        return _cmd_report(args)
    if args.command in {"run", "resume", "status"}:
        return _cmd_state(args)

    return EXIT_USAGE


if __name__ == "__main__":
    sys.exit(main())