#!/usr/bin/env python3 """ Public CLI for the agent comparison benchmark manifest. Usage: python3 scripts/agent_comparison_benchmark.py validate --manifest PATH Exits: 0 - manifest is valid 64 - usage error (missing args, bad flags) 69 - manifest validation failed """ from __future__ import annotations import argparse import sys from pathlib import Path # Ensure the repo root is on sys.path for imports. _REPO_ROOT = Path(__file__).resolve().parent.parent if str(_REPO_ROOT) not in sys.path: sys.path.insert(0, str(_REPO_ROOT)) from scripts.agent_benchmark.manifest import ( ManifestError, load_manifest, ) from scripts.agent_benchmark.attempts import CapabilityUnavailable, RunStore EXIT_VALID = 0 EXIT_USAGE = 64 EXIT_INVALID = 69 class _SanitizedArgumentParser(argparse.ArgumentParser): def error(self, message: str) -> None: print("error: invalid usage", file=sys.stderr) sys.exit(EXIT_USAGE) def _build_parser() -> argparse.ArgumentParser: parser = _SanitizedArgumentParser( prog="agent_comparison_benchmark", description="Agent comparison benchmark manifest tools.", ) sub = parser.add_subparsers(dest="command", required=True) p_validate = sub.add_parser( "validate", help="Validate a benchmark manifest JSON file.", ) p_validate.add_argument( "--manifest", required=True, help="Path to the manifest JSON file.", ) for command in ("run", "resume", "status"): entry = sub.add_parser(command, help=f"Safely {command} benchmark state.") entry.add_argument("--manifest", required=True, help="Path to the manifest JSON file.") if command != "run": entry.add_argument("--run-id", required=True, help="Harness-generated run id.") if command == "resume": entry.add_argument("--retry-failed", action="store_true") return parser def _cmd_validate(args: argparse.Namespace) -> int: manifest_path = Path(args.manifest) if not manifest_path.is_file(): print("error: manifest is unavailable", file=sys.stderr) return EXIT_INVALID try: load_manifest(manifest_path) except ManifestError as exc: print(f"error: {exc}", file=sys.stderr) return EXIT_INVALID except Exception: print("error: manifest validation failed", file=sys.stderr) return EXIT_INVALID print("ok: manifest is valid") return EXIT_VALID def _cmd_state(args: argparse.Namespace) -> int: try: manifest_path = Path(args.manifest) manifest = load_manifest(manifest_path, repo_root=_REPO_ROOT) raw = manifest_path.read_bytes() store = RunStore(_REPO_ROOT) if args.command == "run": # Real caller adapters are deliberately deferred. Preflight before # allocation ensures this creates no run directory or downstream work. raise CapabilityUnavailable("capability-unavailable: caller-adapter") run = store.open(manifest, args.run_id, raw) if args.command == "status": print("ok: " + str(store.status(run, manifest)["attempts"])) return EXIT_VALID raise CapabilityUnavailable("capability-unavailable: caller-adapter") except CapabilityUnavailable: print("error: capability unavailable", file=sys.stderr) except Exception: print("error: benchmark state is unavailable", file=sys.stderr) return EXIT_INVALID def main(argv: list[str] | None = None) -> int: parser = _build_parser() try: args = parser.parse_args(argv) except SystemExit as exc: return exc.code if isinstance(exc.code, int) else EXIT_USAGE if args.command == "validate": return _cmd_validate(args) if args.command in {"run", "resume", "status"}: return _cmd_state(args) return EXIT_USAGE if __name__ == "__main__": sys.exit(main())