toki
91dec43635
caller와 모델 조합을 반복 비교할 때 실행·격리·재개 근거가 흔들리지 않도록 manifest, workspace, lifecycle, append-only attempt 기반과 project-local 진입점을 함께 고정한다.
124 lines
3.9 KiB
Python
124 lines
3.9 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Public CLI for the agent comparison benchmark manifest.
|
|
|
|
Usage:
|
|
python3 scripts/agent_comparison_benchmark.py validate --manifest PATH
|
|
|
|
Exits:
|
|
0 - manifest is valid
|
|
64 - usage error (missing args, bad flags)
|
|
69 - manifest validation failed
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
# Ensure the repo root is on sys.path for imports.
|
|
_REPO_ROOT = Path(__file__).resolve().parent.parent
|
|
if str(_REPO_ROOT) not in sys.path:
|
|
sys.path.insert(0, str(_REPO_ROOT))
|
|
|
|
from scripts.agent_benchmark.manifest import (
|
|
ManifestError,
|
|
load_manifest,
|
|
)
|
|
from scripts.agent_benchmark.attempts import CapabilityUnavailable, RunStore
|
|
|
|
EXIT_VALID = 0
|
|
EXIT_USAGE = 64
|
|
EXIT_INVALID = 69
|
|
|
|
|
|
class _SanitizedArgumentParser(argparse.ArgumentParser):
|
|
def error(self, message: str) -> None:
|
|
print("error: invalid usage", file=sys.stderr)
|
|
sys.exit(EXIT_USAGE)
|
|
|
|
|
|
def _build_parser() -> argparse.ArgumentParser:
|
|
parser = _SanitizedArgumentParser(
|
|
prog="agent_comparison_benchmark",
|
|
description="Agent comparison benchmark manifest tools.",
|
|
)
|
|
sub = parser.add_subparsers(dest="command", required=True)
|
|
p_validate = sub.add_parser(
|
|
"validate",
|
|
help="Validate a benchmark manifest JSON file.",
|
|
)
|
|
p_validate.add_argument(
|
|
"--manifest",
|
|
required=True,
|
|
help="Path to the manifest JSON file.",
|
|
)
|
|
for command in ("run", "resume", "status"):
|
|
entry = sub.add_parser(command, help=f"Safely {command} benchmark state.")
|
|
entry.add_argument("--manifest", required=True, help="Path to the manifest JSON file.")
|
|
if command != "run":
|
|
entry.add_argument("--run-id", required=True, help="Harness-generated run id.")
|
|
if command == "resume":
|
|
entry.add_argument("--retry-failed", action="store_true")
|
|
return parser
|
|
|
|
|
|
def _cmd_validate(args: argparse.Namespace) -> int:
|
|
manifest_path = Path(args.manifest)
|
|
if not manifest_path.is_file():
|
|
print("error: manifest is unavailable", file=sys.stderr)
|
|
return EXIT_INVALID
|
|
|
|
try:
|
|
load_manifest(manifest_path)
|
|
except ManifestError as exc:
|
|
print(f"error: {exc}", file=sys.stderr)
|
|
return EXIT_INVALID
|
|
except Exception:
|
|
print("error: manifest validation failed", file=sys.stderr)
|
|
return EXIT_INVALID
|
|
|
|
print("ok: manifest is valid")
|
|
return EXIT_VALID
|
|
|
|
|
|
def _cmd_state(args: argparse.Namespace) -> int:
|
|
try:
|
|
manifest_path = Path(args.manifest)
|
|
manifest = load_manifest(manifest_path, repo_root=_REPO_ROOT)
|
|
raw = manifest_path.read_bytes()
|
|
store = RunStore(_REPO_ROOT)
|
|
if args.command == "run":
|
|
# Real caller adapters are deliberately deferred. Preflight before
|
|
# allocation ensures this creates no run directory or downstream work.
|
|
raise CapabilityUnavailable("capability-unavailable: caller-adapter")
|
|
run = store.open(manifest, args.run_id, raw)
|
|
if args.command == "status":
|
|
print("ok: " + str(store.status(run, manifest)["attempts"]))
|
|
return EXIT_VALID
|
|
raise CapabilityUnavailable("capability-unavailable: caller-adapter")
|
|
except CapabilityUnavailable:
|
|
print("error: capability unavailable", file=sys.stderr)
|
|
except Exception:
|
|
print("error: benchmark state is unavailable", file=sys.stderr)
|
|
return EXIT_INVALID
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
parser = _build_parser()
|
|
try:
|
|
args = parser.parse_args(argv)
|
|
except SystemExit as exc:
|
|
return exc.code if isinstance(exc.code, int) else EXIT_USAGE
|
|
|
|
if args.command == "validate":
|
|
return _cmd_validate(args)
|
|
if args.command in {"run", "resume", "status"}:
|
|
return _cmd_state(args)
|
|
|
|
return EXIT_USAGE
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|