303 lines
13 KiB
Python
303 lines
13 KiB
Python
"""Regression coverage for the strict benchmark report projection."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import contextlib
|
|
import io
|
|
import tempfile
|
|
import unittest
|
|
from dataclasses import replace
|
|
from pathlib import Path
|
|
from unittest import mock
|
|
|
|
import scripts.agent_comparison_benchmark as cli_module
|
|
from scripts.agent_benchmark import scoring as scoring_module
|
|
from scripts.agent_benchmark.reporting import ReportError, project_report, publish_report, render_report
|
|
from scripts.agent_benchmark import scoring_test
|
|
|
|
|
|
class ReportingTest(unittest.TestCase):
|
|
"""Use the S13 synthetic run builder to cover all S14 terminal states."""
|
|
|
|
def setUp(self) -> None:
|
|
self.fixture = Path(
|
|
"scripts/fixtures/agent-comparison-benchmark-report.expected.md"
|
|
)
|
|
self.harness = scoring_test.ScoringTest()
|
|
self.harness.setUp()
|
|
self.addCleanup(self.harness.doCleanups)
|
|
|
|
def _all_status_run(self) -> None:
|
|
# The RunStore retains every immutable attempt. Scoring sees the latest
|
|
# slot attempt on each invocation, leaving a representative S14 history.
|
|
self.harness._attempt()
|
|
scoring_module.score_run(
|
|
self.harness.store, self.harness.run, self.harness.manifest,
|
|
adapter=scoring_test.FakeScoringAdapter(),
|
|
)
|
|
self.harness._attempt()
|
|
scoring_module.score_run(
|
|
self.harness.store, self.harness.run, self.harness.manifest,
|
|
adapter=scoring_test.FakeScoringAdapter(),
|
|
)
|
|
self.harness._attempt("failed")
|
|
scoring_module.score_run(
|
|
self.harness.store, self.harness.run, self.harness.manifest,
|
|
adapter=scoring_test.FakeScoringAdapter(),
|
|
)
|
|
self.harness._attempt()
|
|
scoring_module.score_run(
|
|
self.harness.store, self.harness.run, self.harness.manifest,
|
|
adapter=scoring_test.FakeScoringAdapter(modes=["malformed"]),
|
|
)
|
|
self.harness._attempt()
|
|
summary = scoring_module.score_run(
|
|
self.harness.store, self.harness.run, self.harness.manifest,
|
|
adapter=scoring_test.FakeScoringAdapter(blocked=True),
|
|
)
|
|
self.assertEqual(summary.blocked, 1)
|
|
|
|
def test_all_status_tie_projection_matches_golden(self) -> None:
|
|
self._all_status_run()
|
|
projection = project_report(
|
|
self.harness.store, self.harness.run, self.harness.manifest
|
|
)
|
|
self.assertEqual(
|
|
[item.score.status for item in projection.attempts],
|
|
["scored", "scored", "unscored", "scoring_failed", "blocked"],
|
|
)
|
|
self.assertEqual(
|
|
[item.score.rank for item in projection.attempts], [1, 1, None, None, None]
|
|
)
|
|
self.assertEqual(
|
|
[
|
|
tuple((category.id, category.score, category.max_score)
|
|
for category in item.score.categories)
|
|
for item in projection.attempts
|
|
],
|
|
[
|
|
(
|
|
("task_fidelity", 24, 25),
|
|
("visual_hierarchy", 25, 25),
|
|
("responsive_composition", 20, 20),
|
|
("typography_readability", 15, 15),
|
|
("polish_consistency", 15, 15),
|
|
),
|
|
(
|
|
("task_fidelity", 24, 25),
|
|
("visual_hierarchy", 25, 25),
|
|
("responsive_composition", 20, 20),
|
|
("typography_readability", 15, 15),
|
|
("polish_consistency", 15, 15),
|
|
),
|
|
(),
|
|
(),
|
|
(),
|
|
],
|
|
)
|
|
self.assertEqual(
|
|
render_report(projection), self.fixture.read_bytes()
|
|
)
|
|
|
|
def test_detail_tables_include_repetition_in_attempt_label(self) -> None:
|
|
self._all_status_run()
|
|
projection = project_report(
|
|
self.harness.store, self.harness.run, self.harness.manifest
|
|
)
|
|
first = projection.attempts[0]
|
|
repeated = replace(
|
|
first,
|
|
attempt=replace(
|
|
first.attempt,
|
|
identity=replace(first.attempt.identity, repetition=2),
|
|
),
|
|
)
|
|
rendered = render_report(
|
|
replace(projection, attempts=(first, repeated))
|
|
).decode("utf-8")
|
|
|
|
for section, next_section in (
|
|
("## Quality score breakdown", "## Timing and token evidence"),
|
|
("## Timing and token evidence", "## Web validation and scoring provenance"),
|
|
("## Web validation and scoring provenance", "## Limitations"),
|
|
):
|
|
table = rendered.split(section, 1)[1].split(next_section, 1)[0]
|
|
self.assertIn("| cell-sentinel/r1/a1 |", table)
|
|
self.assertIn("| cell-sentinel/r2/a1 |", table)
|
|
|
|
def test_unscored_and_score_directory_conflict_is_rejected(self) -> None:
|
|
attempt = self.harness._attempt("failed")
|
|
scoring_module.score_run(
|
|
self.harness.store, self.harness.run, self.harness.manifest,
|
|
adapter=scoring_test.FakeScoringAdapter(),
|
|
)
|
|
(Path(attempt.root) / "scoring" / "score-000001").mkdir()
|
|
with self.assertRaisesRegex(ReportError, "report scoring state is invalid"):
|
|
publish_report(self.harness.store, self.harness.run, self.harness.manifest)
|
|
self.assertFalse((Path(self.harness.run.root) / "report.md").exists())
|
|
|
|
def test_publication_is_idempotent_and_refuses_replacement(self) -> None:
|
|
self._all_status_run()
|
|
path = publish_report(
|
|
self.harness.store, self.harness.run, self.harness.manifest
|
|
)
|
|
expected = self.fixture.read_bytes()
|
|
self.assertEqual(path.read_bytes(), expected)
|
|
self.assertEqual(
|
|
publish_report(self.harness.store, self.harness.run, self.harness.manifest),
|
|
path,
|
|
)
|
|
path.write_bytes(expected + b"changed\n")
|
|
with self.assertRaisesRegex(ReportError, "refused an existing target"):
|
|
publish_report(self.harness.store, self.harness.run, self.harness.manifest)
|
|
self.assertEqual(path.read_bytes(), expected + b"changed\n")
|
|
|
|
def test_corrupt_required_measurement_creates_no_report(self) -> None:
|
|
attempt = self.harness._attempt()
|
|
measurement = Path(attempt.root) / "attempt-measurement.json"
|
|
measurement.write_bytes(b"{}\n")
|
|
with self.assertRaises(ReportError):
|
|
publish_report(self.harness.store, self.harness.run, self.harness.manifest)
|
|
self.assertFalse((Path(self.harness.run.root) / "report.md").exists())
|
|
|
|
def test_symlinked_required_evidence_is_not_a_contained_raw_link(self) -> None:
|
|
attempt = self.harness._attempt()
|
|
measurement = Path(attempt.root) / "attempt-measurement.json"
|
|
measurement.unlink()
|
|
measurement.symlink_to(Path(self.harness.run.root) / "manifest.json")
|
|
with self.assertRaises(ReportError):
|
|
publish_report(self.harness.store, self.harness.run, self.harness.manifest)
|
|
self.assertFalse((Path(self.harness.run.root) / "report.md").exists())
|
|
|
|
def test_report_target_symlink_is_rejected(self) -> None:
|
|
self.harness._attempt()
|
|
path = Path(self.harness.run.root) / "report.md"
|
|
path.symlink_to("manifest.json")
|
|
with self.assertRaisesRegex(ReportError, "publication target is invalid"):
|
|
publish_report(self.harness.store, self.harness.run, self.harness.manifest)
|
|
|
|
def test_markdown_cell_escaping_is_stable(self) -> None:
|
|
from scripts.agent_benchmark.reporting import _markdown
|
|
|
|
self.assertEqual(_markdown("line|next\\tail\nlast"), "line\\|next\\\\tail last")
|
|
|
|
|
|
class ReportCliTest(unittest.TestCase):
|
|
"""Deterministic boundary coverage for the public ``report`` CLI handler."""
|
|
|
|
def setUp(self) -> None:
|
|
self._tmp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self._tmp.cleanup)
|
|
self.manifest_path = Path(self._tmp.name) / "manifest.json"
|
|
self.manifest_bytes = b"{\"version\": \"example\"}\n"
|
|
self.manifest_path.write_bytes(self.manifest_bytes)
|
|
self.report_path = cli_module._REPO_ROOT / "runs" / "example" / "report.md"
|
|
self.report_rel = "runs/example/report.md"
|
|
|
|
def _namespace(self, run_id: str = "run-1") -> argparse.Namespace:
|
|
return argparse.Namespace(manifest=str(self.manifest_path), run_id=run_id)
|
|
|
|
def _patch_boundaries(self) -> dict[str, mock.Mock]:
|
|
stack = contextlib.ExitStack()
|
|
self.addCleanup(stack.close)
|
|
load_manifest = stack.enter_context(
|
|
mock.patch.object(cli_module, "load_manifest", return_value=mock.sentinel.manifest)
|
|
)
|
|
run_store_cls = stack.enter_context(mock.patch.object(cli_module, "RunStore"))
|
|
store = run_store_cls.return_value
|
|
store.open.return_value = mock.sentinel.run
|
|
publish_report_mock = stack.enter_context(
|
|
mock.patch.object(
|
|
cli_module, "publish_report", return_value=self.report_path
|
|
)
|
|
)
|
|
build_registry = stack.enter_context(
|
|
mock.patch.object(cli_module, "build_adapter_registry")
|
|
)
|
|
return {
|
|
"load_manifest": load_manifest,
|
|
"RunStore": run_store_cls,
|
|
"store": store,
|
|
"publish_report": publish_report_mock,
|
|
"build_adapter_registry": build_registry,
|
|
}
|
|
|
|
def _invoke(self, run_id: str = "run-1") -> tuple[int, str, str]:
|
|
out = io.StringIO()
|
|
err = io.StringIO()
|
|
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
|
|
code = cli_module._cmd_report(self._namespace(run_id=run_id))
|
|
return code, out.getvalue(), err.getvalue()
|
|
|
|
def test_success_prints_repo_relative_path_and_exits_zero(self) -> None:
|
|
mocks = self._patch_boundaries()
|
|
code, out, err = self._invoke(run_id="run-1")
|
|
|
|
self.assertEqual(code, cli_module.EXIT_VALID)
|
|
mocks["load_manifest"].assert_called_once_with(
|
|
self.manifest_path, repo_root=cli_module._REPO_ROOT
|
|
)
|
|
mocks["RunStore"].assert_called_once_with(cli_module._REPO_ROOT)
|
|
mocks["store"].open.assert_called_once_with(
|
|
mock.sentinel.manifest, "run-1", self.manifest_bytes
|
|
)
|
|
mocks["publish_report"].assert_called_once_with(
|
|
mocks["store"], mock.sentinel.run, mock.sentinel.manifest
|
|
)
|
|
self.assertEqual(out, f"ok: report run_id=run-1 path={self.report_rel}\n")
|
|
self.assertEqual(err, "")
|
|
mocks["build_adapter_registry"].assert_not_called()
|
|
|
|
def test_typed_failure_emits_only_closed_line_and_exits_invalid(self) -> None:
|
|
mocks = self._patch_boundaries()
|
|
mocks["publish_report"].side_effect = ReportError("projection failed")
|
|
|
|
code, out, err = self._invoke(run_id="run-9")
|
|
|
|
self.assertEqual(code, cli_module.EXIT_INVALID)
|
|
self.assertEqual(out, "")
|
|
self.assertEqual(err, "error: benchmark report is unavailable\n")
|
|
mocks["build_adapter_registry"].assert_not_called()
|
|
|
|
def test_generic_failure_emits_only_closed_line_and_exits_invalid(self) -> None:
|
|
mocks = self._patch_boundaries()
|
|
mocks["publish_report"].side_effect = ValueError("unexpected boundary")
|
|
|
|
code, out, err = self._invoke(run_id="run-9")
|
|
|
|
self.assertEqual(code, cli_module.EXIT_INVALID)
|
|
self.assertEqual(out, "")
|
|
self.assertEqual(err, "error: benchmark report is unavailable\n")
|
|
mocks["build_adapter_registry"].assert_not_called()
|
|
|
|
def test_two_successful_calls_delegate_idempotently_to_reporter(self) -> None:
|
|
mocks = self._patch_boundaries()
|
|
|
|
for _ in range(2):
|
|
code, out, err = self._invoke(run_id="run-1")
|
|
self.assertEqual(code, cli_module.EXIT_VALID)
|
|
self.assertEqual(out, f"ok: report run_id=run-1 path={self.report_rel}\n")
|
|
self.assertEqual(err, "")
|
|
|
|
# Each CLI call delegates exactly once to the strict reporter boundary;
|
|
# the reporter itself remains the sole idempotent publication surface.
|
|
self.assertEqual(mocks["publish_report"].call_count, 2)
|
|
self.assertEqual(mocks["store"].open.call_count, 2)
|
|
mocks["build_adapter_registry"].assert_not_called()
|
|
|
|
def test_adapter_registry_is_never_constructed(self) -> None:
|
|
mocks = self._patch_boundaries()
|
|
|
|
code, _, _ = self._invoke(run_id="run-1")
|
|
self.assertEqual(code, cli_module.EXIT_VALID)
|
|
|
|
mocks["publish_report"].side_effect = ReportError("publication failed")
|
|
code, _, _ = self._invoke(run_id="run-1")
|
|
self.assertEqual(code, cli_module.EXIT_INVALID)
|
|
|
|
mocks["build_adapter_registry"].assert_not_called()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|