from __future__ import annotations import json import os import tempfile import unittest from pathlib import Path from scripts.agent_benchmark.rubric import ( RUBRIC_CATEGORIES, RubricError, canonical_worksheet_bytes, load_worksheet, validate_worksheet, ) def _worksheet() -> dict: categories = [ { "id": ident, "max_score": maximum, "score": maximum, "evidence": f"Evidence for {ident}.", } for ident, maximum in RUBRIC_CATEGORIES ] return { "rubric_version": "landing-quality-v1", "categories": categories, "total": 100, } class RubricTest(unittest.TestCase): def test_exact_categories_and_total_are_accepted(self): worksheet = validate_worksheet(_worksheet()) self.assertEqual(worksheet.total, 100) self.assertEqual( [(item.id, item.max_score) for item in worksheet.categories], list(RUBRIC_CATEGORIES), ) self.assertEqual( validate_worksheet(json.loads(canonical_worksheet_bytes(worksheet))), worksheet, ) def test_missing_extra_reordered_and_out_of_range_values_fail(self): cases = [] missing = _worksheet() missing["categories"] = missing["categories"][:-1] cases.append(missing) extra = _worksheet() extra["automatic_gate_points"] = 1 cases.append(extra) reordered = _worksheet() reordered["categories"] = list(reversed(reordered["categories"])) cases.append(reordered) too_high = _worksheet() too_high["categories"][0]["score"] = 26 too_high["total"] = 101 cases.append(too_high) bad_total = _worksheet() bad_total["total"] = 0 cases.append(bad_total) empty_evidence = _worksheet() empty_evidence["categories"][0]["evidence"] = " " cases.append(empty_evidence) for value in cases: with self.subTest(value=value): with self.assertRaises(RubricError): validate_worksheet(value) def test_loader_refuses_nonregular_and_malformed_files(self): with tempfile.TemporaryDirectory() as temp: root = Path(temp) valid = root / "worksheet.json" valid.write_bytes(json.dumps(_worksheet()).encode()) self.assertEqual(load_worksheet(valid).total, 100) malformed = root / "malformed.json" malformed.write_bytes(b"{}") with self.assertRaises(RubricError): load_worksheet(malformed) link = root / "link.json" os.symlink(valid, link) with self.assertRaises(RubricError): load_worksheet(link) if __name__ == "__main__": unittest.main()