93 lines
2.8 KiB
Python
93 lines
2.8 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
from scripts.agent_benchmark.rubric import (
|
|
RUBRIC_CATEGORIES,
|
|
RubricError,
|
|
canonical_worksheet_bytes,
|
|
load_worksheet,
|
|
validate_worksheet,
|
|
)
|
|
|
|
|
|
def _worksheet() -> dict:
|
|
categories = [
|
|
{
|
|
"id": ident,
|
|
"max_score": maximum,
|
|
"score": maximum,
|
|
"evidence": f"Evidence for {ident}.",
|
|
}
|
|
for ident, maximum in RUBRIC_CATEGORIES
|
|
]
|
|
return {
|
|
"rubric_version": "landing-quality-v1",
|
|
"categories": categories,
|
|
"total": 100,
|
|
}
|
|
|
|
|
|
class RubricTest(unittest.TestCase):
|
|
def test_exact_categories_and_total_are_accepted(self):
|
|
worksheet = validate_worksheet(_worksheet())
|
|
self.assertEqual(worksheet.total, 100)
|
|
self.assertEqual(
|
|
[(item.id, item.max_score) for item in worksheet.categories],
|
|
list(RUBRIC_CATEGORIES),
|
|
)
|
|
self.assertEqual(
|
|
validate_worksheet(json.loads(canonical_worksheet_bytes(worksheet))),
|
|
worksheet,
|
|
)
|
|
|
|
def test_missing_extra_reordered_and_out_of_range_values_fail(self):
|
|
cases = []
|
|
missing = _worksheet()
|
|
missing["categories"] = missing["categories"][:-1]
|
|
cases.append(missing)
|
|
extra = _worksheet()
|
|
extra["automatic_gate_points"] = 1
|
|
cases.append(extra)
|
|
reordered = _worksheet()
|
|
reordered["categories"] = list(reversed(reordered["categories"]))
|
|
cases.append(reordered)
|
|
too_high = _worksheet()
|
|
too_high["categories"][0]["score"] = 26
|
|
too_high["total"] = 101
|
|
cases.append(too_high)
|
|
bad_total = _worksheet()
|
|
bad_total["total"] = 0
|
|
cases.append(bad_total)
|
|
empty_evidence = _worksheet()
|
|
empty_evidence["categories"][0]["evidence"] = " "
|
|
cases.append(empty_evidence)
|
|
for value in cases:
|
|
with self.subTest(value=value):
|
|
with self.assertRaises(RubricError):
|
|
validate_worksheet(value)
|
|
|
|
def test_loader_refuses_nonregular_and_malformed_files(self):
|
|
with tempfile.TemporaryDirectory() as temp:
|
|
root = Path(temp)
|
|
valid = root / "worksheet.json"
|
|
valid.write_bytes(json.dumps(_worksheet()).encode())
|
|
self.assertEqual(load_worksheet(valid).total, 100)
|
|
|
|
malformed = root / "malformed.json"
|
|
malformed.write_bytes(b"{}")
|
|
with self.assertRaises(RubricError):
|
|
load_worksheet(malformed)
|
|
|
|
link = root / "link.json"
|
|
os.symlink(valid, link)
|
|
with self.assertRaises(RubricError):
|
|
load_worksheet(link)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|