"""Strict D12 landing-page worksheet contract. Automatic web gates establish scoring eligibility; they are intentionally not represented in this 100-point worksheet and can never contribute points. """ from __future__ import annotations import json import os import stat from dataclasses import dataclass from pathlib import Path from typing import Any from scripts.agent_benchmark.manifest import RUBRIC_VERSION MAX_WORKSHEET_BYTES = 64 * 1024 MAX_EVIDENCE_CHARS = 4096 RUBRIC_CATEGORIES = ( ("task_fidelity", 25), ("visual_hierarchy", 25), ("responsive_composition", 20), ("typography_readability", 15), ("polish_consistency", 15), ) class RubricError(Exception): """A worksheet is missing, malformed, non-canonical, or out of bounds.""" @dataclass(frozen=True) class CategoryScore: id: str max_score: int score: int evidence: str @dataclass(frozen=True) class Worksheet: rubric_version: str categories: tuple[CategoryScore, ...] total: int def as_dict(self) -> dict[str, Any]: return { "rubric_version": self.rubric_version, "categories": [ { "id": item.id, "max_score": item.max_score, "score": item.score, "evidence": item.evidence, } for item in self.categories ], "total": self.total, } def canonical_worksheet_bytes(worksheet: Worksheet) -> bytes: if not isinstance(worksheet, Worksheet): raise RubricError("worksheet object is invalid") return ( json.dumps( worksheet.as_dict(), sort_keys=True, separators=(",", ":"), ensure_ascii=True, ).encode("ascii") + b"\n" ) def validate_worksheet(value: Any) -> Worksheet: if not isinstance(value, dict) or set(value) != { "rubric_version", "categories", "total", }: raise RubricError("worksheet schema is invalid") if value["rubric_version"] != RUBRIC_VERSION: raise RubricError("worksheet rubric version is invalid") raw_categories = value["categories"] if not isinstance(raw_categories, list) or len(raw_categories) != len( RUBRIC_CATEGORIES ): raise RubricError("worksheet categories are invalid") categories: list[CategoryScore] = [] for raw, (expected_id, expected_max) in zip( raw_categories, RUBRIC_CATEGORIES ): if not isinstance(raw, dict) or set(raw) != { "id", "max_score", "score", "evidence", }: raise RubricError("worksheet category schema is invalid") score = raw["score"] evidence = raw["evidence"] if ( raw["id"] != expected_id or raw["max_score"] != expected_max or isinstance(score, bool) or not isinstance(score, int) or not 0 <= score <= expected_max or not isinstance(evidence, str) or not evidence.strip() or len(evidence) > MAX_EVIDENCE_CHARS or any(ord(char) < 0x20 and char not in "\n\t" for char in evidence) ): raise RubricError("worksheet category is invalid") categories.append( CategoryScore(expected_id, expected_max, score, evidence) ) total = value["total"] expected_total = sum(item.score for item in categories) if ( isinstance(total, bool) or not isinstance(total, int) or total != expected_total or not 0 <= total <= 100 ): raise RubricError("worksheet total is invalid") return Worksheet(RUBRIC_VERSION, tuple(categories), total) def load_worksheet(path: str | Path) -> Worksheet: target = Path(path) flags = os.O_RDONLY | os.O_CLOEXEC | os.O_NONBLOCK if hasattr(os, "O_NOFOLLOW"): flags |= os.O_NOFOLLOW try: fd = os.open(target, flags) except OSError as exc: raise RubricError("worksheet is unavailable") from exc try: info = os.fstat(fd) if not stat.S_ISREG(info.st_mode) or info.st_size > MAX_WORKSHEET_BYTES: raise RubricError("worksheet must be a bounded regular file") raw = bytearray() while len(raw) < info.st_size: chunk = os.read(fd, info.st_size - len(raw)) if not chunk: raise RubricError("worksheet changed while reading") raw.extend(chunk) if os.read(fd, 1): raise RubricError("worksheet changed while reading") except OSError as exc: raise RubricError("worksheet is unavailable") from exc finally: os.close(fd) try: value = json.loads(bytes(raw).decode("utf-8")) except (UnicodeDecodeError, json.JSONDecodeError) as exc: raise RubricError("worksheet JSON is invalid") from exc return validate_worksheet(value)