547 lines
21 KiB
Python
547 lines
21 KiB
Python
from __future__ import annotations
|
|
|
|
import copy
|
|
import hashlib
|
|
import json
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
from unittest import mock
|
|
|
|
from scripts.agent_benchmark import web_validation as web_validation_module
|
|
from scripts.agent_benchmark.browser_cdp import (
|
|
BrowserError,
|
|
RenderObservation,
|
|
ViewportObservation,
|
|
)
|
|
from scripts.agent_benchmark.web_validation import (
|
|
WEB_GATES,
|
|
WebValidationError,
|
|
build_web_validation,
|
|
load_web_validation,
|
|
publish_web_validation,
|
|
validate_web_attempt,
|
|
)
|
|
|
|
|
|
def _digest(data: bytes) -> str:
|
|
return "sha256:" + hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
class WebValidationTest(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.temporary = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.temporary.cleanup)
|
|
self.attempt = Path(self.temporary.name) / "attempt"
|
|
self.attempt.mkdir()
|
|
self.workspace = self.attempt / "workspace"
|
|
self.workspace.mkdir()
|
|
(self.workspace / "assets").mkdir()
|
|
(self.workspace / "brief").mkdir()
|
|
self.asset_content = {
|
|
"assets/a.svg": b"<svg xmlns='http://www.w3.org/2000/svg' width='20' height='20'/>",
|
|
"assets/b.svg": b"<svg xmlns='http://www.w3.org/2000/svg' width='20' height='20'/>",
|
|
"brief/reference.txt": b"reference\n",
|
|
}
|
|
for relative, content in self.asset_content.items():
|
|
(self.workspace / relative).write_bytes(content)
|
|
(self.workspace / "index.html").write_text(
|
|
"<main><h1>Ready</h1><img src='assets/a.svg' alt='A'>"
|
|
"<img src='assets/b.svg' alt='B'><a href='#x'>go</a>"
|
|
"<script src='script.js'></script></main>",
|
|
encoding="utf-8",
|
|
)
|
|
(self.workspace / "styles.css").write_text(
|
|
"body{color:#111;background:#fff}img{width:20px}"
|
|
"a:focus{outline:2px solid #05f}",
|
|
encoding="utf-8",
|
|
)
|
|
(self.workspace / "script.js").write_text(
|
|
"document.body.dataset.ready='1';", encoding="utf-8"
|
|
)
|
|
(self.attempt / "attempt-measurement.json").write_bytes(b"measurement\n")
|
|
|
|
def _manifest(self):
|
|
assets = tuple(
|
|
SimpleNamespace(workspace_path=path, content=content)
|
|
for path, content in sorted(self.asset_content.items())
|
|
)
|
|
return SimpleNamespace(
|
|
digest="sha256:" + "a" * 64,
|
|
fixture=SimpleNamespace(
|
|
checksum="sha256:" + "b" * 64,
|
|
assets=assets,
|
|
),
|
|
viewports=(
|
|
SimpleNamespace(id="desktop", width=800, height=600),
|
|
SimpleNamespace(id="mobile", width=375, height=600),
|
|
),
|
|
timeout=SimpleNamespace(run_seconds=20),
|
|
)
|
|
|
|
@staticmethod
|
|
def _measurement(reason: str = "success"):
|
|
return SimpleNamespace(
|
|
run_id="run",
|
|
cell_id="cell",
|
|
repetition=1,
|
|
attempt=1,
|
|
terminal_reason=reason,
|
|
)
|
|
|
|
def _view(self, ident: str, width: int, *, suffix: str = "") -> ViewportObservation:
|
|
screenshot = f"screenshot-{ident}.png"
|
|
png = b"\x89PNG\r\n\x1a\n" + ident.encode("ascii") + suffix.encode("ascii")
|
|
(self.attempt / screenshot).write_bytes(png)
|
|
images = tuple(
|
|
{
|
|
"src": path,
|
|
"alt": path,
|
|
"complete": True,
|
|
"natural_width": 20,
|
|
"natural_height": 20,
|
|
"visible": True,
|
|
"rect": {
|
|
"x": 0,
|
|
"y": 0,
|
|
"width": 20,
|
|
"height": 20,
|
|
"right": 20,
|
|
"bottom": 20,
|
|
},
|
|
}
|
|
for path in ("assets/a.svg", "assets/b.svg")
|
|
)
|
|
accessibility = {
|
|
"h1_count": 1,
|
|
"headings": [1],
|
|
"heading_progression": True,
|
|
"main_count": 1,
|
|
"landmarks": 1,
|
|
"controls": [
|
|
{
|
|
"name": True,
|
|
"tab_index": 0,
|
|
"focused": True,
|
|
"focus_visible": True,
|
|
"contrast": 7.0,
|
|
}
|
|
],
|
|
"ax": {"nodes": 4, "non_ignored": 3, "named": 2},
|
|
}
|
|
return ViewportObservation(
|
|
ident,
|
|
width,
|
|
600,
|
|
screenshot,
|
|
_digest(png),
|
|
len(png),
|
|
images,
|
|
{
|
|
"scroll_width": width,
|
|
"client_width": width,
|
|
"clipped": 0,
|
|
"overlaps": 0,
|
|
},
|
|
accessibility,
|
|
)
|
|
|
|
def _render(self) -> RenderObservation:
|
|
return RenderObservation(
|
|
"Chromium/Test",
|
|
"http://127.0.0.1:12345",
|
|
(
|
|
{"kind": "local", "path": "/index.html", "allowed": True, "status": 200},
|
|
{"kind": "local", "path": "/assets/a.svg", "allowed": True, "status": 200},
|
|
{"kind": "local", "path": "/assets/b.svg", "allowed": True, "status": 200},
|
|
),
|
|
(),
|
|
(self._view("desktop", 800), self._view("mobile", 375)),
|
|
)
|
|
|
|
def _build(self, render=None, reason: str = "success"):
|
|
return build_web_validation(
|
|
self._manifest(),
|
|
self.workspace,
|
|
self._measurement(reason),
|
|
self._render() if render is None else render,
|
|
)
|
|
|
|
def _publish_valid(self):
|
|
record = self._build()
|
|
publish_web_validation(self.attempt, record)
|
|
return record
|
|
|
|
def _rewrite_record(self, mutate) -> None:
|
|
path = self.attempt / "web-validation.json"
|
|
record = json.loads(path.read_text(encoding="ascii"))
|
|
mutate(record)
|
|
path.write_text(
|
|
json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n",
|
|
encoding="ascii",
|
|
)
|
|
|
|
def _reproject_runtime_gates(self, record) -> None:
|
|
manifest = self._manifest()
|
|
render = SimpleNamespace(
|
|
requests=tuple(record["requests"]),
|
|
console=tuple(record["console"]),
|
|
viewports=tuple(
|
|
SimpleNamespace(
|
|
id=item["id"],
|
|
width=item["width"],
|
|
height=item["height"],
|
|
image_facts=tuple(item["images"]),
|
|
layout=item["layout"],
|
|
accessibility=item["accessibility"],
|
|
)
|
|
for item in record["viewports"]
|
|
),
|
|
)
|
|
gates = web_validation_module._runtime_gates(manifest, render)
|
|
generated = web_validation_module._generated_gate(record["workspace"])
|
|
gates["generated_files"] = generated
|
|
gates["static_safety"] = web_validation_module._static_gate(
|
|
self.workspace, generated, manifest
|
|
)
|
|
record["screenshots"] = [
|
|
{"id": item["id"], **item["screenshot"]}
|
|
for item in record["viewports"]
|
|
]
|
|
record["gates"] = [gates[ident] for ident in WEB_GATES]
|
|
passed = all(item["passed"] for item in record["gates"])
|
|
record["status"] = "passed" if passed else "failed"
|
|
record["reason"] = "" if passed else next(
|
|
item["reason"] for item in record["gates"] if not item["passed"]
|
|
)
|
|
|
|
def test_valid_record_binds_complete_evidence_and_is_immutable(self):
|
|
record = self._publish_valid()
|
|
loaded = load_web_validation(self.attempt, manifest=self._manifest())
|
|
self.assertEqual(loaded.status, "passed")
|
|
self.assertEqual([item["id"] for item in loaded.record["gates"]], list(WEB_GATES))
|
|
self.assertEqual(len(loaded.record["workspace"]["inputs"]), 3)
|
|
self.assertEqual(len(loaded.record["viewports"]), 2)
|
|
self.assertEqual(len(loaded.record["screenshots"]), 2)
|
|
self.assertEqual(record.record, loaded.record)
|
|
before = (self.attempt / "web-validation.json").read_bytes()
|
|
with self.assertRaises(WebValidationError):
|
|
publish_web_validation(self.attempt, record)
|
|
self.assertEqual((self.attempt / "web-validation.json").read_bytes(), before)
|
|
|
|
def test_static_workspace_fault_matrix_is_recorded_failed(self):
|
|
cases = {
|
|
"fixture_mutation": lambda: (self.workspace / "assets/a.svg").write_bytes(b"changed"),
|
|
"fixture_symlink": self._replace_fixture_with_symlink,
|
|
"extra_nested": lambda: (self.workspace / "assets/extra.svg").write_text("extra"),
|
|
"external_reference": lambda: (self.workspace / "index.html").write_text(
|
|
"<main><h1>x</h1><img src='https://example.invalid/a.png' alt='x'></main>"
|
|
),
|
|
}
|
|
for name, mutate in cases.items():
|
|
with self.subTest(name=name):
|
|
self.tearDown()
|
|
self.setUp()
|
|
mutate()
|
|
record = self._build()
|
|
self.assertEqual(record.status, "failed")
|
|
self.assertFalse(all(item["passed"] for item in record.record["gates"]))
|
|
|
|
def _replace_fixture_with_symlink(self) -> None:
|
|
target = self.workspace / "assets/a.svg"
|
|
target.unlink()
|
|
target.symlink_to(self.workspace / "assets/b.svg")
|
|
|
|
def test_runtime_gate_one_fault_matrix(self):
|
|
cases = {}
|
|
render = self._render()
|
|
image_views = list(render.viewports)
|
|
image_facts = [dict(item) for item in image_views[0].image_facts]
|
|
image_facts[0]["complete"] = False
|
|
image_views[0] = SimpleNamespace(**{
|
|
**image_views[0].__dict__, "image_facts": tuple(image_facts)
|
|
})
|
|
cases["images"] = SimpleNamespace(**{**render.__dict__, "viewports": tuple(image_views)})
|
|
|
|
render = self._render()
|
|
cases["network"] = SimpleNamespace(**{
|
|
**render.__dict__,
|
|
"requests": (*render.requests, {"kind": "external", "url_digest": _digest(b"x"), "allowed": False, "status": 0}),
|
|
})
|
|
render = self._render()
|
|
cases["console"] = SimpleNamespace(**{
|
|
**render.__dict__, "console": ({"kind": "exception", "level": "error"},)
|
|
})
|
|
render = self._render()
|
|
views = list(render.viewports)
|
|
views[0] = SimpleNamespace(**{
|
|
**views[0].__dict__,
|
|
"layout": {"scroll_width": 801, "client_width": 800, "clipped": 1, "overlaps": 0},
|
|
})
|
|
cases["responsive"] = SimpleNamespace(**{**render.__dict__, "viewports": tuple(views)})
|
|
render = self._render()
|
|
views = list(render.viewports)
|
|
accessibility = copy.deepcopy(views[0].accessibility)
|
|
accessibility["controls"][0]["focus_visible"] = False
|
|
views[0] = SimpleNamespace(**{**views[0].__dict__, "accessibility": accessibility})
|
|
cases["accessibility"] = SimpleNamespace(**{**render.__dict__, "viewports": tuple(views)})
|
|
|
|
for gate, faulty in cases.items():
|
|
with self.subTest(gate=gate):
|
|
record = self._build(faulty)
|
|
self.assertEqual(record.status, "failed")
|
|
gates = {item["id"]: item for item in record.record["gates"]}
|
|
self.assertFalse(gates[gate]["passed"])
|
|
|
|
def test_lifecycle_non_success_is_not_run_without_browser(self):
|
|
for reason in ("nonzero_exit", "timed_out", "cancelled", "controller_lost"):
|
|
with self.subTest(reason=reason):
|
|
record = build_web_validation(
|
|
self._manifest(), self.workspace, self._measurement(reason), None
|
|
)
|
|
self.assertEqual(record.status, "not_run")
|
|
self.assertEqual(record.record["reason"], f"lifecycle_{reason}")
|
|
self.assertFalse(any(item["passed"] for item in record.record["gates"]))
|
|
|
|
def test_browser_discovery_or_start_failure_is_blocked(self):
|
|
prepared = SimpleNamespace(
|
|
workspace_dir=str(self.workspace), attempt_root=str(self.attempt)
|
|
)
|
|
for error in (FileNotFoundError("missing"), OSError("start")):
|
|
with self.subTest(error=type(error).__name__), mock.patch(
|
|
"scripts.agent_benchmark.web_validation.BrowserRenderer.render",
|
|
side_effect=error,
|
|
):
|
|
record = validate_web_attempt(
|
|
self._manifest(),
|
|
self.attempt,
|
|
prepared,
|
|
self._measurement(),
|
|
)
|
|
self.assertEqual(record.status, "blocked")
|
|
self.assertFalse(record.record["screenshots"])
|
|
|
|
with mock.patch(
|
|
"scripts.agent_benchmark.web_validation.BrowserRenderer.render",
|
|
side_effect=BrowserError("screenshot_collision"),
|
|
), self.assertRaises(WebValidationError):
|
|
validate_web_attempt(
|
|
self._manifest(),
|
|
self.attempt,
|
|
prepared,
|
|
self._measurement(),
|
|
)
|
|
|
|
def test_missing_generated_files_are_failed_not_not_run(self):
|
|
(self.workspace / "index.html").unlink()
|
|
record = build_web_validation(
|
|
self._manifest(), self.workspace, self._measurement(), None
|
|
)
|
|
self.assertEqual(record.status, "failed")
|
|
self.assertFalse(record.record["gates"][0]["passed"])
|
|
|
|
def test_status_gate_contradiction_and_unknown_nested_field_are_rejected(self):
|
|
self._publish_valid()
|
|
self._rewrite_record(
|
|
lambda record: record["gates"][0].update(
|
|
{"passed": False, "reason": "generated_missing"}
|
|
)
|
|
)
|
|
with self.assertRaises(WebValidationError):
|
|
load_web_validation(self.attempt)
|
|
|
|
(self.attempt / "web-validation.json").unlink()
|
|
self._publish_valid()
|
|
self._rewrite_record(lambda record: record["browser"].update({"unknown": 1}))
|
|
with self.assertRaises(WebValidationError):
|
|
load_web_validation(self.attempt)
|
|
|
|
(self.attempt / "web-validation.json").unlink()
|
|
self._publish_valid()
|
|
self._rewrite_record(
|
|
lambda record: record["viewports"][0]["images"][0].update(
|
|
{"complete": False}
|
|
)
|
|
)
|
|
with self.assertRaises(WebValidationError):
|
|
load_web_validation(self.attempt)
|
|
|
|
(self.attempt / "web-validation.json").unlink()
|
|
self._publish_valid()
|
|
self._rewrite_record(
|
|
lambda record: record["gates"][0]["evidence"].append("invented")
|
|
)
|
|
with self.assertRaises(WebValidationError):
|
|
load_web_validation(self.attempt)
|
|
|
|
def test_screenshot_corruption_missing_nonregular_and_symlink_are_rejected(self):
|
|
variants = ("corrupt", "missing", "directory", "symlink", "extra")
|
|
for variant in variants:
|
|
with self.subTest(variant=variant):
|
|
self.tearDown()
|
|
self.setUp()
|
|
record = self._publish_valid()
|
|
target = self.attempt / record.record["screenshots"][0]["file"]
|
|
if variant == "corrupt":
|
|
target.write_bytes(b"not-png")
|
|
elif variant == "extra":
|
|
(self.attempt / "screenshot-extra.png").write_bytes(
|
|
b"\x89PNG\r\n\x1a\nextra"
|
|
)
|
|
else:
|
|
target.unlink()
|
|
if variant == "directory":
|
|
target.mkdir()
|
|
elif variant == "symlink":
|
|
target.symlink_to(self.attempt / record.record["screenshots"][1]["file"])
|
|
with self.assertRaises(WebValidationError):
|
|
load_web_validation(self.attempt)
|
|
|
|
def test_fixture_generated_extra_and_measurement_changes_are_rejected(self):
|
|
variants = ("fixture", "generated", "extra", "measurement")
|
|
for variant in variants:
|
|
with self.subTest(variant=variant):
|
|
self.tearDown()
|
|
self.setUp()
|
|
self._publish_valid()
|
|
if variant == "fixture":
|
|
(self.workspace / "assets/a.svg").write_bytes(b"changed")
|
|
elif variant == "generated":
|
|
(self.workspace / "script.js").write_text("changed")
|
|
elif variant == "extra":
|
|
(self.workspace / "extra.txt").write_text("extra")
|
|
else:
|
|
(self.attempt / "attempt-measurement.json").write_bytes(b"changed")
|
|
with self.assertRaises(WebValidationError):
|
|
load_web_validation(self.attempt)
|
|
|
|
def test_manifest_fixture_and_viewport_binding_are_strict(self):
|
|
self._publish_valid()
|
|
manifest = self._manifest()
|
|
manifest.fixture.checksum = "sha256:" + "c" * 64
|
|
with self.assertRaises(WebValidationError):
|
|
load_web_validation(self.attempt, manifest=manifest)
|
|
|
|
manifest = self._manifest()
|
|
manifest.viewports = tuple(reversed(manifest.viewports))
|
|
with self.assertRaises(WebValidationError):
|
|
load_web_validation(self.attempt, manifest=manifest)
|
|
|
|
def test_observed_records_require_exact_manifest_viewports(self):
|
|
def failed_render():
|
|
render = self._render()
|
|
return SimpleNamespace(
|
|
**{
|
|
**render.__dict__,
|
|
"console": ({"kind": "exception", "level": "error"},),
|
|
}
|
|
)
|
|
|
|
valid_failed = self._build(failed_render())
|
|
self.assertEqual(valid_failed.status, "failed")
|
|
publish_web_validation(self.attempt, valid_failed)
|
|
loaded = load_web_validation(self.attempt, manifest=self._manifest())
|
|
self.assertEqual(loaded.status, "failed")
|
|
self.assertEqual(len(loaded.record["viewports"]), 2)
|
|
|
|
def missing(record):
|
|
removed = record["viewports"].pop()
|
|
(self.attempt / removed["screenshot"]["file"]).unlink()
|
|
|
|
def empty(record):
|
|
for item in record["viewports"]:
|
|
(self.attempt / item["screenshot"]["file"]).unlink()
|
|
record["viewports"] = []
|
|
|
|
def duplicate(record):
|
|
removed = record["viewports"][1]
|
|
(self.attempt / removed["screenshot"]["file"]).unlink()
|
|
record["viewports"] = [
|
|
copy.deepcopy(record["viewports"][0]),
|
|
copy.deepcopy(record["viewports"][0]),
|
|
]
|
|
|
|
def reordered(record):
|
|
record["viewports"].reverse()
|
|
|
|
def foreign(record):
|
|
item = record["viewports"][1]
|
|
old = self.attempt / item["screenshot"]["file"]
|
|
item["id"] = "foreign"
|
|
item["screenshot"]["file"] = "screenshot-foreign.png"
|
|
old.rename(self.attempt / item["screenshot"]["file"])
|
|
|
|
for name, mutate in (
|
|
("missing", missing),
|
|
("empty", empty),
|
|
("duplicate", duplicate),
|
|
("reordered", reordered),
|
|
("foreign", foreign),
|
|
):
|
|
with self.subTest(case=name):
|
|
self.tearDown()
|
|
self.setUp()
|
|
record = self._build(failed_render())
|
|
publish_web_validation(self.attempt, record)
|
|
path = self.attempt / "web-validation.json"
|
|
raw = json.loads(path.read_text(encoding="ascii"))
|
|
mutate(raw)
|
|
self._reproject_runtime_gates(raw)
|
|
path.write_text(
|
|
json.dumps(raw, sort_keys=True, separators=(",", ":")) + "\n",
|
|
encoding="ascii",
|
|
)
|
|
before = path.read_bytes()
|
|
with self.assertRaises(WebValidationError):
|
|
load_web_validation(self.attempt, manifest=self._manifest())
|
|
self.assertEqual(path.read_bytes(), before)
|
|
|
|
def test_manifest_viewport_id_grammar_round_trips(self):
|
|
manifest = self._manifest()
|
|
manifest.viewports = (
|
|
SimpleNamespace(id="mobile.small+wide", width=375, height=600),
|
|
)
|
|
render = RenderObservation(
|
|
"Chromium/Test",
|
|
"http://127.0.0.1:12345",
|
|
(
|
|
{
|
|
"kind": "local",
|
|
"path": "/index.html",
|
|
"allowed": True,
|
|
"status": 200,
|
|
},
|
|
{
|
|
"kind": "local",
|
|
"path": "/assets/a.svg",
|
|
"allowed": True,
|
|
"status": 200,
|
|
},
|
|
{
|
|
"kind": "local",
|
|
"path": "/assets/b.svg",
|
|
"allowed": True,
|
|
"status": 200,
|
|
},
|
|
),
|
|
(),
|
|
(self._view("mobile.small+wide", 375),),
|
|
)
|
|
record = build_web_validation(
|
|
manifest,
|
|
self.workspace,
|
|
self._measurement(),
|
|
render,
|
|
)
|
|
self.assertEqual(record.status, "passed")
|
|
publish_web_validation(self.attempt, record)
|
|
loaded = load_web_validation(self.attempt, manifest=manifest)
|
|
self.assertEqual(
|
|
[item["id"] for item in loaded.record["viewports"]],
|
|
["mobile.small+wide"],
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|