1615 lines
63 KiB
Python
1615 lines
63 KiB
Python
from __future__ import annotations
|
|
|
|
import datetime
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import socket
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import unittest
|
|
from pathlib import Path
|
|
from unittest import mock
|
|
|
|
from scripts.agent_benchmark.attempts import (
|
|
AttemptStateError,
|
|
PreflightObservation,
|
|
RunStore,
|
|
Slot,
|
|
)
|
|
from scripts.agent_benchmark.browser_cdp import RenderObservation, ViewportObservation
|
|
from scripts.agent_benchmark.connectivity import (
|
|
CallerCapability,
|
|
ConnectivityIssue,
|
|
EffectiveBinding,
|
|
ISSUE_RESUME_CODES,
|
|
RequestedEffectiveBinding,
|
|
make_result,
|
|
)
|
|
from scripts.agent_benchmark.lifecycle import (
|
|
COMPLETION_EXIT_AFTER_IDLE,
|
|
CLOCK_HARNESS_MONOTONIC,
|
|
METRIC_NAMES,
|
|
SOURCE_HARNESS,
|
|
SOURCE_WORKSPACE_POLL,
|
|
SUBMISSION_STDIN_ONCE,
|
|
UNIT_NANOSECONDS,
|
|
InvocationSpec,
|
|
SupervisorLocator,
|
|
env_pairs,
|
|
recover_invocation,
|
|
run_invocation,
|
|
spec_digest,
|
|
)
|
|
from scripts.agent_benchmark.manifest import AssetMapping, digest_workspace_inputs, load_manifest
|
|
from scripts.agent_benchmark.measurement import (
|
|
AttemptMeasurement,
|
|
REASON_NOT_OBSERVED,
|
|
REASON_NOT_REPORTED,
|
|
WorkspaceWriteObservation,
|
|
observed,
|
|
publish_measurement,
|
|
unavailable,
|
|
)
|
|
from scripts.agent_benchmark.rubric import RUBRIC_CATEGORIES
|
|
from scripts.agent_benchmark import scoring as scoring_module
|
|
from scripts.agent_benchmark.scoring import (
|
|
BlindWorkspace,
|
|
ScoringEvidenceFinalization,
|
|
ScoringError,
|
|
ScoringInvocationResult,
|
|
score_run,
|
|
)
|
|
from scripts.agent_benchmark.web_validation import (
|
|
WEB_GATES,
|
|
build_web_validation,
|
|
publish_web_validation,
|
|
)
|
|
|
|
|
|
def _digest(data: bytes) -> str:
|
|
return "sha256:" + hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
def _worksheet(total_delta: int = 0) -> dict:
|
|
categories = []
|
|
for index, (ident, maximum) in enumerate(RUBRIC_CATEGORIES):
|
|
score = maximum - (1 if index == 0 else 0)
|
|
categories.append(
|
|
{
|
|
"id": ident,
|
|
"max_score": maximum,
|
|
"score": score,
|
|
"evidence": f"Anonymous evidence for {ident}.",
|
|
}
|
|
)
|
|
return {
|
|
"rubric_version": "landing-quality-v1",
|
|
"categories": categories,
|
|
"total": sum(item["score"] for item in categories) + total_delta,
|
|
}
|
|
|
|
|
|
class FakeScoringAdapter:
|
|
capability = CallerCapability(
|
|
"codex", ("direct", "execution_preset"), ("xhigh",)
|
|
)
|
|
|
|
def __init__(
|
|
self,
|
|
*,
|
|
blocked: bool = False,
|
|
modes: list[str] | None = None,
|
|
sensitive_value: str = "",
|
|
):
|
|
self.blocked = blocked
|
|
self.modes = list(modes or ["success"])
|
|
self.preflights = 0
|
|
self.invocations: list[tuple[BlindWorkspace, bytes]] = []
|
|
self.sensitive_value = sensitive_value
|
|
self.last_mode = ""
|
|
|
|
def preflight(self, cell):
|
|
self.preflights += 1
|
|
iop = cell.iop
|
|
requested = RequestedEffectiveBinding(
|
|
cell.id,
|
|
cell.caller,
|
|
iop.route_kind,
|
|
iop.route_id,
|
|
iop.request_model,
|
|
iop.requested_effort,
|
|
)
|
|
issues = ()
|
|
if self.blocked:
|
|
issues = (
|
|
ConnectivityIssue(
|
|
"credential_missing",
|
|
ISSUE_RESUME_CODES["credential_missing"],
|
|
),
|
|
)
|
|
else:
|
|
requested = RequestedEffectiveBinding(
|
|
cell.id,
|
|
cell.caller,
|
|
iop.route_kind,
|
|
iop.route_id,
|
|
iop.request_model,
|
|
iop.requested_effort,
|
|
iop.route_kind,
|
|
iop.route_id,
|
|
iop.request_model,
|
|
iop.requested_effort,
|
|
tuple(
|
|
EffectiveBinding(item.stage, item.model, item.effort)
|
|
for item in iop.expected_bindings
|
|
),
|
|
)
|
|
result = make_result(cell, self.capability, requested, issues)
|
|
return PreflightObservation(
|
|
result, "sha256:" + "1" * 64, "sha256:" + "2" * 64
|
|
)
|
|
|
|
def _publish_lifecycle(self, cell, blind, mode, on_started):
|
|
output = Path(blind.output_dir)
|
|
control = output / "fake-control"
|
|
control.mkdir()
|
|
locator = SupervisorLocator(
|
|
os.getpid(),
|
|
"fake-start-identity",
|
|
str(control / "control.sock"),
|
|
"fake-challenge-" + blind.blind_id,
|
|
str(control),
|
|
"2026-08-11T00:00:00+00:00",
|
|
)
|
|
locator_payload = {
|
|
"supervisor_pid": locator.supervisor_pid,
|
|
"start_identity": locator.start_identity,
|
|
"socket_path": locator.socket_path,
|
|
"challenge": locator.challenge,
|
|
"control_dir": locator.control_dir,
|
|
"created_at": locator.created_at,
|
|
}
|
|
(control / "locator.json").write_text(
|
|
json.dumps(locator_payload), encoding="utf-8"
|
|
)
|
|
invocation_digest = "sha256:" + "4" * 64
|
|
on_started(locator, invocation_digest)
|
|
terminal_reason = "nonzero_exit" if mode == "raise" else "success"
|
|
receipt = {
|
|
"receipt_version": 1,
|
|
"supervisor_pid": locator.supervisor_pid,
|
|
"challenge_digest": hashlib.sha256(
|
|
locator.challenge.encode("utf-8")
|
|
).hexdigest(),
|
|
"reason": terminal_reason,
|
|
"exit_code": 1 if mode == "raise" else 0,
|
|
"signal": None,
|
|
"caller_launched": True,
|
|
"cleanup_complete": True,
|
|
"process_group_alive": False,
|
|
"completed_at": "2026-08-11T00:00:01+00:00",
|
|
}
|
|
(control / "cleanup-receipt.json").write_text(
|
|
json.dumps(receipt), encoding="utf-8"
|
|
)
|
|
public_locator = {
|
|
key: value
|
|
for key, value in locator_payload.items()
|
|
if key != "challenge"
|
|
}
|
|
public_locator["challenge_digest"] = receipt["challenge_digest"]
|
|
lifecycle = {
|
|
"record": "result",
|
|
"success": terminal_reason == "success",
|
|
"terminal_reason": terminal_reason,
|
|
"cleanup_complete": True,
|
|
"process_group_alive": False,
|
|
"spec_digest": invocation_digest,
|
|
"locator": public_locator,
|
|
"effective_binding": {
|
|
"route_kind": cell.iop.route_kind,
|
|
"route_id": cell.iop.route_id,
|
|
"model": cell.iop.request_model,
|
|
"effort": cell.iop.requested_effort,
|
|
},
|
|
}
|
|
(output / "lifecycle-result.json").write_text(
|
|
json.dumps(lifecycle), encoding="utf-8"
|
|
)
|
|
journal = (
|
|
json.dumps(
|
|
{"record": "header", "spec_digest": invocation_digest}
|
|
)
|
|
+ "\n"
|
|
+ json.dumps(
|
|
{
|
|
"record": "terminal",
|
|
"terminal_reason": terminal_reason,
|
|
"cleanup_complete": True,
|
|
"process_group_alive": False,
|
|
}
|
|
)
|
|
+ "\n"
|
|
)
|
|
(output / "lifecycle-journal.jsonl").write_text(
|
|
journal, encoding="utf-8"
|
|
)
|
|
|
|
def invoke(self, cell, blind, task_payload, timeout, on_started):
|
|
self.invocations.append((blind, task_payload))
|
|
mode = self.modes.pop(0) if self.modes else "success"
|
|
self.last_mode = mode
|
|
output = Path(blind.output_dir)
|
|
self._publish_lifecycle(cell, blind, mode, on_started)
|
|
if mode == "mutate":
|
|
target = Path(blind.input_dir) / "index.html"
|
|
target.chmod(0o600)
|
|
target.write_text("<main>mutated</main>", encoding="utf-8")
|
|
if mode == "raise":
|
|
raise RuntimeError("fake evaluator failed")
|
|
if mode == "malformed":
|
|
(output / "worksheet.json").write_text("{}", encoding="utf-8")
|
|
else:
|
|
worksheet = _worksheet()
|
|
if mode == "secret":
|
|
worksheet["categories"][0]["evidence"] = self.sensitive_value
|
|
(output / "worksheet.json").write_text(
|
|
json.dumps(worksheet, sort_keys=True, separators=(",", ":"))
|
|
+ "\n",
|
|
encoding="ascii",
|
|
)
|
|
binding = (
|
|
cell.iop.route_kind,
|
|
cell.iop.route_id,
|
|
cell.iop.request_model,
|
|
cell.iop.requested_effort,
|
|
)
|
|
if mode == "binding":
|
|
binding = (binding[0], "substituted", binding[2], binding[3])
|
|
return ScoringInvocationResult(mode not in {"failed", "binding"}, "success" if mode not in {"failed", "binding"} else "failed", binding)
|
|
|
|
def finalize_evidence(self, blind):
|
|
leaked = False
|
|
if self.sensitive_value:
|
|
sensitive = self.sensitive_value.encode("utf-8")
|
|
for path in Path(blind.root).rglob("*"):
|
|
if path.is_file() and sensitive in path.read_bytes():
|
|
path.unlink()
|
|
leaked = True
|
|
return ScoringEvidenceFinalization(
|
|
not leaked, "" if not leaked else "runtime_secret_leak"
|
|
)
|
|
|
|
|
|
class ScoringTest(unittest.TestCase):
|
|
def setUp(self):
|
|
temporary = tempfile.TemporaryDirectory(dir="/tmp", prefix="iop-score-")
|
|
self.addCleanup(temporary.cleanup)
|
|
self.root = Path(temporary.name)
|
|
(self.root / "Makefile").write_text("test:\n\t@true\n")
|
|
fixture_root = self.root / "scripts" / "fixtures" / "bench"
|
|
fixture_root.mkdir(parents=True)
|
|
(fixture_root / "prompt.md").write_text("Build the page.", encoding="utf-8")
|
|
self.asset_bytes = {
|
|
"scripts/fixtures/bench/reference.txt": b"anonymous reference\n",
|
|
"scripts/fixtures/bench/a.svg": b"<svg xmlns='http://www.w3.org/2000/svg' width='20' height='20'/>",
|
|
"scripts/fixtures/bench/b.svg": b"<svg xmlns='http://www.w3.org/2000/svg' width='20' height='20'/>",
|
|
}
|
|
for relative, data in self.asset_bytes.items():
|
|
path = self.root / relative
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_bytes(data)
|
|
assets = (
|
|
AssetMapping(
|
|
"scripts/fixtures/bench/reference.txt",
|
|
"brief/reference.txt",
|
|
self.asset_bytes["scripts/fixtures/bench/reference.txt"],
|
|
),
|
|
AssetMapping(
|
|
"scripts/fixtures/bench/a.svg",
|
|
"assets/a.svg",
|
|
self.asset_bytes["scripts/fixtures/bench/a.svg"],
|
|
),
|
|
AssetMapping(
|
|
"scripts/fixtures/bench/b.svg",
|
|
"assets/b.svg",
|
|
self.asset_bytes["scripts/fixtures/bench/b.svg"],
|
|
),
|
|
)
|
|
raw = {
|
|
"pipeline_version": "2",
|
|
"environment": "dev",
|
|
"testbed": "../iop-s2",
|
|
"repetitions": 1,
|
|
"session_policy": "fresh",
|
|
"setup_cache_policy": "isolated",
|
|
"timeout": {
|
|
"run_seconds": 5,
|
|
"idle_seconds": 1,
|
|
"quiet_seconds": 1,
|
|
"cleanup_grace_seconds": 1,
|
|
},
|
|
"viewports": [
|
|
{"id": "desktop", "width": 800, "height": 600},
|
|
{"id": "mobile", "width": 375, "height": 600},
|
|
],
|
|
"rubric_version": "landing-quality-v1",
|
|
"evaluator": {
|
|
"caller": "codex",
|
|
"iop": {
|
|
"request_model": "judge-model",
|
|
"requested_effort": "xhigh",
|
|
"route_kind": "direct",
|
|
"route_id": "judge-route",
|
|
"expected_bindings": [
|
|
{
|
|
"stage": "request",
|
|
"model": "judge-model",
|
|
"effort": "xhigh",
|
|
}
|
|
],
|
|
},
|
|
},
|
|
"output_root": "agent-test/runs/anonymous-bench",
|
|
"fixture": {
|
|
"version": "landing-v1",
|
|
"prompt": "scripts/fixtures/bench/prompt.md",
|
|
"assets": [
|
|
{
|
|
"source": item.source,
|
|
"workspace_path": item.workspace_path,
|
|
}
|
|
for item in assets
|
|
],
|
|
"checksum": digest_workspace_inputs(assets),
|
|
},
|
|
"matrix": [
|
|
{
|
|
"id": "cell-sentinel",
|
|
"caller": "claude",
|
|
"iop": {
|
|
"request_model": "source-model",
|
|
"requested_effort": "high",
|
|
"route_kind": "direct",
|
|
"route_id": "source-route",
|
|
"expected_bindings": [
|
|
{
|
|
"stage": "request",
|
|
"model": "source-model",
|
|
"effort": "high",
|
|
}
|
|
],
|
|
},
|
|
}
|
|
],
|
|
}
|
|
self.manifest_path = self.root / "manifest.json"
|
|
self.manifest_path.write_text(json.dumps(raw), encoding="utf-8")
|
|
self.manifest = load_manifest(self.manifest_path, repo_root=self.root)
|
|
tokens = iter(("123456abcdef", "234567abcdef", "345678abcdef"))
|
|
self.store = RunStore(
|
|
self.root,
|
|
clock=lambda: datetime.datetime(
|
|
2026, 8, 11, 1, 2, 3, tzinfo=datetime.timezone.utc
|
|
),
|
|
token_hex=lambda _n: next(tokens),
|
|
)
|
|
self.run = self.store.create(self.manifest, self.manifest_path.read_bytes())
|
|
|
|
@staticmethod
|
|
def _measurement(attempt, terminal_reason: str) -> AttemptMeasurement:
|
|
timeline = {
|
|
"submitted_at": unavailable(REASON_NOT_OBSERVED, SOURCE_HARNESS),
|
|
"first_output_at": unavailable(REASON_NOT_OBSERVED, SOURCE_HARNESS),
|
|
"first_write_observed_at": unavailable(
|
|
REASON_NOT_OBSERVED, SOURCE_WORKSPACE_POLL
|
|
),
|
|
"first_write_mtime": unavailable(
|
|
REASON_NOT_OBSERVED, SOURCE_WORKSPACE_POLL
|
|
),
|
|
"total_duration": observed(
|
|
1, UNIT_NANOSECONDS, CLOCK_HARNESS_MONOTONIC, SOURCE_HARNESS
|
|
),
|
|
}
|
|
usage = {
|
|
name: unavailable(REASON_NOT_REPORTED, SOURCE_HARNESS)
|
|
for name in METRIC_NAMES
|
|
}
|
|
return AttemptMeasurement(
|
|
attempt.identity.run_id,
|
|
attempt.identity.cell_id,
|
|
attempt.identity.repetition,
|
|
attempt.identity.attempt,
|
|
"claude",
|
|
"sha256:" + "3" * 64,
|
|
terminal_reason,
|
|
timeline,
|
|
usage,
|
|
WorkspaceWriteObservation(
|
|
False, None, None, "", 1, 0, REASON_NOT_OBSERVED
|
|
),
|
|
(),
|
|
)
|
|
|
|
def _view(self, root: Path, ident: str, width: int) -> ViewportObservation:
|
|
screenshot = f"screenshot-{ident}.png"
|
|
png = b"\x89PNG\r\n\x1a\n" + ident.encode("ascii")
|
|
(root / screenshot).write_bytes(png)
|
|
images = tuple(
|
|
{
|
|
"src": path,
|
|
"alt": path,
|
|
"complete": True,
|
|
"natural_width": 20,
|
|
"natural_height": 20,
|
|
"visible": True,
|
|
"rect": {
|
|
"x": 0,
|
|
"y": 0,
|
|
"width": 20,
|
|
"height": 20,
|
|
"right": 20,
|
|
"bottom": 20,
|
|
},
|
|
}
|
|
for path in ("assets/a.svg", "assets/b.svg")
|
|
)
|
|
return ViewportObservation(
|
|
ident,
|
|
width,
|
|
600,
|
|
screenshot,
|
|
_digest(png),
|
|
len(png),
|
|
images,
|
|
{
|
|
"scroll_width": width,
|
|
"client_width": width,
|
|
"clipped": 0,
|
|
"overlaps": 0,
|
|
},
|
|
{
|
|
"h1_count": 1,
|
|
"headings": [1],
|
|
"heading_progression": True,
|
|
"main_count": 1,
|
|
"landmarks": 1,
|
|
"controls": [
|
|
{
|
|
"name": True,
|
|
"tab_index": 0,
|
|
"focused": True,
|
|
"focus_visible": True,
|
|
"contrast": 7.0,
|
|
}
|
|
],
|
|
"ax": {"nodes": 4, "non_ignored": 3, "named": 2},
|
|
},
|
|
)
|
|
|
|
def _attempt(
|
|
self, state: str = "success", *, leaked: bool = False,
|
|
rendered: bool = True, leaked_identity: str | None = None,
|
|
):
|
|
with self.store.writer(self.run):
|
|
attempt = self.store.allocate(self.run, Slot("cell-sentinel", 1))
|
|
workspace = Path(attempt.root) / "workspace"
|
|
(workspace / "assets").mkdir(parents=True)
|
|
(workspace / "brief").mkdir()
|
|
for asset in self.manifest.fixture.assets:
|
|
(workspace / asset.workspace_path).write_bytes(asset.content)
|
|
heading = leaked_identity or ("source-route" if leaked else "Ready")
|
|
(workspace / "index.html").write_text(
|
|
f"<main><h1>{heading}</h1><img src='assets/a.svg' alt='A'>"
|
|
"<img src='assets/b.svg' alt='B'><a href='#x'>go</a>"
|
|
"<script src='script.js'></script></main>",
|
|
encoding="utf-8",
|
|
)
|
|
(workspace / "styles.css").write_text(
|
|
"body{color:#111;background:#fff}img{width:20px}"
|
|
"a:focus{outline:2px solid #05f}",
|
|
encoding="utf-8",
|
|
)
|
|
(workspace / "script.js").write_text(
|
|
"document.body.dataset.ready='1';", encoding="utf-8"
|
|
)
|
|
terminal_reason = "success" if state == "success" else state
|
|
measurement = self._measurement(attempt, terminal_reason)
|
|
publish_measurement(attempt.root, measurement)
|
|
if state == "success" and rendered:
|
|
render = RenderObservation(
|
|
"Chromium/Test",
|
|
"http://127.0.0.1:12345",
|
|
(
|
|
{"kind": "local", "path": "/index.html", "allowed": True, "status": 200},
|
|
{"kind": "local", "path": "/assets/a.svg", "allowed": True, "status": 200},
|
|
{"kind": "local", "path": "/assets/b.svg", "allowed": True, "status": 200},
|
|
),
|
|
(),
|
|
(
|
|
self._view(Path(attempt.root), "desktop", 800),
|
|
self._view(Path(attempt.root), "mobile", 375),
|
|
),
|
|
)
|
|
else:
|
|
render = None
|
|
web = build_web_validation(
|
|
self.manifest, workspace, measurement, render
|
|
)
|
|
publish_web_validation(attempt.root, web)
|
|
terminal = self.store.publish_terminal(
|
|
attempt, state, result={"terminal_reason": terminal_reason}
|
|
)
|
|
return terminal
|
|
|
|
def test_scored_attempt_is_blind_exactly_once_and_strict(self):
|
|
attempt = self._attempt()
|
|
adapter = FakeScoringAdapter()
|
|
summary = score_run(
|
|
self.store, self.run, self.manifest, adapter=adapter
|
|
)
|
|
self.assertEqual(
|
|
(summary.scored, summary.unscored, summary.scoring_failed, summary.blocked),
|
|
(1, 0, 0, 0),
|
|
)
|
|
self.assertEqual(adapter.preflights, 1)
|
|
self.assertEqual(len(adapter.invocations), 1)
|
|
blind, prompt = adapter.invocations[0]
|
|
self.assertNotIn("cell-sentinel", blind.root)
|
|
self.assertNotIn("source-route", blind.root)
|
|
self.assertNotIn(str(Path(attempt.root).resolve()), blind.root)
|
|
visible = prompt
|
|
for path in Path(blind.root).rglob("*"):
|
|
if path.is_file():
|
|
visible += b"\n" + str(path).encode() + b"\n" + path.read_bytes()
|
|
for sentinel in (
|
|
b"cell-sentinel",
|
|
b"source-route",
|
|
b"source-model",
|
|
str(Path(attempt.root).resolve()).encode(),
|
|
):
|
|
self.assertNotIn(sentinel, visible)
|
|
self.assertEqual(
|
|
sorted(path.relative_to(blind.input_dir).as_posix() for path in Path(blind.input_dir).rglob("*") if path.is_file()),
|
|
[
|
|
"assets/a.svg",
|
|
"assets/b.svg",
|
|
"index.html",
|
|
"screenshots/screenshot-desktop.png",
|
|
"screenshots/screenshot-mobile.png",
|
|
"script.js",
|
|
"styles.css",
|
|
],
|
|
)
|
|
result_path = Path(attempt.root) / "scoring" / "score-000001" / "result.json"
|
|
result = json.loads(result_path.read_text(encoding="ascii"))
|
|
self.assertEqual(result["status"], "scored")
|
|
self.assertEqual(result["worksheet"]["total"], 99)
|
|
self.assertNotIn("gates", result["worksheet"])
|
|
mappings = tuple((Path(self.run.root) / "blind-mappings").glob("*.json"))
|
|
self.assertEqual(len(mappings), 1)
|
|
mapping = json.loads(mappings[0].read_text(encoding="ascii"))
|
|
self.assertEqual(mapping["attempt"]["cell_id"], "cell-sentinel")
|
|
self.assertFalse(mappings[0].is_relative_to(Path(blind.root)))
|
|
|
|
before = {path: path.read_bytes() for path in Path(attempt.root).rglob("*") if path.is_file()}
|
|
second = score_run(self.store, self.run, self.manifest, adapter=adapter)
|
|
self.assertEqual(second.scored, 1)
|
|
self.assertEqual(len(adapter.invocations), 1)
|
|
self.assertEqual(before, {path: path.read_bytes() for path in Path(attempt.root).rglob("*") if path.is_file()})
|
|
|
|
def test_ineligible_attempt_is_unscored_without_preflight_or_zero(self):
|
|
attempt = self._attempt("failed")
|
|
adapter = FakeScoringAdapter()
|
|
summary = score_run(self.store, self.run, self.manifest, adapter=adapter)
|
|
self.assertEqual((summary.unscored, summary.scored), (1, 0))
|
|
self.assertEqual((adapter.preflights, adapter.invocations), (0, []))
|
|
path = Path(attempt.root) / "scoring" / "unscored.json"
|
|
before = path.read_bytes()
|
|
record = json.loads(before)
|
|
self.assertEqual(record["status"], "unscored")
|
|
self.assertEqual(record["reasons"], ["lifecycle_failed"])
|
|
self.assertFalse(set(record) & {"score", "total", "worksheet"})
|
|
score_run(self.store, self.run, self.manifest, adapter=adapter)
|
|
self.assertEqual(path.read_bytes(), before)
|
|
|
|
def test_not_run_web_gates_are_all_unscored_without_evaluator(self):
|
|
attempt = self._attempt(rendered=False)
|
|
adapter = FakeScoringAdapter()
|
|
summary = score_run(self.store, self.run, self.manifest, adapter=adapter)
|
|
self.assertEqual((summary.unscored, summary.scored), (1, 0))
|
|
self.assertEqual((adapter.preflights, adapter.invocations), (0, []))
|
|
record = json.loads(
|
|
(Path(attempt.root) / "scoring" / "unscored.json").read_text()
|
|
)
|
|
self.assertEqual(record["status"], "unscored")
|
|
web = json.loads(
|
|
(Path(attempt.root) / "web-validation.json").read_text()
|
|
)
|
|
failed_gates = {
|
|
f"gate_{item['id']}" for item in web["gates"] if not item["passed"]
|
|
}
|
|
self.assertTrue(failed_gates)
|
|
self.assertTrue(failed_gates.issubset(record["reasons"]))
|
|
self.assertEqual(
|
|
[item["id"] for item in web["gates"]], list(WEB_GATES)
|
|
)
|
|
self.assertFalse(set(record) & {"score", "total", "worksheet"})
|
|
|
|
def test_symlinked_generated_input_fails_before_evaluator_invocation(self):
|
|
attempt = self._attempt()
|
|
workspace = Path(attempt.root) / "workspace"
|
|
(workspace / "index.html").unlink()
|
|
(workspace / "index.html").symlink_to("styles.css")
|
|
adapter = FakeScoringAdapter()
|
|
with self.assertRaises(AttemptStateError):
|
|
score_run(self.store, self.run, self.manifest, adapter=adapter)
|
|
self.assertEqual(adapter.invocations, [])
|
|
self.assertFalse((Path(attempt.root) / "scoring").exists())
|
|
|
|
def test_preflight_blocker_allocates_no_score(self):
|
|
attempt = self._attempt()
|
|
adapter = FakeScoringAdapter(blocked=True)
|
|
summary = score_run(self.store, self.run, self.manifest, adapter=adapter)
|
|
self.assertEqual(summary.blocked, 1)
|
|
self.assertEqual(adapter.invocations, [])
|
|
self.assertFalse((Path(attempt.root) / "scoring").exists())
|
|
self.assertEqual(
|
|
len(list((Path(self.run.root) / "scoring-preflight").iterdir())), 1
|
|
)
|
|
|
|
def test_failed_score_retries_only_with_new_id_and_fresh_session(self):
|
|
attempt = self._attempt()
|
|
adapter = FakeScoringAdapter(modes=["malformed", "success"])
|
|
first = score_run(self.store, self.run, self.manifest, adapter=adapter)
|
|
self.assertEqual(first.scoring_failed, 1)
|
|
failed = Path(attempt.root) / "scoring" / "score-000001"
|
|
failed_bytes = {path: path.read_bytes() for path in failed.rglob("*") if path.is_file()}
|
|
|
|
second = score_run(self.store, self.run, self.manifest, adapter=adapter)
|
|
self.assertEqual(second.scoring_failed, 1)
|
|
self.assertEqual(len(adapter.invocations), 1)
|
|
self.assertFalse((Path(attempt.root) / "scoring" / "score-000002").exists())
|
|
|
|
third = score_run(
|
|
self.store,
|
|
self.run,
|
|
self.manifest,
|
|
adapter=adapter,
|
|
retry_scoring_failed=True,
|
|
)
|
|
self.assertEqual(third.scored, 1)
|
|
self.assertEqual(len(adapter.invocations), 2)
|
|
self.assertEqual(
|
|
failed_bytes,
|
|
{path: path.read_bytes() for path in failed.rglob("*") if path.is_file()},
|
|
)
|
|
allocations = [
|
|
json.loads((Path(attempt.root) / "scoring" / f"score-{index:06d}" / "allocation.json").read_text())
|
|
for index in (1, 2)
|
|
]
|
|
self.assertNotEqual(allocations[0]["blind_id"], allocations[1]["blind_id"])
|
|
self.assertNotEqual(
|
|
allocations[0]["session_identity"], allocations[1]["session_identity"]
|
|
)
|
|
|
|
def test_invalid_binding_and_identity_leak_fail_without_fallback(self):
|
|
attempt = self._attempt()
|
|
binding = FakeScoringAdapter(modes=["binding"])
|
|
summary = score_run(self.store, self.run, self.manifest, adapter=binding)
|
|
self.assertEqual(summary.scoring_failed, 1)
|
|
result = json.loads(
|
|
(Path(attempt.root) / "scoring" / "score-000001" / "result.json").read_text()
|
|
)
|
|
self.assertEqual(result["status"], "scoring_failed")
|
|
self.assertNotIn("worksheet", result)
|
|
|
|
# A separate run proves an identity sentinel in retained page bytes is
|
|
# rejected before the evaluator is invoked.
|
|
self.run = self.store.create(self.manifest, self.manifest_path.read_bytes())
|
|
leaked = self._attempt(leaked=True)
|
|
adapter = FakeScoringAdapter()
|
|
leaked_summary = score_run(
|
|
self.store, self.run, self.manifest, adapter=adapter
|
|
)
|
|
self.assertEqual(leaked_summary.scoring_failed, 1)
|
|
self.assertEqual(adapter.invocations, [])
|
|
leaked_result = json.loads(
|
|
(Path(leaked.root) / "scoring" / "score-000001" / "result.json").read_text()
|
|
)
|
|
self.assertEqual(leaked_result["reason"], "blind_preparation_failed")
|
|
|
|
def test_shared_evaluator_binding_is_allowed_but_short_caller_leak_fails(self):
|
|
raw = json.loads(self.manifest_path.read_text())
|
|
raw["matrix"][0]["caller"] = "agy"
|
|
raw["matrix"][0]["iop"] = {
|
|
"request_model": "judge-model",
|
|
"requested_effort": "xhigh",
|
|
"route_kind": "direct",
|
|
"route_id": "judge-route",
|
|
"expected_bindings": [
|
|
{
|
|
"stage": "request",
|
|
"model": "judge-model",
|
|
"effort": "xhigh",
|
|
}
|
|
],
|
|
}
|
|
raw["output_root"] = "agent-test/runs/shared-binding"
|
|
path = self.root / "shared.json"
|
|
path.write_text(json.dumps(raw), encoding="utf-8")
|
|
self.manifest_path = path
|
|
self.manifest = load_manifest(path, repo_root=self.root)
|
|
self.run = self.store.create(self.manifest, path.read_bytes())
|
|
self._attempt()
|
|
shared = FakeScoringAdapter()
|
|
summary = score_run(
|
|
self.store, self.run, self.manifest, adapter=shared
|
|
)
|
|
self.assertEqual((summary.scored, summary.scoring_failed), (1, 0))
|
|
|
|
self.run = self.store.create(self.manifest, path.read_bytes())
|
|
leaked = self._attempt(leaked_identity="agy")
|
|
rejected = FakeScoringAdapter()
|
|
summary = score_run(
|
|
self.store, self.run, self.manifest, adapter=rejected
|
|
)
|
|
self.assertEqual((summary.scored, summary.scoring_failed), (0, 1))
|
|
self.assertEqual(rejected.invocations, [])
|
|
result = json.loads(
|
|
(
|
|
Path(leaked.root)
|
|
/ "scoring"
|
|
/ "score-000001"
|
|
/ "result.json"
|
|
).read_text(encoding="ascii")
|
|
)
|
|
self.assertEqual(result["reason"], "blind_preparation_failed")
|
|
|
|
def test_delimited_short_caller_and_cell_identity_leaks_fail(self):
|
|
identity = scoring_module.ProducerIdentity(
|
|
exact_tokens=("agy", "cell-sentinel"),
|
|
path_tokens=(),
|
|
producer_tokens=(),
|
|
evaluator_shared_tokens=(),
|
|
)
|
|
for value in (
|
|
b"agy",
|
|
b"caller=agy",
|
|
b"agy-output",
|
|
b"agy_output",
|
|
b"cell-sentinel-output",
|
|
b"cell-sentinel_output",
|
|
):
|
|
with self.subTest(value=value):
|
|
self.assertTrue(scoring_module._contains_identity(value, identity))
|
|
for value in (
|
|
b"strategy",
|
|
b"agyextended",
|
|
b"mycell-sentinel",
|
|
):
|
|
with self.subTest(value=value):
|
|
self.assertFalse(scoring_module._contains_identity(value, identity))
|
|
|
|
def test_binary_identity_boundaries_do_not_disappear(self):
|
|
identity = scoring_module.ProducerIdentity(
|
|
exact_tokens=("agy", "cell-sentinel"),
|
|
path_tokens=(),
|
|
producer_tokens=("producer-model", "shared-model"),
|
|
evaluator_shared_tokens=("shared-model",),
|
|
)
|
|
for value in (
|
|
b"\x89PNG\r\n\x1a\nx\xffagy\x00",
|
|
b"\x89PNG\r\n\x1a\nagy\xffx",
|
|
b"\x89PNG\r\n\x1a\nx\xffcell-sentinel\x00",
|
|
b"\x89PNG\r\n\x1a\ncell-sentinel\xffx",
|
|
b"\x89PNG\r\n\x1a\nx\xffproducer-model\x00",
|
|
b"\x89PNG\r\n\x1a\nproducer-model\xffx",
|
|
):
|
|
with self.subTest(value=value):
|
|
self.assertTrue(scoring_module._contains_identity(value, identity))
|
|
for value in (
|
|
b"strategy",
|
|
b"xagy",
|
|
b"agyx",
|
|
b"xproducer-model",
|
|
b"producer-modelx",
|
|
b"x\xffshared-model\x00",
|
|
):
|
|
with self.subTest(value=value):
|
|
self.assertFalse(scoring_module._contains_identity(value, identity))
|
|
|
|
def test_invalid_filesystem_bytes_do_not_bypass_identity_scan(self):
|
|
if os.name != "posix":
|
|
self.skipTest("raw invalid filesystem bytes require POSIX paths")
|
|
|
|
identity = scoring_module.ProducerIdentity(
|
|
exact_tokens=("agy", "cell-sentinel"),
|
|
path_tokens=(),
|
|
producer_tokens=("source-route", "source-model"),
|
|
evaluator_shared_tokens=(),
|
|
)
|
|
|
|
def write_raw(directory: Path, name: bytes, data: bytes = b"safe") -> None:
|
|
descriptor = os.open(
|
|
os.fsencode(directory) + b"/" + name,
|
|
os.O_WRONLY | os.O_CREAT | os.O_EXCL,
|
|
0o600,
|
|
)
|
|
try:
|
|
os.write(descriptor, data)
|
|
finally:
|
|
os.close(descriptor)
|
|
|
|
for index, name in enumerate(
|
|
(
|
|
b"x\xffagy\xfe.png",
|
|
b"x\xffcell-sentinel\xfe.png",
|
|
b"x\xffsource-route\xfe.png",
|
|
b"x\xffsource-model\xfe.png",
|
|
)
|
|
):
|
|
with self.subTest(name=name):
|
|
visible_root = self.root / f"invalid-visible-{index}"
|
|
visible_root.mkdir()
|
|
write_raw(visible_root, name)
|
|
with self.assertRaisesRegex(
|
|
ScoringError, "evaluator-visible evidence leaks execution identity"
|
|
):
|
|
scoring_module._scan_visible_tree(visible_root, identity)
|
|
|
|
class RawFilenameAdapter(FakeScoringAdapter):
|
|
def __init__(self, raw_name: bytes):
|
|
super().__init__()
|
|
self.raw_name = raw_name
|
|
|
|
def invoke(self, cell, blind, task_payload, timeout, on_started):
|
|
result = super().invoke(
|
|
cell, blind, task_payload, timeout, on_started
|
|
)
|
|
write_raw(Path(blind.output_dir), self.raw_name)
|
|
return result
|
|
|
|
leaked_attempt = self._attempt()
|
|
leaked_adapter = RawFilenameAdapter(
|
|
b"x\xffcell-sentinel\xfe-output.bin"
|
|
)
|
|
leaked_summary = score_run(
|
|
self.store, self.run, self.manifest, adapter=leaked_adapter
|
|
)
|
|
self.assertEqual(
|
|
(leaked_summary.scored, leaked_summary.scoring_failed), (0, 1)
|
|
)
|
|
leaked_root = (
|
|
Path(leaked_attempt.root) / "scoring" / "score-000001"
|
|
)
|
|
leaked_result = json.loads(
|
|
(leaked_root / "result.json").read_text(encoding="ascii")
|
|
)
|
|
self.assertEqual(leaked_result["status"], "scoring_failed")
|
|
self.assertEqual(leaked_result["reason"], "evaluator_output_leak")
|
|
leaked_bytes = {
|
|
path: path.read_bytes()
|
|
for path in leaked_root.rglob("*")
|
|
if path.is_file()
|
|
}
|
|
retained = score_run(
|
|
self.store, self.run, self.manifest, adapter=leaked_adapter
|
|
)
|
|
self.assertEqual(retained.scoring_failed, 1)
|
|
self.assertEqual(len(leaked_adapter.invocations), 1)
|
|
self.assertEqual(
|
|
leaked_bytes,
|
|
{
|
|
path: path.read_bytes()
|
|
for path in leaked_root.rglob("*")
|
|
if path.is_file()
|
|
},
|
|
)
|
|
|
|
self.run = self.store.create(
|
|
self.manifest, self.manifest_path.read_bytes()
|
|
)
|
|
safe_attempt = self._attempt()
|
|
safe_adapter = RawFilenameAdapter(b"x\xffanonymous\xfe-output.bin")
|
|
safe_summary = score_run(
|
|
self.store, self.run, self.manifest, adapter=safe_adapter
|
|
)
|
|
self.assertEqual(
|
|
(safe_summary.scored, safe_summary.scoring_failed), (1, 0)
|
|
)
|
|
safe_blind = Path(safe_adapter.invocations[0][0].root)
|
|
first_digest = scoring_module._blind_tree_digest(safe_blind)
|
|
self.assertEqual(
|
|
first_digest, scoring_module._blind_tree_digest(safe_blind)
|
|
)
|
|
safe_result = json.loads(
|
|
(
|
|
Path(safe_attempt.root)
|
|
/ "scoring"
|
|
/ "score-000001"
|
|
/ "result.json"
|
|
).read_text(encoding="ascii")
|
|
)
|
|
self.assertEqual(safe_result["status"], "scored")
|
|
self.assertEqual(safe_result["post_tree_digest"], first_digest)
|
|
|
|
ordinary_path = "input/caf\N{LATIN SMALL LETTER E WITH ACUTE}.txt"
|
|
ordinary_data = b"ordinary"
|
|
framed = bytearray(b"IOP-BENCH-BLIND-INPUT-V1\0")
|
|
ordinary_bytes = ordinary_path.encode("utf-8")
|
|
framed += len(ordinary_bytes).to_bytes(8, "big") + ordinary_bytes
|
|
framed += len(ordinary_data).to_bytes(8, "big") + ordinary_data
|
|
self.assertEqual(
|
|
scoring_module._input_digest([(ordinary_path, ordinary_data)]),
|
|
_digest(bytes(framed)),
|
|
)
|
|
|
|
def unusable_fsencode(value):
|
|
raise AssertionError("path framing must not use the filesystem codec")
|
|
|
|
with mock.patch.object(scoring_module.os, "fsencode", unusable_fsencode):
|
|
self.assertEqual(
|
|
scoring_module._input_digest([(ordinary_path, ordinary_data)]),
|
|
_digest(bytes(framed)),
|
|
)
|
|
self.assertEqual(
|
|
scoring_module._path_bytes(ordinary_path),
|
|
ordinary_path.encode("utf-8"),
|
|
)
|
|
self.assertEqual(
|
|
scoring_module._path_bytes("input/x\udcff.txt"),
|
|
b"input/x\xff.txt",
|
|
)
|
|
with self.assertRaisesRegex(ScoringError, "scoring path is invalid"):
|
|
scoring_module._path_bytes("input/x\ud800.txt")
|
|
|
|
surrogate_path = "input/x\udcff.txt"
|
|
surrogate_data = b"raw"
|
|
surrogate_framed = bytearray(b"IOP-BENCH-BLIND-INPUT-V1\0")
|
|
surrogate_bytes = b"input/x\xff.txt"
|
|
surrogate_framed += len(surrogate_bytes).to_bytes(8, "big") + surrogate_bytes
|
|
surrogate_framed += len(surrogate_data).to_bytes(8, "big") + surrogate_data
|
|
self.assertEqual(
|
|
scoring_module._input_digest([(surrogate_path, surrogate_data)]),
|
|
_digest(bytes(surrogate_framed)),
|
|
)
|
|
|
|
def test_receipt_only_recovery_waits_for_lifecycle_quiescence(self):
|
|
blind_root = self.root / "receipt-only" / "blind" / "blind-paused"
|
|
for name in ("input", "session", "output"):
|
|
(blind_root / name).mkdir(parents=True, exist_ok=True)
|
|
output = blind_root / "output"
|
|
control = output / "paused-control"
|
|
control.mkdir()
|
|
alias = Path(tempfile.gettempdir()) / (
|
|
"iop-score-paused-"
|
|
+ hashlib.sha256(str(output).encode("utf-8")).hexdigest()[:16]
|
|
)
|
|
os.symlink(output, alias, target_is_directory=True)
|
|
self.addCleanup(
|
|
lambda: alias.unlink()
|
|
if alias.exists() or alias.is_symlink()
|
|
else None
|
|
)
|
|
locator = SupervisorLocator(
|
|
os.getpid(),
|
|
"paused-start-identity",
|
|
str(alias / "paused-control" / "control.sock"),
|
|
"paused-challenge",
|
|
str(alias / "paused-control"),
|
|
"2026-08-11T00:00:00+00:00",
|
|
)
|
|
locator_payload = {
|
|
"supervisor_pid": locator.supervisor_pid,
|
|
"start_identity": locator.start_identity,
|
|
"socket_path": locator.socket_path,
|
|
"challenge": locator.challenge,
|
|
"control_dir": locator.control_dir,
|
|
"created_at": locator.created_at,
|
|
}
|
|
(control / "locator.json").write_text(
|
|
json.dumps(locator_payload), encoding="utf-8"
|
|
)
|
|
receipt = {
|
|
"receipt_version": 1,
|
|
"supervisor_pid": locator.supervisor_pid,
|
|
"challenge_digest": hashlib.sha256(
|
|
locator.challenge.encode("utf-8")
|
|
).hexdigest(),
|
|
"reason": "success",
|
|
"exit_code": 0,
|
|
"signal": None,
|
|
"caller_launched": True,
|
|
"cleanup_complete": True,
|
|
"process_group_alive": False,
|
|
"completed_at": "2026-08-11T00:00:01+00:00",
|
|
}
|
|
(control / "cleanup-receipt.json").write_text(
|
|
json.dumps(receipt), encoding="utf-8"
|
|
)
|
|
invocation_digest = "sha256:" + "9" * 64
|
|
public_locator = {
|
|
key: value for key, value in locator_payload.items() if key != "challenge"
|
|
}
|
|
public_locator["challenge_digest"] = receipt["challenge_digest"]
|
|
lifecycle = {
|
|
"record": "result",
|
|
"success": True,
|
|
"terminal_reason": "success",
|
|
"cleanup_complete": True,
|
|
"process_group_alive": False,
|
|
"spec_digest": invocation_digest,
|
|
"locator": public_locator,
|
|
}
|
|
journal = (
|
|
json.dumps({"record": "header", "spec_digest": invocation_digest})
|
|
+ "\n"
|
|
+ json.dumps(
|
|
{
|
|
"record": "terminal",
|
|
"terminal_reason": "success",
|
|
"cleanup_complete": True,
|
|
"process_group_alive": False,
|
|
}
|
|
)
|
|
+ "\n"
|
|
)
|
|
prior = {
|
|
path: path.read_bytes()
|
|
for path in (control / "locator.json", control / "cleanup-receipt.json")
|
|
}
|
|
recovered: list[tuple[str | None, str]] = []
|
|
errors: list[BaseException] = []
|
|
successor_started = threading.Event()
|
|
|
|
def recover() -> None:
|
|
try:
|
|
recovered.append(
|
|
scoring_module._recover_runner(
|
|
blind_root,
|
|
locator,
|
|
invocation_digest,
|
|
control_target=control,
|
|
)
|
|
)
|
|
scoring_module._release_runner_alias(
|
|
{
|
|
"control_alias": str(alias),
|
|
"control_target": str(control),
|
|
}
|
|
)
|
|
successor_started.set()
|
|
except BaseException as exc:
|
|
errors.append(exc)
|
|
|
|
worker = threading.Thread(target=recover)
|
|
started = time.monotonic()
|
|
worker.start()
|
|
time.sleep(0.35)
|
|
self.assertTrue(worker.is_alive())
|
|
self.assertFalse(successor_started.is_set())
|
|
self.assertTrue(alias.is_symlink())
|
|
self.assertFalse((output / "lifecycle-result.json").exists())
|
|
|
|
(output / "lifecycle-journal.jsonl").write_text(journal, encoding="utf-8")
|
|
(output / "lifecycle-result.json").write_text(
|
|
json.dumps(lifecycle), encoding="utf-8"
|
|
)
|
|
time.sleep(0.05)
|
|
self.assertTrue(worker.is_alive())
|
|
worker.join(5)
|
|
self.assertFalse(worker.is_alive())
|
|
self.assertGreaterEqual(time.monotonic() - started, 0.5)
|
|
self.assertEqual(errors, [])
|
|
self.assertTrue(successor_started.is_set())
|
|
self.assertEqual(len(recovered), 1)
|
|
self.assertRegex(recovered[0][0] or "", r"^sha256:[0-9a-f]{64}$")
|
|
self.assertRegex(recovered[0][1], r"^sha256:[0-9a-f]{64}$")
|
|
self.assertFalse(alias.exists() or alias.is_symlink())
|
|
self.assertTrue((output / "lifecycle-result.json").is_file())
|
|
self.assertTrue((output / "lifecycle-journal.jsonl").is_file())
|
|
for path, data in prior.items():
|
|
self.assertEqual(path.read_bytes(), data)
|
|
|
|
def prepare_prepublished(name: str):
|
|
root = self.root / "receipt-only" / "blind" / name
|
|
for directory in ("input", "session", "output"):
|
|
(root / directory).mkdir(parents=True, exist_ok=True)
|
|
local_output = root / "output"
|
|
local_control = self.root / f"{name}-control"
|
|
local_control.mkdir()
|
|
local_alias = Path(tempfile.gettempdir()) / (
|
|
f"iop-score-{name}-"
|
|
+ hashlib.sha256(str(local_output).encode("utf-8")).hexdigest()[:16]
|
|
)
|
|
os.symlink(
|
|
local_control.parent, local_alias, target_is_directory=True
|
|
)
|
|
self.addCleanup(
|
|
lambda: local_alias.unlink()
|
|
if local_alias.exists() or local_alias.is_symlink()
|
|
else None
|
|
)
|
|
local_locator = SupervisorLocator(
|
|
os.getpid(),
|
|
f"{name}-start-identity",
|
|
str(local_alias / local_control.name / "control.sock"),
|
|
f"{name}-challenge",
|
|
str(local_alias / local_control.name),
|
|
"2026-08-11T00:00:00+00:00",
|
|
)
|
|
local_receipt = dict(receipt)
|
|
local_receipt["challenge_digest"] = hashlib.sha256(
|
|
local_locator.challenge.encode("utf-8")
|
|
).hexdigest()
|
|
(local_control / "cleanup-receipt.json").write_text(
|
|
json.dumps(local_receipt), encoding="utf-8"
|
|
)
|
|
local_public_locator = {
|
|
"supervisor_pid": local_locator.supervisor_pid,
|
|
"start_identity": local_locator.start_identity,
|
|
"socket_path": local_locator.socket_path,
|
|
"control_dir": local_locator.control_dir,
|
|
"created_at": local_locator.created_at,
|
|
"challenge_digest": local_receipt["challenge_digest"],
|
|
}
|
|
local_lifecycle = dict(lifecycle)
|
|
local_lifecycle["locator"] = local_public_locator
|
|
(local_output / "lifecycle-journal.jsonl").write_text(
|
|
journal, encoding="utf-8"
|
|
)
|
|
(local_output / "lifecycle-result.json").write_text(
|
|
json.dumps(local_lifecycle), encoding="utf-8"
|
|
)
|
|
control_socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
|
control_socket.bind(str(local_control / "control.sock"))
|
|
self.addCleanup(control_socket.close)
|
|
return (
|
|
root,
|
|
local_output,
|
|
local_control,
|
|
local_alias,
|
|
local_locator,
|
|
local_lifecycle,
|
|
)
|
|
|
|
def start_prepublished_recovery(root, control, alias, local_locator):
|
|
local_recovered: list[tuple[str | None, str]] = []
|
|
local_errors: list[BaseException] = []
|
|
local_successor_started = threading.Event()
|
|
|
|
def local_recover() -> None:
|
|
try:
|
|
local_recovered.append(
|
|
scoring_module._recover_runner(
|
|
root,
|
|
local_locator,
|
|
invocation_digest,
|
|
control_target=control,
|
|
)
|
|
)
|
|
scoring_module._release_runner_alias(
|
|
{
|
|
"control_alias": str(alias),
|
|
"control_target": str(control),
|
|
}
|
|
)
|
|
local_successor_started.set()
|
|
except BaseException as exc:
|
|
local_errors.append(exc)
|
|
|
|
local_worker = threading.Thread(target=local_recover)
|
|
local_worker.start()
|
|
return (
|
|
local_worker,
|
|
local_recovered,
|
|
local_errors,
|
|
local_successor_started,
|
|
)
|
|
|
|
(
|
|
stable_root,
|
|
_stable_output,
|
|
stable_control,
|
|
stable_alias,
|
|
stable_locator,
|
|
_stable_lifecycle,
|
|
) = prepare_prepublished("prepublished-stable")
|
|
stable_quiet_started = threading.Event()
|
|
original_wait = scoring_module._wait_post_cleanup_quiet
|
|
|
|
def observe_stable_quiet(*args, **kwargs):
|
|
stable_quiet_started.set()
|
|
return original_wait(*args, **kwargs)
|
|
|
|
with mock.patch.object(
|
|
scoring_module, "_POST_CLEANUP_TIMEOUT_SECONDS", 0.5
|
|
), mock.patch.object(
|
|
scoring_module, "_POST_CLEANUP_QUIET_SECONDS", 0.1
|
|
), mock.patch.object(
|
|
scoring_module, "_POST_CLEANUP_POLL_SECONDS", 0.01
|
|
), mock.patch.object(
|
|
scoring_module,
|
|
"_wait_post_cleanup_quiet",
|
|
side_effect=observe_stable_quiet,
|
|
):
|
|
(
|
|
stable_worker,
|
|
stable_recovered,
|
|
stable_errors,
|
|
stable_successor_started,
|
|
) = start_prepublished_recovery(
|
|
stable_root, stable_control, stable_alias, stable_locator
|
|
)
|
|
self.assertTrue(stable_quiet_started.wait(1))
|
|
time.sleep(0.03)
|
|
self.assertTrue(stable_worker.is_alive())
|
|
self.assertTrue(stable_alias.is_symlink())
|
|
self.assertTrue((stable_control / "control.sock").exists())
|
|
self.assertFalse(stable_successor_started.is_set())
|
|
stable_worker.join(2)
|
|
self.assertFalse(stable_worker.is_alive())
|
|
self.assertEqual(stable_errors, [])
|
|
self.assertEqual(len(stable_recovered), 1)
|
|
self.assertFalse(stable_alias.exists() or stable_alias.is_symlink())
|
|
self.assertFalse((stable_control / "control.sock").exists())
|
|
self.assertTrue(stable_successor_started.is_set())
|
|
|
|
(
|
|
changed_root,
|
|
changed_output,
|
|
changed_control,
|
|
changed_alias,
|
|
changed_locator,
|
|
changed_lifecycle,
|
|
) = prepare_prepublished("prepublished-changed")
|
|
changed_quiet_started = threading.Event()
|
|
|
|
def observe_changed_quiet(*args, **kwargs):
|
|
changed_quiet_started.set()
|
|
return original_wait(*args, **kwargs)
|
|
|
|
with mock.patch.object(
|
|
scoring_module, "_POST_CLEANUP_TIMEOUT_SECONDS", 0.5
|
|
), mock.patch.object(
|
|
scoring_module, "_POST_CLEANUP_QUIET_SECONDS", 0.1
|
|
), mock.patch.object(
|
|
scoring_module, "_POST_CLEANUP_POLL_SECONDS", 0.01
|
|
), mock.patch.object(
|
|
scoring_module,
|
|
"_wait_post_cleanup_quiet",
|
|
side_effect=observe_changed_quiet,
|
|
):
|
|
(
|
|
changed_worker,
|
|
changed_recovered,
|
|
changed_errors,
|
|
changed_successor_started,
|
|
) = start_prepublished_recovery(
|
|
changed_root, changed_control, changed_alias, changed_locator
|
|
)
|
|
self.assertTrue(changed_quiet_started.wait(1))
|
|
mutated_lifecycle = dict(changed_lifecycle)
|
|
mutated_lifecycle["publication_revision"] = 2
|
|
staged_lifecycle = changed_output / "lifecycle-result.changed.tmp"
|
|
staged_lifecycle.write_text(
|
|
json.dumps(mutated_lifecycle), encoding="utf-8"
|
|
)
|
|
os.replace(
|
|
staged_lifecycle, changed_output / "lifecycle-result.json"
|
|
)
|
|
changed_worker.join(2)
|
|
self.assertFalse(changed_worker.is_alive())
|
|
self.assertEqual(changed_recovered, [])
|
|
self.assertEqual(len(changed_errors), 1)
|
|
self.assertIsInstance(changed_errors[0], ScoringError)
|
|
self.assertIn("lifecycle publication changed", str(changed_errors[0]))
|
|
self.assertTrue(changed_alias.is_symlink())
|
|
self.assertTrue((changed_control / "control.sock").exists())
|
|
self.assertFalse(changed_successor_started.is_set())
|
|
|
|
missing_root = self.root / "receipt-only" / "blind" / "blind-missing"
|
|
for name in ("input", "session", "output"):
|
|
(missing_root / name).mkdir(parents=True, exist_ok=True)
|
|
missing_output = missing_root / "output"
|
|
missing_control = missing_output / "missing-control"
|
|
missing_control.mkdir()
|
|
missing_alias = Path(tempfile.gettempdir()) / (
|
|
"iop-score-missing-"
|
|
+ hashlib.sha256(str(missing_output).encode("utf-8")).hexdigest()[:16]
|
|
)
|
|
os.symlink(missing_output, missing_alias, target_is_directory=True)
|
|
self.addCleanup(
|
|
lambda: missing_alias.unlink()
|
|
if missing_alias.exists() or missing_alias.is_symlink()
|
|
else None
|
|
)
|
|
missing_locator = SupervisorLocator(
|
|
os.getpid(),
|
|
"missing-start-identity",
|
|
str(missing_alias / "missing-control" / "control.sock"),
|
|
"missing-challenge",
|
|
str(missing_alias / "missing-control"),
|
|
"2026-08-11T00:00:00+00:00",
|
|
)
|
|
missing_receipt = dict(receipt)
|
|
missing_receipt["challenge_digest"] = hashlib.sha256(
|
|
missing_locator.challenge.encode("utf-8")
|
|
).hexdigest()
|
|
(missing_control / "cleanup-receipt.json").write_text(
|
|
json.dumps(missing_receipt), encoding="utf-8"
|
|
)
|
|
with mock.patch.object(
|
|
scoring_module, "_POST_CLEANUP_TIMEOUT_SECONDS", 0.05
|
|
), mock.patch.object(
|
|
scoring_module, "_POST_CLEANUP_QUIET_SECONDS", 0.01
|
|
):
|
|
with self.assertRaisesRegex(
|
|
ScoringError, "lifecycle publication is incomplete"
|
|
):
|
|
scoring_module._recover_runner(
|
|
missing_root,
|
|
missing_locator,
|
|
invocation_digest,
|
|
control_target=missing_control,
|
|
)
|
|
self.assertTrue(missing_alias.is_symlink())
|
|
self.assertFalse((missing_output / "lifecycle-result.json").exists())
|
|
|
|
def test_interrupted_evaluator_is_stopped_before_retry(self):
|
|
attempt = self._attempt()
|
|
|
|
class InterruptingAdapter(FakeScoringAdapter):
|
|
def __init__(self):
|
|
super().__init__()
|
|
self.first = True
|
|
self.worker: threading.Thread | None = None
|
|
self.worker_result = None
|
|
self.worker_error: BaseException | None = None
|
|
self.successor_started = False
|
|
self.locator: SupervisorLocator | None = None
|
|
self.callback_error: BaseException | None = None
|
|
self.control_alias: Path | None = None
|
|
|
|
def invoke(self, cell, blind, task_payload, timeout, on_started):
|
|
if not self.first:
|
|
self.successor_started = True
|
|
if self.worker is not None and self.worker.is_alive():
|
|
raise AssertionError(
|
|
"successor started before survivor cleanup"
|
|
)
|
|
return super().invoke(
|
|
cell, blind, task_payload, timeout, on_started
|
|
)
|
|
self.first = False
|
|
self.invocations.append((blind, task_payload))
|
|
ready = threading.Event()
|
|
locator_box: list[SupervisorLocator] = []
|
|
alias = Path(tempfile.gettempdir()) / (
|
|
"iop-score-test-"
|
|
+ hashlib.sha256(blind.output_dir.encode("utf-8")).hexdigest()[:16]
|
|
)
|
|
os.symlink(blind.output_dir, alias, target_is_directory=True)
|
|
self.control_alias = alias
|
|
spec = InvocationSpec(
|
|
argv=(
|
|
sys.executable,
|
|
"-u",
|
|
"-c",
|
|
"import sys,time; sys.stdin.buffer.read(); "
|
|
"print('START', flush=True); time.sleep(30)",
|
|
),
|
|
cwd=blind.root,
|
|
env=env_pairs(
|
|
{"PATH": os.environ.get("PATH", "/usr/bin:/bin")}
|
|
),
|
|
submission_mode=SUBMISSION_STDIN_ONCE,
|
|
completion_mode=COMPLETION_EXIT_AFTER_IDLE,
|
|
timeout=timeout,
|
|
evidence_dir=blind.output_dir,
|
|
task_payload=b"evaluate",
|
|
control_dir=str(alias / "live-control"),
|
|
)
|
|
|
|
def run() -> None:
|
|
try:
|
|
def commit(locator):
|
|
try:
|
|
on_started(locator, spec_digest(spec))
|
|
locator_box.append(locator)
|
|
except BaseException as exc:
|
|
self.callback_error = exc
|
|
finally:
|
|
ready.set()
|
|
if self.callback_error is not None:
|
|
raise self.callback_error
|
|
|
|
self.worker_result = run_invocation(
|
|
spec,
|
|
parse_event=lambda _stream, _line: None,
|
|
on_started=commit,
|
|
)
|
|
except BaseException as exc:
|
|
self.worker_error = exc
|
|
|
|
self.worker = threading.Thread(target=run)
|
|
self.worker.start()
|
|
if not ready.wait(15):
|
|
raise KeyboardInterrupt(
|
|
"evaluator locator was not published: "
|
|
+ repr(self.worker_error)
|
|
)
|
|
if self.callback_error is not None:
|
|
raise KeyboardInterrupt(repr(self.callback_error))
|
|
deadline = time.monotonic() + 5
|
|
while not recover_invocation(
|
|
locator_box[0], stop=False
|
|
).caller_launched:
|
|
if time.monotonic() >= deadline:
|
|
raise AssertionError("evaluator did not launch")
|
|
time.sleep(0.01)
|
|
self.locator = locator_box[0]
|
|
raise KeyboardInterrupt("simulated scoring controller loss")
|
|
|
|
adapter = InterruptingAdapter()
|
|
|
|
def cleanup() -> None:
|
|
if adapter.worker is not None and adapter.worker.is_alive():
|
|
if adapter.locator is not None:
|
|
try:
|
|
recover_invocation(adapter.locator, stop=True)
|
|
except Exception:
|
|
pass
|
|
adapter.worker.join(5)
|
|
if adapter.control_alias is not None and adapter.control_alias.is_symlink():
|
|
adapter.control_alias.unlink()
|
|
|
|
self.addCleanup(cleanup)
|
|
with self.assertRaises(KeyboardInterrupt):
|
|
score_run(self.store, self.run, self.manifest, adapter=adapter)
|
|
self.assertIsNone(adapter.callback_error, repr(adapter.callback_error))
|
|
self.assertIsNone(adapter.worker_error, repr(adapter.worker_error))
|
|
self.assertIsNone(adapter.worker_result, repr(adapter.worker_result))
|
|
first_root = Path(attempt.root) / "scoring" / "score-000001"
|
|
self.assertTrue((first_root / "runner.json").is_file())
|
|
self.assertFalse((first_root / "result.json").exists())
|
|
prior = {
|
|
path: path.read_bytes() for path in first_root.rglob("*") if path.is_file()
|
|
}
|
|
|
|
summary = score_run(
|
|
self.store,
|
|
self.run,
|
|
self.manifest,
|
|
adapter=adapter,
|
|
retry_scoring_failed=True,
|
|
)
|
|
self.assertEqual((summary.scored, summary.scoring_failed), (1, 0))
|
|
self.assertTrue(adapter.successor_started)
|
|
self.assertIsNotNone(adapter.worker)
|
|
adapter.worker.join(5) # type: ignore[union-attr]
|
|
self.assertFalse(adapter.worker.is_alive()) # type: ignore[union-attr]
|
|
self.assertIsNone(adapter.worker_error)
|
|
self.assertIsNotNone(adapter.worker_result)
|
|
self.assertTrue(adapter.worker_result.cleanup_complete)
|
|
self.assertFalse(adapter.worker_result.process_group_alive)
|
|
receipt = json.loads(
|
|
(
|
|
Path(adapter.invocations[0][0].output_dir)
|
|
/ "live-control"
|
|
/ "cleanup-receipt.json"
|
|
).read_text()
|
|
)
|
|
self.assertEqual(receipt["reason"], "recovered_stop")
|
|
self.assertTrue(receipt["cleanup_complete"])
|
|
self.assertFalse(receipt["process_group_alive"])
|
|
for path, data in prior.items():
|
|
self.assertEqual(path.read_bytes(), data)
|
|
self.assertTrue(
|
|
(Path(attempt.root) / "scoring" / "score-000002").is_dir()
|
|
)
|
|
|
|
def test_mutated_input_and_runtime_secret_fail_before_scored(self):
|
|
attempt = self._attempt()
|
|
mutated = FakeScoringAdapter(modes=["mutate", "success"])
|
|
first = score_run(
|
|
self.store, self.run, self.manifest, adapter=mutated
|
|
)
|
|
self.assertEqual((first.scored, first.scoring_failed), (0, 1))
|
|
first_root = Path(attempt.root) / "scoring" / "score-000001"
|
|
first_bytes = {
|
|
path: path.read_bytes() for path in first_root.rglob("*") if path.is_file()
|
|
}
|
|
result = json.loads((first_root / "result.json").read_text())
|
|
self.assertEqual(result["reason"], "input_mutated")
|
|
self.assertNotIn("worksheet", result)
|
|
|
|
retry = score_run(
|
|
self.store,
|
|
self.run,
|
|
self.manifest,
|
|
adapter=mutated,
|
|
retry_scoring_failed=True,
|
|
)
|
|
self.assertEqual((retry.scored, retry.scoring_failed), (1, 0))
|
|
self.assertEqual(
|
|
first_bytes,
|
|
{
|
|
path: path.read_bytes()
|
|
for path in first_root.rglob("*")
|
|
if path.is_file()
|
|
},
|
|
)
|
|
|
|
secret = "runtime-evaluator-secret-literal"
|
|
self.run = self.store.create(self.manifest, self.manifest_path.read_bytes())
|
|
secret_attempt = self._attempt()
|
|
leaking = FakeScoringAdapter(
|
|
modes=["secret"], sensitive_value=secret
|
|
)
|
|
leaked = score_run(
|
|
self.store, self.run, self.manifest, adapter=leaking
|
|
)
|
|
self.assertEqual((leaked.scored, leaked.scoring_failed), (0, 1))
|
|
durable = b"".join(
|
|
path.read_bytes()
|
|
for path in Path(self.run.root).rglob("*")
|
|
if path.is_file()
|
|
)
|
|
self.assertNotIn(secret.encode("utf-8"), durable)
|
|
secret_result = json.loads(
|
|
(
|
|
Path(secret_attempt.root)
|
|
/ "scoring"
|
|
/ "score-000001"
|
|
/ "result.json"
|
|
).read_text()
|
|
)
|
|
self.assertEqual(secret_result["reason"], "runtime_secret_leak")
|
|
self.assertNotIn("worksheet", secret_result)
|
|
|
|
def test_tampered_score_fails_closed_without_rewrite(self):
|
|
attempt = self._attempt()
|
|
score_run(self.store, self.run, self.manifest, adapter=FakeScoringAdapter())
|
|
result = Path(attempt.root) / "scoring" / "score-000001" / "result.json"
|
|
record = json.loads(result.read_text())
|
|
record["worksheet"]["total"] = 0
|
|
result.write_text(
|
|
json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n"
|
|
)
|
|
before = result.read_bytes()
|
|
with self.assertRaises(ScoringError):
|
|
score_run(self.store, self.run, self.manifest, adapter=FakeScoringAdapter())
|
|
self.assertEqual(result.read_bytes(), before)
|
|
|
|
def test_execution_preset_evaluator_uses_manifest_stage_binding(self):
|
|
raw = json.loads(self.manifest_path.read_text())
|
|
raw["evaluator"]["iop"] = {
|
|
"request_model": "judge-model",
|
|
"requested_effort": "xhigh",
|
|
"route_kind": "execution_preset",
|
|
"route_id": "judge-preset",
|
|
"expected_bindings": [
|
|
{"stage": stage, "model": "judge-model"}
|
|
for stage in ("selector", "plan", "work", "review")
|
|
],
|
|
}
|
|
raw["output_root"] = "agent-test/runs/preset-bench"
|
|
path = self.root / "preset.json"
|
|
path.write_text(json.dumps(raw))
|
|
self.manifest_path = path
|
|
self.manifest = load_manifest(path, repo_root=self.root)
|
|
self.run = self.store.create(self.manifest, path.read_bytes())
|
|
self._attempt()
|
|
adapter = FakeScoringAdapter()
|
|
summary = score_run(self.store, self.run, self.manifest, adapter=adapter)
|
|
self.assertEqual(summary.scored, 1)
|
|
self.assertEqual(adapter.preflights, 1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|