toki
91dec43635
caller와 모델 조합을 반복 비교할 때 실행·격리·재개 근거가 흔들리지 않도록 manifest, workspace, lifecycle, append-only attempt 기반과 project-local 진입점을 함께 고정한다.
755 lines
31 KiB
Python
755 lines
31 KiB
Python
"""
|
|
workspace_test.py - Comprehensive tests for workspace materialization and isolation.
|
|
|
|
Covers exact run/cell/repetition/attempt grammar, path containment, symlink/collision rejection,
|
|
asset mapping, prompt exclusion, checksum verification, session freshness, testbed provenance,
|
|
and cross-attempt isolation.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
from scripts.agent_benchmark.manifest import (
|
|
AssetMapping,
|
|
Fixture,
|
|
IopCell,
|
|
Manifest,
|
|
MatrixCell,
|
|
Timeout,
|
|
Viewport,
|
|
digest_manifest_and_resolved_inputs,
|
|
digest_workspace_inputs,
|
|
load_manifest,
|
|
)
|
|
from scripts.agent_benchmark.workspace import (
|
|
AttemptIdentity,
|
|
PreparedWorkspace,
|
|
TestbedError,
|
|
TestbedProvenance,
|
|
WorkspaceChecksumError,
|
|
WorkspaceError,
|
|
WorkspacePathError,
|
|
WorkspaceValidationError,
|
|
inspect_testbed_provenance,
|
|
prepare_workspace,
|
|
validate_attempt_identity,
|
|
)
|
|
|
|
|
|
class BaseWorkspaceTest(unittest.TestCase):
|
|
"""Base test class providing temporary Git repositories and testbed fixtures."""
|
|
|
|
def setUp(self) -> None:
|
|
self.tmp_dir_obj = tempfile.TemporaryDirectory()
|
|
self.tmp_dir = Path(self.tmp_dir_obj.name).resolve()
|
|
|
|
self.repo_root = self.tmp_dir / "repo"
|
|
self.repo_root.mkdir()
|
|
|
|
# Initialize git repo in repo_root
|
|
subprocess.run(["git", "init"], cwd=self.repo_root, capture_output=True, check=True)
|
|
subprocess.run(
|
|
["git", "config", "user.name", "Test"], cwd=self.repo_root, capture_output=True, check=True
|
|
)
|
|
subprocess.run(
|
|
["git", "config", "user.email", "test@example.com"],
|
|
cwd=self.repo_root,
|
|
capture_output=True,
|
|
check=True,
|
|
)
|
|
|
|
# Initialize git repo in testbed (iop-s2)
|
|
self.testbed_dir = self.tmp_dir / "iop-s2"
|
|
self.testbed_dir.mkdir()
|
|
(self.testbed_dir / "README.md").write_text("testbed content", encoding="utf-8")
|
|
subprocess.run(["git", "init"], cwd=self.testbed_dir, capture_output=True, check=True)
|
|
subprocess.run(
|
|
["git", "config", "user.name", "Test"], cwd=self.testbed_dir, capture_output=True, check=True
|
|
)
|
|
subprocess.run(
|
|
["git", "config", "user.email", "test@example.com"],
|
|
cwd=self.testbed_dir,
|
|
capture_output=True,
|
|
check=True,
|
|
)
|
|
subprocess.run(["git", "add", "."], cwd=self.testbed_dir, capture_output=True, check=True)
|
|
subprocess.run(
|
|
["git", "commit", "-m", "initial testbed commit"],
|
|
cwd=self.testbed_dir,
|
|
capture_output=True,
|
|
check=True,
|
|
)
|
|
|
|
# Create fixture files under repo_root
|
|
self.fixture_dir = self.repo_root / "scripts" / "fixtures" / "bench"
|
|
self.fixture_dir.mkdir(parents=True)
|
|
|
|
self.prompt_rel = "scripts/fixtures/bench/prompt.md"
|
|
self.prompt_file = self.repo_root / self.prompt_rel
|
|
self.prompt_content = b"# Test Prompt\nDo task.\n"
|
|
self.prompt_file.write_bytes(self.prompt_content)
|
|
|
|
self.ref_rel = "scripts/fixtures/bench/ref.txt"
|
|
self.ref_file = self.repo_root / self.ref_rel
|
|
self.ref_content = b"Reference data content\n"
|
|
self.ref_file.write_bytes(self.ref_content)
|
|
|
|
subprocess.run(["git", "add", "."], cwd=self.repo_root, capture_output=True, check=True)
|
|
subprocess.run(
|
|
["git", "commit", "-m", "add fixture files"],
|
|
cwd=self.repo_root,
|
|
capture_output=True,
|
|
check=True,
|
|
)
|
|
|
|
# Assets list (initially only ref.txt is an asset, prompt.md is separate prompt file)
|
|
self.assets = [
|
|
AssetMapping(source=self.ref_rel, workspace_path="data/ref.txt", content=self.ref_content)
|
|
]
|
|
self.fixture_checksum = digest_workspace_inputs(self.assets)
|
|
|
|
self.run_id = "run-20260809T161730Z-0123456789ab"
|
|
self.output_root_rel = "agent-test/runs/bench-01"
|
|
|
|
self.manifest_raw = {
|
|
"pipeline_version": "1",
|
|
"environment": "dev",
|
|
"testbed": "../iop-s2",
|
|
"repetitions": 2,
|
|
"session_policy": "fresh",
|
|
"setup_cache_policy": "isolated",
|
|
"timeout": {
|
|
"run_seconds": 300,
|
|
"idle_seconds": 30,
|
|
"quiet_seconds": 10,
|
|
"cleanup_grace_seconds": 5,
|
|
},
|
|
"viewports": [{"id": "desktop_1080", "width": 1920, "height": 1080}],
|
|
"rubric_version": "v1.0",
|
|
"output_root": self.output_root_rel,
|
|
"fixture": {
|
|
"version": "v1.0",
|
|
"prompt": self.prompt_rel,
|
|
"assets": [{"source": self.ref_rel, "workspace_path": "data/ref.txt"}],
|
|
"checksum": self.fixture_checksum,
|
|
},
|
|
"matrix": [
|
|
{
|
|
"id": "cell-1",
|
|
"caller": "claude",
|
|
"iop": {
|
|
"request_model": "claude-sonnet-4-20250514",
|
|
"requested_effort": "high",
|
|
"route_kind": "direct",
|
|
"route_id": "claude-direct",
|
|
"expected_bindings": [
|
|
{"stage": "request", "model": "claude-sonnet-4-20250514", "effort": "high"}
|
|
],
|
|
},
|
|
},
|
|
{
|
|
"id": "cell-2",
|
|
"caller": "agy",
|
|
"iop": {
|
|
"request_model": "gemini-2.0-flash",
|
|
"requested_effort": "high",
|
|
"route_kind": "direct",
|
|
"route_id": "agy-direct",
|
|
"expected_bindings": [
|
|
{"stage": "request", "model": "gemini-2.0-flash", "effort": "high"}
|
|
],
|
|
},
|
|
},
|
|
],
|
|
}
|
|
|
|
self.manifest_file = self.repo_root / "manifest.json"
|
|
self.manifest_file.write_text(json.dumps(self.manifest_raw, indent=2), encoding="utf-8")
|
|
self.manifest = load_manifest(self.manifest_file, repo_root=self.repo_root)
|
|
|
|
def tearDown(self) -> None:
|
|
self.tmp_dir_obj.cleanup()
|
|
|
|
def make_attempt_root(
|
|
self, cell_id: str = "cell-1", repetition: int = 1, attempt: int = 1
|
|
) -> tuple[AttemptIdentity, Path]:
|
|
identity = AttemptIdentity(
|
|
run_id=self.run_id, cell_id=cell_id, repetition=repetition, attempt=attempt
|
|
)
|
|
rep_segment = f"repetition-{repetition:04d}"
|
|
att_segment = f"attempt-{attempt:06d}"
|
|
attempt_root = (
|
|
self.repo_root
|
|
/ self.output_root_rel
|
|
/ self.run_id
|
|
/ "cells"
|
|
/ cell_id
|
|
/ rep_segment
|
|
/ att_segment
|
|
)
|
|
attempt_root.mkdir(parents=True, exist_ok=True)
|
|
return identity, attempt_root
|
|
|
|
|
|
class TestAttemptIdentityValidation(BaseWorkspaceTest):
|
|
"""Tests for AttemptIdentity format and boundary validation."""
|
|
|
|
def test_valid_identity(self) -> None:
|
|
identity = AttemptIdentity(
|
|
run_id=self.run_id, cell_id="cell-1", repetition=1, attempt=1
|
|
)
|
|
validate_attempt_identity(identity, self.manifest)
|
|
|
|
def test_invalid_run_id_rejected(self) -> None:
|
|
bad_run_ids = [
|
|
"invalid_run_id",
|
|
"run-20260809161730Z-0123456789ab", # missing T
|
|
"run-20260809T161730Z-0123456789aG", # uppercase G
|
|
"run-20260809T161730Z-short",
|
|
]
|
|
for run_id in bad_run_ids:
|
|
identity = AttemptIdentity(run_id=run_id, cell_id="cell-1", repetition=1, attempt=1)
|
|
with self.assertRaises(WorkspaceValidationError):
|
|
validate_attempt_identity(identity, self.manifest)
|
|
|
|
def test_invalid_cell_id_rejected(self) -> None:
|
|
# Invalid format or missing from manifest
|
|
identity_bad_fmt = AttemptIdentity(
|
|
run_id=self.run_id, cell_id="-invalid", repetition=1, attempt=1
|
|
)
|
|
with self.assertRaises(WorkspaceValidationError):
|
|
validate_attempt_identity(identity_bad_fmt, self.manifest)
|
|
|
|
identity_missing = AttemptIdentity(
|
|
run_id=self.run_id, cell_id="cell-nonexistent", repetition=1, attempt=1
|
|
)
|
|
with self.assertRaises(WorkspaceValidationError):
|
|
validate_attempt_identity(identity_missing, self.manifest)
|
|
|
|
def test_invalid_repetition_rejected(self) -> None:
|
|
# Non-positive or exceeding manifest repetitions (2)
|
|
for rep in [0, -1, 3, True]:
|
|
identity = AttemptIdentity(
|
|
run_id=self.run_id, cell_id="cell-1", repetition=rep, attempt=1
|
|
)
|
|
with self.assertRaises(WorkspaceValidationError):
|
|
validate_attempt_identity(identity, self.manifest)
|
|
|
|
def test_invalid_attempt_rejected(self) -> None:
|
|
for att in [0, -1, True]:
|
|
identity = AttemptIdentity(
|
|
run_id=self.run_id, cell_id="cell-1", repetition=1, attempt=att
|
|
)
|
|
with self.assertRaises(WorkspaceValidationError):
|
|
validate_attempt_identity(identity, self.manifest)
|
|
|
|
|
|
class TestAttemptRootPathRules(BaseWorkspaceTest):
|
|
"""Tests path rules for attempt_root."""
|
|
|
|
def test_attempt_root_does_not_exist(self) -> None:
|
|
identity = AttemptIdentity(
|
|
run_id=self.run_id, cell_id="cell-1", repetition=1, attempt=1
|
|
)
|
|
non_existent = (
|
|
self.repo_root
|
|
/ self.output_root_rel
|
|
/ self.run_id
|
|
/ "cells"
|
|
/ "cell-1"
|
|
/ "repetition-0001"
|
|
/ "attempt-000001"
|
|
)
|
|
with self.assertRaises(WorkspacePathError):
|
|
prepare_workspace(
|
|
self.manifest, non_existent, identity, repo_root=self.repo_root
|
|
)
|
|
|
|
def test_attempt_root_is_file(self) -> None:
|
|
identity, attempt_root = self.make_attempt_root()
|
|
attempt_root.rmdir()
|
|
attempt_root.write_text("file instead of dir", encoding="utf-8")
|
|
with self.assertRaises(WorkspacePathError):
|
|
prepare_workspace(
|
|
self.manifest, attempt_root, identity, repo_root=self.repo_root
|
|
)
|
|
|
|
def test_attempt_root_is_symlink(self) -> None:
|
|
identity, attempt_root = self.make_attempt_root()
|
|
attempt_root.rmdir()
|
|
real_target = self.tmp_dir / "real_target"
|
|
real_target.mkdir()
|
|
attempt_root.symlink_to(real_target)
|
|
with self.assertRaises(WorkspacePathError):
|
|
prepare_workspace(
|
|
self.manifest, attempt_root, identity, repo_root=self.repo_root
|
|
)
|
|
|
|
def test_attempt_root_parent_is_symlink(self) -> None:
|
|
identity = AttemptIdentity(
|
|
run_id=self.run_id, cell_id="cell-1", repetition=1, attempt=1
|
|
)
|
|
parent_dir = (
|
|
self.repo_root
|
|
/ self.output_root_rel
|
|
/ self.run_id
|
|
/ "cells"
|
|
/ "cell-1"
|
|
/ "repetition-0001"
|
|
)
|
|
parent_dir.mkdir(parents=True, exist_ok=True)
|
|
real_parent = self.tmp_dir / "real_parent"
|
|
real_parent.mkdir()
|
|
attempt_root_real = real_parent / "attempt-000001"
|
|
attempt_root_real.mkdir()
|
|
|
|
symlink_att = parent_dir / "attempt-000001"
|
|
symlink_att.symlink_to(attempt_root_real)
|
|
|
|
with self.assertRaises(WorkspacePathError):
|
|
prepare_workspace(
|
|
self.manifest, symlink_att, identity, repo_root=self.repo_root
|
|
)
|
|
|
|
def test_attempt_root_not_empty(self) -> None:
|
|
identity, attempt_root = self.make_attempt_root()
|
|
(attempt_root / "existing.txt").write_text("existing", encoding="utf-8")
|
|
with self.assertRaises(WorkspacePathError):
|
|
prepare_workspace(
|
|
self.manifest, attempt_root, identity, repo_root=self.repo_root
|
|
)
|
|
|
|
def test_attempt_root_canonical_path_mismatch(self) -> None:
|
|
identity = AttemptIdentity(
|
|
run_id=self.run_id, cell_id="cell-1", repetition=1, attempt=1
|
|
)
|
|
wrong_path = (
|
|
self.repo_root
|
|
/ self.output_root_rel
|
|
/ self.run_id
|
|
/ "cells"
|
|
/ "cell-1"
|
|
/ "repetition-0002" # mismatched repetition
|
|
/ "attempt-000001"
|
|
)
|
|
wrong_path.mkdir(parents=True, exist_ok=True)
|
|
with self.assertRaises(WorkspacePathError):
|
|
prepare_workspace(self.manifest, wrong_path, identity, repo_root=self.repo_root)
|
|
|
|
def test_exclusive_child_collision(self) -> None:
|
|
identity, attempt_root = self.make_attempt_root()
|
|
(attempt_root / "workspace").mkdir()
|
|
with self.assertRaises(WorkspacePathError):
|
|
prepare_workspace(
|
|
self.manifest, attempt_root, identity, repo_root=self.repo_root
|
|
)
|
|
|
|
|
|
class TestWorkspaceMaterialization(BaseWorkspaceTest):
|
|
"""Tests for workspace asset copying, prompt exclusion, checksums, and metadata."""
|
|
|
|
def test_successful_workspace_preparation(self) -> None:
|
|
identity, attempt_root = self.make_attempt_root()
|
|
prepared = prepare_workspace(
|
|
self.manifest, attempt_root, identity, repo_root=self.repo_root
|
|
)
|
|
|
|
self.assertEqual(prepared.identity, identity)
|
|
self.assertEqual(prepared.workspace_checksum, self.manifest.fixture.checksum)
|
|
self.assertTrue(prepared.session_is_fresh)
|
|
self.assertEqual(prepared.setup_cache_policy, "isolated")
|
|
|
|
# Verify directories exist
|
|
ws_path = Path(prepared.workspace_dir)
|
|
session_path = Path(prepared.session_dir)
|
|
self.assertTrue(ws_path.is_dir())
|
|
self.assertTrue(session_path.is_dir())
|
|
|
|
# Verify session directory is empty
|
|
self.assertEqual(list(session_path.iterdir()), [])
|
|
|
|
# Verify asset materialization
|
|
asset_file = ws_path / "data" / "ref.txt"
|
|
self.assertTrue(asset_file.is_file())
|
|
self.assertEqual(asset_file.read_bytes(), self.ref_content)
|
|
|
|
# Verify prepared.json content
|
|
prep_json = Path(prepared.attempt_root) / "prepared.json"
|
|
self.assertTrue(prep_json.is_file())
|
|
data = json.loads(prep_json.read_text(encoding="utf-8"))
|
|
self.assertEqual(data["session_id"], prepared.session_id)
|
|
self.assertEqual(data["workspace_checksum"], prepared.workspace_checksum)
|
|
|
|
def test_prompt_exclusion_when_not_declared(self) -> None:
|
|
# Prompt file exists in fixture.prompt, but is not listed in fixture.assets
|
|
identity, attempt_root = self.make_attempt_root()
|
|
prepared = prepare_workspace(
|
|
self.manifest, attempt_root, identity, repo_root=self.repo_root
|
|
)
|
|
ws_path = Path(prepared.workspace_dir)
|
|
|
|
# Prompt should not exist in workspace
|
|
prompt_ws = ws_path / self.prompt_rel
|
|
self.assertFalse(prompt_ws.exists())
|
|
|
|
def test_prompt_included_when_declared_as_asset(self) -> None:
|
|
# Update manifest to declare prompt.md as an asset too
|
|
new_assets = [
|
|
{"source": self.ref_rel, "workspace_path": "data/ref.txt"},
|
|
{"source": self.prompt_rel, "workspace_path": "prompt.md"},
|
|
]
|
|
asset_objs = [
|
|
AssetMapping(source=self.ref_rel, workspace_path="data/ref.txt", content=self.ref_content),
|
|
AssetMapping(source=self.prompt_rel, workspace_path="prompt.md", content=self.prompt_content),
|
|
]
|
|
checksum = digest_workspace_inputs(asset_objs)
|
|
self.manifest_raw["fixture"]["assets"] = new_assets
|
|
self.manifest_raw["fixture"]["checksum"] = checksum
|
|
|
|
self.manifest_file.write_text(json.dumps(self.manifest_raw, indent=2), encoding="utf-8")
|
|
manifest = load_manifest(self.manifest_file, repo_root=self.repo_root)
|
|
|
|
identity, attempt_root = self.make_attempt_root()
|
|
prepared = prepare_workspace(manifest, attempt_root, identity, repo_root=self.repo_root)
|
|
ws_path = Path(prepared.workspace_dir)
|
|
|
|
prompt_ws = ws_path / "prompt.md"
|
|
self.assertTrue(prompt_ws.is_file())
|
|
self.assertEqual(prompt_ws.read_bytes(), self.prompt_content)
|
|
|
|
def test_fixture_checksum_mismatch_rejected(self) -> None:
|
|
# Corrupt declared checksum in manifest
|
|
self.manifest_raw["fixture"]["checksum"] = "sha256:" + "0" * 64
|
|
self.manifest_file.write_text(json.dumps(self.manifest_raw, indent=2), encoding="utf-8")
|
|
|
|
# Manually create manifest with corrupt checksum to bypass load_manifest validation
|
|
dummy_manifest = Manifest(
|
|
pipeline_version=self.manifest.pipeline_version,
|
|
environment=self.manifest.environment,
|
|
testbed=self.manifest.testbed,
|
|
repetitions=self.manifest.repetitions,
|
|
session_policy=self.manifest.session_policy,
|
|
setup_cache_policy=self.manifest.setup_cache_policy,
|
|
timeout=self.manifest.timeout,
|
|
viewports=self.manifest.viewports,
|
|
rubric_version=self.manifest.rubric_version,
|
|
output_root=self.manifest.output_root,
|
|
fixture=Fixture(
|
|
version=self.manifest.fixture.version,
|
|
prompt=self.manifest.fixture.prompt,
|
|
assets=self.manifest.fixture.assets,
|
|
checksum="sha256:" + "0" * 64,
|
|
prompt_content=self.manifest.fixture.prompt_content,
|
|
),
|
|
matrix=self.manifest.matrix,
|
|
digest=self.manifest.digest,
|
|
)
|
|
|
|
identity, attempt_root = self.make_attempt_root()
|
|
with self.assertRaises(WorkspaceChecksumError):
|
|
prepare_workspace(dummy_manifest, attempt_root, identity, repo_root=self.repo_root)
|
|
|
|
def test_symlink_asset_source_rejected(self) -> None:
|
|
symlink_source = self.repo_root / "scripts" / "fixtures" / "bench" / "symlink.txt"
|
|
symlink_source.symlink_to(self.ref_file)
|
|
|
|
symlink_rel = "scripts/fixtures/bench/symlink.txt"
|
|
self.manifest_raw["fixture"]["assets"] = [
|
|
{"source": symlink_rel, "workspace_path": "data/symlink.txt"}
|
|
]
|
|
asset_objs = [
|
|
AssetMapping(source=symlink_rel, workspace_path="data/symlink.txt", content=self.ref_content)
|
|
]
|
|
self.manifest_raw["fixture"]["checksum"] = digest_workspace_inputs(asset_objs)
|
|
self.manifest_file.write_text(json.dumps(self.manifest_raw, indent=2), encoding="utf-8")
|
|
|
|
# Create manifest directly to bypass load_manifest symlink check
|
|
dummy_manifest = Manifest(
|
|
pipeline_version=self.manifest.pipeline_version,
|
|
environment=self.manifest.environment,
|
|
testbed=self.manifest.testbed,
|
|
repetitions=self.manifest.repetitions,
|
|
session_policy=self.manifest.session_policy,
|
|
setup_cache_policy=self.manifest.setup_cache_policy,
|
|
timeout=self.manifest.timeout,
|
|
viewports=self.manifest.viewports,
|
|
rubric_version=self.manifest.rubric_version,
|
|
output_root=self.manifest.output_root,
|
|
fixture=Fixture(
|
|
version=self.manifest.fixture.version,
|
|
prompt=self.manifest.fixture.prompt,
|
|
assets=(AssetMapping(source=symlink_rel, workspace_path="data/symlink.txt", content=self.ref_content),),
|
|
checksum=self.manifest_raw["fixture"]["checksum"],
|
|
prompt_content=self.manifest.fixture.prompt_content,
|
|
),
|
|
matrix=self.manifest.matrix,
|
|
digest=self.manifest.digest,
|
|
)
|
|
|
|
identity, attempt_root = self.make_attempt_root()
|
|
with self.assertRaises(WorkspacePathError):
|
|
prepare_workspace(dummy_manifest, attempt_root, identity, repo_root=self.repo_root)
|
|
|
|
def test_escaping_workspace_path_rejected(self) -> None:
|
|
bad_assets = [
|
|
{"source": self.ref_rel, "workspace_path": "../data/ref.txt"},
|
|
]
|
|
self.manifest_raw["fixture"]["assets"] = bad_assets
|
|
self.manifest_file.write_text(json.dumps(self.manifest_raw, indent=2), encoding="utf-8")
|
|
|
|
with self.assertRaises(Exception):
|
|
load_manifest(self.manifest_file, repo_root=self.repo_root)
|
|
|
|
def test_source_drift_failure_leaves_attempt_root_empty_and_retryable(self) -> None:
|
|
"""R1: Mutate a fixture source after manifest load, prove rollback and retry."""
|
|
identity, attempt_root = self.make_attempt_root()
|
|
|
|
# Capture original ref.txt content
|
|
original_ref_content = self.ref_file.read_bytes()
|
|
|
|
# Mutate the source file after manifest load (simulates source drift)
|
|
self.ref_file.write_bytes(b"corrupted reference data\n")
|
|
|
|
with self.assertRaises(WorkspaceChecksumError):
|
|
prepare_workspace(
|
|
self.manifest, attempt_root, identity, repo_root=self.repo_root
|
|
)
|
|
|
|
# Assert the attempt root is empty (rollback succeeded)
|
|
self.assertEqual(list(attempt_root.iterdir()), [])
|
|
|
|
# Restore the original source
|
|
self.ref_file.write_bytes(original_ref_content)
|
|
|
|
# Prove the same attempt root can succeed on retry
|
|
# (need to recreate it since rollback removed it)
|
|
identity2, attempt_root2 = self.make_attempt_root()
|
|
prepared = prepare_workspace(
|
|
self.manifest, attempt_root2, identity2, repo_root=self.repo_root
|
|
)
|
|
self.assertEqual(prepared.identity, identity2)
|
|
self.assertTrue(Path(prepared.workspace_dir).is_dir())
|
|
self.assertTrue(Path(prepared.session_dir).is_dir())
|
|
|
|
def test_postflight_failure_leaves_attempt_root_empty(self) -> None:
|
|
"""R1: Deterministic mocked postflight failure proves rollback of all owned entries."""
|
|
import unittest.mock
|
|
|
|
identity, attempt_root = self.make_attempt_root()
|
|
|
|
# Mock inspect_testbed_provenance to succeed on first call (preflight) but fail on second (postflight)
|
|
original_inspect = inspect_testbed_provenance
|
|
call_count = {"n": 0}
|
|
|
|
def mock_inspect(path):
|
|
call_count["n"] += 1
|
|
if call_count["n"] == 1:
|
|
return original_inspect(path) # preflight succeeds
|
|
# postflight raises TestbedError
|
|
from scripts.agent_benchmark.workspace import TestbedError as TE
|
|
raise TE("mocked postflight: testbed modified")
|
|
|
|
with unittest.mock.patch(
|
|
"scripts.agent_benchmark.workspace.inspect_testbed_provenance",
|
|
side_effect=mock_inspect,
|
|
):
|
|
with self.assertRaises(TestbedError):
|
|
prepare_workspace(
|
|
self.manifest, attempt_root, identity, repo_root=self.repo_root
|
|
)
|
|
|
|
# Assert the attempt root is empty (rollback succeeded)
|
|
self.assertEqual(list(attempt_root.iterdir()), [])
|
|
|
|
# Assert prepared.json does not exist
|
|
prepared_json = attempt_root / "prepared.json"
|
|
self.assertFalse(prepared_json.exists())
|
|
|
|
def test_ancestor_destination_collision_rejected_before_mutation(self) -> None:
|
|
"""R1: Asset destinations with ancestor/file conflict are rejected before mutation."""
|
|
new_assets = [
|
|
{"source": self.ref_rel, "workspace_path": "data"},
|
|
{"source": self.prompt_rel, "workspace_path": "data/prompt.md"},
|
|
]
|
|
asset_objs = [
|
|
AssetMapping(source=self.ref_rel, workspace_path="data", content=self.ref_content),
|
|
AssetMapping(source=self.prompt_rel, workspace_path="data/prompt.md", content=self.prompt_content),
|
|
]
|
|
checksum = digest_workspace_inputs(asset_objs)
|
|
self.manifest_raw["fixture"]["assets"] = new_assets
|
|
self.manifest_raw["fixture"]["checksum"] = checksum
|
|
self.manifest_file.write_text(json.dumps(self.manifest_raw, indent=2), encoding="utf-8")
|
|
manifest = load_manifest(self.manifest_file, repo_root=self.repo_root)
|
|
|
|
identity, attempt_root = self.make_attempt_root()
|
|
|
|
with self.assertRaises(WorkspacePathError):
|
|
prepare_workspace(manifest, attempt_root, identity, repo_root=self.repo_root)
|
|
|
|
# Assert attempt_root remains completely empty (no staging, workspace, or session created)
|
|
self.assertEqual(list(attempt_root.iterdir()), [])
|
|
|
|
def test_concurrent_collision_preserves_unrelated_entries(self) -> None:
|
|
"""R1: Concurrent collision content not created by this preparation is preserved on rollback."""
|
|
import unittest.mock
|
|
import scripts.agent_benchmark.workspace
|
|
|
|
identity, attempt_root = self.make_attempt_root()
|
|
|
|
def mock_verify_with_collision(staging, manifest):
|
|
# Simulate a caller/external collision creating workspace/caller-owned.txt after validation
|
|
caller_ws = attempt_root / "workspace"
|
|
caller_ws.mkdir(exist_ok=True)
|
|
sentinel = caller_ws / "caller-owned.txt"
|
|
sentinel.write_text("caller owned content", encoding="utf-8")
|
|
raise TestbedError("mocked failure during preparation")
|
|
|
|
with unittest.mock.patch(
|
|
"scripts.agent_benchmark.workspace._verify_staged_preparation",
|
|
side_effect=mock_verify_with_collision,
|
|
):
|
|
with self.assertRaises(TestbedError):
|
|
prepare_workspace(self.manifest, attempt_root, identity, repo_root=self.repo_root)
|
|
|
|
# Assert caller-owned collision file was preserved on rollback
|
|
sentinel = attempt_root / "workspace" / "caller-owned.txt"
|
|
self.assertTrue(sentinel.is_file())
|
|
self.assertEqual(sentinel.read_text(encoding="utf-8"), "caller owned content")
|
|
|
|
def test_empty_publication_collision_preserves_unrelated_directory(self) -> None:
|
|
"""R1: Empty concurrent collision directory created before final publication is preserved on rollback."""
|
|
import unittest.mock
|
|
import scripts.agent_benchmark.workspace
|
|
|
|
identity, attempt_root = self.make_attempt_root()
|
|
|
|
empty_inode: int | None = None
|
|
orig_publish = scripts.agent_benchmark.workspace._publish_owned_directory
|
|
|
|
def mock_publish_with_empty_collision(staging_dir, final_dir, owned, collision_message):
|
|
nonlocal empty_inode
|
|
if final_dir.name == "workspace":
|
|
final_dir.mkdir(exist_ok=False)
|
|
empty_inode = final_dir.stat().st_ino
|
|
return orig_publish(staging_dir, final_dir, owned, collision_message)
|
|
|
|
with unittest.mock.patch(
|
|
"scripts.agent_benchmark.workspace._publish_owned_directory",
|
|
side_effect=mock_publish_with_empty_collision,
|
|
):
|
|
with self.assertRaises(WorkspacePathError):
|
|
prepare_workspace(self.manifest, attempt_root, identity, repo_root=self.repo_root)
|
|
|
|
caller_ws = attempt_root / "workspace"
|
|
self.assertTrue(caller_ws.is_dir())
|
|
self.assertIsNotNone(empty_inode)
|
|
self.assertEqual(caller_ws.stat().st_ino, empty_inode)
|
|
self.assertEqual(list(caller_ws.iterdir()), [])
|
|
self.assertFalse((attempt_root / "session").exists())
|
|
self.assertFalse((attempt_root / "prepared.json").exists())
|
|
self.assertEqual(list(attempt_root.iterdir()), [caller_ws])
|
|
|
|
|
|
class TestTestbedProvenanceAndNonMutation(BaseWorkspaceTest):
|
|
"""Tests for runtime testbed provenance checking and non-mutation."""
|
|
|
|
def test_clean_testbed_provenance(self) -> None:
|
|
prov = inspect_testbed_provenance(self.testbed_dir)
|
|
self.assertTrue(prov.clean)
|
|
self.assertTrue(prov.status_digest.startswith("sha256:"))
|
|
self.assertTrue(len(prov.head) > 0)
|
|
|
|
def test_dirty_testbed_rejected(self) -> None:
|
|
# Create an uncommitted file in testbed_dir
|
|
(self.testbed_dir / "dirty.txt").write_text("dirty", encoding="utf-8")
|
|
|
|
with self.assertRaises(TestbedError):
|
|
inspect_testbed_provenance(self.testbed_dir)
|
|
|
|
identity, attempt_root = self.make_attempt_root()
|
|
with self.assertRaises(TestbedError):
|
|
prepare_workspace(self.manifest, attempt_root, identity, repo_root=self.repo_root)
|
|
|
|
def test_testbed_unaffected_by_preparation(self) -> None:
|
|
prov_before = inspect_testbed_provenance(self.testbed_dir)
|
|
|
|
identity, attempt_root = self.make_attempt_root()
|
|
prepared = prepare_workspace(
|
|
self.manifest, attempt_root, identity, repo_root=self.repo_root
|
|
)
|
|
|
|
prov_after = inspect_testbed_provenance(self.testbed_dir)
|
|
self.assertEqual(prov_before, prov_after)
|
|
|
|
# Assert no file from testbed_dir was copied into workspace_dir
|
|
ws_files = list(Path(prepared.workspace_dir).rglob("*"))
|
|
ws_rel_paths = {f.name for f in ws_files}
|
|
self.assertNotIn("README.md", ws_rel_paths) # testbed README.md is not in workspace
|
|
|
|
|
|
class TestCrossAttemptIsolation(BaseWorkspaceTest):
|
|
"""API-2: Prove cross-attempt isolation and source integrity."""
|
|
|
|
def test_cross_attempt_isolation_and_source_integrity(self) -> None:
|
|
# Prepare 4 attempt roots (2 cells x 2 repetitions)
|
|
testbed_before = inspect_testbed_provenance(self.testbed_dir)
|
|
|
|
attempts_config = [
|
|
("cell-1", 1, 1),
|
|
("cell-1", 2, 1),
|
|
("cell-2", 1, 1),
|
|
("cell-2", 2, 1),
|
|
]
|
|
|
|
prepared_list: list[PreparedWorkspace] = []
|
|
for cell_id, rep, att in attempts_config:
|
|
identity, attempt_root = self.make_attempt_root(
|
|
cell_id=cell_id, repetition=rep, attempt=att
|
|
)
|
|
prep = prepare_workspace(
|
|
self.manifest, attempt_root, identity, repo_root=self.repo_root
|
|
)
|
|
prepared_list.append(prep)
|
|
|
|
# 1. Assert four distinct workspace/session identities
|
|
session_ids = {p.session_id for p in prepared_list}
|
|
self.assertEqual(len(session_ids), 4)
|
|
|
|
attempt_roots = {p.attempt_root for p in prepared_list}
|
|
self.assertEqual(len(attempt_roots), 4)
|
|
|
|
# 2. Assert identical initial workspace digests matching manifest.fixture.checksum
|
|
workspace_checksums = {p.workspace_checksum for p in prepared_list}
|
|
self.assertEqual(workspace_checksums, {self.manifest.fixture.checksum})
|
|
|
|
# 3. Mutate one session (session #1) and prove peer sessions remain empty
|
|
session1_path = Path(prepared_list[0].session_dir)
|
|
sentinel = session1_path / "history-sentinel"
|
|
sentinel.write_text("owned", encoding="utf-8")
|
|
self.assertTrue(sentinel.is_file())
|
|
|
|
for peer in prepared_list[1:]:
|
|
self.assertEqual(list(Path(peer.session_dir).iterdir()), [])
|
|
|
|
# 4. Capture testbed branch/HEAD/status digest before and after and assert equality
|
|
testbed_after = inspect_testbed_provenance(self.testbed_dir)
|
|
self.assertEqual(testbed_before, testbed_after)
|
|
|
|
# 5. Assert no path under testbed appears beneath any workspace
|
|
testbed_files = {f.name for f in self.testbed_dir.rglob("*") if f.is_file()}
|
|
for prep in prepared_list:
|
|
ws_files = {f.name for f in Path(prep.workspace_dir).rglob("*") if f.is_file()}
|
|
# Intersection of testbed files and workspace files should be empty
|
|
# (testbed contains README.md; workspace contains data/ref.txt)
|
|
self.assertEqual(testbed_files.intersection(ws_files), set())
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|