diff --git a/Makefile b/Makefile index abeeedc4..76f0ddf6 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: all build build-local build-edge build-edge-host build-node build-node-target build-node-targets pack-node-target pack-edge archive-edge tidy test test-e2e test-control-plane-edge-wire test-credential-slot-smoke test-openai-ollama test-openai-lemonade test-openai-glm-coding test-hot-path-agent-smoke-self-test test-hot-path-agent-smoke-preflight test-hot-path-agent-smoke test-single-request-claude-smoke-self-test test-single-request-claude-smoke-preflight test-single-request-claude-smoke-validate test-single-request-claude-smoke readability-audit proto proto-dart client-test client-build-web clean test-agent-comparison-benchmark +.PHONY: all build build-local build-edge build-edge-host build-node build-node-target build-node-targets pack-node-target pack-edge archive-edge tidy test test-e2e test-control-plane-edge-wire test-credential-slot-smoke test-openai-ollama test-openai-lemonade test-openai-glm-coding test-hot-path-agent-smoke-self-test test-hot-path-agent-smoke-preflight test-hot-path-agent-smoke test-single-request-claude-smoke-self-test test-single-request-claude-smoke-preflight test-single-request-claude-smoke-validate test-single-request-claude-smoke readability-audit proto proto-dart client-test client-build-web clean GOFLAGS ?= -trimpath BUILD_DIR ?= build @@ -78,13 +78,6 @@ test: readability-audit: python3 scripts/readability_audit.py --check --input-mode worktree --output build/readability-audit.json -# Deterministic, credential-free benchmark manifest tests. -# Fresh unittest discovery for *_test.py plus the tracked example validation. -test-agent-comparison-benchmark: - cd $(CURDIR) && PYTHONPATH=$(CURDIR) python3 -m unittest discover -s scripts/agent_benchmark -p '*_test.py' -v - python3 scripts/agent_comparison_benchmark.py validate \ - --manifest scripts/fixtures/agent-comparison-benchmark-manifest.example.json - test-e2e: @echo "NOTE: test-e2e runs auxiliary smoke (Edge-Node + OpenAI) plus Control Plane-Edge wire smoke; completion still requires user-flow verification when changing runtime paths." ./scripts/e2e-smoke.sh diff --git a/README.md b/README.md index 876658f5..239b6b43 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,5 @@ Start with `agent-contract/index.md` for protocol and runtime contracts, and `ag Operator and client setup guides: - [Edge Local Quickstart](docs/edge-local-dev-guide.md) -- [Agent Comparison Benchmark Dev Guide](docs/agent-comparison-benchmark-dev-guide.md) - [dev OpenCode Settings Guide](docs/dev-opencode-settings-guide.md) - [dev-corp Pi Settings Guide](docs/dev-corp-pi-settings-guide.md) diff --git a/agent-contract/index.md b/agent-contract/index.md index f1fc51e3..8b328a3e 100644 --- a/agent-contract/index.md +++ b/agent-contract/index.md @@ -14,7 +14,7 @@ |----|-----------|-----------|------| | `iop.openai-compatible-api` | OpenAI-compatible API, Responses API, Chat Completions, legacy Completions, error envelope/SSE terminal error, `model` route, managed projection principal auth and slot-route binding, managed-versus-legacy provider credential selection, model-driven passthrough/normalized routing, provider-pool admission/unavailable error, safe credential-slot attribution, standard metadata, and provider-native extension fields such as `chat_template_kwargs` | `apps/edge/internal/openai/*`, `apps/edge/internal/authprojection/*`, `apps/edge/internal/service/provider_tunnel.go`, `packages/go/config/config.go`, `configs/edge.yaml` | `agent-contract/outer/openai-compatible-api.md` | | `iop.anthropic-compatible-api` | Anthropic Messages API, count_tokens, models list, bearer or `X-Api-Key` principal auth, active managed projection auth and slot-route binding, `anthropic-version` routing, native Anthropic tunnel, Chat bridge, provider-pool-only admission, profile capability checks, managed-versus-legacy provider credentials, marked-preset single-request admission with Edge-owned internal Plan/Review template customization that leaves caller I/O unchanged, and current no-OpenAI-metric status | `apps/edge/internal/openai/anthropic_handler.go`, `apps/edge/internal/openai/anthropic_native.go`, `apps/edge/internal/openai/anthropic_bridge.go`, `apps/edge/internal/openai/anthropic_stream.go`, `apps/edge/internal/openai/anthropic_types.go`, `apps/edge/internal/openai/routes.go`, `apps/edge/internal/openai/principal.go`, `apps/edge/internal/authprojection/*`, `apps/edge/internal/openai/provider_tunnel.go`, `apps/edge/internal/openai/provider_model_rewrite.go`, `apps/edge/internal/openai/single_request_preset_binding.go`, `apps/edge/internal/openai/single_request_plan_stage.go`, `apps/edge/internal/openai/single_request_review_stage.go`, `packages/go/singlerequesttemplate/template.go`, `packages/go/config/protocol_profile.go` | `agent-contract/outer/anthropic-compatible-api.md` | -| `iop.gemini-compatible-api` | Gemini Developer API `streamGenerateContent`, route-qualified Gemini-native ingress, `x-goog-api-key` principal auth, `GOOGLE_GEMINI_BASE_URL`, official agy 1.1.12 API-key transport, Gemini function calls/thought signatures/SSE, and direct-versus-execution-preset binding | `apps/edge/internal/openai/routes.go`, `apps/edge/internal/openai/principal.go`, `apps/edge/internal/openai/gemini_handler.go`, `apps/edge/internal/openai/gemini_bridge.go`, `apps/edge/internal/openai/gemini_types.go`, `scripts/agent_benchmark/agy_iop.py`, `scripts/agent_benchmark/live_iop.py` | `agent-contract/outer/gemini-compatible-api.md` | +| `iop.gemini-compatible-api` | Gemini Developer API `streamGenerateContent`, route-qualified Gemini-native ingress, `x-goog-api-key` principal auth, `GOOGLE_GEMINI_BASE_URL`, official agy 1.1.12 API-key transport, Gemini function calls/thought signatures/SSE, and direct-versus-execution-preset binding | `apps/edge/internal/openai/routes.go`, `apps/edge/internal/openai/principal.go`, `apps/edge/internal/openai/gemini_handler.go`, `apps/edge/internal/openai/gemini_bridge.go`, `apps/edge/internal/openai/gemini_types.go` | `agent-contract/outer/gemini-compatible-api.md` | | `iop.a2a-json-rpc-api` | A2A JSON-RPC API, `message/send`, `tasks/get`, `tasks/cancel`, A2A task state, agent card, `a2a.bearer_token`, Edge A2A input surface | `apps/edge/internal/input/a2a/*`, `packages/go/config/config.go`, `configs/edge.yaml` | `agent-contract/outer/a2a-json-rpc-api.md` | ## Inner Contracts diff --git a/agent-contract/outer/gemini-compatible-api.md b/agent-contract/outer/gemini-compatible-api.md index 307c9588..b17a367c 100644 --- a/agent-contract/outer/gemini-compatible-api.md +++ b/agent-contract/outer/gemini-compatible-api.md @@ -9,14 +9,12 @@ - `apps/edge/internal/openai/routes.go` - `apps/edge/internal/openai/principal.go` - `apps/edge/internal/openai/chat_handler.go` - - `scripts/agent_benchmark/agy_iop.py` - - `scripts/agent_benchmark/live_iop.py` - external caller surface: official Antigravity CLI `agy` 1.1.12 Gemini API-key provider ## 읽는 조건 - Gemini Developer API `streamGenerateContent`, `x-goog-api-key`, `GOOGLE_GEMINI_BASE_URL`, `GEMINI_API_KEY`, `agy` API-key provider, Gemini-native tool/function call, 또는 Gemini-native SSE ingress를 구현·검증할 때 읽는다. -- IOP execution preset을 Gemini-native caller에 노출하거나 `agy` benchmark transport를 변경할 때 읽는다. +- IOP execution preset을 Gemini-native caller에 노출하거나 공식 `agy` transport를 변경할 때 읽는다. ## 범위 @@ -41,7 +39,7 @@ x-goog-api-key: - `GEMINI_API_KEY`: upstream provider key가 아니라 IOP principal token이다. - `GOOGLE_GEMINI_BASE_URL`: `https:///gemini/{route-id}`다. - 사설 dev CA를 사용하는 경우 caller child에는 표준 `SSL_CERT_FILE`과 `NODE_EXTRA_CA_CERTS`만 명시적으로 전달한다. -- `--model`: 공식 CLI가 인식하는 Gemini 모델 label을 사용한다. benchmark의 Gemini 3.6 Flash 호출은 `Gemini 3.6 Flash`다. +- `--model`: 공식 CLI가 인식하는 Gemini 모델 label을 사용한다. dev 호환 확인에서 사용하는 label은 `Gemini 3.6 Flash`다. - `--effort`는 API-key provider 호출에 전달하지 않는다. 요청된 high effort는 인증된 IOP route/preset의 effective binding으로 검증한다. `GEMINI_BASE_URL`, `AGY_PROVIDER`, `AGY_OPENAI_BASE_URL`, `AGY_OPENAI_API_KEY`는 이 계약의 transport가 아니다. @@ -75,7 +73,7 @@ Edge는 이를 기존 Chat/preset ingress의 system/user/assistant/tool message, - provider-reported usage가 있으면 `usageMetadata.promptTokenCount`, `candidatesTokenCount`, `thoughtsTokenCount`, `cachedContentTokenCount`, `totalTokenCount`의 존재하는 값만 투영한다. 누락 값을 0으로 발명하지 않는다. - caller disconnect는 기존 request cancellation 경계를 사용하며 이후 frame을 쓰지 않는다. -## 공식 agy stream-json lifecycle +## 공식 agy 수동 호출 확인 기준 `agy` 1.1.12의 각 JSONL record는 `event` discriminator와 같은 이름의 중첩 payload를 사용한다. @@ -83,7 +81,7 @@ Edge는 이를 기존 Chat/preset ingress의 system/user/assistant/tool message, - step: `{"event":"step_update","step_update":{"state":...,"step_type":...,"usage":{...}}}` - terminal: `{"event":"result","result":{"status":"SUCCESS","duration_seconds":...,"num_turns":...,"usage":{...}}}` -benchmark adapter는 중첩 payload만 파싱하며 `result.status=SUCCESS` 한 건과 process exit/quiet를 success terminal로 인정한다. 구조가 유효한 `result.status=ERROR`는 stream incompatibility가 아니므로 success terminal을 만들지 않고 caller process의 non-zero exit를 lifecycle failure authority로 보존한다. `response`, `text_delta`, tool payload, conversation id는 durable evidence에 보존하지 않는다. usage는 caller가 제공한 `input_tokens`, `cache_read_tokens`, `output_tokens`, `thinking_tokens`, `total_tokens`만 원래 단위의 count로 기록하고 누락값을 합성하지 않는다. +수동 확인에서는 중첩 `result.status=SUCCESS` 한 건과 process exit 0을 성공 terminal로 본다. 구조가 유효한 `result.status=ERROR` 또는 non-zero exit는 실패로 남기며 성공으로 재해석하지 않는다. 확인 기록에는 raw 응답, tool payload, conversation id, credential을 남기지 않고 caller가 제공한 usage만 원래 단위로 요약한다. ## 오류 @@ -101,9 +99,8 @@ HTTP commit 전 오류는 다음 Gemini envelope 한 건으로 반환한다. - route/auth: `apps/edge/internal/openai/routes.go`, `apps/edge/internal/openai/principal.go` - Gemini request/SSE bridge: `apps/edge/internal/openai/gemini_handler.go`, `apps/edge/internal/openai/gemini_bridge.go`, `apps/edge/internal/openai/gemini_types.go` - Edge regression: `apps/edge/internal/openai/gemini_handler_test.go`, existing Chat/preset/auth tests -- caller adapter: `scripts/agent_benchmark/agy_iop.py`, `scripts/agent_benchmark/agy_iop_test.py`, `scripts/agent_benchmark/live_iop.py` - live proof: official `agy --output-format stream-json` through the dev Edge route-specific base URL, with direct and execution-preset effective binding evidence 2026-08-12 dev 검증에서 normal/boundary/auth/tool/SSE 회귀 테스트와 공식 `agy` 1.1.12 direct·hybrid 실호출이 통과해 이 계약을 active로 전환했다. -2026-08-12 benchmark 실측 보정에서 official agy planner의 structured-output schema field를 Chat `response_format`으로 변환하고, 구조가 유효한 ERROR result를 parser failure와 분리했다. +2026-08-12 실호출 보정에서 official agy planner의 structured-output schema field를 Chat `response_format`으로 변환했다. diff --git a/agent-ops/rules/project/rules.md b/agent-ops/rules/project/rules.md index 3fed6198..7be98f42 100644 --- a/agent-ops/rules/project/rules.md +++ b/agent-ops/rules/project/rules.md @@ -111,5 +111,3 @@ - field 테스트 포트, artifact/bootstrap HTTP, 외부 테스트 환경: `agent-test/local/rules.md`를 따른다. - bootstrap/install UX, Agent Bootstrap, specialized agent 등록, Control Plane enrollment: `testing` domain rule과 `agent-test/local/rules.md`를 따른다. - 반복 작업이 확인되면 `agent-ops/skills/project//SKILL.md`를 생성하고 이 표에 등록한다. -- 벤치마크 매니페스트 검증/실행/재개/상태 확인/익명 채점/리포트: `agent-ops/skills/project/iop-agent-comparison-benchmark/SKILL.md` -- 벤치마크 validate, run, resume, status, score, report 요청, 매니페스트 검증 요청, 벤치마크 실행 요청, 벤치마크 상태 확인 요청, 벤치마크 익명 채점 요청, 벤치마크 리포트 요청 diff --git a/agent-ops/skills/project/iop-agent-comparison-benchmark/SKILL.md b/agent-ops/skills/project/iop-agent-comparison-benchmark/SKILL.md deleted file mode 100644 index 6b08cce8..00000000 --- a/agent-ops/skills/project/iop-agent-comparison-benchmark/SKILL.md +++ /dev/null @@ -1,211 +0,0 @@ ---- -name: iop-agent-comparison-benchmark -description: Recognize benchmark validate/preflight/run/resume/status/score/report requests, delegate supported operations to the deterministic CLI, and fail closed at execution or scoring blockers. ---- - -# iop-agent-comparison-benchmark - -## Purpose - -Route agent comparison benchmark requests to the deterministic CLI while enforcing append-only evidence, no-substitution, capability, and safety boundaries. The skill is routing and documentation, not a second implementation or orchestration dispatcher. - -## When to use - -- User requests benchmark manifest validation: `validate`, `validate manifest`, `manifest 검증` -- User requests route readiness: `preflight`, `preflight benchmark`, `연결 사전 점검` -- User requests benchmark execution: `run`, `run benchmark`, `벤치마크 실행`, `시작해` -- User requests benchmark resume: `resume`, `resume benchmark`, `재개`, `계속해` -- User requests benchmark status: `status`, `status benchmark`, `상태 확인`, `어디까지 왔어` -- User requests blind scoring: `score`, `score benchmark`, `익명 채점`, `품질 채점` -- User requests report or output: `report`, `report output`, `결과 보고`, `리포트`, `리포트 보여줘` - -## Inputs - -- `manifest`: Path to the benchmark manifest JSON file. (required for validate, preflight, run, resume, status, score, report) -- `run_id`: Harness-generated run id. (required for resume, status, score, report) -- `retry_failed`: Boolean flag for resume. (optional, default: false) -- `retry_scoring_failed`: Boolean flag for score. (optional, default: false) - -## Preflight - -- [ ] Confirm the request matches one of the supported trigger cases above. -- [ ] For validate/preflight/run/resume/status/score/report: confirm a manifest path is provided. If missing, return `error: manifest path is required`. -- [ ] For resume/status/score/report: confirm a harness-issued run id is provided. If missing, return `error: run id is required`. -- [ ] Confirm the CLI exists: `scripts/agent_comparison_benchmark.py` is present at the repo root. -- [ ] Confirm the manifest file exists and is readable before delegating. - -## Procedure - -1. **Classify the request** - - Map the user request to one of: `validate`, `preflight`, `run`, `resume`, `status`, `score`, `report`. - - If the request does not match any trigger, report that the benchmark pipeline skill does not cover the request and route to the appropriate skill. - -2. **Delegate report to the CLI** - - Run: `python3 scripts/agent_comparison_benchmark.py report --manifest --run-id ` - - The CLI opens immutable run state and delegates to the strict reporter with no caller adapter path. - - On exit 0, report `ok: report run_id= path=`. - - On exit 69, report `error: benchmark report is unavailable` from stderr. - -3. **Delegate validate to the CLI** - - Run: `python3 scripts/agent_comparison_benchmark.py validate --manifest ` - - Report the CLI exit code and stdout/stderr verbatim. - - On exit 0, report `ok: manifest is valid`. - - On exit 69, report the validation error from stderr. - - On exit 64, report the usage error from stderr. - -4. **Delegate preflight to the CLI** - - Run: `python3 scripts/agent_comparison_benchmark.py preflight --manifest ` - - The CLI records one fresh live observation for every immutable matrix cell, including direct and execution-preset routes, in canonical matrix order in one append-only run record. - - On exit 0, report the exact closed `ready` summary from stdout. - - On exit 69, report the exact `registration_required` or `implementation_gap` summary from stderr and stop. Never bypass the blocker, substitute a route/model/effort, or treat local manifest validation as live readiness. - -5. **Delegate run to the CLI** - - Run: `python3 scripts/agent_comparison_benchmark.py run --manifest ` - - On missing or invalid manifest, the CLI prints `error: benchmark state is unavailable` to stderr with exit 69 (or `error: invalid usage` with exit 64) before creating execution state. - - The CLI creates one run and uses its single writer to append a fresh all-cell preflight before attempt allocation. - - On `registration_required` or `implementation_gap`, it prints `error: preflight blocked ...` to stderr with exit 69, allocates no attempt, and preserves the run id for a later resume. - - On `ready`, it binds the exact caller, cell, fresh workspace, session, and attempt identity, then must invoke each eligible cell exactly once with the fixture task. - - Exit 0 when every manifest slot has complete terminal evidence (`unresolved=0`); independent failure counts remain in stdout and are classified by `score`. Exit 69 only for preflight blockers or incomplete evidence (absent/running slots). Never performs an implicit retry of a failed gate. - -6. **Delegate resume to the CLI** - - Run: `python3 scripts/agent_comparison_benchmark.py resume --manifest --run-id [--retry-failed]` - - On missing or invalid manifest or state, the CLI prints `error: benchmark state is unavailable` to stderr with exit 69 (or `error: invalid usage` with exit 64) before changing the run. - - The CLI opens the exact immutable run and uses its single writer to append a fresh all-cell preflight before attempt allocation. - - On `registration_required` or `implementation_gap`, it prints `error: preflight blocked ...` to stderr with exit 69 and allocates no attempt. - - On `ready`, it reconciles interrupted state, skips only slots whose latest product/harness/process/artifact gates all pass, preserves prior attempt bytes, and allocates a new attempt only for eligible work. `--retry-failed` admits a new attempt for any latest terminal attempt whose independent gates do not all pass. - - Exit 0 when every manifest slot has complete terminal evidence (`unresolved=0`); independent failure counts remain in stdout and are classified by `score`. Exit 69 only for preflight blockers or incomplete evidence. Never performs an implicit retry of a failed gate. - - It must invoke each eligible cell exactly once with a new workspace and session identity. - -7. **Delegate status to the CLI** - - Run: `python3 scripts/agent_comparison_benchmark.py status --manifest --run-id ` - - On missing or invalid manifest or state, the CLI prints `error: benchmark state is unavailable` to stderr with exit 69 (or `error: invalid usage` with exit 64). - - On success, the CLI prints the controller counts, product/harness/process/artifact counts, and `unresolved=` to stdout with exit 0. - - Report the exact CLI output. - -8. **Delegate score to the CLI** - - Run: `python3 scripts/agent_comparison_benchmark.py score --manifest --run-id [--retry-scoring-failed]` - - The CLI classifies each failed product, harness, process, or required artifact gate with its own immutable `unscored` reason, without invoking the evaluator or assigning zero. - - Eligible attempts receive an opaque blind workspace, one manifest-bound fresh Codex evaluator session, and the exact immutable manifest-selected rubric from the closed supported catalog (`landing-quality-v1`, `one-shot-agent-comparison-v1`); no substitute rubric or reinterpretation is permitted. - - A prior `scored` result is terminal. A prior `scoring_failed` result is retried only with `--retry-scoring-failed`, which allocates a new score id and preserves every prior byte. - - On exit 0, report the exact closed `scored`, `unscored`, `scoring_failed`, and `blocked` counts from stdout. - - On exit 69, report the exact closed counts or unavailable-state line from stderr. Never substitute evaluator route/model/effort, fabricate a worksheet, or turn failure into zero. - -9. **Report the result** - - Report the command executed, the exact CLI exit code, and the full stdout/stderr. - - Do not summarize, paraphrase, or fabricate CLI output. - -## Validation - -- [ ] The CLI command was executed and the exit code matches the documented contract. -- [ ] The reported stdout/stderr matches the CLI output exactly. -- [ ] Preflight evidence is append-only, covers every immutable matrix cell in canonical order, and uses only closed status/count fields. -- [ ] A preflight blocker created no scored attempt and was not bypassed. -- [ ] An ineligible execution attempt became `unscored` without an evaluator invocation or a zero score. -- [ ] Product, harness, process, and artifact outcomes remain separately visible in run/resume/status output and report rows. -- [ ] Each eligible score id used one opaque blind workspace and one fresh evaluator session; retry preserved prior bytes and used a new id. -- [ ] `scoring_failed` used no fallback, synthetic worksheet, or implicit retry. -- [ ] No caller or provider was invoked outside the deterministic CLI. -- [ ] No report or output was fabricated for report requests; the CLI produced the deterministic artifact. -- [ ] No public `prepare` operation was exposed or referenced. -- [ ] Run/resume exit 0 requires `unresolved=0` (every slot has complete terminal evidence); independent failure axes remain visible and are classified by `score`. -- [ ] Retry is explicit only (`--retry-failed`); a failed terminal gate is never reinterpreted as success. -- If validation fails, report the mismatch and stop without fallback. - -## Output format - -For validate/run/resume/status: - -``` -command: -exit_code: -stdout: -stderr: -``` - -For preflight: - -``` -command: preflight -exit_code: <0|69> -stdout: -stderr: -``` - -For score: - -``` -command: score -exit_code: <0|69> -stdout: -stderr: -``` - -For report: - -``` -command: report -exit_code: <0|69> -stdout: -stderr: -``` - -For run/resume ready completion: - -``` -command: -exit_code: 0 -stdout: ok: run_id= executed= unresolved=0 completed= timed_out= cancelled= interrupted= running=0 product_succeeded= product_failed= product_unknown= harness_passed= harness_failed= process_exited= process_signalled= process_timed_out= process_cancelled= process_not_started= artifact_passed= artifact_failed= artifact_blocked= artifact_not_run=0 -stderr: (none) -``` - -Note: `unresolved=0` means every manifest slot has complete terminal evidence (web validation present). Independent failure axes (`product_failed`, `artifact_failed`, etc.) remain visible in the summary and are classified by `score` as `unscored` without invoking the evaluator or assigning zero. - -For run/resume blocker or execution failure: - -``` -command: -exit_code: 69 -stdout: (none) -stderr: -``` - -## Safety rules - -- The skill delegates every supported stateful operation verbatim to `scripts/agent_comparison_benchmark.py`. It does not reproduce pipeline policy in prose. -- Durable run/attempt state and preflight evidence are persisted only under the validated run root (`agent-test/runs///`). -- Caller sessions, output workspaces, and caches are fresh and isolated for every cell, repetition, and attempt; session or cache state is never shared within a run or across runs. -- Read-only testbed/fixture inputs (such as `../iop-s2`) are not copied back or mutated; no writes occur outside the validated run root. -- Preflight never allocates a scored attempt. Every immutable matrix cell requires its own fresh live observation. -- Run/resume append a fresh all-cell preflight under the run writer before any attempt allocation; a blocker allocates no attempt. -- Ready execution binds one exact cell and immutable attempt identity to one fresh workspace/session and one task submission. -- Product, harness, process, and artifact are independent gates. Controller state `completed` only means the invocation controller reached a terminal state. -- Resolution (`unresolved=0`) requires every manifest slot to have complete terminal evidence (web validation present); it does not require every gate to pass. Failed gates remain visible as independent failure counts and are classified by `score` as `unscored`. -- Release qualification runs the five-cell direct manifest as one unscored canary and requires fresh `ready=5`, exactly five fresh attempts, `unresolved=0`, `running=0`, `interrupted=0`, terminal controller/product/harness/process/web-validation evidence for every slot, and no exhausted browser/CDP infrastructure block before a fresh nine-cell preflight. Product failure, upstream HTTP rejection, generated-missing after caller failure, and timeout remain measured outcomes and do not trigger an implicit retry. Edge pre-ingress incompatibility or an exhausted browser/CDP infrastructure block stops qualification; this step does not allocate hybrid or scored execution. -- Scoring copies only anonymous generated files, two local images, and screenshots into an opaque run-owned blind tree; the identity mapping remains outside that tree. -- Scoring records `unscored`, `scored`, and `scoring_failed` append-only, and a retry always allocates a fresh score id/session. -- The internal workspace API (`RunStore`, `Manifest`, etc.) is not a user command. Do not expose it. - -## Stop conditions - -- Stop immediately on a preflight `registration_required` or `implementation_gap` result. Do not substitute an alias, change an effort, or continue to attempt allocation. -- Stop immediately on a run/resume preflight blocker without allocating an attempt or invoking another execution path. -- Do not resume or retry a retained execution failure unless the user explicitly requests resume with `--retry-failed`. A release-qualification run with complete terminal evidence may continue to the fresh nine-cell preflight under the Safety rules without retrying or reinterpreting the failed result. -- Stop after `scoring_failed` unless the user explicitly requests score with `--retry-scoring-failed`. -- Stop immediately and report the unavailable-state line for report requests that cannot project the deterministic report. -- Stop immediately if the manifest path is missing or the file is not readable. -- Stop immediately if the run id is missing for resume/status. -- Stop without fallback, fabricated evidence, ad-hoc provider calls, subagents, or orchestration dispatchers. - -## Prohibitions - -- Do not expose a public `prepare` operation. -- Do not invoke a caller or provider outside the deterministic benchmark CLI. -- Do not bypass a preflight blocker or substitute caller, route, model, effort, or preset. -- Do not retry scoring implicitly, replace the manifest evaluator, fabricate a worksheet, or convert `unscored`/`scoring_failed` to zero. -- Do not claim execution-preset fixture validation as live readiness. -- Do not use subagents, orchestration dispatchers, or dispatch.py-equivalent tools for benchmark execution. -- Do not fabricate benchmark results, reports, or output. -- Do not reproduce pipeline policy, state machine, or allocation logic in prose. -- Do not allow user override of the read-only `../iop-s2` testbed provenance. -- Do not write output or ad-hoc state outside the validated run root (`agent-test/runs///`). -- Do not share session or cache state within a run across cells, repetitions, or attempts, or across run invocations. diff --git a/agent-roadmap/archive/phase/knowledge-tool-optimization-extension/milestones/iop-one-shot-agent-model-comparison.md b/agent-roadmap/archive/phase/knowledge-tool-optimization-extension/milestones/iop-one-shot-agent-model-comparison.md new file mode 100644 index 00000000..35c73b78 --- /dev/null +++ b/agent-roadmap/archive/phase/knowledge-tool-optimization-extension/milestones/iop-one-shot-agent-model-comparison.md @@ -0,0 +1,56 @@ +# Milestone: [bench-02] IOP 원샷 Agent 모델 비교 벤치마크 + +## 위치 + +- Roadmap: [ROADMAP.md](../../../../ROADMAP.md) +- Phase: [PHASE.md](../../../../phase/knowledge-tool-optimization-extension/PHASE.md) +- SDD: [SDD.md](../../../sdd/knowledge-tool-optimization-extension/iop-one-shot-agent-model-comparison/SDD.md) + +## 목표 + +동일한 정적 웹 과제를 9개 IOP 경유 caller/model 조합으로 실행해 속도·usage·품질을 비교하는 전용 benchmark를 수행하려 했다. + +## 상태 + +[폐기] + +## 구현 잠금 + +- 상태: 잠금 +- SDD: 필요 +- SDD 문서: [폐기된 SDD](../../../sdd/knowledge-tool-optimization-extension/iop-one-shot-agent-model-comparison/SDD.md) +- SDD 상태: 폐기 +- SDD 잠금: 잠금 +- SDD 사용자 리뷰: 없음 +- 결정 필요: 없음 + +## 범위 + +- 전용 manifest/runner, 격리·재개·복구, 자동 웹 검증, 익명 채점과 deterministic report를 사용한 C01-C09 비교였다. +- 2026-08-13 사용자 결정으로 제품 안정성과 측정 경계를 분리하기 위해 전용 harness와 함께 현재 실행 범위에서 철회했다. + +## 기능 + +### Epic: [benchmark-rollback] 전용 측정 경계 폐기 + +- [x] [harness-retirement] 전용 benchmark 구현, 공식 CLI/skill, manifest/schema, 자동 검증·채점·보고와 현재 living spec을 제거했다. +- [x] [milestone-retirement] 제품 안정성 전수 재검증을 이 측정 Milestone의 대체 작업으로 만들지 않고, 구체적 결함만 해당 제품 소유 영역에서 처리하도록 분리했다. + +## 완료 리뷰 + +- 상태: 폐기 +- 요청일: 2026-08-13 +- 완료 근거: 사용자 요청과 전용 harness 삭제 diff +- 검토 항목: 없음 +- 리뷰 코멘트: 기존 2/9 scored run과 후속 복구 계획은 완료 근거가 아니며 재개하지 않는다. + +## 범위 제외 + +- IOP 제품 ingress, route/preset, stream terminal과 공식 caller 호환 코드의 롤백 +- 구체적 제품 결함의 소유 영역별 수정과 회귀 검증 + +## 작업 컨텍스트 + +- 폐기 사유: 측정 시스템의 정합성과 복구가 제품 안정성보다 우선되는 목적 역전 +- 후속 제품 Milestone: 없음. 현재 IOP 호출부를 기준선으로 유지하고 구체적 회귀만 국소 처리한다. +- 복구 가능성: 삭제된 구현과 이전 문서는 Git 이력에서 복구할 수 있다. diff --git a/agent-roadmap/archive/sdd/knowledge-tool-optimization-extension/iop-one-shot-agent-model-comparison/SDD.md b/agent-roadmap/archive/sdd/knowledge-tool-optimization-extension/iop-one-shot-agent-model-comparison/SDD.md new file mode 100644 index 00000000..39fe0ca8 --- /dev/null +++ b/agent-roadmap/archive/sdd/knowledge-tool-optimization-extension/iop-one-shot-agent-model-comparison/SDD.md @@ -0,0 +1,80 @@ +# SDD: IOP 원샷 Agent 모델 비교 벤치마크 + +## 위치 + +- Milestone: [폐기된 Milestone](../../../phase/knowledge-tool-optimization-extension/milestones/iop-one-shot-agent-model-comparison.md) +- Phase: [PHASE.md](../../../../phase/knowledge-tool-optimization-extension/PHASE.md) + +## 상태 + +[폐기] + +## SDD 잠금 + +- 상태: 잠금 +- 사용자 리뷰: 없음 +- 잠금 항목: + - 없음. 설계 자체를 폐기했다. + +## 문제 / 비목표 + +- 문제: 전용 측정 harness의 lifecycle, 복구, 증거와 채점 정합성이 IOP 제품과 caller/model 호출 안정성보다 우선되는 목적 역전이 발생했다. +- 비목표: + - 이 SDD를 제품 안정성 설계로 재사용한다. + +## Source of Truth + +| 영역 | 기준 | 메모 | +|------|------|------| +| Roadmap | [폐기된 Milestone](../../../phase/knowledge-tool-optimization-extension/milestones/iop-one-shot-agent-model-comparison.md) | 폐기 결정 원장 | +| User Decision | 2026-08-13 하네스 폐기와 경계 분리 지시 | 측정 설계 철회 근거 | + +## State Machine + +| 상태 | 진입 조건 | 다음 상태 | 근거 | +|------|-----------|-----------|------| +| `retired` | 전용 harness와 Milestone 폐기 | 종료 | 삭제 diff와 archive 상태 | + +## Interface Contract + +- 계약 원문: 없음 +- 입력: + - 없음 +- 출력: + - 없음 +- 금지: + - 이 폐기된 SDD를 현재 실행 또는 완료 조건으로 사용한다. + +## Acceptance Scenarios + +| ID | Milestone Task | Given | When | Then | +|----|----------------|-------|------|------| +| S01 | `harness-retirement` | 전용 측정 구현과 공식 포인터 | 폐기 반영 | 활성 구현과 라우팅에서 제거된다. | +| S02 | `milestone-retirement` | 제품과 측정이 섞인 현재 Milestone | 경계 분리 | 측정 Milestone은 폐기되고 전수 안정성 대체 Milestone을 만들지 않는다. | + +## Evidence Map + +| Scenario | Required Evidence | `agent-task` 연결 | 완료 Evidence 기대 | +|----------|-------------------|------------------|---------------------------| +| S01 | 삭제 diff와 dead-reference 검색 | 해당 없음 | 전용 harness 표면 없음 | +| S02 | archive, current, Phase와 queue diff | 해당 없음 | 대체 전수 검증 gate 없음 | + +## Cross-repo Dependencies + +- 폐기된 `[bench-02]`를 가리키던 workspace lock은 제거하며 대체 안정성 dependency를 만들지 않는다. + +## Drift Check + +- [x] Milestone의 폐기 상태와 일치한다. +- [x] 현재 제품 계약을 이 SDD에 복제하지 않았다. +- [x] 사용자 결정이 반영됐다. +- [x] 전수 재검증을 요구하는 대체 Milestone과 실행 차단을 만들지 않았다. + +## 사용자 리뷰 이력 + +- 2026-08-13: 사용자가 전용 harness 폐기, benchmark Milestone 롤백, IOP 제품 안정성 우선과 측정 경계 분리를 확정했다. + +## 작업 컨텍스트 + +- 표준선: 폐기된 측정 상태 머신과 완료 조건을 후속 제품 작업에서 재사용하지 않는다. +- 후속 SDD: 없음. 구체적 결함은 해당 제품 소유 영역에서 국소 처리한다. diff --git a/agent-roadmap/phase/knowledge-tool-optimization-extension/PHASE.md b/agent-roadmap/phase/knowledge-tool-optimization-extension/PHASE.md index 6c9d25b3..ae644eaf 100644 --- a/agent-roadmap/phase/knowledge-tool-optimization-extension/PHASE.md +++ b/agent-roadmap/phase/knowledge-tool-optimization-extension/PHASE.md @@ -12,7 +12,7 @@ Ollama serving 경로와 운영 기반이 안정화된 뒤, execution preset, cloud-first route evidence가 충분히 쌓이면 동일한 mode decision contract를 쓰는 RAG 기반 local routing model을 shadow/canary로 검증해 운영 기본 경로로 점진 전환한다. caller-neutral 누적 요청 컨텍스트 최적화, repository 장기 기억 RAG, advisor와 Context Hook은 routing evidence RAG와 서로 다른 후속 기능으로 분리한다. 이 Phase는 특정 Agent Shell에 종속되지 않고 OpenAI-compatible, A2A, IOP native protocol 중 맞는 표면에서 공통 최적화 책임을 제공하는 방향을 다룬다. -단일 요청 Agent 실행의 정식 smoke 이후 비교 검증은 별도 benchmark lane에서 수행하며, 준비 pipeline은 병렬 구축하고 실제 scored 비교는 route-02 완료 뒤 실행한다. +단일 요청 Agent 실행은 현재 완료된 제품 상태를 기준선으로 유지한다. 구체적 결함이 재현되면 해당 제품 소유 영역에서 국소 수정·검증하며, 측정 도구는 제품 완료 조건이나 실행 차단을 소유하지 않는다. ## Milestone 흐름 @@ -57,9 +57,17 @@ Phase를 가로지르는 실제 다음 작업 선택은 [전역 마일스톤 실 - 경로: [[bench-01] Agent 비교 벤치마크 파이프라인 준비](../../archive/phase/knowledge-tool-optimization-extension/milestones/agent-comparison-benchmark-pipeline.md) - 요약: 모델·caller·prompt·반복 횟수를 manifest로 바꾸고 Claude Code, agy, Codex의 IOP 연결부터 finish/idle, 시간·token·웹 검증·익명 채점·Markdown 보고까지 같은 pipeline으로 재현한다. -- [진행중] [bench-02] IOP 원샷 Agent 모델 비교 벤치마크 - - 경로: [[bench-02] IOP 원샷 Agent 모델 비교 벤치마크](milestones/iop-one-shot-agent-model-comparison.md) - - 요약: route-02 정식 smoke와 benchmark pipeline 준비 뒤 dev `../iop-s2`에서 동일 정적 웹 fixture로 9개 IOP 경유 단독·하이브리드 caller 조합을 각각 한 번 실행해 속도·token·품질을 비교한다. +- [폐기] [bench-02] IOP 원샷 Agent 모델 비교 벤치마크 + - 경로: [[bench-02] IOP 원샷 Agent 모델 비교 벤치마크](../../archive/phase/knowledge-tool-optimization-extension/milestones/iop-one-shot-agent-model-comparison.md) + - 요약: 전용 harness의 정합성과 복구가 제품 안정성보다 우선되는 목적 역전으로 2026-08-13 폐기했다. 기존 결과와 계획은 재개하지 않는다. + +- [진행중] [bench-route-01] 벤치 경로 최소 HTML 스모크 + - 경로: [[bench-route-01] 벤치 경로 최소 HTML 스모크](milestones/benchmark-route-minimal-html-smoke.md) + - 요약: 벤치에 사용할 9개 caller/model/route 조합을 고정된 최소 `index.html` 생성 요청으로 한 번씩 직접 호출하고, 실패한 경로만 귀속·국소 수정·재검증한다. + +- [계획] [bench-lite-01] 초경량 Agent 모델 비교 + - 경로: [[bench-lite-01] 초경량 Agent 모델 비교](milestones/thin-agent-model-comparison-benchmark.md) + - 요약: 최소 HTML 스모크를 통과한 동일 경로를 복구·재개·자동 채점 없는 단일 시도로 실행하고, 성공 여부·경과 시간·제공된 usage·산출물만 한 표에 기록한다. - [스케치] [output-03] OpenAI-compatible Runtime Output Integrity Filter - 경로: [[output-03] OpenAI-compatible Runtime Output Integrity Filter](milestones/openai-compatible-runtime-output-integrity-filter.md) @@ -108,6 +116,6 @@ Phase를 가로지르는 실제 다음 작업 선택은 [전역 마일스톤 실 - plan-bearing one-shot mode는 IOP Node가 승인된 workspace root 아래 `.iop/job//plan.md`와 `review.md`를 직접 생성·읽기·갱신·정리한다. 내부 model tool call/result는 IOP coordinator가 소비하며 Claude에 후속 tool result 요청을 요구하지 않는다. - 각 stage의 routing, plan, work, review, defect와 repair는 outer stream에 redacted 진행 요약으로만 투영한다. 내부 provider reasoning, control prompt, tool protocol·argument/result, credential과 stage terminal은 공개하지 않고 최종 사용자 결과와 outer terminal만 완결된 응답으로 반환한다. - target agent나 외부 workflow 제품의 process/state를 실행 의존성으로 연결하지 않는다. 범용 interactive shell과 장기 workflow는 제외하지만, execution preset의 request-scoped workspace/tool executor는 IOP가 소유한다. -- benchmark skill/pipeline은 제품 coordinator가 아니라 dev 검증 harness다. 준비 작업은 route-02와 병렬일 수 있지만 실제 scored 비교는 route-02 정식 기능·필수 smoke와 benchmark pipeline 완료 뒤 별도 Milestone에서 수행한다. +- 공식 caller 경로는 현재 제품 기준선을 유지한다. 벤치 준비에서는 벤치 대상 경로만 최소 HTML로 얕게 확인하고, 구체적 회귀가 발생한 경로만 해당 제품 소유 영역에서 검증한다. 전수 재검증 campaign이나 측정 도구를 제품 완료 gate로 두지 않는다. - cloud model은 초기 semantic judge/teacher 역할을 하고, 충분한 정제 evidence가 쌓인 뒤 RAG local router로 운영 기본을 전환한다. 두 경우 모두 최종 권한은 deterministic hard gate를 적용하는 Edge arbiter에 남는다. - routing evidence RAG는 route 판정 전용이고, repository 장기 기억 RAG·누적 요청 context·advisor·Context Hook과 corpus/index/평가를 공유하지 않는다. diff --git a/agent-roadmap/phase/knowledge-tool-optimization-extension/milestones/benchmark-route-minimal-html-smoke.md b/agent-roadmap/phase/knowledge-tool-optimization-extension/milestones/benchmark-route-minimal-html-smoke.md new file mode 100644 index 00000000..4a87e0aa --- /dev/null +++ b/agent-roadmap/phase/knowledge-tool-optimization-extension/milestones/benchmark-route-minimal-html-smoke.md @@ -0,0 +1,77 @@ +# Milestone: [bench-route-01] 벤치 경로 최소 HTML 스모크 + +## 위치 + +- Roadmap: [ROADMAP.md](../../../ROADMAP.md) +- Phase: [PHASE.md](../PHASE.md) + +## 목표 + +벤치마크에 사용할 caller/model/route 조합만 고정된 최소 HTML 생성 요청으로 빠르게 확인한다. +IOP 전체 안정성을 처음부터 재검증하지 않고, 실패가 재현된 경로만 제품·caller·provider·환경 중 한 경계에 귀속해 국소 수정한 뒤 그 경로만 다시 확인한다. + +## 상태 + +[진행중] + +## 구현 잠금 + +- 상태: 해제 +- SDD: 불필요 +- SDD 문서: 없음 +- SDD 사유: 기존 공개 호출 경로를 임시 workspace에서 수동 확인하는 test-only 작업이며 API, wire, config, schema, lifecycle 또는 retry 계약을 새로 만들지 않는다. +- 결정 필요: 없음 + +## 범위 + +- 아래 9개 기존 벤치 대상 조합을 순서와 무관하게 한 경로씩 직접 실행한다. + - Claude Code → Claude direct + - Claude Code → Gemini direct + - agy → Gemini direct + - Claude Code → GPT direct + - Codex → GPT direct + - Claude Code → Gemini execution preset + - agy → Gemini execution preset + - Claude Code → GPT execution preset + - Codex → GPT execution preset +- 모든 경로에 같은 요청을 사용한다: 외부 asset과 JavaScript 없이 exact marker가 있는 단일 `index.html`을 생성하고 종료한다. +- 각 경로는 최초 1회만 실행하고 120초 안에 terminal과 `index.html` 생성 여부를 확인한다. +- 실패하면 자동 retry나 전체 재실행을 하지 않는다. sanitized terminal/error와 IOP request stage만 확인해 소유 경계를 정하고, 변경된 원인이 있을 때 실패 경로만 1회 재검증한다. +- 제품 코드를 수정한 결함에는 해당 소유 package의 focused regression test를 추가하고 관련 Go test를 실행한다. + +## 기능 + +### Epic: [route-smoke] 벤치 경로 초경량 확인 + +- [ ] [minimal-html-calls] 9개 조합에 동일한 최소 HTML 요청을 한 번씩 직접 실행하고, 경로별 caller/model/route, terminal, 경과 시간, `index.html` marker 확인 결과를 한 개의 Markdown 표에 기록한다. 검증: 새 runner/manifest 없이 각 행에 실제 호출 결과가 하나만 있어야 한다. +- [ ] [failed-path-fixes] 실패한 조합마다 제품·caller·provider·환경 중 소유 경계를 기록하고, IOP 제품 결함이 재현된 경우에만 국소 수정과 focused regression을 수행한 뒤 해당 조합만 다시 호출한다. 검증: 성공한 조합의 반복 실행이 없고, 재실행 행에는 변경된 원인과 연결된 수정·테스트 근거가 있어야 한다. +- [ ] [thin-bench-handoff] 9개 조합의 통과 또는 구체적 외부 차단 상태를 짧게 정리해 `[bench-lite-01]` 실행 가능 여부를 남긴다. 검증: 비교 점수나 순위가 아니라 호출 가능 여부와 남은 소유자만 기록한다. + +## 완료 리뷰 + +- 상태: 없음 +- 요청일: 없음 +- 완료 근거: 최소 HTML 호출 evidence가 아직 없다. +- 검토 항목: + - [ ] 새 benchmark script, runner, manifest, state store가 생성되지 않았다. + - [ ] 성공 경로는 한 번만 실행했고 실패 경로만 변경된 원인 뒤 재검증했다. + - [ ] 제품 수정은 재현된 결함과 focused regression으로 한정됐다. +- agent-ui 상태 반영: 해당 없음 +- 리뷰 코멘트: 없음 + +## 범위 제외 + +- IOP ingress/provider/preset 전체 전수 안정화 +- benchmark runner, manifest/schema, resume/recovery store, 자동 retry +- browser/CDP, screenshot, accessibility/network gate, 자동 품질 채점 +- 반복 측정, 통계, 순위, token 정규화 +- benchmark 성공을 다른 제품 Milestone이나 프로젝트의 완료 gate로 사용하는 것 + +## 작업 컨텍스트 + +- 계획 범위: 짧은 단일 test plan으로 즉시 실행할 수 있게 유지한다. 별도 설계·SDD·다단계 복구 계획으로 확장하지 않는다. +- 실행 방식: 기존 공식 caller 명령을 한 번씩 직접 실행한다. 공통화가 필요해 보여도 이 Milestone에서는 script로 승격하지 않는다. +- evidence 위치: `agent-test/dev/iop-benchmark-route-minimal-html-smoke.md` +- 현재 사전 확인: 2026-08-13에 Claude Code 2.1.228, agy 1.1.12, Codex 0.147.0 실행 파일과 CA/token 파일의 존재·mode를 확인했으나, 기존 `token/.iop-bench`로 public `/v1/models`가 HTTP 401을 반환했다. 현재 public Edge 활성 config에는 이 token hash 매핑이 0건이며 caller/model 호출은 시작하지 않았다. +- 재개 조건: 기존 5개 route를 볼 수 있는 유효한 dev-corp principal token을 operator-private 경로에 준비하거나 기존 test token을 active Edge에 안전하게 매핑·재시작하고, token 원문을 출력하지 않은 `/v1/models` 확인이 200이어야 한다. +- 후속 측정: [초경량 Agent 모델 비교](thin-agent-model-comparison-benchmark.md) diff --git a/agent-roadmap/phase/knowledge-tool-optimization-extension/milestones/iop-one-shot-agent-model-comparison.md b/agent-roadmap/phase/knowledge-tool-optimization-extension/milestones/iop-one-shot-agent-model-comparison.md deleted file mode 100644 index 49c5cbf6..00000000 --- a/agent-roadmap/phase/knowledge-tool-optimization-extension/milestones/iop-one-shot-agent-model-comparison.md +++ /dev/null @@ -1,111 +0,0 @@ -# Milestone: [bench-02] IOP 원샷 Agent 모델 비교 벤치마크 - -## 위치 - -- Roadmap: [ROADMAP.md](../../../ROADMAP.md) -- Phase: [PHASE.md](../PHASE.md) -- SDD: [SDD.md](../../../sdd/knowledge-tool-optimization-extension/iop-one-shot-agent-model-comparison/SDD.md) - -## 목표 - -`[route-02]`의 정식 기능과 필수 smoke가 완료된 뒤, 동일한 정적 웹페이지 과제를 IOP를 경유하는 3개 단독 모델과 Gemini/GPT 하이브리드 구조의 9개 caller 조합으로 각각 한 번 실행한다. -첫 output·model/tool·전체시간, 호출 횟수와 세부 token, 자동 웹 검증과 익명 100점 품질 평가를 함께 비교하고 재현 가능한 Markdown 보고서를 현재 프로젝트에 남긴다. - -## 상태 - -[진행중] - -## 구현 잠금 - -- 상태: 해제 -- SDD: 필요 -- SDD 문서: [IOP 원샷 Agent 모델 비교 벤치마크 SDD](../../../sdd/knowledge-tool-optimization-extension/iop-one-shot-agent-model-comparison/SDD.md) -- SDD 사유: 실제 dev provider/credential과 외부 CLI를 사용하는 field benchmark이며 실행 순서, 비용, 실패·재실행, secret-safe evidence와 비교 공정성을 고정해야 한다. -- SDD 상태: 승인됨 -- SDD 잠금: 해제 -- SDD 사용자 리뷰: 없음 -- 잠금 해제 조건: 아래 체크리스트 - - [x] SDD 잠금이 해제되어 있다. - - [x] SDD 사용자 리뷰가 없거나 승인/해결되었다. - - [x] Acceptance Scenario가 Milestone 기능 Task와 연결되어 있다. - - [x] Evidence Map이 완료 시 `complete.log`의 `milestone-task` id별 집계와 최종 검증 evidence로 검증 가능하게 연결되어 있다. -- 결정 필요: 없음 - -## 범위 - -- 선행 조건은 `[bench-01]` benchmark pipeline 준비 완료와 `[route-02]` 정식 기능·필수 Claude smoke 완료다. -- 모든 scored 실행은 dev 환경의 `../iop-s2` IOP runtime을 경유하고, 동일 checksum의 이미지 2장과 vanilla HTML/CSS/JS 단일 페이지 prompt를 run별 clean workspace와 fresh caller session에 제공한다. -- 원샷은 사용자 작업 제출 1회 뒤 사람의 중간 feedback·수동 수정·재시작 없이 caller가 finish/complete event 후 idle이 될 때까지를 뜻하며 model/tool 호출 횟수는 제한하지 않고 측정한다. -- 초기 benchmark는 아래 9개 비교군을 각각 1회 실행한다. - -| ID | 유형 | Caller | IOP 실행 구성 | -|----|------|--------|---------------| -| C01 | Claude 단독 | Claude Code | Claude Sonnet 5 최고 effort | -| C02 | Gemini 단독 | Claude Code | Gemini 3.6 Flash high | -| C03 | Gemini 단독 | agy | Gemini 3.6 Flash high | -| C04 | GPT 단독 | Claude Code | GPT-5.6 luna xhigh | -| C05 | GPT 단독 | Codex | GPT-5.6 luna xhigh | -| C06 | Gemini 하이브리드 | Claude Code | Gemini plan → ornith-fast work → Gemini review/repair | -| C07 | Gemini 하이브리드 | agy | Gemini plan → ornith-fast work → Gemini review/repair | -| C08 | GPT 하이브리드 | Claude Code | GPT plan → ornith-fast work → GPT review/repair | -| C09 | GPT 하이브리드 | Codex | GPT plan → ornith-fast work → GPT review/repair | - -- provider가 제공하는 input/output/reasoning/cached/total token을 model·stage별로 기록하고, 제공되지 않는 값은 추정 원본과 섞지 않고 `미제공`으로 표시한다. -- 결과물 identity를 가린 뒤 Codex가 동일 rubric으로 품질을 채점하고 자동 검증 결과와 분리해 보고한다. - -## 기능 - -### Epic: [benchmark-readiness] 비교 입력과 실행 준비 고정 - -실행 전에 공정한 fixture와 실제 IOP route/credential 상태를 고정한다. - -- [x] [fixture-lock] 이미지 2장, 동일 one-page 요구사항, vanilla HTML/CSS/JS 초기 workspace, viewport와 자동 검증·100점 rubric을 checksum/version과 함께 고정한다. -- [x] [agy-iop-compatibility] 공식 `agy 1.1.12`의 Gemini API-key provider가 route별 `GOOGLE_GEMINI_BASE_URL`과 `GEMINI_API_KEY=`으로 Edge의 Gemini-native `streamGenerateContent` ingress를 호출하고, Gemini request/tool/SSE를 기존 direct·execution-preset 실행에 연결하며 실제 `stream-json` lifecycle/usage를 benchmark adapter가 수집하도록 구현한다. `--effort`와 비공식 custom model에 의존하지 않고 high effort는 IOP effective binding으로 검증한다. -- [x] [managed-credential-dev] dev Control Plane·Edge·Node에 CA-signed mTLS, Edge HTTPS, at-rest/issuer/recipient key material, principal projection과 Gemini/Claude/GPT slot-route를 operator-owned secret 경로로 구성한다. legacy static credential source를 제거하고 marked hybrid preset이 같은 principal의 고정 stage route를 managed lease로 실행하는지 secret-safe smoke로 확인한다. -- [x] [route-readiness] dev `../iop-s2`에서 Claude Code·Codex의 호환 ingress와 공식 agy의 Gemini API-key transport가 각각 IOP 인증 경계를 통과하고, Sonnet/Gemini/GPT route, Gemini/GPT hybrid preset, effort와 stream/finish/idle이 모두 실제 caller preflight를 통과했는지 확인한다. 존재하지 않는 caller 환경 변수나 합성 event fixture는 live 호환 근거로 인정하지 않는다. -- [x] [matrix-lock] C01-C09의 caller, IOP route/preset, model/effort, 반복 횟수 1, 실행 순서 seed, fresh-session과 setup/cache 정책 및 timeout을 immutable run manifest로 확정한다. - -### Epic: [comparison-runs] 9개 원샷 실행 - -각 비교군을 clean workspace에서 한 번 실행하고 실패를 포함한 attempt evidence를 보존한다. - -- [ ] [claude-standalone] C01 Claude Code→IOP→Claude Sonnet 5 최고 effort 단독 원샷을 실행한다. -- [ ] [gemini-standalone] C02 Claude Code와 C03 agy가 각각 IOP→Gemini 3.6 Flash high 단독 원샷을 실행한다. -- [ ] [gpt-standalone] C04 Claude Code와 C05 Codex가 각각 IOP→GPT-5.6 luna xhigh 단독 원샷을 실행한다. -- [ ] [gemini-hybrid] C06 Claude Code와 C07 agy가 각각 IOP의 Gemini plan→ornith-fast work→Gemini review/repair 원샷을 실행한다. -- [ ] [gpt-hybrid] C08 Claude Code와 C09 Codex가 각각 IOP의 GPT plan→ornith-fast work→GPT review/repair 원샷을 실행한다. - -### Epic: [comparison-report] 검증·채점·보고서 - -정량 evidence와 익명 품질 평가를 결합하되 원본 수치와 해석을 분리한다. - -- [ ] [objective-validation] 복구 후 새 C01-C09 full run의 각 결과에 build/serve, desktop·mobile screenshot, 이미지·asset, console 오류, 요구사항·반응형·접근성 gate와 최종 workspace 상태를 자동 검증한다. 생성물 존재·정적 안전·asset·render hard gate는 모두 통과해야 하며 품질 gate 실패는 익명 채점의 감점 evidence로 전달한다. -- [ ] [quality-scoring] 익명화된 9개 결과 모두에 요구사항 25, 시각 완성도 25, 반응형·접근성 15, 이미지·디테일 10, 안정성 10, 코드 품질 10, 자체 검증 5의 동일 100점 rubric으로 Codex가 점수를 기록한다. 완료 집계는 `scored=9`, `unscored=0`, `scoring_failed=0`, `blocked=0`이어야 한다. -- [ ] [performance-usage] 첫 output·첫 file write·model 호출별·tool·queue·전체 finish/idle 시간, 호출 횟수와 model/stage별 input/output/reasoning/cached/total token을 clock/source·미제공 여부와 함께 비교하고 중첩 구간이나 미관측 overhead를 임의 산술 분해하지 않는다. -- [ ] [benchmark-report] 9개 결과의 속도·품질·token 표, 실행 조건·버전·실패·한계·raw evidence 링크를 포함한 날짜별 Markdown 보고서를 `agent-test/dev/`에 남긴다. - -## 완료 리뷰 - -- 상태: 없음 -- 요청일: 없음 -- 완료 근거: 사용자 확정 9개 비교군과 post-smoke 실행·평가 기준을 SDD와 기능 Task로 정리했으며 실제 비교 evidence는 아직 없다. -- 검토 항목: 없음 -- 리뷰 코멘트: 없음 - -## 범위 제외 - -- `[route-02]` 정식 기능이나 필수 smoke의 완료 여부를 이 비교 점수로 대체하거나 소급 변경하는 작업 -- 첫 보고서에서 비교군별 2회 이상 반복하는 실행 -- React/Vite 등 dependency 설치와 cache가 속도에 섞이는 frontend framework 과제 -- provider가 보고하지 않은 reasoning token을 exact 값처럼 추정하거나 서로 다른 tokenizer 수치를 무보정 단일 합계로 단정하는 방식 -- 실패 attempt를 삭제하고 성공 재실행만 대표값으로 선택하는 방식 - -## 작업 컨텍스트 - -- 관련 경로: `agent-test/dev/`, `agent-test/runs/`, `../iop-s2` -- 표준선: preflight는 scored attempt와 분리하고, scored 실행이 시작된 뒤의 실패는 결과로 보존하며 재실행이 필요하면 새 attempt로 기록한다. -- 표준선: IOP credential/model route가 없으면 안전한 등록을 요청하고, alias/effort를 임의 대체하지 않는다. -- 현재 차단: 보존 run `run-20260813T081326Z-4e1ac5152c6c`은 2개 scored/7개 unscored이므로 완료 근거가 아니다. Claude/agy caller·IOP route와 scoring eligibility를 복구하고 짧은 route별 smoke를 통과한 뒤, 기존 run과 다른 identity의 새 C01-C09 full run을 실행해 9개 모두 success/hard-gate-pass/scored가 되어야 한다. 이전 run과 attempt는 resume/retry/수정하지 않는다. -- 실행 순서와 차단 관계: [전역 마일스톤 실행 순서](../../../priority-queue.md) -- 관련 Milestone: [[bench-01] Agent 비교 벤치마크 파이프라인 준비](agent-comparison-benchmark-pipeline.md), [[route-02] IOP 단일 요청 Agent 실행](../../../archive/phase/knowledge-tool-optimization-extension/milestones/iop-owned-single-request-agent-execution.md) -- 확인 필요: 없음 diff --git a/agent-roadmap/phase/knowledge-tool-optimization-extension/milestones/thin-agent-model-comparison-benchmark.md b/agent-roadmap/phase/knowledge-tool-optimization-extension/milestones/thin-agent-model-comparison-benchmark.md new file mode 100644 index 00000000..b8f6e466 --- /dev/null +++ b/agent-roadmap/phase/knowledge-tool-optimization-extension/milestones/thin-agent-model-comparison-benchmark.md @@ -0,0 +1,65 @@ +# Milestone: [bench-lite-01] 초경량 Agent 모델 비교 + +## 위치 + +- Roadmap: [ROADMAP.md](../../../ROADMAP.md) +- Phase: [PHASE.md](../PHASE.md) + +## 목표 + +`[bench-route-01]`에서 호출 가능성이 확인된 동일 9개 조합을 고정된 실제 비교 과제로 한 번씩 실행해 최소 비교 결과를 남긴다. +측정 지원 코드는 만들지 않고, caller 실행과 결과 기록만 측정 경계로 둔다. + +## 상태 + +[계획] + +## 구현 잠금 + +- 상태: 해제 +- SDD: 불필요 +- SDD 문서: 없음 +- SDD 사유: 기존 caller와 제품 경로를 단일 시도로 관측하는 test-only 작업이며 제품 API, 상태 머신, retry 또는 schema를 변경하지 않는다. +- 결정 필요: 없음 + +## 범위 + +- `[bench-route-01]`과 동일한 9개 caller/model/route 조합 +- 모든 조합에 같은 고정 비교 prompt와 같은 빈 임시 workspace 사용 +- 조합별 정확히 1회 실행 +- 성공 여부, 전체 경과 시간, caller가 직접 제공한 usage, 산출물 경로와 짧은 수동 관찰만 기록 +- 실패한 조합은 실패로 기록하고 같은 측정 안에서 retry, resume 또는 대체 run을 하지 않음 + +## 기능 + +### Epic: [thin-run] 단일 시도 비교 + +- [ ] [single-attempt-matrix] 9개 조합을 같은 prompt와 초기 상태에서 정확히 한 번씩 실행한다. 검증: 조합별 producer attempt가 하나이며 retry/resume/recovery 기록이 없어야 한다. +- [ ] [minimal-result-table] 성공 여부, 경과 시간, caller 제공 usage, 산출물 경로와 짧은 관찰을 단일 Markdown 표로 기록한다. 제공되지 않은 usage는 `미제공`으로 두고 추정하거나 0으로 바꾸지 않는다. +- [ ] [bounded-conclusion] 성공한 결과만 비교하고 실패·미제공 데이터를 점수 0으로 취급하지 않는 짧은 결론을 남긴다. 자동 채점이나 통계적 일반화는 하지 않는다. + +## 완료 리뷰 + +- 상태: 없음 +- 요청일: 없음 +- 완료 근거: `[bench-route-01]`과 단일 시도 결과가 아직 없다. +- 검토 항목: + - [ ] `[bench-route-01]`이 통과 또는 사용자 승인된 외부 차단 상태다. + - [ ] 새 benchmark script와 자동화 state가 없다. + - [ ] 조합별 정확히 한 번의 실행과 최소 결과 표만 남았다. +- agent-ui 상태 반영: 해당 없음 +- 리뷰 코멘트: 없음 + +## 범위 제외 + +- manifest runner, orchestration framework, lifecycle store +- retry, resume, recovery qualification, stale-run repair +- browser/CDP 자동 검증, screenshot gate, accessibility/network gate +- 익명 LLM 채점, 자동 순위, 반복·분산·유의성 분석 +- 이 측정을 IOP 제품 안정성 또는 다른 Milestone의 완료 조건으로 승격하는 것 + +## 작업 컨텍스트 + +- 선행 작업: [벤치 경로 최소 HTML 스모크](benchmark-route-minimal-html-smoke.md) +- 실행 방식: 기존 공식 caller 명령을 한 번씩 직접 실행하며 공통 runner를 만들지 않는다. +- 결과 위치: `agent-test/dev/iop-thin-agent-model-comparison.md` diff --git a/agent-roadmap/priority-queue.md b/agent-roadmap/priority-queue.md index 868d17e5..7dc29495 100644 --- a/agent-roadmap/priority-queue.md +++ b/agent-roadmap/priority-queue.md @@ -4,11 +4,21 @@ ## 실행 순서 +### bench-route + +1. [[bench-route-01] 벤치 경로 최소 HTML 스모크](phase/knowledge-tool-optimization-extension/milestones/benchmark-route-minimal-html-smoke.md) + 벤치 대상 9개 caller/model/route 조합에 고정된 최소 `index.html` 생성 요청을 한 번씩 직접 보내고 실패 경로만 국소 수정한다. + +### bench-lite + +1. [[bench-lite-01] 초경량 Agent 모델 비교](phase/knowledge-tool-optimization-extension/milestones/thin-agent-model-comparison-benchmark.md) + 통과한 동일 경로를 복구·재개·자동 채점 없이 한 번씩 실행하고 최소 비교 표만 남긴다. + - 선행 차단: `[bench-route-01]` + ### route 3. [[route-03] Heavy Plan/Review 실행과 검증 MVP](phase/knowledge-tool-optimization-extension/milestones/knowledge-tool-validation-optimization.md) Hot Path의 lightweight Plan/Review를 장기 작업용 `heavy` mode로 확장해 `heavy-only` preset에서 재계획·검증·review/repair·resume 경계를 먼저 검증한다. - - 선행 차단: `[bench-02]` 4. [[route-04] Execution Preset 하이브리드 Mode 라우팅](phase/knowledge-tool-optimization-extension/milestones/openai-compatible-hybrid-request-execution-routing.md) cloud model advisory와 deterministic hard gate를 결합해 Edge가 외부 model에 매핑된 preset의 허용 mode 중 요청 난이도에 맞는 실행 경로를 고르고 route evidence를 축적한다. @@ -17,11 +27,6 @@ cloud-first route evidence가 품질·규모 gate를 통과하면 RAG local router를 shadow/canary로 검증해 운영 기본 경로로 점진 전환한다. - 선행 차단: `[observe-03]`, `[provider-02]` -### bench - -2. [[bench-02] IOP 원샷 Agent 모델 비교 벤치마크](phase/knowledge-tool-optimization-extension/milestones/iop-one-shot-agent-model-comparison.md) - `[route-02]` 정식 smoke 뒤 동일 정적 웹 fixture로 Sonnet/Gemini/GPT 단독과 Gemini/GPT 하이브리드의 9개 IOP 경유 조합을 각각 한 번 비교한다. - ### output 1. [[output-01] OpenAI-compatible 출력 검증 필터](phase/knowledge-tool-optimization-extension/milestones/openai-compatible-output-validation-filters.md) diff --git a/agent-roadmap/sdd/knowledge-tool-optimization-extension/iop-one-shot-agent-model-comparison/SDD.md b/agent-roadmap/sdd/knowledge-tool-optimization-extension/iop-one-shot-agent-model-comparison/SDD.md deleted file mode 100644 index de22385d..00000000 --- a/agent-roadmap/sdd/knowledge-tool-optimization-extension/iop-one-shot-agent-model-comparison/SDD.md +++ /dev/null @@ -1,169 +0,0 @@ -# SDD: [bench-02] IOP 원샷 Agent 모델 비교 벤치마크 - -## 위치 - -- Milestone: [IOP 원샷 Agent 모델 비교 벤치마크](../../../phase/knowledge-tool-optimization-extension/milestones/iop-one-shot-agent-model-comparison.md) -- Phase: [PHASE.md](../../../phase/knowledge-tool-optimization-extension/PHASE.md) - -## 상태 - -[승인됨] - -## SDD 잠금 - -- 상태: 해제 -- 사용자 리뷰: 없음 -- 잠금 항목: - - [x] [D01] 실제 비교는 `[route-02]` 정식 기능·필수 smoke와 `[bench-01]` pipeline 준비가 끝난 뒤 시작한다. - - [x] [D02] 9개 scored 비교군은 모두 dev `../iop-s2` IOP runtime을 경유한다. - - [x] [D03] 단독군은 Sonnet 5 최고, Gemini 3.6 Flash high, GPT-5.6 luna xhigh이며 Gemini/GPT는 Claude Code와 전용 caller(agy/Codex)를 각각 비교한다. - - [x] [D04] 하이브리드는 Gemini 또는 GPT가 plan/review/repair를, ornith-fast가 work를 담당하고 각각 Claude Code와 전용 caller를 비교한다. - - [x] [D05] 동일 이미지 2장과 vanilla HTML/CSS/JS 한 페이지 fixture를 clean workspace에 제공한다. - - [x] [D06] 각 승인된 scored run의 repetitions는 cell별 1이며 clean workspace와 fresh caller session에서 사용자 작업 제출 1회부터 finish/complete 후 idle까지 사람 개입 없이 실행한다. caller launch 전 harness 결함으로 중단된 과거 run 뒤 사용자가 새 실행을 명시적으로 승인하면, 과거 run을 resume/retry하지 않고 새 run identity로 같은 immutable manifest를 한 번 실행할 수 있다. - - [x] [D07] 시간은 첫 output, 첫 file write, model/stage별 작업, tool, queue와 전체 finish/idle을 clock/source와 함께 기록하고 중첩 구간이나 미관측 overhead를 임의 산술 분해하지 않는다. - - [x] [D08] token은 input/output/reasoning/cached/total과 source를 model/stage별로 기록하고 미제공 값을 exact로 추정하지 않는다. - - [x] [D09] 결과 identity를 가린 뒤 동일 100점 rubric으로 Codex가 채점하고 자동 검증과 수동 점수를 분리한다. - - [x] [D10] scored failure는 보존하고 같은 run의 재실행은 명시적 retry의 새 attempt로만 기록하며 성공 결과만 골라 대표하지 않는다. 새 run 승인은 기존 실패 run/attempt를 대체하지 않는 별도 비교 cycle이며, 보고서에는 이전 실패 run과 새 run의 관계 및 한계를 함께 남긴다. - - [x] [D11] 공식 `agy 1.1.12`는 Gemini API-key provider의 route별 `GOOGLE_GEMINI_BASE_URL`을 IOP Edge로 지정하고 `GEMINI_API_KEY`에는 upstream key가 아닌 IOP principal token을 넣는다. `--effort`와 비공식 custom model은 사용하지 않고 high effort는 IOP effective binding으로 검증한다. - - [x] [D12] marked hybrid preset은 dev managed credential plane의 fresh projection, 고정 stage authorization과 sealed provider lease가 준비된 뒤에만 실행하며 legacy credential fallback을 허용하지 않는다. - - [x] [D13] 배포 qualification의 5-cell direct 진단은 all-success가 아니라 terminal-evidence completeness를 판정한다. fresh `ready=5`, 정확히 5개의 fresh attempt, `unresolved=0`, `running=0`, `interrupted=0`, 모든 slot의 controller/product/harness/process/web-validation terminal evidence와 exhausted browser/CDP infrastructure block 없음이 필요하다. 제품 실패·provider rejection·caller failure 뒤 generated-missing·timeout은 보존할 benchmark 결과이며 암묵 재시도하지 않는다. Edge pre-ingress incompatibility 또는 exhausted browser/CDP infrastructure block만 qualification을 막는다. 이 unscored 진단은 D06/D10의 유일한 새 scored C01-C09 run identity를 소비하지 않는다. - - [x] [D14] Milestone 완료에는 복구 후 승인된 새 C01-C09 full run에서 9개 cell 모두 `product=succeeded`, `harness=passed`, `process=exited/0`, 신뢰 가능한 생성물과 viewport screenshot을 남기고 익명 rubric 결과가 `scored=9`, `unscored=0`, `scoring_failed=0`, `blocked=0`이어야 한다. terminal failure를 보존했다는 사실만으로 Milestone이나 최종 벤치 task를 완료하지 않는다. - - [x] [D15] 생성 파일·정적 안전·asset·render처럼 evaluator 입력의 존재와 신뢰성을 보장하는 gate는 hard eligibility로 유지한다. console·responsive·accessibility처럼 생성물의 품질을 평가하는 gate 실패는 신뢰 가능한 페이지가 존재하면 unscored 사유가 아니라 익명 rubric의 감점 evidence로 전달한다. - -## 문제 / 비목표 - -- 문제: `[route-02]` 하이브리드 원샷의 실사용 가치와 overhead를 판단하려면 같은 IOP 경계, task fixture와 평가 기준에서 단독 모델·caller agent 조합과 속도·token·품질을 함께 비교해야 한다. 단일 성공 smoke만으로는 모델·agent·coordinator 차이를 설명할 수 없다. -- 비목표: - - `[route-02]` 완료 smoke를 대신하거나 benchmark 점수로 완료 상태를 소급 변경 - - 첫 보고서에서 통계적 다회 반복이나 장기/heavy 작업 평가 - - framework 설치·cache 성능 비교 - - model/provider 가격표를 billing-grade 비용으로 확정 - -## Source of Truth - -| 영역 | 기준 | 메모 | -|------|------|------| -| Roadmap | [Milestone 문서](../../../phase/knowledge-tool-optimization-extension/milestones/iop-one-shot-agent-model-comparison.md) | 9개 비교군과 완료 상태 원장 | -| Pipeline | [bench-01 Milestone](../../../phase/knowledge-tool-optimization-extension/milestones/agent-comparison-benchmark-pipeline.md)의 승인된 manifest/runner/report contract | 실행·측정·보고 구현 원본 | -| Testbed | `../iop-s2` dev IOP runtime | 모든 scored model 호출의 IOP 경유 대상 | -| Fixture | versioned prompt, 이미지 2장과 vanilla workspace checksum | 모든 cell의 동일 입력 기준 | -| Evidence | `agent-test/runs//` | attempt별 timeline, usage, validation, screenshot와 score | -| Report | `agent-test/dev/iop-one-shot-agent-comparison-.md` | 현재 프로젝트의 사람이 읽는 비교 결과 | -| API Contract | [Anthropic-Compatible Messages API](../../../../agent-contract/outer/anthropic-compatible-api.md), [OpenAI-Compatible API](../../../../agent-contract/outer/openai-compatible-api.md), [Gemini-Compatible API](../../../../agent-contract/outer/gemini-compatible-api.md) | caller ingress, stream/terminal과 usage 기준 | -| Config Contract | [Edge Config And Runtime Refresh](../../../../agent-contract/inner/edge-config-runtime-refresh.md) | model route, preset, protocol profile과 credential 경계 | -| User Decision | D01-D10 | 2026-08-06 확정 방향, 추가 사용자 결정 없음 | - -## State Machine - -| 상태 | 진입 조건 | 다음 상태 | 근거 | -|------|-----------|-----------|------| -| `blocked` | `[route-02]` smoke 또는 `[bench-01]` 완료 전 | `preflighting`, 종료 | active Milestone 상태와 pipeline evidence | -| `preflighting` | 선행 조건 충족, execution-day caller/route/credential 점검 | `ready`, `blocked` | official caller transport, managed projection/lease와 redacted preflight matrix | -| `ready` | fixture와 C01-C09 immutable manifest 확정 | `running`, `cancelled` | manifest/fixture/rubric digest | -| `running` | seed 순서에 따라 각 cell에 사용자 작업 1회 제출 | `validating`, `failed`, `timed_out`, `cancelled` | cell/attempt event timeline | -| `validating` | cell finish/complete 후 idle 확정 | `scoring`, `failed` | workspace, build/render/test evidence | -| `scoring` | C01-C09 결과 identity 제거 완료 | `analyzing`, `failed` | 9개 blind mapping과 rubric worksheet | -| `analyzing` | 자동 gate·시간·usage와 9개 score 완비 | `reported`, `failed` | `scored=9`, `unscored=0`, `scoring_failed=0`, `blocked=0` 및 comparison table | -| `reported` | Markdown과 raw evidence 포인터 생성 | 종료 | report path와 digest | -| `failed` | cell 실행·검증·채점·보고 실패 | `analyzing`, 종료 | 보존된 실패 attempt; 누락 없는 matrix | -| `timed_out` | cell timeout | `analyzing`, 종료 | timeout/cancel/cleanup evidence | -| `cancelled` | 명시 중단 | 종료 | 실행된 cell과 미실행 cell 상태 | - -State invariant: - -- C01-C09는 승인된 run마다 동일 fixture checksum, viewport, rubric version, fresh caller session, setup/cache policy와 repetitions=1을 사용한다. 새 run은 기존 실패 run과 다른 run identity를 가지며 기존 run을 resume/retry하거나 대표 결과에서 삭제하지 않는다. -- execution order는 고정 seed로 생성해 보고서에 남기고 결과에 따라 재정렬하지 않는다. -- preflight와 setup usage/time은 scored measurement에 합산하지 않지만 별도 기록한다. -- 한 cell의 사용자 작업은 한 번 제출하며 사람의 feedback, manual edit, restart가 없다. -- model/tool 호출 횟수는 제약이 아니라 측정 대상이며 finish event 뒤 idle까지가 wall-clock terminal이다. -- 실패 cell도 report matrix에 남고 재실행 결과는 원래 attempt를 대체하지 않는다. -- 배포 qualification의 5-cell direct 진단은 scored C01-C09 run과 별개다. terminal evidence가 완결된 제품 실패·provider rejection·timeout을 acceptance failure로 재해석하지 않고, Edge pre-ingress incompatibility 또는 최대 renderer 재시도 뒤 browser/CDP infrastructure block만 다음 단계 진입을 막는다. -- 복구 후 final full run은 9개 cell 모두 성공 산출물과 익명 점수를 가져야 한다. 어느 한 cell이라도 product/harness/process/hard artifact gate가 실패하거나 `unscored`, `scoring_failed`, `blocked`이면 보고서는 진단 산출물로 보존하되 Milestone 완료로 판정하지 않는다. -- 품질 gate 실패는 hard trust gate와 분리한다. 생성물이 신뢰 가능하면 console·responsive·accessibility 실패도 evaluator에 전달해 해당 rubric 항목에서 감점한다. - -## Interface Contract - -- 계약 원문: [Anthropic-Compatible Messages API](../../../../agent-contract/outer/anthropic-compatible-api.md), [OpenAI-Compatible API](../../../../agent-contract/outer/openai-compatible-api.md), [Gemini-Compatible API](../../../../agent-contract/outer/gemini-compatible-api.md), [Edge Config And Runtime Refresh](../../../../agent-contract/inner/edge-config-runtime-refresh.md) -- 입력: - - `fixture`: 동일 이미지 2장, one-page 요구사항, vanilla HTML/CSS/JS initial workspace와 checksum이다. - - `cells`: C01-C09의 caller, IOP route/preset, expected model/stage와 effort binding이다. - - `repetitions=1`, `session_policy=fresh`, `setup_cache_policy`: 초기 scored attempt 수, conversation/resume 격리와 공통 setup/cache 기준이다. - - `environment=dev`, `testbed=../iop-s2`: 실제 IOP runtime 선택이다. - - `completion`: caller별 finish/complete event와 idle 판정 규칙이다. - - `agy`: `modelProvider=gemini`, route별 `GOOGLE_GEMINI_BASE_URL`, `GEMINI_API_KEY=`과 official `stream-json` event다. API-key provider가 지원하지 않는 `--effort`는 전달하지 않는다. -- 측정 출력: - - timestamp: submitted, first output, first file write, model/stage start/end, tool start/end, finish, idle의 monotonic 값과 observation source다. overlap과 unavailable을 명시한다. - - usage: call count, input/output/reasoning/cached/total token과 source다. - - validation: requirement, build/serve, desktop/mobile, asset/console, responsive/accessibility 결과다. - - score: rubric version, 항목별 점수/근거와 총점이며 identity mapping과 분리한다. -- 100점 rubric: - - 요구사항 충족 25, 시각 완성도 25, 반응형·접근성 15, 이미지 활용·디테일 10, 동작 안정성 10, 코드 품질 10, 자체 검증 완결성 5. -- 금지: - - IOP를 우회한 model 호출을 scored cell로 인정한다. - - cell마다 prompt, asset, initial workspace나 viewport를 다르게 사용한다. - - unavailable token을 0으로 기록하거나 estimated 값을 provider-reported와 합친다. - - evaluator가 identity를 본 상태에서 점수를 조정하거나 결과를 수동 수정한다. - -## Acceptance Scenarios - -| ID | Milestone Task | Given | When | Then | -|----|----------------|-------|------|------| -| S01 | `fixture-lock` | 이미지 2장과 one-page benchmark brief | fixture 확정 | prompt/assets/workspace/viewports/rubric의 checksum과 version이 모든 cell에 동일하다. | -| S02 | `route-readiness` | 5-cell direct 진단과 C01-C09 caller·dev IOP | execution-day qualification과 preflight | direct는 fresh `ready=5`, 정확히 5개 attempt, 모든 terminal evidence와 infrastructure block 없음으로 admission되고, 이후 C01-C09 auth/model/preset/effort/stream/finish/idle이 모두 확인되거나 exact blocker로 중단된다. | -| S03 | `matrix-lock` | 선행 gate가 통과한 9개 cell | scored manifest 생성 | repetitions=1, 실행 순서 seed, fresh-session/setup-cache 정책, timeout과 expected binding이 immutable하게 기록된다. | -| S04 | `claude-standalone` | C01 clean workspace | Claude Code 사용자 작업 1회 | IOP→Sonnet 최고 effort 결과와 complete/idle evidence가 생성된다. | -| S05 | `gemini-standalone` | C02-C03 clean workspace | Claude Code와 agy 사용자 작업을 각각 1회 제출 | 두 caller 모두 IOP→Gemini high 결과와 caller별 timing/usage를 남긴다. | -| S06 | `gpt-standalone` | C04-C05 clean workspace | Claude Code와 Codex 사용자 작업을 각각 1회 제출 | 두 caller 모두 IOP→GPT xhigh 결과와 caller별 timing/usage를 남긴다. | -| S07 | `gemini-hybrid` | C06-C07 clean workspace | Claude Code와 agy 사용자 작업을 각각 1회 제출 | IOP Gemini plan→ornith work→Gemini review/repair의 stage evidence와 최종 결과를 남긴다. | -| S08 | `gpt-hybrid` | C08-C09 clean workspace | Claude Code와 Codex 사용자 작업을 각각 1회 제출 | IOP GPT plan→ornith work→GPT review/repair의 stage evidence와 최종 결과를 남긴다. | -| S09 | `objective-validation` | 복구 후 C01-C09 성공 workspace | 자동 웹 검증 | 각 cell의 동일 hard trust gate, 품질 gate와 desktop/mobile screenshot이 누락 없이 생성되며 hard gate는 모두 통과한다. | -| S10 | `quality-scoring` | identity가 제거된 9개 신뢰 가능 결과 | Codex rubric 평가 | 9개 모두 항목별 점수/근거와 총점이 기록되고 `scored=9`, `unscored=0`, `scoring_failed=0`, `blocked=0`이다. 품질 gate 실패는 해당 항목의 감점 evidence다. | -| S11 | `performance-usage` | 모든 attempt timeline/usage | 비교 집계 | 첫 output·첫 write·model/tool/queue/total 시간의 clock/source·overlap, 호출 수와 token/source가 cell·stage별 표가 된다. | -| S12 | `benchmark-report` | S01-S11 evidence | 보고서 생성 | 조건·버전·9개 결과·속도·token·품질·실패·한계와 raw evidence 링크가 Markdown에 남는다. | -| S13 | `agy-iop-compatibility` | official `agy 1.1.12`와 dev Edge | direct·hybrid route별 Gemini base URL로 실제 API-key 호출 | 두 호출 모두 `x-goog-api-key` IOP principal auth, Gemini-native request/tool/SSE, official `stream-json` finish/exit와 config-owned effective binding evidence를 남기고 upstream key 직접 호출이나 합성 event에 의존하지 않는다. | -| S14 | `managed-credential-dev` | dev Control Plane·Edge·Node와 operator-owned security material | managed credential profile로 재기동하고 slot/route를 등록 | CA-signed mTLS·Edge HTTPS·fresh projection·sealed lease가 확인되고 legacy credential source나 cross-route fallback 없이 direct와 marked preset stage가 실행된다. | - -## Evidence Map - -| Scenario | Required Evidence | `agent-task` 연결 | 완료 Evidence 기대 | -|----------|-------------------|------------------|---------------------------| -| S01 | fixture prompt/assets/workspace/rubric digest | `agent-task/m-iop-one-shot-agent-model-comparison/fixture-lock/` | `fixture-lock` identical-input evidence | -| S02 | redacted direct `ready=5`·5-slot terminal evidence·infrastructure 판정과 C01-C09 preflight matrix | `agent-task/m-iop-one-shot-agent-model-comparison/route-readiness/` | `route-readiness` direct admission과 auth/route/effort/terminal evidence | -| S03 | immutable scored manifest와 order seed | `agent-task/m-iop-one-shot-agent-model-comparison/matrix-lock/` | `matrix-lock` 9-cell/repetitions=1 evidence | -| S04 | C01 event/timing/usage/workspace evidence | `agent-task/m-iop-one-shot-agent-model-comparison/claude-standalone/` | `claude-standalone` one-submission/IOP evidence | -| S05 | C02-C03 caller별 event/timing/usage/workspace evidence | `agent-task/m-iop-one-shot-agent-model-comparison/gemini-standalone/` | `gemini-standalone` two-caller evidence | -| S06 | C04-C05 caller별 event/timing/usage/workspace evidence | `agent-task/m-iop-one-shot-agent-model-comparison/gpt-standalone/` | `gpt-standalone` two-caller evidence | -| S07 | C06-C07 Gemini/ornith stage와 terminal evidence | `agent-task/m-iop-one-shot-agent-model-comparison/gemini-hybrid/` | `gemini-hybrid` two-caller stage evidence | -| S08 | C08-C09 GPT/ornith stage와 terminal evidence | `agent-task/m-iop-one-shot-agent-model-comparison/gpt-hybrid/` | `gpt-hybrid` two-caller stage evidence | -| S09 | build/render/viewport/asset/console/accessibility result와 screenshot | `agent-task/m-iop-one-shot-agent-model-comparison/objective-validation/` | `objective-validation` uniform gate evidence | -| S10 | blind mapping 분리와 Codex rubric worksheet | `agent-task/m-iop-one-shot-agent-model-comparison/quality-scoring/` | `quality-scoring` 100-point evidence | -| S11 | cell/stage별 normalized timeline, calls와 token-source table | `agent-task/m-iop-one-shot-agent-model-comparison/performance-usage/` | `performance-usage` speed/token evidence | -| S12 | `agent-test/dev/` Markdown과 raw run links | `agent-task/m-iop-one-shot-agent-model-comparison/benchmark-report/` | `benchmark-report` complete comparison evidence | -| S13 | official agy request-shape capture, Edge Gemini bridge tests, direct·hybrid live preflight와 sanitized lifecycle/usage | `agent-task/m-iop-one-shot-agent-model-comparison/06+05_comparison_runs/` | `agy-iop-compatibility` official 1.1.12 IOP transport evidence | -| S14 | dev config check, TLS/workload identity, projection generation, slot-route/lease attribution과 post-revoke no-fallback smoke | `agent-task/m-iop-one-shot-agent-model-comparison/06+05_comparison_runs/` | `managed-credential-dev` secure composition and hybrid admission evidence | - -공통 완료 검증은 복구 후 승인된 새 C01-C09 full run에서 9개 모두 success terminal evidence, hard trust gate 통과, desktop/mobile screenshot, blind score와 timing/usage source를 가지며 최종 집계가 `scored=9`, `unscored=0`, `scoring_failed=0`, `blocked=0`인지 확인한다. 이전 실패 run은 원인·한계 evidence로 보존하지만 완료 근거를 대체하지 않는다. 필수 credential/model이 없으면 raw secret을 요구하거나 기록하지 않고 운영 절차로 등록을 요청한다. - -## Cross-repo Dependencies - -- 없음. 같은 IOP 프로젝트의 `[route-02]`와 `[bench-01]` 실행 순서는 [전역 마일스톤 실행 순서](../../../priority-queue.md)에서 관리한다. - -## Drift Check - -- [x] Milestone 기능 Task와 Acceptance Scenario가 일치한다. -- [x] Evidence Map이 code-review/complete.log에서 검증 가능하다. -- [x] agent-contract를 쓰는 경우 SDD에 계약 원문을 복제하지 않았다. -- [x] 사용자 리뷰가 필요한 항목은 없고 확정된 D01-D13을 반영했다. - -## 사용자 리뷰 이력 - -- 2026-08-06: 사용자가 Sonnet/Gemini/GPT 단독과 Gemini/GPT 하이브리드의 9개 IOP 경유 비교군, Claude Code·agy·Codex caller, finish/idle 원샷, 초기 1회, dev `../iop-s2`, 동일 정적 웹 fixture와 시간·token·Codex 품질 평가를 확정했다. -- 2026-08-12: 공식 `agy 1.1.12` API-key provider의 실제 Gemini-native 요청과 `stream-json` event를 확인했고, 사용자의 provider 직접 설정 지시에 따라 upstream key와 IOP principal token을 분리하며 dev managed credential plane까지 구성하는 D11-D12를 기술 보강했다. -- 2026-08-13: 사용자가 terminal outcome과 dispatcher 환경 보완 뒤 다음 벤치까지 계속 실행하도록 승인했다. 이에 기존 실패 run을 보존하고 resume/retry하지 않은 채, 동일 immutable C01-C09 manifest로 repetitions=1인 새 scored run identity를 한 번 생성하는 D06/D10 경계를 확정했다. -- 2026-08-13: 승인된 후속 packet에 따라 direct 배포 qualification을 terminal-evidence admission으로 분리했다. 제품 실패·provider rejection·timeout은 측정 결과로 보존하고, Edge pre-ingress incompatibility와 exhausted browser/CDP infrastructure block만 qualification을 막는 D13을 추가했다. D06/D10의 scored-run uniqueness는 유지한다. -- 2026-08-13: 사용자가 2개 scored/7개 unscored 상태에서 종료하지 말고 9개 모두 측정되도록 원인을 해소하라고 명시했다. 이에 기존 실패 run은 보존하되 완료 조건을 복구 후 새 full run의 `9/9 scored`로 강화하고, 품질 gate와 hard trust gate를 분리하는 D14-D15를 추가했다. - -## 작업 컨텍스트 - -- 표준선: 이 비교는 `[route-02]` 완료 이후의 추가 검증이며 정식 smoke의 일부나 대체 evidence가 아니다. -- 후속 SDD: 없음 diff --git a/agent-spec/archive/testing/agent-comparison-benchmark.log b/agent-spec/archive/testing/agent-comparison-benchmark.log new file mode 100644 index 00000000..266f5125 --- /dev/null +++ b/agent-spec/archive/testing/agent-comparison-benchmark.log @@ -0,0 +1,17 @@ +--- +spec_doc_type: archive-log +spec_id: testing/agent-comparison-benchmark +status: 폐기됨 +source_evidence: + - type: user + path: null + notes: 2026-08-13 전용 하네스를 폐기하고 공식 caller별 수동 개별 검증을 우선하도록 결정 + - type: roadmap + path: agent-roadmap/archive/phase/knowledge-tool-optimization-extension/milestones/iop-one-shot-agent-model-comparison.md + notes: 폐기된 측정 Milestone과 경계 분리 근거 +--- + +# 폐기 기록: Agent 비교 벤치마크 파이프라인 + +전용 manifest runner, 재개·복구 저장소, 자동 웹 검증, 익명 채점과 보고 파이프라인은 2026-08-13 폐기했다. +현재 IOP 호출부는 완료된 제품 기준선으로 유지한다. 구체적 결함이 재현되면 해당 코드와 로드맵 소유 영역에서 국소 처리하며, 전용 benchmark CLI나 전수 안정성 campaign은 현재 구현·완료 표면이 아니다. diff --git a/agent-spec/index.md b/agent-spec/index.md index 63a81005..7b406f0e 100644 --- a/agent-spec/index.md +++ b/agent-spec/index.md @@ -27,7 +27,6 @@ AI agent가 작업 전에 읽는 지도이기도 하지만, 사람도 "지금 - 출력 검증 런타임: staged response-start, evidence hold/release, filter arbitration, bounded recovery/rebuild, raw-free observation은 `runtime/stream-evidence-gate`에서 본다. - 외부 HTTP 입력: OpenAI-compatible 호출, Anthropic-compatible Messages 호출, managed principal route/slot binding, model-driven raw tunnel은 `input/openai-compatible-surface`, A2A JSON-RPC 호출은 `input/a2a-json-rpc-surface`에서 본다. - 운영 제어: Control Plane, credential HTTPS/host-local bootstrap, mTLS Edge enrollment, projection/lease flow, fleet/edge status, Flutter Client 상태 소비는 `control/control-plane-operations`에서 본다. -- 테스트 도구: Claude Code, agy, Codex의 IOP 경유 비교를 manifest로 실행하고 격리·측정·웹 검증·익명 채점·보고하는 현재 harness는 `testing/agent-comparison-benchmark`에서 본다. ## 스펙 목록 @@ -39,7 +38,6 @@ AI agent가 작업 전에 읽는 지도이기도 하지만, 사람도 "지금 | `input/openai-compatible-surface` | 부분 | `/v1/models`, `/v1/chat/completions`, `/v1/responses`, `/v1/messages`, `/v1/messages/count_tokens`, `/anthropic/v1/models`, managed projection/slot routing, OpenAI-compatible auth/metadata/tool handling, Anthropic bearer/`X-Api-Key` auth, provider-pool native/bridge admission, safe slot attribution, marked single-request 내부 stage template과 caller-visible I/O 경계, and OpenAI-only usage metrics를 확인할 때 | `agent-spec/input/openai-compatible-surface.md` | `agent-contract/outer/openai-compatible-api.md`, `agent-contract/outer/anthropic-compatible-api.md`, `apps/edge/internal/openai/chat_handler.go`, `apps/edge/internal/openai/anthropic_handler.go`, `apps/edge/internal/openai/anthropic_bridge.go`, `apps/edge/internal/openai/normalized_sse.go`, `apps/edge/internal/openai/usage_metrics.go` | | `input/a2a-json-rpc-surface` | 부분 | Edge A2A JSON-RPC, `message/send`, `tasks/get`, `tasks/cancel`, A2A task store와 bearer auth를 확인할 때 | `agent-spec/input/a2a-json-rpc-surface.md` | `agent-contract/outer/a2a-json-rpc-api.md`, `apps/edge/internal/input/a2a/server.go`, `apps/edge/internal/input/a2a/task_store.go` | | `control/control-plane-operations` | 부분 | credential HTTPS and host-local bootstrap, Control Plane-Edge mTLS projection/lease wire, Client-Control Plane wire, Control Plane HTTP Edge/fleet status view, Flutter Client status consumer를 확인할 때 | `agent-spec/control/control-plane-operations.md` | `agent-contract/inner/control-plane-edge-wire.md`, `agent-contract/inner/client-control-plane-wire.md`, `apps/control-plane/internal/wire/edge_server.go`, `apps/control-plane/internal/credentiallease/service.go` | -| `testing/agent-comparison-benchmark` | 구현됨 | Claude Code, agy, Codex의 IOP 경유 benchmark manifest, preflight, run/resume, 격리, timing/usage, 웹 gate, 익명 채점과 Markdown report를 확인할 때 | `agent-spec/testing/agent-comparison-benchmark.md` | `scripts/agent_comparison_benchmark.py`, `scripts/agent_benchmark/attempts.py`, `scripts/agent_benchmark/scoring.py`, `scripts/agent_benchmark/reporting.py` | ## 작성 규칙 diff --git a/agent-spec/testing/agent-comparison-benchmark.md b/agent-spec/testing/agent-comparison-benchmark.md deleted file mode 100644 index d6c8a83e..00000000 --- a/agent-spec/testing/agent-comparison-benchmark.md +++ /dev/null @@ -1,143 +0,0 @@ ---- -spec_doc_type: spec -spec_id: testing/agent-comparison-benchmark -status: 구현됨 -source_evidence: - - type: code - path: scripts/agent_comparison_benchmark.py - notes: validate, preflight, run, resume, status, score, report 공개 CLI - - type: code - path: scripts/agent_benchmark/manifest.py - notes: manifest 정규화, 경로 격리, fixture와 입력 digest 검증 - - type: code - path: scripts/agent_benchmark/attempts.py - notes: append-only run/attempt 저장과 preflight, 실행, 재개 lifecycle - - type: code - path: scripts/agent_benchmark/connectivity.py - notes: caller capability, requested/effective binding과 blocker 분류 - - type: code - path: scripts/agent_benchmark/measurement.py - notes: source-aware timing과 usage 정규화 - - type: code - path: scripts/agent_benchmark/web_validation.py - notes: 정적 웹 산출물과 desktop/mobile 자동 gate 검증 - - type: code - path: scripts/agent_benchmark/browser_cdp.py - notes: Chromium/CDP 단일 시도 격리와 닫힌 transient class의 최대 3회 fresh 재시작 - - type: code - path: scripts/agent_benchmark/scoring.py - notes: 익명화 입력, fresh evaluator와 scoring attempt 처리 - - type: code - path: scripts/agent_benchmark/reporting.py - notes: deterministic Markdown 보고서 생성 - - type: test - path: scripts/agent_benchmark/connectivity_integration_test.py - notes: 세 caller의 IOP binding, lifecycle, 격리와 실패 경계 통합 검증 - - type: test - path: scripts/agent_benchmark/scoring_test.py - notes: unscored, scoring_failed, 새 scoring attempt와 익명화 검증 - - type: test - path: scripts/agent_benchmark/reporting_test.py - notes: all-status, 동점과 raw evidence 포인터 보고 검증 - - type: test - path: scripts/agent_benchmark/skill_contract_test.py - notes: project-local skill과 공개 CLI 계약 검증 - - type: test - path: scripts/agent_benchmark/browser_cdp_test.py - notes: transient 재시도·소진·비재시도와 process/screenshot 정리 검증 - - type: contract - path: agent-contract/outer/openai-compatible-api.md - notes: agy와 Codex가 사용하는 IOP OpenAI-compatible ingress 계약 - - type: contract - path: agent-contract/outer/anthropic-compatible-api.md - notes: Claude Code가 사용하는 IOP Anthropic-compatible ingress 계약 - - type: contract - path: agent-contract/inner/edge-config-runtime-refresh.md - notes: model, route, execution preset과 protocol profile 설정 계약 - - type: roadmap - path: agent-roadmap/archive/phase/knowledge-tool-optimization-extension/milestones/agent-comparison-benchmark-pipeline.md - notes: bench-01 완료 범위와 evidence 집계 - - type: sdd - path: agent-roadmap/archive/sdd/knowledge-tool-optimization-extension/agent-comparison-benchmark-pipeline/SDD.md - notes: benchmark lifecycle, 실패 보존, 채점과 secret-safe evidence 결정 ---- - -# 스펙: Agent 비교 벤치마크 파이프라인 - -## 목적 - -Claude Code, agy, Codex가 IOP를 경유해 수행하는 동일 과업을 설정만 바꿔 반복 실행하고, 연결 상태부터 실행·검증·채점·보고까지 재현 가능한 evidence로 남기는 현재 benchmark harness를 설명한다. - -## 기능 목록 - -| 기능 | 설명 | -|------|------| -| manifest 검증 | caller, IOP direct/preset route, model, effort, fixture, 반복 횟수, timeout, evaluator와 `agent-test/runs/` 경로를 검증하고 canonical digest를 만든다. | -| 연결 preflight | Claude Code, agy, Codex의 binary/config와 IOP endpoint·auth·model·effort·stream binding을 확인하고 `ready`, `registration_required`, `implementation_gap`으로 분류한다. | -| 격리 실행과 재개 | 각 cell/repetition을 동일 checksum의 clean workspace와 fresh caller session에서 실행하며, 제품 결과·harness 정합성·process 종료를 독립 결과로 보존한다. top-level state는 `running`, `completed`, `timed_out`, `cancelled`, `interrupted`의 controller 상태만 나타내며, 실패한 attempt는 덮어쓰지 않는다. | -| 측정과 evidence | 제출, 첫 출력, 첫 파일 쓰기, model/tool/queue, finish/idle 시간을 관측 source와 함께 정규화한다. token은 보고 주체와 미제공 상태를 보존하며 임의 추정값을 authoritative 값과 섞지 않는다. | -| 웹 자동 검증 | product/harness 성공 여부와 무관하게 모든 terminal workspace에서 필수 HTML/CSS/JS와 로컬 이미지, 외부 asset 금지, desktop/mobile render, console/asset 오류, 반응형·접근성 gate와 screenshot을 확인한다. | -| 익명 품질 채점 | 필수 자동 gate를 통과한 결과만 identity를 가린 뒤 manifest에 고정된 fresh evaluator로 100점 rubric을 평가한다. 부적격 결과는 `unscored`, 평가 실패는 `scoring_failed`로 남기며 retry는 새 scoring attempt id를 사용한다. | -| 상태와 보고 | `validate`, `preflight`, `run`, `resume`, `status`, `score`, `report` CLI를 제공하고, controller/product/harness/process/artifact/scoring 축과 동점을 raw evidence 포인터와 함께 deterministic Markdown으로 만든다. | - -## 범위 - -- 포함: benchmark manifest, caller adapter binding, run/attempt 저장, 격리 workspace, source-aware timing/usage, 정적 웹 gate, 익명 채점, Markdown 보고. -- 제외: IOP runtime 자체 구현, provider credential 등록, 실제 9개 비교군 실행과 모델 우열 결론. 해당 실행과 결론은 `[bench-02]`가 소유한다. - -## 주요 흐름 - -```mermaid -flowchart LR - Operator[사용자 또는 project skill] --> CLI[benchmark CLI] - CLI --> Manifest[manifest 검증] - Manifest --> Preflight[caller와 IOP preflight] - Preflight --> Store[append-only run store] - Store --> Attempt[clean workspace와 fresh session attempt] - Attempt --> Evidence[timing, usage와 web evidence] - Evidence --> Score[익명 evaluator scoring] - Score --> Report[deterministic Markdown report] -``` - -## 계약 - -- OpenAI-compatible caller ingress는 [OpenAI-Compatible API](../../agent-contract/outer/openai-compatible-api.md)를 따른다. -- Claude Code ingress는 [Anthropic-Compatible Messages API](../../agent-contract/outer/anthropic-compatible-api.md)를 따른다. -- model, direct/preset route와 protocol profile의 기준은 [Edge Config And Runtime Refresh](../../agent-contract/inner/edge-config-runtime-refresh.md)를 따른다. -- 사용자-facing orchestration과 안전한 실행 순서는 [IOP Agent Comparison Benchmark skill](../../agent-ops/skills/project/iop-agent-comparison-benchmark/SKILL.md)이 소유하고, 제품 호출과 evidence 생성은 deterministic Python pipeline이 소유한다. - -## 설정/데이터/이벤트 - -- manifest의 matrix cell은 stable id, caller, IOP route kind, requested/effective model과 effort를 가진다. unsupported alias나 effort는 다른 값으로 대체하지 않고 fail-closed한다. -- run state는 `agent-test/runs///` 아래에 격리되며 manifest digest가 다른 상태를 재개하지 않는다. -- preflight는 scored attempt가 아니며, 실행 중 실패·timeout·cancel과 scoring 실패는 기존 attempt를 수정하지 않고 보존한다. -- caller parser는 raw terminal 문자열 대신 `CallerEvent(finish|idle)`, `CallerTerminal(succeeded|failed)`와 typed metric만 반환한다. Claude result가 마지막 active assistant snapshot을 직접 완성하면 adapter가 typed finish와 idle을 함께 투영하고, assistant가 이미 finish를 냈으면 result는 idle만 투영한다. synthetic API error와 agy ERROR result는 `product=failed`, `harness=passed`가 될 수 있으며 parser malformed는 `product=unknown`, `harness=failed`로 구분한다. -- durable lifecycle/measurement/attempt evidence는 `product`, `harness`, `process` 객체를 그대로 저장한다. `unresolved`은 수집/검증 완결성(모든 슬롯이 웹 검증 증거 보유)이며, `passed`는 전체 gate 성공으로 유지되고 retry/skip를 제어한다. scoring eligibility는 변경없으며, terminal failure는 `unscored` report row로 유지된다. run/resume exit 0은 `unresolved=0`을 요구하며, 독립 실패 축은 stdout에 남고 `score`로 분류된다. -- 배포 qualification은 동일 clean source에서 5-cell direct manifest의 fresh preflight `ready=5`를 확인하고 정확히 5개의 fresh attempt를 한 번씩 실행한다. `unresolved=0`, `running=0`, `interrupted=0`, 모든 slot의 controller/product/harness/process/web-validation terminal evidence, exhausted browser/CDP infrastructure block 없음이 admission 조건이다. product failure, upstream HTTP rejection, caller failure 뒤 generated-missing과 timeout은 측정 결과로 보존하고 암묵 재시도하지 않는다. Edge pre-ingress incompatibility 또는 exhausted browser/CDP infrastructure block만 qualification을 막는다. admission 뒤 fresh C01-C09 preflight `ready=9`까지만 수행하고 hybrid 또는 scored C01-C09 실행은 후속 승인 전에는 할당하지 않는다. -- Chromium/CDP renderer는 source/product validation을 재실행하지 않고, process start·handshake·socket loss의 닫힌 transient class에만 fresh browser/profile로 최대 3회 시도한다. 각 실패 시도는 process group과 부분 screenshot을 정리하며 세 번째 실패는 `artifact=blocked`로 유지한다. -- lifecycle supervisor는 exit watcher와 출력 reader를 join한 뒤 하나의 child return code를 동결해 lifecycle result와 cleanup receipt가 동일한 exit/signal을 갖게 한다. 불일치 evidence는 resume에서 fail-closed한다. -- raw credential과 private endpoint는 tracked manifest, event, log, screenshot과 report에 기록하지 않는다. -- report는 run state의 canonical evidence에서 생성되며 성공하지 않은 결과를 0점으로 변환하거나 동점에 임의 순위를 부여하지 않는다. - -## 검증 - -- `python3 -m unittest discover -s scripts/agent_benchmark -p '*_test.py'` - 전체 benchmark unit/integration 계약이 통과한다. -- `python3 -m unittest scripts.agent_benchmark.reporting_test scripts.agent_benchmark.skill_contract_test` - 공개 보고 CLI와 project-local skill 계약이 통과한다. -- `python3 scripts/agent_comparison_benchmark.py validate --manifest scripts/fixtures/agent-comparison-benchmark-manifest.example.json` - 표준 manifest가 유효하다. -- `python3 scripts/agent_comparison_benchmark.py validate --manifest scripts/fixtures/agent-comparison-benchmark-supported-direct.example.json` - 지원 direct route fixture가 유효하다. -- `python3 scripts/agent_comparison_benchmark.py validate --manifest scripts/fixtures/agent-comparison-benchmark-direct-preflight.example.json` - direct preflight fixture가 유효하다. - -## 한계와 주의사항 - -- example manifest와 fake adapter 테스트 통과는 실제 provider credential, endpoint 또는 preset의 live readiness를 의미하지 않는다. 실제 실행 전 redacted preflight가 필요하다. -- 현재 living spec은 비교 파이프라인의 구현 상태만 다룬다. 9개 scored cell 실행, 비교 결과와 결론은 후속 `[bench-02]` evidence로 관리한다. -- caller나 IOP가 제공하지 않은 usage는 `unavailable`로 남으며, 서로 다른 clock/source의 중첩 구간을 임의로 합산하지 않는다. -- benchmark 결과는 허용된 `agent-test/runs/` 경계 안에만 생성한다. - -## 변경 기록 - -- 2026-08-13: direct 배포 qualification을 all-success에서 terminal-evidence admission으로 분리했다. fresh `ready=5`, 정확히 5개 attempt와 완결 evidence는 요구하되 제품 실패·timeout은 결과로 보존하고, Edge pre-ingress incompatibility 또는 exhausted browser/CDP infrastructure block만 qualification을 막는다. Chromium/CDP 닫힌 transient class에는 최대 3회 fresh renderer 시도를 추가했다. -- 2026-08-13: `unresolved`을 수집/검증 완결성으로 정의하고 `passed`(전체 gate 성공)와 분리했다. run/resume exit 0은 `unresolved=0`(모든 슬롯이 웹 검증 증거 보유)을 요구하며, 독립 실패 축은 stdout에 남고 `score`로 분류된다. 기존 attempt 바이트 변경 없음. -- 2026-08-12: caller terminal을 closed typed observation으로 바꾸고 product/harness/process 결과, failure-inclusive artifact gate, 독립 CLI/report/scoring gate와 direct-first qualification을 구현했다. -- 2026-08-12: official Claude result-direct/API-error 및 agy ERROR terminal을 lifecycle 계약에 맞게 분리하고, timeout cleanup result/receipt가 같은 child exit snapshot을 사용하도록 동기화했다. -- 2026-08-12: `[bench-01]` 종료 감사에서 확인한 421개 benchmark test, manifest/CLI 계약과 구현 evidence를 기준으로 생성했다. diff --git a/agent-task/m-iop-one-shot-agent-model-comparison/16+15_all_cell_measurement_recovery/CODE_REVIEW-cloud-G10.md b/agent-task/m-iop-one-shot-agent-model-comparison/16+15_all_cell_measurement_recovery/CODE_REVIEW-cloud-G10.md deleted file mode 100644 index 87ede29c..00000000 --- a/agent-task/m-iop-one-shot-agent-model-comparison/16+15_all_cell_measurement_recovery/CODE_REVIEW-cloud-G10.md +++ /dev/null @@ -1,98 +0,0 @@ - - -# Code Review Reference - REPAIR - -> **[IMPLEMENTING AGENT — READ FIRST] Filling in this file is the mandatory final step of implementation.** Complete the fixed checklist, record actual output, leave active files in place, and report ready. Do not decide the verdict, create follow-up plans, archive files, write `complete.log`, modify roadmap state, ask the user, or start the dependent final benchmark. A qualification failure must be recorded without retry; official review owns diagnosis and the next exact fix. - -## Overview - -date=2026-08-13 -task=m-iop-one-shot-agent-model-comparison/16+15_all_cell_measurement_recovery, plan=0, tag=REPAIR - -## Archive Evidence Snapshot - -- Predecessor: `agent-task/archive/2026/08/m-iop-one-shot-agent-model-comparison/15+14_scored_benchmark_and_report/complete.log`. -- Preserved run: `run-20260813T081326Z-4e1ac5152c6c`, two scored/seven unscored; immutable diagnostic evidence, not completion. -- Latest merged baseline: `origin/dev@6e2f4ff8`; deployed runtime was stale before this task. - -## For the Review Agent - -> **[REVIEW AGENT ONLY]** Rerun safe deterministic checks and reconstruct the release/qualification evidence. PASS requires all six qualification paths to meet the plan's success/hard-gate oracle. If any path fails, do not PASS or let the final task start: collect the exact sanitized terminal/request-stage evidence, select one concrete root-cause fix with exact files/tests, and generate the required follow-up plan through the plan/finalize-routing flow. - -## Implementation Item Completion - -| Item | Status | -|---|---| -| REPAIR-1 Hard/quality scoring boundary | [ ] | -| REPAIR-2 Five-minute manifests | [ ] | -| REPAIR-3 Central dev deployment | [ ] | -| REPAIR-4 Six-path qualification | [ ] | -| REPAIR-5 Recovery report | [ ] | - -## Implementation Checklist - -- [ ] [REPAIR-1] Separate hard artifact eligibility gates from quality-scoring gates and add fail-closed regression coverage. -- [ ] [REPAIR-2] Set the compact benchmark and diagnostic per-cell ceiling to 300 seconds and lock both manifests in tests. -- [ ] [REPAIR-3] Commit/push the exact feature, merge it into central dev, and complete the private dev-runtime deployment procedure with a safe restart and source/build identity proof. -- [ ] [REPAIR-4] Run exactly one fresh non-scored six-path qualification and require all six caller/route paths to produce successful terminal evidence, generated files, screenshots, and all hard gates. -- [ ] [REPAIR-5] Publish a sanitized deterministic recovery report without secrets or claims about the not-yet-run final benchmark. -- [ ] Fill implementation-owned sections in CODE_REVIEW-*-G??.md with actual implementation notes and verification output. - -## Review-Only Checklist - -> **[REVIEW AGENT ONLY]** Implementing agents must not modify or check this section. - -- [ ] Append one PASS/WARN/FAIL verdict with verified routing signals. -- [ ] Re-run focused/full tests and validate both manifests. -- [ ] Verify the scoring split remains fail-closed for hard evidence and quality evidence reaches the evaluator. -- [ ] Verify exact central dev release/source/build/runtime identity and safe restart evidence. -- [ ] Verify exactly one six-path qualification, no retry/resume/score/final run, six successes, 12 screenshots, and 24 passing hard gates. -- [ ] On live failure, diagnose and select an exact fix before generating a follow-up; do not write `complete.log`. -- [ ] On PASS, archive the pair, preserve milestone metadata, write `complete.log`, move the task directory, and leave the dependent task ready. -- [ ] Reconcile `.gitignore`, work log, active parent, and no duplicate execution. - -## Deviations from Plan - -_Implementer: replace with `None` or exact deviation, reason, and replacement command._ - -## Key Design Decisions - -_Implementer: record the implemented hard/quality partition, manifest identity, release identity, and no-retry boundary._ - -## Reviewer Checkpoints - -- D14 completion is not met here; this is recovery admission only. -- D15 partition is exact: hard=`generated_files,static_safety,images,network`; quality=`console,responsive,accessibility`. -- Runtime warnings alone are not caller failure; terminal product/harness/process and hard artifact evidence are authoritative. -- No raw secret, credential, provider payload, or identity map enters tracked evidence. - -## Verification Results - -### Focused and full deterministic tests - -_Implementer: paste exact commands, output summaries, and exits._ - -### Central dev deployment - -_Implementer: record sanitized feature/dev/release/tag SHA/tree, test counts, binary identities, restart/ports/nodes/providers/capacity results._ - -### Six-path qualification - -_Implementer: record exact run id, preflight count, per-cell terminal/hard/quality/screenshot matrix, and no-retry audit._ - -### Diff and provenance audit - -_Implementer: record manifest digests, contained report links, `git diff --check`, and status._ - ---- - -> **[IMPLEMENTING AGENT — BEFORE SAVING]** Fill every implementation-owned section and check the final checklist item. Leave review-only sections unchanged. - -## Section Ownership - -| Section | Owner | Note | -|---|---|---| -| Header, Overview, Archive Snapshot, Review instructions | Fixed | Do not modify | -| Implementation completion/checklist | Implementer | Check status only | -| Review-Only Checklist and verdict | Reviewer | Implementer must not modify | -| Deviations, decisions, verification results | Implementer then reviewer | Record actual evidence | diff --git a/agent-task/m-iop-one-shot-agent-model-comparison/16+15_all_cell_measurement_recovery/PLAN-cloud-G10.md b/agent-task/m-iop-one-shot-agent-model-comparison/16+15_all_cell_measurement_recovery/PLAN-cloud-G10.md deleted file mode 100644 index 2aabf900..00000000 --- a/agent-task/m-iop-one-shot-agent-model-comparison/16+15_all_cell_measurement_recovery/PLAN-cloud-G10.md +++ /dev/null @@ -1,267 +0,0 @@ - - -# Recover every caller path before the final nine-cell measurement - -## For the Implementing Agent - -Implement only the closed scoring/timeout/qualification packet below. Run every verification, fill the paired `CODE_REVIEW-cloud-G10.md` with actual output, keep the active files in place, and report ready for official review. Final verdict, follow-up planning, archive moves, `complete.log`, and roadmap synchronization belong only to official review. If the six-path qualification still fails, do not retry it or start the final benchmark; record the exact sanitized terminal evidence and resume condition so the reviewer can select the next concrete fix. - -## Background - -The preserved run `run-20260813T081326Z-4e1ac5152c6c` ended with two scored and seven unscored cells, so it is diagnostic evidence rather than milestone completion. C02 produced a trustworthy rendered page but accessibility alone made it ineligible; the other failures are concentrated in Claude and official agy caller paths against a stale dev runtime. This task first fixes the confirmed eligibility defect, adopts the user-approved 300-second per-cell ceiling, deploys the already-merged latest dev stream-liveness baseline, and runs one non-scored six-path qualification. It must not allocate the final C01-C09 run. - -## Archive Evidence Snapshot - -- Predecessor: `agent-task/archive/2026/08/m-iop-one-shot-agent-model-comparison/15+14_scored_benchmark_and_report/complete.log`; internal verdict PASS, but its final `2 scored/7 unscored` state is not the user-level completion condition. -- Preserved evidence: C02 has `product=succeeded`, `harness=passed`, `process=exited/0`, both screenshots, and only `accessibility=false`; C01 has `parser_error`; C03 returned Gemini `400 INVALID_ARGUMENT` after a first tool turn; C04/C06/C08 returned Claude caller errors; C07 timed out. -- Latest central baseline merged into this feature: `origin/dev@6e2f4ff8`, including the stream terminal-liveness recovery absent from the deployed `dev-974` runtime. -- Do not modify, resume, retry, delete, or score the preserved run. - -## Analysis - -### Files Read - -- `agent-roadmap/phase/knowledge-tool-optimization-extension/milestones/iop-one-shot-agent-model-comparison.md` -- `agent-roadmap/sdd/knowledge-tool-optimization-extension/iop-one-shot-agent-model-comparison/SDD.md` -- `agent-contract/outer/anthropic-compatible-api.md` -- `agent-contract/outer/gemini-compatible-api.md` -- `scripts/agent_benchmark/scoring.py` -- `scripts/agent_benchmark/scoring_test.py` -- `scripts/agent_benchmark/web_validation.py` -- `scripts/agent_benchmark/agy_iop.py` -- `scripts/agent_benchmark/agy_iop_test.py` -- `scripts/agent_benchmark/claude_iop.py` -- `scripts/agent_benchmark/claude_iop_test.py` -- `scripts/fixtures/agent-comparison-benchmark-iop-one-shot.json` -- `scripts/fixtures/agent-comparison-benchmark-direct-preflight.example.json` -- `apps/edge/internal/openai/gemini_handler.go` -- `apps/edge/internal/openai/gemini_bridge.go` -- `apps/edge/internal/openai/gemini_handler_test.go` -- `apps/edge/internal/openai/anthropic_stream.go` -- `apps/edge/internal/openai/single_request_anthropic_stream.go` -- `apps/edge/internal/openai/single_request_anthropic_stream_test.go` - -### SDD Criteria - -- SDD: `agent-roadmap/sdd/knowledge-tool-optimization-extension/iop-one-shot-agent-model-comparison/SDD.md`, `[승인됨]`, lock released. -- First-line milestone task ids: `claude-standalone,gemini-standalone,gpt-standalone,gemini-hybrid,gpt-hybrid,objective-validation,quality-scoring`. -- Target scenarios: S04-S10. Evidence Map rows require per-caller lifecycle/workspace evidence, uniform hard gates/screenshots, and anonymous scoring eligibility. -- D14 makes 9/9 successful and scored the only final completion condition. D15 makes `generated_files`, `static_safety`, `images`, and `network` hard eligibility gates while `console`, `responsive`, and `accessibility` remain quality evidence. Those rows define REPAIR-1 and the six-path admission gate in REPAIR-4. - -### Verification Context - -- No external handoff was supplied. Repository evidence and the preserved run were read directly. -- Current local feature contains `origin/dev@6e2f4ff8`; live preflight is `ready=9`. The remote dev runner `/Users/toki/agent-work/iop-dev` was clean at `019500a5`, twelve commits behind `origin/dev`, with an approximately seven-hour-old Edge/Node runtime listening on 18082/18083/18084/19101. -- External runner: `toki@toki-labs.com`, repo `/Users/toki/agent-work/iop-dev`, macOS/arm64. Before mutation, re-run the preflight in the project `dev-runtime-deploy` skill and require clean refs, exact fetched feature SHA, toolchain, runtime identity, and ports. -- Use `agent-ops/skills/private/dev-runtime-deploy/SKILL.md` (or the project fallback when private is absent) for the complete feature-to-dev merge, release, tests, rebuild, safe restart, capacity smoke, finish, and atomic push. Do not shortcut its gates. -- Constraint: qualification is append-only and non-scored; exactly one fresh run is allowed. Final C01-C09 execution belongs to the dependent task. -- Gap: latest-dev live behavior has not yet been observed. Qualification failure is valid evidence, not authority for an implementation agent to choose an unplanned runtime fix. Official review must diagnose it and create a concrete follow-up plan. -- Confidence: high for the scoring false positive and stale-runtime mismatch; medium for whether the latest baseline alone closes all six caller paths. - -### Test Coverage Gaps - -- Existing scoring tests cover lifecycle and web failure eligibility, but not the D15 hard-versus-quality split; add regression cases. -- Existing manifest tests pin `run_seconds=180`; update them to the approved 300 seconds and validate the diagnostic manifest identity. -- Existing adapter/Edge unit tests cannot prove the deployed official Claude/agy process combination. The fresh six-path qualification supplies that external coverage. - -### Symbol References - -- No public symbol rename or removal. -- Add private scoring gate constants only within `scripts/agent_benchmark/scoring.py`; `_eligibility` is the sole call site whose classification changes. - -### Split Judgment - -- This is split subtask `16+15_all_cell_measurement_recovery`. -- Predecessor `15+14_scored_benchmark_and_report` is satisfied by `agent-task/archive/2026/08/m-iop-one-shot-agent-model-comparison/15+14_scored_benchmark_and_report/complete.log`. -- Stable output contract: a deployed exact dev release plus one six-path qualification where every cell has successful product/harness/process and all four hard gates. Dependent task `17+16_full_nine_cell_scored_benchmark` may start only after this task has archived PASS evidence. - -### Scope Rationale - -- Included: scoring eligibility, manifest timeout/tests, one diagnostic manifest/report, central dev deployment, and six-path live qualification. -- Excluded: preserved run mutation, final C01-C09 allocation/scoring/reporting, fixture prompt/assets/rubric changes, caller/model substitution, credential changes, and speculative Edge/adapter edits. A remaining live defect must be owned by a reviewer-selected follow-up packet. - -### Final Routing - -- `evaluation_mode=first-pass`; `finalizer=finalize-task-policy.sh`, mode `pair`. -- Build and review closures are all true: scope/context/verification/evidence/ownership/decision are closed by D14-D15, the exact dev runner, and a one-run qualification oracle. -- Build scores: scope 2, state 2, blast 2, evidence 2, verification 2 => G10; base/route basis `grade-boundary`, cloud, `worker/cloud/G10`. -- Review scores: 2/2/2/2/2 => G10; `official-review`, cloud, `review/cloud/G10`. -- `large_indivisible_context=false`; matched risks: `temporal_state`, `boundary_contract`, `structured_interpretation`, `variant_product`; count 4. `review_rework_count=0`, `evidence_integrity_failure=false`. -- Canonical files: `PLAN-cloud-G10.md`, `CODE_REVIEW-cloud-G10.md`. - -## Implementation Checklist - -- [ ] [REPAIR-1] Separate hard artifact eligibility gates from quality-scoring gates and add fail-closed regression coverage. -- [ ] [REPAIR-2] Set the compact benchmark and diagnostic per-cell ceiling to 300 seconds and lock both manifests in tests. -- [ ] [REPAIR-3] Commit/push the exact feature, merge it into central dev, and complete the private dev-runtime deployment procedure with a safe restart and source/build identity proof. -- [ ] [REPAIR-4] Run exactly one fresh non-scored six-path qualification and require all six caller/route paths to produce successful terminal evidence, generated files, screenshots, and all hard gates. -- [ ] [REPAIR-5] Publish a sanitized deterministic recovery report without secrets or claims about the not-yet-run final benchmark. -- [ ] Fill implementation-owned sections in CODE_REVIEW-*-G??.md with actual implementation notes and verification output. - -### [REPAIR-1] Split eligibility from quality evidence - -**Problem:** `scripts/agent_benchmark/scoring.py:1056-1067` appends the aggregate `web_failed` reason and every failed web gate, so a trustworthy page with only accessibility/responsive/console findings is unscored even though those dimensions belong to the rubric. - -**Solution:** Replace aggregate web-status rejection with explicit hard-gate evaluation. Keep record-shape validation for all ordered gates; reject missing/invalid evidence and failures of `generated_files`, `static_safety`, `images`, or `network`; allow failures of `console`, `responsive`, and `accessibility` to reach the unchanged anonymous evaluator input. - -Before (`scoring.py:1056`): - -```python -if web.status != "passed": - reasons.append(f"web_{web.status}") -reasons.extend(f"gate_{item['id']}" for item in gates if not item["passed"]) -``` - -After: - -```python -hard_failures = [item for item in gates if item["id"] in HARD_ELIGIBILITY_GATES and not item["passed"]] -reasons.extend(f"gate_{item['id']}" for item in hard_failures) -``` - -#### Modified Files and Checklist - -- [ ] `scripts/agent_benchmark/scoring.py`: define the closed hard/quality partition and use only hard failures for eligibility. -- [ ] `scripts/agent_benchmark/scoring_test.py`: prove each hard failure remains unscored, each quality-only failure remains score-eligible, mixed failures reject, and malformed/missing web evidence still fails closed. - -#### Test Strategy - -Add table-driven regression coverage in `scripts/agent_benchmark/scoring_test.py` using its existing web-validation fixture helpers. No production run is used. - -#### Verification - -```bash -python3 -m unittest scripts.agent_benchmark.scoring_test -``` - -Expected: all tests pass; quality-only failures invoke the evaluator, hard failures do not. - -### [REPAIR-2] Lock the five-minute compact ceiling and diagnostic matrix - -**Problem:** `scripts/fixtures/agent-comparison-benchmark-iop-one-shot.json:10` and `scripts/agent_benchmark/manifest_test.py:207-280` still pin 180 seconds after the user approved a 300-second ceiling. There is no isolated manifest for testing only the six previously failing caller/route paths without consuming the final run. - -**Solution:** Change only `timeout.run_seconds` to 300 in the immutable full manifest. Add a recovery qualification manifest with the same fixture checksum, viewport/rubric/session/cache policies and only C01, C03, C04, C06, C07, C08 equivalents, a distinct seed, and `output_root=agent-test/runs/bench-02-recovery`. Lock its exact matrix and non-scored purpose in manifest tests. - -#### Modified Files and Checklist - -- [ ] `scripts/fixtures/agent-comparison-benchmark-iop-one-shot.json`: set `run_seconds=300`; change nothing else. -- [ ] `scripts/fixtures/agent-comparison-benchmark-recovery-qualification.json`: add the closed six-path diagnostic manifest. -- [ ] `scripts/agent_benchmark/manifest_test.py`: update the full-manifest timeout oracle and add exact diagnostic identity/matrix assertions. - -#### Test Strategy - -Use parser round-trip tests to prove only timeout differs in the final manifest and that qualification cannot be confused with the final output root or nine-cell seed. - -#### Verification - -```bash -python3 -m unittest scripts.agent_benchmark.manifest_test -python3 scripts/agent_comparison_benchmark.py validate --manifest scripts/fixtures/agent-comparison-benchmark-iop-one-shot.json -python3 scripts/agent_comparison_benchmark.py validate --manifest scripts/fixtures/agent-comparison-benchmark-recovery-qualification.json -``` - -Expected: tests and both validations pass. - -### [REPAIR-3] Deploy the exact central dev baseline safely - -**Problem:** the live dev runtime predates `origin/dev@6e2f4ff8`, while this feature already includes it. Running qualification against that process would repeat stale stream-liveness behavior. - -**Solution:** Use `commit-push` to publish this task's exact feature SHA. Then follow `agent-ops/skills/private/dev-runtime-deploy/SKILL.md` completely: fetch the exact feature on the declared runner, merge it into `dev`, use the commit-count release, pass sequential pre/post tests, rebuild all four binaries, restart Edge then nodes, verify ports/snapshots/routes/capacity, finish only after every release gate, and atomically push main/dev/tag. Preserve secrets and ignored runtime configuration. - -#### Modified Files and Checklist - -- [ ] `agent-test/dev/iop-one-shot-agent-comparison-recovery-2026-08-13.md`: record only sanitized release/source/build/process/port and qualification evidence. -- [ ] Run the exact private deploy skill without editing central common rules, dispatcher code, or agent runtime code. - -#### Test Strategy - -The deployment skill's full pre/post sequential Go tests, binary provenance, config checks, listener/node/provider checks, and capacity smokes are mandatory. A failure keeps the release unfinished and blocks REPAIR-4. - -#### Verification - -```bash -ssh toki@toki-labs.com 'cd /Users/toki/agent-work/iop-dev && git status --short --branch && git rev-parse HEAD && git rev-parse origin/dev' -``` - -Expected after successful finish: clean `dev`, local HEAD equals `origin/dev`, and the recorded deployed binary source equals the finished release tag tree. - -### [REPAIR-4] Qualify all previously failing caller/route families once - -**Problem:** unit/preflight success does not prove official Claude/agy execution against the rebuilt runtime. Reusing the final manifest would spend the only authorized final cycle before recovery is proven. - -**Solution:** With `/tmp/iop-bench-13-env` and the declared dev environment, append one all-cell preflight for the six-path diagnostic manifest and execute it exactly once. Do not score, resume, or retry it. Require `ready=6`, exactly six attempts, `product_succeeded=6`, `harness_passed=6`, `process_exited=6`, exit code zero for each, two screenshots per cell, and all four hard gates true. Quality gates may be false and must be recorded as quality evidence. - -#### Modified Files and Checklist - -- [ ] `agent-test/dev/iop-one-shot-agent-comparison-recovery-2026-08-13.md`: record run id, terminal counts, hard/quality gate matrix, and contained raw-run link. - -#### Test Strategy - -One live qualification run only. Any failure is preserved and handed to official review; no manual workspace edits, provider substitution, resume, retry, or final benchmark start. - -#### Verification - -```bash -/bin/bash /tmp/iop-bench-13-env python3 scripts/agent_comparison_benchmark.py preflight --manifest scripts/fixtures/agent-comparison-benchmark-recovery-qualification.json -/bin/bash /tmp/iop-bench-13-env python3 scripts/agent_comparison_benchmark.py run --manifest scripts/fixtures/agent-comparison-benchmark-recovery-qualification.json -``` - -Expected: one new qualification run; `ready=6`; all six terminals and hard artifacts satisfy the criteria above. - -### [REPAIR-5] Publish bounded recovery evidence - -**Problem:** without a deterministic summary, a later task could mistake a diagnostic run or a warning-only log for final benchmark completion. - -**Solution:** Write `agent-test/dev/iop-one-shot-agent-comparison-recovery-2026-08-13.md` with exact source/release/run identities, six rows, hard/quality gates, failures if any, and the explicit statement that it is non-scored and cannot satisfy D14. - -#### Modified Files and Checklist - -- [ ] `agent-test/dev/iop-one-shot-agent-comparison-recovery-2026-08-13.md`: publish sanitized evidence and no secrets. - -#### Test Strategy - -Reviewer reconstructs all six rows from the exact run and verifies every relative link remains inside its run root. - -#### Verification - -```bash -git diff --check -- . ':(exclude)agent-task/archive/**' -git status --short --branch -``` - -Expected: no whitespace errors; only declared task changes before commit/final review. - -## Dependencies and Execution Order - -1. Archived predecessor `15+14_scored_benchmark_and_report` is satisfied by the exact `complete.log` cited above. -2. REPAIR-1 and REPAIR-2 run before any central deployment. -3. REPAIR-3 must fully finish before REPAIR-4. -4. REPAIR-4 is exactly once. REPAIR-5 records it. A failed qualification blocks task completion and the dependent final benchmark. - -## Modified Files Summary - -| File | Items | -|---|---| -| `scripts/agent_benchmark/scoring.py` | REPAIR-1 | -| `scripts/agent_benchmark/scoring_test.py` | REPAIR-1 | -| `scripts/agent_benchmark/manifest_test.py` | REPAIR-2 | -| `scripts/fixtures/agent-comparison-benchmark-iop-one-shot.json` | REPAIR-2 | -| `scripts/fixtures/agent-comparison-benchmark-recovery-qualification.json` | REPAIR-2, REPAIR-4 | -| `agent-test/dev/iop-one-shot-agent-comparison-recovery-2026-08-13.md` | REPAIR-3..5 | -| `agent-task/m-iop-one-shot-agent-model-comparison/16+15_all_cell_measurement_recovery/CODE_REVIEW-cloud-G10.md` | REPAIR-1..5 | - -## Final Verification - -```bash -python3 -m unittest scripts.agent_benchmark.scoring_test scripts.agent_benchmark.manifest_test -python3 -m unittest discover -s scripts/agent_benchmark -p '*_test.py' -python3 scripts/agent_comparison_benchmark.py validate --manifest scripts/fixtures/agent-comparison-benchmark-iop-one-shot.json -python3 scripts/agent_comparison_benchmark.py validate --manifest scripts/fixtures/agent-comparison-benchmark-recovery-qualification.json -/bin/bash /tmp/iop-bench-13-env python3 scripts/agent_comparison_benchmark.py status --manifest scripts/fixtures/agent-comparison-benchmark-recovery-qualification.json --run-id -git diff --check -- . ':(exclude)agent-task/archive/**' -git status --short --branch -``` - -Expected: deterministic suite passes; final manifest remains nine cells with a 300-second ceiling; diagnostic manifest remains six cells; the one exact qualification has six successful product/harness/process outcomes, two screenshots and four hard gates per cell; no final nine-cell run was allocated. - -After completing all code changes, fill implementation-owned sections in `CODE_REVIEW-*-G??.md`. diff --git a/agent-test/dev/iop-benchmark-route-minimal-html-smoke.md b/agent-test/dev/iop-benchmark-route-minimal-html-smoke.md new file mode 100644 index 00000000..3983b747 --- /dev/null +++ b/agent-test/dev/iop-benchmark-route-minimal-html-smoke.md @@ -0,0 +1,37 @@ +# IOP 벤치 경로 최소 HTML 스모크 + +## 목적 + +벤치 대상 9개 caller/model/route 조합에 같은 최소 `index.html` 생성 요청을 한 번씩 직접 보내 호출 경로만 빠르게 확인한다. 전용 runner, manifest, 자동 retry, browser gate와 품질 채점은 사용하지 않는다. + +## 고정 요청 + +빈 임시 workspace에 외부 asset과 JavaScript 없이 단일 `index.html`을 만든다. 문서에는 ``, `IOP Route Smoke`, `

IOP_ROUTE_SMOKE_OK

`이 정확히 한 번씩 있어야 하며 파일 생성 뒤 종료한다. + +## 사전 확인 — 2026-08-13 + +| 항목 | 결과 | +|---|---| +| Claude Code | 2.1.228 확인 | +| agy | 1.1.12 확인 | +| Codex | 0.147.0 확인 | +| managed CA | 파일 존재 확인 | +| 기존 benchmark principal token | 파일 존재, mode `0600`, 65 bytes 확인 | +| public `/v1/models` | HTTP 401 | +| active public Edge token hash 매핑 | 0건 | +| direct Edge `:18086` | 현재 host에서 연결 불가 | +| model 호출 | 시작하지 않음 | + +## 현재 분류 + +- 소유 경계: 환경/credential +- 근거: caller를 시작하기 전 공통 인증 preflight에서 HTTP 401이 발생했고, 현재 public Edge 활성 config의 `principal_tokens`에 test token hash가 없다. +- IOP 제품 결함 판정: 아직 아님 +- caller/model 결함 판정: 아직 아님 +- benchmark script 결함 판정: 해당 없음. script를 사용하지 않았다. + +## 재개 조건 + +기존 direct·execution-preset route를 볼 수 있는 유효한 dev-corp principal token을 operator-private 경로에 준비하거나 기존 test token을 active Edge에 안전하게 매핑·재시작한다. token 원문을 출력하지 않은 `/v1/models` 확인이 HTTP 200이고 `claude-sonnet-5`, `gemini-3.6-flash`, `gpt-5.6-luna`, `gemini-hybrid`, `gpt-hybrid`가 존재하면 9개 최소 호출을 각 1회 시작한다. + +성공한 경로는 반복하지 않는다. 실패한 경로는 원인이 변경된 경우에만 해당 경로를 1회 재검증한다. diff --git a/agent-test/dev/iop-one-shot-agent-comparison-2026-08-13.md b/agent-test/dev/iop-one-shot-agent-comparison-2026-08-13.md deleted file mode 100644 index d9b642ce..00000000 --- a/agent-test/dev/iop-one-shot-agent-comparison-2026-08-13.md +++ /dev/null @@ -1,114 +0,0 @@ -# IOP 원샷 Agent 모델 비교 결과 — 2026-08-13 - -## 결론 - -`run-20260813T081326Z-4e1ac5152c6c`은 C01-C09 모두에 대해 한 번씩 실행한 terminal evidence를 보존했다. 최종 상태는 `unresolved=0`, `scored=2`, `unscored=7`, `scoring_failed=0`, `blocked=0`이다. - -자동 웹 gate를 모두 통과해 익명 품질 채점이 가능했던 C05 GPT 단독과 C09 GPT 하이브리드는 각각 93점으로 공동 1위다. 나머지 7개 결과는 0점이 아니라 `unscored`이며 순위를 부여하지 않는다. 따라서 이 1회 실행만으로 caller 또는 model 전체의 우열을 일반화할 수 없다. - -- 결정적 원본 보고서: [run report](../runs/bench-02/run-20260813T081326Z-4e1ac5152c6c/report.md) -- 이전 `run-20260813T071816Z-755d136c3e2b`은 fixture/scoring 진단 run으로만 보존하며 이 결과나 순위에 포함하지 않았다. -- 최종 run은 9개 cell마다 producer attempt가 정확히 1개다. producer caller를 retry하거나 새 run으로 교체하지 않았다. - -## 실행 조건 - -| 항목 | 값 | -|---|---| -| environment | `dev` | -| pipeline version | `2` | -| manifest digest | `sha256:38e48ef35beaa6ecc1aee0460df443e0ccf101a89aa6b91ce1f4b198aa84022a` | -| fixture | `product-card-v2` (`sha256:fb16198fd4c3576f880f047ed7de54dddc160b0f70c2ba55b435cf078c61828e`) | -| rubric | `one-shot-agent-comparison-v1` | -| session/cache | `fresh` / `isolated` | -| evaluator | `codex/gpt-5.6-luna/xhigh` | -| execution preflight | sequence 1, `ready=9` | -| repetition / timeout | cell별 1회 / cell별 180초 | - -## 9개 결과 - -표의 순서는 immutable seed가 정한 실제 실행 순서다. - -| 순서 | cell | 구성 | controller | product | harness | process | 자동 웹 gate | 채점 | 점수 | 순위 | -|---:|---|---|---|---|---|---|---|---|---:|---:| -| 1 | C02 | Claude Code → Gemini 단독 | completed | succeeded | passed | exited | accessibility 실패 | unscored | — | — | -| 2 | C05 | Codex → GPT 단독 | completed | succeeded | passed | exited | 통과 | scored | 93 | 1 | -| 3 | C03 | agy → Gemini 단독 | completed | failed | passed | exited | 산출물 없음 | unscored | — | — | -| 4 | C06 | Claude Code → Gemini 하이브리드 | completed | failed | passed | exited | 산출물 없음 | unscored | — | — | -| 5 | C08 | Claude Code → GPT 하이브리드 | completed | failed | passed | exited | 산출물 없음 | unscored | — | — | -| 6 | C09 | Codex → GPT 하이브리드 | completed | succeeded | passed | exited | 통과 | scored | 93 | 1 | -| 7 | C01 | Claude Code → Claude 단독 | completed | unknown | failed | exited | 산출물 없음 | unscored | — | — | -| 8 | C07 | agy → Gemini 하이브리드 | timed_out | unknown | failed | timed_out | 산출물 없음 | unscored | — | — | -| 9 | C04 | Claude Code → GPT 단독 | completed | failed | passed | exited | 산출물 없음 | unscored | — | — | - -집계는 controller `completed=8`, `timed_out=1`; product `succeeded=3`, `failed=4`, `unknown=2`; harness `passed=7`, `failed=2`; process `exited=8`, `timed_out=1`; artifact `passed=2`, `failed=7`이다. - -## 익명 품질 점수 - -| 항목 | 최대 | C05 GPT 단독 | C09 GPT 하이브리드 | -|---|---:|---:|---:| -| 요구사항 충족 | 25 | 24 | 23 | -| 시각 완성도 | 25 | 24 | 24 | -| 반응형·접근성 | 15 | 13 | 13 | -| 이미지 활용·디테일 | 10 | 10 | 10 | -| 동작 안정성 | 10 | 9 | 9 | -| 코드 품질 | 10 | 9 | 9 | -| 자체 검증 완결성 | 5 | 4 | 5 | -| 합계 | 100 | 93 | 93 | - -자동 웹 gate는 채점 eligibility만 결정하며 점수에 합산되지 않았다. C05는 `score-000002`, C09는 provenance 경계 수정 후 새로 할당한 `score-000003`의 결과다. - -## 시간 비교 - -모든 시간은 harness monotonic clock 기준이며, first write는 workspace poll로 관측했다. `first output`과 `first write`는 제출 시점부터의 경과 시간이다. - -| cell | 전체 시간 | first output | first write | 관측 결과 | -|---|---:|---:|---:|---| -| C01 | 65.048초 | 1.285초 | 미관측 | parser error 뒤 product unknown | -| C02 | 127.718초 | 1.363초 | 95.441초 | product 성공, accessibility gate 실패 | -| C03 | 17.661초 | 0.662초 | 미관측 | caller error | -| C04 | 12.139초 | 1.069초 | 미관측 | caller error | -| C05 | 150.131초 | 0.791초 | 미제공 (`observer_unavailable`) | 통과·채점 | -| C06 | 24.394초 | 1.368초 | 미관측 | caller error | -| C07 | 180.284초 | 0.699초 | 미관측 | 180초 cell timeout | -| C08 | 16.606초 | 1.477초 | 미관측 | caller error | -| C09 | 179.132초 | 0.785초 | 77.213초 | 통과·채점 | - -180초는 전체 9개 matrix의 제한이 아니라 각 cell의 실행 제한이다. cell들은 순차 실행되므로 전체 matrix가 3분 안에 끝났다고 해석할 수 없다. 위 시간은 cell별 attempt 관측치이며 preflight, cell 사이 overhead, scoring과 report 시간을 포함한 run 전체 wall clock으로 합산하지 않는다. - -## 호출과 token 비교 - -caller가 제공한 값만 기록한다. 미제공 값을 0으로 바꾸거나 서로 다른 tokenizer의 값을 합산하지 않았다. - -| cell | model calls | tool calls | input | cached input | cache write | output | reasoning | total | -|---|---:|---:|---:|---:|---:|---:|---:|---| -| C02 | 12 | 미제공 | 182,484 | 97,822 | 0 | 3,701 | 미제공 | 미제공 | -| C05 | 1 | 9 | 248,818 | 217,034 | 31,751 | 20,721 | 5,156 | 미제공 | -| C09 | 1 | 7 | 247,278 | 218,156 | 28,649 | 12,134 | 4,317 | 미제공 | -| C01, C03, C04, C06, C07, C08 | 미제공 | 미제공 | 미제공 | 미제공 | 미제공 | 미제공 | 미제공 | 미제공 | - -C02의 caller-reported model duration은 116.081초, caller-reported total duration은 116.760초다. 이 구간은 harness 전체 시간과 clock/source가 다르고 중첩되므로 임의로 더하거나 overhead로 분해하지 않았다. 나머지 cell의 model/tool/queue duration은 미제공이다. - -## 실패와 한계 - -- C02는 product가 성공하고 두 viewport screenshot도 생성했지만 accessibility gate가 실패해 unscored다. -- C03, C04, C06, C08은 caller error와 non-zero exit 뒤 필수 산출물이 없어 unscored다. -- C01은 parser error로 product가 unknown이고 harness가 실패했으며, C07은 180초 timeout으로 product가 unknown이다. -- C05와 C09만 모든 자동 gate를 통과했다. 두 결과의 동점은 이 두 eligible artifact 사이의 1회 평가 결과일 뿐, 실패한 7개 조합의 품질이 0점이라는 의미가 아니다. -- provider/caller가 제공하지 않은 timing·usage는 `미제공`으로 유지했다. stage별, queue, tool duration과 total token은 비교할 수 없다. -- scored 결과는 각 1회뿐이므로 분산, 재현성, 비용 일반화나 통계적 유의성을 제공하지 않는다. - -## Raw evidence - -모든 링크는 preserved run 내부만 가리킨다. - -| cell | 실행 | 측정 | 웹 검증 | 채점 | screenshot | -|---|---|---|---|---|---| -| C01 | [attempt](../runs/bench-02/run-20260813T081326Z-4e1ac5152c6c/cells/c01-claude-sonnet-direct/repetition-0001/attempt-000001/attempt.json) | [measurement](../runs/bench-02/run-20260813T081326Z-4e1ac5152c6c/cells/c01-claude-sonnet-direct/repetition-0001/attempt-000001/attempt-measurement.json) | [web](../runs/bench-02/run-20260813T081326Z-4e1ac5152c6c/cells/c01-claude-sonnet-direct/repetition-0001/attempt-000001/web-validation.json) | [unscored](../runs/bench-02/run-20260813T081326Z-4e1ac5152c6c/cells/c01-claude-sonnet-direct/repetition-0001/attempt-000001/scoring/unscored.json) | — | -| C02 | [attempt](../runs/bench-02/run-20260813T081326Z-4e1ac5152c6c/cells/c02-claude-gemini-direct/repetition-0001/attempt-000001/attempt.json) | [measurement](../runs/bench-02/run-20260813T081326Z-4e1ac5152c6c/cells/c02-claude-gemini-direct/repetition-0001/attempt-000001/attempt-measurement.json) | [web](../runs/bench-02/run-20260813T081326Z-4e1ac5152c6c/cells/c02-claude-gemini-direct/repetition-0001/attempt-000001/web-validation.json) | [unscored](../runs/bench-02/run-20260813T081326Z-4e1ac5152c6c/cells/c02-claude-gemini-direct/repetition-0001/attempt-000001/scoring/unscored.json) | [desktop](../runs/bench-02/run-20260813T081326Z-4e1ac5152c6c/cells/c02-claude-gemini-direct/repetition-0001/attempt-000001/screenshot-desktop_1080.png), [mobile](../runs/bench-02/run-20260813T081326Z-4e1ac5152c6c/cells/c02-claude-gemini-direct/repetition-0001/attempt-000001/screenshot-mobile_375.png) | -| C03 | [attempt](../runs/bench-02/run-20260813T081326Z-4e1ac5152c6c/cells/c03-agy-gemini-direct/repetition-0001/attempt-000001/attempt.json) | [measurement](../runs/bench-02/run-20260813T081326Z-4e1ac5152c6c/cells/c03-agy-gemini-direct/repetition-0001/attempt-000001/attempt-measurement.json) | [web](../runs/bench-02/run-20260813T081326Z-4e1ac5152c6c/cells/c03-agy-gemini-direct/repetition-0001/attempt-000001/web-validation.json) | [unscored](../runs/bench-02/run-20260813T081326Z-4e1ac5152c6c/cells/c03-agy-gemini-direct/repetition-0001/attempt-000001/scoring/unscored.json) | — | -| C04 | [attempt](../runs/bench-02/run-20260813T081326Z-4e1ac5152c6c/cells/c04-claude-gpt-direct/repetition-0001/attempt-000001/attempt.json) | [measurement](../runs/bench-02/run-20260813T081326Z-4e1ac5152c6c/cells/c04-claude-gpt-direct/repetition-0001/attempt-000001/attempt-measurement.json) | [web](../runs/bench-02/run-20260813T081326Z-4e1ac5152c6c/cells/c04-claude-gpt-direct/repetition-0001/attempt-000001/web-validation.json) | [unscored](../runs/bench-02/run-20260813T081326Z-4e1ac5152c6c/cells/c04-claude-gpt-direct/repetition-0001/attempt-000001/scoring/unscored.json) | — | -| C05 | [attempt](../runs/bench-02/run-20260813T081326Z-4e1ac5152c6c/cells/c05-codex-gpt-direct/repetition-0001/attempt-000001/attempt.json) | [measurement](../runs/bench-02/run-20260813T081326Z-4e1ac5152c6c/cells/c05-codex-gpt-direct/repetition-0001/attempt-000001/attempt-measurement.json) | [web](../runs/bench-02/run-20260813T081326Z-4e1ac5152c6c/cells/c05-codex-gpt-direct/repetition-0001/attempt-000001/web-validation.json) | [score-000002](../runs/bench-02/run-20260813T081326Z-4e1ac5152c6c/cells/c05-codex-gpt-direct/repetition-0001/attempt-000001/scoring/score-000002/result.json) | [desktop](../runs/bench-02/run-20260813T081326Z-4e1ac5152c6c/cells/c05-codex-gpt-direct/repetition-0001/attempt-000001/screenshot-desktop_1080.png), [mobile](../runs/bench-02/run-20260813T081326Z-4e1ac5152c6c/cells/c05-codex-gpt-direct/repetition-0001/attempt-000001/screenshot-mobile_375.png) | -| C06 | [attempt](../runs/bench-02/run-20260813T081326Z-4e1ac5152c6c/cells/c06-claude-gemini-hybrid/repetition-0001/attempt-000001/attempt.json) | [measurement](../runs/bench-02/run-20260813T081326Z-4e1ac5152c6c/cells/c06-claude-gemini-hybrid/repetition-0001/attempt-000001/attempt-measurement.json) | [web](../runs/bench-02/run-20260813T081326Z-4e1ac5152c6c/cells/c06-claude-gemini-hybrid/repetition-0001/attempt-000001/web-validation.json) | [unscored](../runs/bench-02/run-20260813T081326Z-4e1ac5152c6c/cells/c06-claude-gemini-hybrid/repetition-0001/attempt-000001/scoring/unscored.json) | — | -| C07 | [attempt](../runs/bench-02/run-20260813T081326Z-4e1ac5152c6c/cells/c07-agy-gemini-hybrid/repetition-0001/attempt-000001/attempt.json) | [measurement](../runs/bench-02/run-20260813T081326Z-4e1ac5152c6c/cells/c07-agy-gemini-hybrid/repetition-0001/attempt-000001/attempt-measurement.json) | [web](../runs/bench-02/run-20260813T081326Z-4e1ac5152c6c/cells/c07-agy-gemini-hybrid/repetition-0001/attempt-000001/web-validation.json) | [unscored](../runs/bench-02/run-20260813T081326Z-4e1ac5152c6c/cells/c07-agy-gemini-hybrid/repetition-0001/attempt-000001/scoring/unscored.json) | — | -| C08 | [attempt](../runs/bench-02/run-20260813T081326Z-4e1ac5152c6c/cells/c08-claude-gpt-hybrid/repetition-0001/attempt-000001/attempt.json) | [measurement](../runs/bench-02/run-20260813T081326Z-4e1ac5152c6c/cells/c08-claude-gpt-hybrid/repetition-0001/attempt-000001/attempt-measurement.json) | [web](../runs/bench-02/run-20260813T081326Z-4e1ac5152c6c/cells/c08-claude-gpt-hybrid/repetition-0001/attempt-000001/web-validation.json) | [unscored](../runs/bench-02/run-20260813T081326Z-4e1ac5152c6c/cells/c08-claude-gpt-hybrid/repetition-0001/attempt-000001/scoring/unscored.json) | — | -| C09 | [attempt](../runs/bench-02/run-20260813T081326Z-4e1ac5152c6c/cells/c09-codex-gpt-hybrid/repetition-0001/attempt-000001/attempt.json) | [measurement](../runs/bench-02/run-20260813T081326Z-4e1ac5152c6c/cells/c09-codex-gpt-hybrid/repetition-0001/attempt-000001/attempt-measurement.json) | [web](../runs/bench-02/run-20260813T081326Z-4e1ac5152c6c/cells/c09-codex-gpt-hybrid/repetition-0001/attempt-000001/web-validation.json) | [score-000003](../runs/bench-02/run-20260813T081326Z-4e1ac5152c6c/cells/c09-codex-gpt-hybrid/repetition-0001/attempt-000001/scoring/score-000003/result.json) | [desktop](../runs/bench-02/run-20260813T081326Z-4e1ac5152c6c/cells/c09-codex-gpt-hybrid/repetition-0001/attempt-000001/screenshot-desktop_1080.png), [mobile](../runs/bench-02/run-20260813T081326Z-4e1ac5152c6c/cells/c09-codex-gpt-hybrid/repetition-0001/attempt-000001/screenshot-mobile_375.png) | diff --git a/agent-test/inventory-dev.yaml b/agent-test/inventory-dev.yaml index 1e271304..6acdafb3 100644 --- a/agent-test/inventory-dev.yaml +++ b/agent-test/inventory-dev.yaml @@ -70,7 +70,7 @@ model: aliases: "claude-sonnet-5": observed_at: "2026-08-10" - status: active_edge_model_group_benchmark_target_observed + status: active_edge_model_group_observed display_name: Claude Sonnet 5 capacity_total: 1 providers: @@ -90,7 +90,7 @@ model: provider_snapshot: healthy_idle "gpt-5.6-luna": observed_at: "2026-08-10" - status: active_edge_model_group_benchmark_target_observed + status: active_edge_model_group_observed display_name: GPT-5.6 Luna capacity_total: 1 providers: diff --git a/docs/agent-comparison-benchmark-dev-guide.md b/docs/agent-comparison-benchmark-dev-guide.md deleted file mode 100644 index 73e46098..00000000 --- a/docs/agent-comparison-benchmark-dev-guide.md +++ /dev/null @@ -1,583 +0,0 @@ -# Agent Comparison Benchmark Dev Guide - -이 문서는 IOP one-shot agent/model comparison benchmark의 현재 구현, dev 환경 구성, caller 연결 방식, managed credential 경계, 실행 절차와 장애 대응 기준을 한곳에 모은 운영 가이드다. - -raw token, provider API key, private key, slot alias, lease id와 개인 endpoint는 이 문서에 기록하지 않는다. 실제 host, checkout, Node/provider endpoint와 최신 process 상태는 아래 source of truth에서 확인한다. - -- benchmark manifest: `scripts/fixtures/agent-comparison-benchmark-iop-one-shot.json` -- public CLI: `scripts/agent_comparison_benchmark.py` -- dev environment: `agent-test/dev/rules.md` -- machine-readable dev inventory: `agent-test/inventory-dev.yaml` -- Edge/Node verification: `agent-test/dev/edge-smoke.md`, `agent-test/dev/node-smoke.md` -- API contracts: `agent-contract/outer/anthropic-compatible-api.md`, `agent-contract/outer/openai-compatible-api.md`, `agent-contract/outer/gemini-compatible-api.md` -- benchmark SDD: `agent-roadmap/sdd/knowledge-tool-optimization-extension/iop-one-shot-agent-model-comparison/SDD.md` - -이 문서는 위 계약과 manifest를 설명할 뿐 덮어쓰지 않는다. 값이 다르면 manifest, contract, inventory, environment rule 순서로 최신 상태를 확인한다. - -## 1. 현재 상태 - -2026-08-12 기준 상태는 다음과 같다. - -| 영역 | 상태 | -|---|---| -| Gemini-native Edge ingress | 구현 완료. route-qualified `streamGenerateContent`, `x-goog-api-key` IOP principal auth, request/tool/SSE 변환을 지원한다. | -| official agy adapter | 구현 완료. `agy 1.1.12`, Gemini API-key provider, real `init/step_update/result` JSONL을 사용한다. | -| managed dev credential runtime | 구성 완료. Control Plane projection, credential slot/route, sealed lease, Edge HTTPS, CP↔Edge/Edge↔Node mTLS를 사용한다. | -| caller readiness | 최근 완료된 readiness evidence에서 C01-C09 `ready=9`를 확인했다. 실제 실행일에는 fresh preflight가 다시 필요하다. | -| controller recovery | control socket symlink와 caller launch 전 interruption 회귀 수정 및 테스트가 완료됐다. | -| Codex production JSONL | `cache_write_input_tokens` 수용과 config-owned effective binding 정합화가 완료됐다. | -| deterministic tests | focused 41 tests, 전체 benchmark 429 tests가 통과했다. 숫자는 당시 snapshot이며 현재 suite 결과는 fresh 실행으로 판단한다. | -| C01-C09 scored execution | 아직 완료되지 않았다. 과거 incomplete/failed run은 append-only evidence로 보존하며 정상 결과로 간주하지 않는다. | -| blind scoring/report | 유효한 9-cell execution이 생긴 뒤 수행하는 후속 단계다. | - -과거 실행 실패는 새 run을 정당화하는 완료 evidence가 아니다. 이전 run tree를 수정하거나 old run을 암묵적으로 `resume`/retry하지 않는다. 새 scored execution은 그 실행을 명시적으로 소유하는 현재 plan과 사용자 권한이 있을 때만 한 번 수행한다. 이 문서 자체는 지속적 실행 승인을 부여하지 않는다. - -## 2. 시스템 구성 - -```text -Benchmark runner - ├─ Claude Code ── Anthropic Messages ───────────────┐ - ├─ agy ───────── Gemini streamGenerateContent ─────┤ - └─ Codex ─────── OpenAI Responses ─────────────────┤ - v - IOP Edge HTTPS - │ - ┌───────────────────────────┴───────────────────────────┐ - │ │ - direct model route execution preset - │ selector/plan/work/review/repair - └───────────────────────────┬───────────────────────────┘ - v - managed credential lease - Control Plane projection + Node sealed lease - │ - v - Node-owned provider -``` - -핵심 보안 경계는 다음과 같다. - -- benchmark caller는 하나의 IOP principal token으로 Edge만 인증한다. -- caller token은 upstream provider credential이 아니다. -- provider credential은 Control Plane에 등록된 slot과 Node 대상 sealed lease에서만 온다. -- Edge는 provider credential을 복호화하지 않고 caller가 보낸 token을 upstream auth로 재사용하지 않는다. -- managed mode는 legacy static principal/provider credential과 혼용하지 않는다. -- config observation은 secret이 아닌 route/model/stage binding의 독립 증거다. - -## 3. 환경 프로필 - -### 3.1 Benchmark runner - -현재 검증된 runner class는 Linux/AArch64다. 명령은 repository root에서 실행한다. - -필수 command: - -```bash -command -v python3 -command -v git -command -v claude -command -v agy -command -v codex -``` - -2026-08-12 확인 snapshot: - -| Tool | 확인된 버전 | 정책 | -|---|---|---| -| Claude Code | `2.1.228` | 고정 버전으로 추정하지 않고 매 execution preflight에서 `--version`/`--help`를 확인한다. | -| agy | `1.1.12` | adapter가 이 버전을 명시적으로 gate한다. 다른 버전은 재검증 전 fail closed한다. | -| Codex CLI | `0.147.0` | 고정 버전으로 추정하지 않고 매 execution preflight에서 `--version`과 `exec --help`를 확인한다. | - -### 3.2 Testbed - -- path: `../iop-s2` -- 현재 확인 branch: `dev` -- 현재 확인 상태: clean -- benchmark는 testbed를 read-only provenance로 취급한다. -- caller별 workspace와 session은 run tree 아래에 새로 만들며 서로 공유하지 않는다. -- testbed를 benchmark 결과로 수정하거나 결과 파일을 다시 복사하지 않는다. - -fixture checksum과 source file 목록은 manifest가 고정한다. testbed HEAD와 clean 상태는 실행일에 다시 확인한다. - -### 3.3 Dev runtime - -현재 검증된 runtime class는 macOS/ARM64 remote dev runner다. exact SSH target과 checkout은 `agent-test/dev/rules.md` 및 `agent-test/inventory-dev.yaml`을 따른다. - -benchmark 관련 runtime 역할: - -| Port | 역할 | -|---:|---| -| `18082` | Edge artifact/bootstrap HTTP | -| `18083` | managed Edge public HTTPS; Anthropic/OpenAI/Gemini caller ingress | -| `18084` | native dev-runtime Edge↔Node TCP | -| `19093` | Edge admin/config refresh | -| `19101` | Edge metrics | - -2026-08-12 read-only 확인에서 위 listener와 managed Edge process는 모두 active였고 remote checkout은 clean release 상태였다. exact commit, binary checksum, process id와 endpoint는 실행 evidence에만 기록하고 이 가이드에 고정하지 않는다. - -최근 완료된 live readiness evidence는 4 connected Nodes와 8 healthy/available providers를 확인했다. 최신 Node/provider 세부와 접속 위치는 반드시 `agent-test/inventory-dev.yaml`에서 다시 확인한다. - -이 benchmark는 compose dev stack의 Edge-Node TCP `19003`이 아니라 native dev-runtime provider pool의 `18084`를 사용한다. compose와 native profile은 포트, process와 판정 evidence가 서로 다르므로 한 실행에서 섞지 않는다. 현재 inventory가 가리키는 배포 산출물은 다음과 같다. - -| Artifact | Path | -|---|---| -| native Edge config | `build/dev-runtime/edge.yaml` | -| Edge binary | `build/dev-runtime/bin/edge` | -| macOS Node binary | `build/dev-runtime/bin/iop-node` | -| Linux ARM64 Node binary | `build/dev-runtime/bin/iop-node-linux-arm64` | -| Windows AMD64 Node binary | `build/dev-runtime/bin/iop-node-windows-amd64.exe` | - -모든 Edge/Node binary는 scored execution 전에 동일 source ref로 rebuild·redeploy·restart한다. `build/dev-runtime/**`의 runtime config와 untracked credential material은 원격 runner가 소유하며 tracked 문서나 testbed로 복사하지 않는다. - -## 4. 보호 파일과 credential 역할 - -benchmark runner의 `token/` 아래에는 다음 파일이 준비돼 있다. 파일 존재와 mode만 확인하며 내용을 출력하지 않는다. - -| Path | 역할 | 실행 시 사용 | -|---|---|---| -| `token/.iop-bench` | benchmark용 IOP principal token | preflight/run/score caller가 Edge를 인증할 때 사용 | -| `token/iop-dev-ca.pem` | managed dev Edge HTTPS CA certificate | `SSL_CERT_FILE`, `NODE_EXTRA_CA_CERTS`로 전달 | -| `token/.claude` | Claude provider credential의 초기 provisioning source | provider slot 등록 시에만 사용; benchmark caller에 전달하지 않음 | -| `token/.gemini` | Gemini provider credential의 초기 provisioning source | provider slot 등록 시에만 사용; benchmark caller에 전달하지 않음 | -| `token/.gpt` | GPT provider credential의 초기 provisioning source | provider slot 등록 시에만 사용; benchmark caller에 전달하지 않음 | - -현재 secret source 파일은 `0600`, CA certificate는 `0644`로 확인됐다. CA certificate는 public trust material이지만 private key는 아니다. - -안전 확인: - -```bash -for benchmark_secret_file in token/.iop-bench token/.claude token/.gemini token/.gpt; do - test -f "$benchmark_secret_file" - test "$(stat -c '%a' "$benchmark_secret_file")" = 600 -done -test -f token/iop-dev-ca.pem -``` - -macOS에서 동일 검사를 수행할 때는 BSD `stat` 문법을 사용한다. 어떤 경우에도 `cat`, `echo`, shell tracing(`set -x`)으로 secret 내용을 출력하지 않는다. - -## 5. Provider와 managed credential 설정 - -현재 dev 구성은 다음 절차로 만들어졌다. - -1. Control Plane credential plane용 CA, workload certificate, at-rest keyring과 lease issuer/recipient key를 operator-owned untracked 경로에 생성했다. -2. Control Plane, Edge와 각 Node에 role/name-bound mTLS identity를 배치했다. -3. Edge public ingress를 HTTPS로 구성하고 benchmark runner에 CA certificate만 전달했다. -4. benchmark principal을 bootstrap하고 one-time token을 `token/.iop-bench`에 저장했다. -5. `token/.claude`, `token/.gemini`, `token/.gpt`의 raw provider key를 credential HTTPS request body로 직접 등록했다. command argument, YAML, tracked docs나 task evidence에는 넣지 않았다. -6. credential slot과 public route를 별도로 생성하고 principal projection에 direct route와 hybrid preset stage authorization을 연결했다. -7. Control Plane → Edge → Nodes 순서로 bounded restart하고 fresh projection, sealed lease, route revision과 no-fallback 동작을 검증했다. - -재구성이 필요하면 `docs/edge-local-dev-guide.md`의 “Managed credential plane and TLS startup”과 “Safe slot lifecycle”을 따른다. 실제 slot id, alias, revision과 lease id는 운영 상태이므로 이 문서에 복사하지 않는다. - -## 6. Caller별 연결 방식 - -| Caller | Edge surface | Child 설정 | 중요한 제한 | -|---|---|---|---| -| Claude Code | Anthropic-compatible Messages | `ANTHROPIC_BASE_URL`, `ANTHROPIC_API_KEY`; `--bare --print --verbose --output-format stream-json --no-session-persistence --permission-mode dontAsk --tools Read,Write,Edit --allowedTools Read,Write,Edit` | task는 stdin으로 한 번 제출한다. network/shell 도구 없이 격리 workspace 파일 작업만 허용한다. model/effort는 manifest 값을 그대로 전달한다. | -| agy | Gemini-native `streamGenerateContent` | fresh session `HOME`, `GOOGLE_GEMINI_BASE_URL`, `GEMINI_API_KEY`; `--sandbox --output-format stream-json --model ... --print ...` | `agy 1.1.12`만 승인된다. API-key provider에 `--effort`를 전달하지 않으며 ambient user config를 읽지 않는다. | -| Codex | OpenAI-compatible Responses | fresh session `HOME`, isolated `iop_benchmark` provider override, `IOP_BENCHMARK_API_KEY`; `exec --sandbox workspace-write --json --ephemeral --ignore-user-config --strict-config` | user config를 읽지 않고 격리 workspace에만 쓸 수 있다. current adapter effort는 `xhigh`; caller binding event가 없으면 admitted config binding을 사용하고, 보고된 mismatch는 거부한다. | - -세 caller child에는 필요한 `PATH`와 CA 변수만 allowlist로 전달한다. parent의 caller/provider 설정이나 unrelated secret은 상속하지 않는다. - -agy의 public base는 adapter가 cell별로 다음처럼 route-qualified 한다. - -```text -/gemini/ -``` - -`GEMINI_BASE_URL`, `AGY_PROVIDER`, `AGY_OPENAI_BASE_URL`, `AGY_OPENAI_API_KEY`는 이 benchmark transport가 아니다. - -official agy planner가 Gemini `generationConfig.responseMimeType`과 `responseSchema` 또는 `responseJsonSchema`를 보내면 Edge는 이를 기존 Chat `response_format`으로 변환한다. function declaration은 official SDK가 사용하는 `parameters`와 `parametersJsonSchema` 표기 중 정확히 하나를 허용한다. 동의어 필드가 동시에 있거나 schema가 JSON object가 아니면 provider dispatch 전에 거부한다. 구조가 유효한 agy `result.status=ERROR`는 stream parser 오류로 바꾸지 않고 caller process 실패로 기록한다. - -## 7. Benchmark 고정 설정 - -| 설정 | 값 | -|---|---| -| pipeline version | `2` | -| environment | `dev` | -| execution seed | `bench-02-c01-c09-v1` | -| repetitions | `1` | -| session policy | `fresh` | -| setup cache policy | `isolated` | -| run timeout | 300 seconds | -| idle timeout | 30 seconds | -| quiet window | 10 seconds | -| cleanup grace | 5 seconds | -| desktop viewport | `1920x1080` | -| mobile viewport | `375x812` | -| output root | `agent-test/runs/bench-02` | -| rubric | `one-shot-agent-comparison-v1` | - -execution seed로 결정되는 현재 slot 순서는 다음과 같다. 표의 C 번호 순서와 실제 실행 순서는 다르다. - -1. C02 Claude→Gemini direct -2. C05 Codex→GPT direct -3. C03 agy→Gemini direct -4. C06 Claude→Gemini hybrid -5. C08 Claude→GPT hybrid -6. C09 Codex→GPT hybrid -7. C01 Claude→Sonnet direct -8. C07 agy→Gemini hybrid -9. C04 Claude→GPT direct - -## 8. C01-C09 matrix와 route binding - -| Cell | Caller | Route kind/id | Requested model/effort | Effective stage binding | -|---|---|---|---|---| -| C01 | Claude | direct / `claude-sonnet-5` | `claude-sonnet-5` / `max` | request=`claude-sonnet-5` max | -| C02 | Claude | direct / `gemini-3.6-flash` | `gemini-3.6-flash` / `high` | request=`gemini-3.6-flash` high | -| C03 | agy | direct / `gemini-3.6-flash` | `gemini-3.6-flash` / `high` | request=`gemini-3.6-flash` high | -| C04 | Claude | direct / `gpt-5.6-luna` | `gpt-5.6-luna` / `xhigh` | request=`gpt-5.6-luna` xhigh | -| C05 | Codex | direct / `gpt-5.6-luna` | `gpt-5.6-luna` / `xhigh` | request=`gpt-5.6-luna` xhigh | -| C06 | Claude | preset / `gemini-hybrid` | `gemini-hybrid` / `high` | selector/plan/review/repair=`gemini-3.6-flash` high; work=`ornith-fast` | -| C07 | agy | preset / `gemini-hybrid` | `gemini-hybrid` / `high` | selector/plan/review/repair=`gemini-3.6-flash` high; work=`ornith-fast` | -| C08 | Claude | preset / `gpt-hybrid` | `gpt-hybrid` / `xhigh` | selector/plan/review/repair=`gpt-5.6-terra` high; work=`ornith-fast` | -| C09 | Codex | preset / `gpt-hybrid` | `gpt-hybrid` / `xhigh` | selector/plan/review/repair=`gpt-5.6-terra` high; work=`ornith-fast` | - -hybrid preset은 caller가 stage를 따로 호출하는 구조가 아니다. 하나의 caller request 안에서 Edge가 selector/plan/work/review/repair를 소유한다. - -## 9. Fixture와 결과 조건 - -공통 task는 fictional product “Lumen Atlas”의 responsive one-page landing page다. - -- prompt: `scripts/fixtures/agent-comparison-benchmark/prompt.md` -- copy: `scripts/fixtures/agent-comparison-benchmark/reference.txt` -- images: `aurora-grid.svg`, `orbit-rings.svg` -- 생성 파일: workspace root의 `index.html`, `styles.css`, `script.js` 정확히 세 개 -- 외부 asset, framework, package manager, build tool, analytics와 network dependency 금지 -- desktop/mobile responsive, semantic HTML, focus/contrast/accessibility 요구 -- 각 attempt는 fresh workspace/session에서 task를 한 번만 제출 - -fixture checksum은 manifest의 값이 유일한 기준이다. prompt나 asset을 변경하면 기존 run과 비교하지 말고 manifest/version/checksum을 함께 갱신하는 별도 작업으로 처리한다. - -## 10. Process environment 준비 - -다음은 value를 출력하지 않는 process-local 예시다. ``를 문서에 실제 값으로 치환하지 말고 실행 환경에서만 주입한다. - -Public live registry가 소비하는 environment contract는 다음과 같다. - -| Variable | 값/의미 | Durable evidence | -|---|---|---| -| `IOP_BENCH_CLAUDE_BASE_URL` | managed Edge HTTPS origin | raw 값 금지; endpoint digest만 허용 | -| `IOP_BENCH_AGY_BASE_URL` | managed Edge HTTPS origin; adapter가 `/gemini/`를 추가 | raw 값 금지; endpoint digest만 허용 | -| `IOP_BENCH_CODEX_BASE_URL` | managed Edge OpenAI-compatible `/v1` base | raw 값 금지; endpoint digest만 허용 | -| `IOP_BENCH_CLAUDE_SECRET_ENV` | Claude가 사용할 secret-bearing variable 이름 | variable 이름만 허용 | -| `IOP_BENCH_AGY_SECRET_ENV` | agy가 사용할 secret-bearing variable 이름 | variable 이름만 허용 | -| `IOP_BENCH_CODEX_SECRET_ENV` | Codex가 사용할 secret-bearing variable 이름 | variable 이름만 허용 | -| `IOP_BENCH_SHARED_TOKEN` | 이 가이드 예시의 secret-bearing variable | 값 기록 금지 | -| `SSL_CERT_FILE` | dev Edge HTTPS CA certificate path | repository-relative file reference만 허용 | -| `NODE_EXTRA_CA_CERTS` | Node.js caller용 동일 CA certificate path | repository-relative file reference만 허용 | -| `IOP_BENCH_CONFIG_OBSERVATION_ENV` | config JSON을 보유한 variable 이름 | variable 이름만 허용 | -| `BENCH_CONFIG` | schema v1 route/model/stage observation JSON | secret은 없지만 runtime과 일치하는 canonical digest만 evidence에 기록 | -| `PATH` | caller binary resolution | resolved executable path/version만 preflight에서 확인 | - -Child adapter가 내부적으로 만드는 값은 caller별로 격리된다. - -- Claude: `ANTHROPIC_BASE_URL`, `ANTHROPIC_API_KEY`, restricted `Read,Write,Edit`, nonessential traffic/autoupdater disable flags와 CA variables -- agy: fresh session `HOME`, `GOOGLE_GEMINI_BASE_URL`, `GEMINI_API_KEY`, `LANG=C.UTF-8`, `LC_ALL=C.UTF-8`, `TZ=UTC`와 CA variables -- Codex: fresh session `HOME`, `IOP_BENCHMARK_API_KEY`, strict ephemeral provider override와 CA variables - -이 child variable은 사용자가 별도로 준비할 값이 아니다. live registry가 위 public contract에서 파생하며, parent의 같은 이름 값을 그대로 신뢰하거나 상속하지 않는다. - -```bash -set -euo pipefail - -benchmark_edge_origin="${BENCHMARK_EDGE_ORIGIN:?set BENCHMARK_EDGE_ORIGIN to the managed dev Edge HTTPS origin}" -read -r IOP_BENCH_SHARED_TOKEN < token/.iop-bench -export IOP_BENCH_SHARED_TOKEN - -export IOP_BENCH_CLAUDE_BASE_URL="$benchmark_edge_origin" -export IOP_BENCH_AGY_BASE_URL="$benchmark_edge_origin" -export IOP_BENCH_CODEX_BASE_URL="$benchmark_edge_origin/v1" - -export IOP_BENCH_CLAUDE_SECRET_ENV=IOP_BENCH_SHARED_TOKEN -export IOP_BENCH_AGY_SECRET_ENV=IOP_BENCH_SHARED_TOKEN -export IOP_BENCH_CODEX_SECRET_ENV=IOP_BENCH_SHARED_TOKEN - -export SSL_CERT_FILE="$PWD/token/iop-dev-ca.pem" -export NODE_EXTRA_CA_CERTS="$PWD/token/iop-dev-ca.pem" - -export IOP_BENCH_CONFIG_OBSERVATION_ENV=BENCH_CONFIG -BENCH_CONFIG="$(python3 - <<'PY' -import json - -routes = [ - {"route_kind":"direct","route_id":"claude-sonnet-5","model":"claude-sonnet-5","bindings":[{"stage":"request","model":"claude-sonnet-5","effort":"max"}]}, - {"route_kind":"direct","route_id":"gemini-3.6-flash","model":"gemini-3.6-flash","bindings":[{"stage":"request","model":"gemini-3.6-flash","effort":"high"}]}, - {"route_kind":"direct","route_id":"gpt-5.6-luna","model":"gpt-5.6-luna","bindings":[{"stage":"request","model":"gpt-5.6-luna","effort":"xhigh"}]}, - {"route_kind":"execution_preset","route_id":"gemini-hybrid","model":"gemini-hybrid","bindings":[{"stage":"selector","model":"gemini-3.6-flash","effort":"high"},{"stage":"plan","model":"gemini-3.6-flash","effort":"high"},{"stage":"work","model":"ornith-fast","effort":None},{"stage":"review","model":"gemini-3.6-flash","effort":"high"},{"stage":"repair","model":"gemini-3.6-flash","effort":"high"}]}, - {"route_kind":"execution_preset","route_id":"gpt-hybrid","model":"gpt-hybrid","bindings":[{"stage":"selector","model":"gpt-5.6-terra","effort":"high"},{"stage":"plan","model":"gpt-5.6-terra","effort":"high"},{"stage":"work","model":"ornith-fast","effort":None},{"stage":"review","model":"gpt-5.6-terra","effort":"high"},{"stage":"repair","model":"gpt-5.6-terra","effort":"high"}]}, -] -print(json.dumps({"schema_version":"1","routes":routes}, separators=(",",":"))) -PY -)" -export BENCH_CONFIG -``` - -중요한 의미: - -- `IOP_BENCH_*_SECRET_ENV`의 값은 secret이 아니라 실제 secret을 보유한 environment variable의 이름이다. -- 세 caller는 같은 IOP principal을 사용하지만 서로 다른 protocol base를 받는다. -- `BENCH_CONFIG`는 runtime에서 독립적으로 확인한 route/stage snapshot이어야 한다. manifest를 보고 임의 합성한 값을 live readiness evidence로 사용하면 안 된다. -- endpoint와 config는 evidence에 raw 값 대신 digest identity로만 남는다. - -작업 후에는 같은 shell에서 다음 변수를 제거한다. - -```bash -unset IOP_BENCH_SHARED_TOKEN -unset IOP_BENCH_CLAUDE_BASE_URL IOP_BENCH_AGY_BASE_URL IOP_BENCH_CODEX_BASE_URL -unset IOP_BENCH_CLAUDE_SECRET_ENV IOP_BENCH_AGY_SECRET_ENV IOP_BENCH_CODEX_SECRET_ENV -unset IOP_BENCH_CONFIG_OBSERVATION_ENV BENCH_CONFIG -unset SSL_CERT_FILE NODE_EXTRA_CA_CERTS -unset BENCHMARK_EDGE_ORIGIN benchmark_edge_origin -``` - -## 11. 실행 절차 - -manifest path는 모든 명령에서 동일하게 사용한다. - -```bash -benchmark_manifest=scripts/fixtures/agent-comparison-benchmark-iop-one-shot.json -``` - -### 11.1 Source와 deterministic verification - -```bash -python3 -m unittest scripts.agent_benchmark.codex_iop_test scripts.agent_benchmark.connectivity_integration_test -python3 -m unittest discover -s scripts/agent_benchmark -p '*_test.py' -python3 scripts/agent_comparison_benchmark.py validate --manifest "$benchmark_manifest" -git diff --check -``` - -필요하면 변경 범위에 따라 Go tests와 managed credential qualification도 실행한다. - -```bash -go test -count=1 ./... -credential_smoke_parent="$(mktemp -d /tmp/iop-benchmark-credential.XXXXXX)" -TMPDIR="$credential_smoke_parent" make test-credential-slot-smoke -rmdir "$credential_smoke_parent" -``` - -### 11.2 Direct-first qualification - -동일 clean source ref를 모든 runtime binary에 배포하고 4/4 Node와 provider health를 확인한 다음, 먼저 기존 5-cell direct manifest를 사용한다. - -```bash -direct_manifest="scripts/fixtures/agent-comparison-benchmark-direct-preflight.example.json" -python3 scripts/agent_comparison_benchmark.py preflight --manifest "$direct_manifest" -python3 scripts/agent_comparison_benchmark.py run --manifest "$direct_manifest" -``` - -direct canary는 unscored 진단이다. fresh preflight가 `ready=5`여야 하고, 정확히 5개의 fresh attempt를 한 번씩 할당하며, `unresolved=0`, `running=0`, `interrupted=0`과 함께 각 slot의 controller/product/harness/process/web-validation terminal evidence가 있어야 한다. 최대 3회 Chromium/CDP 재시도 뒤에도 `browser_*` 또는 `cdp_*` infrastructure block이 남거나 Edge pre-ingress incompatibility가 확인되면 qualification을 중단한다. - -`product=failed`, upstream HTTP rejection, caller failure 뒤 `generated_missing`, timeout은 terminal evidence가 완결되면 측정 결과다. 이를 성공으로 바꾸거나 암묵 재시도하지 않으며, 그 자체로 다음 구현 cycle을 만들지 않는다. scoring eligibility는 각 독립 gate로 별도 판정한다. - -### 11.3 Public nine-cell preflight - -10절의 환경을 같은 shell에 준비한 뒤 실행한다. - -```bash -python3 scripts/agent_comparison_benchmark.py preflight --manifest "$benchmark_manifest" -``` - -성공 조건: - -```text -status=ready ready=9 registration_required=0 implementation_gap=0 -``` - -preflight는 다음을 함께 확인한다. - -- caller binary/version/help -- Edge model catalog -- principal auth와 endpoint compatibility -- direct/preset route 존재 -- requested model/effort -- exact stage binding과 order -- official agy transport capability - -`registration_required` 또는 `implementation_gap`이면 즉시 중단한다. alias, model, effort, route나 caller를 대체하지 않는다. preflight-only run root는 evidence이므로 삭제하지 않는다. - -terminal-evidence direct canary admission 뒤 fresh one-shot preflight가 `ready=9`인지 확인하고 멈춘다. 이 qualification 단계에서는 hybrid canary나 C01-C09 `run`을 호출하지 않는다. - -### 11.4 Scored execution - -fresh preflight와 명시적 실행 권한이 있는 현재 plan에서만 다음 명령을 한 번 호출한다. - -```bash -python3 scripts/agent_comparison_benchmark.py run --manifest "$benchmark_manifest" -``` - -- direct CLI stdout/stderr와 exit code를 보존한다. -- CLI가 출력한 canonical `run-...` id만 이후 `status`, `score`, `report`에 사용한다. -- command가 nonzero여도 같은 plan에서 `run`을 다시 호출하지 않는다. -- CLI가 run id를 출력하지 않으면 임의 id나 성공 pointer를 만들지 않는다. -- caller나 provider를 CLI 밖에서 별도로 호출해 scored result를 보충하지 않는다. - -### 11.5 Status - -```bash -python3 scripts/agent_comparison_benchmark.py status \ - --manifest "$benchmark_manifest" \ - --run-id "$benchmark_run_id" -``` - -현재 comparison execution 완료 조건: - -- controller terminal 수 `completed + timed_out + cancelled + interrupted = 9` -- `running = 0` -- `interrupted = 0` -- 각 cell/repetition에 retained terminal attempt가 존재 -- 모든 최신 attempt에 controller/product/harness/process/web-validation terminal evidence가 있고 `unresolved=0` - -`completed`는 controller 종료만 뜻하며 product 성공을 뜻하지 않는다. product/harness/process/artifact 실패는 scoring eligibility와 최종 비교에서 각각 별도로 표시되고, terminal 실패를 성공 결과로 바꾸거나 암묵 재시도하지 않는다. - -### 11.6 Blind scoring - -유효한 execution run에 대해서만 수행한다. - -```bash -python3 scripts/agent_comparison_benchmark.py score \ - --manifest "$benchmark_manifest" \ - --run-id "$benchmark_run_id" -``` - -evaluator는 Codex→`gpt-5.6-luna` xhigh direct route다. identity가 제거된 blind workspace만 보며 source cell identity mapping은 blind tree 밖에 유지한다. - -product, harness, acceptable process 또는 artifact gate 실패는 각각의 reason을 가진 `unscored`이고 0점으로 바꾸지 않는다. `scoring_failed`도 명시적 `--retry-scoring-failed` 권한 없이 재시도하지 않는다. - -### 11.7 Report - -```bash -python3 scripts/agent_comparison_benchmark.py report \ - --manifest "$benchmark_manifest" \ - --run-id "$benchmark_run_id" -``` - -report는 run root의 immutable evidence를 읽어 idempotent `report.md`를 만든다. 기존 report 내용과 새 projection이 다르면 덮어쓰지 않고 실패한다. - -## 12. 수집 evidence - -각 attempt는 다음 범주의 evidence를 가진다. - -| 범주 | 내용 | -|---|---| -| lifecycle | typed caller terminal, submission, first output, finish, idle, quiet와 독립 product/harness/process 결과 | -| timeline | submitted, first output, first workspace write observation/mtime, total duration | -| usage | input/output/reasoning/cache read/cache write/total tokens, model/tool calls와 duration | -| workspace | fresh session identity, fixture checksum, testbed provenance, generated file tree | -| web validation | product/harness 결과와 무관하게 모든 terminal workspace에서 생성되는 generated files, static safety, images, network, console, responsive, accessibility | -| screenshots | desktop `1920x1080`, mobile `375x812` | -| scoring | eligibility, blind allocation, rubric worksheet, score status | - -caller가 보고하지 않은 metric은 0으로 만들지 않고 `unavailable`과 reason/source를 보존한다. - -Codex current usage mapping: - -| Caller JSONL field | Canonical metric | -|---|---| -| `input_tokens` | `input_tokens` | -| `cached_input_tokens` | `cached_input_tokens` | -| `cache_write_input_tokens` | `cache_write_tokens` | -| `output_tokens` | `output_tokens` | -| `reasoning_output_tokens` | `reasoning_tokens` | -| `total_tokens` | `total_tokens` | - -누락된 `total_tokens`는 하위 category 합으로 재구성하지 않는다. - -## 13. 자동 gate와 100점 rubric - -automatic web gates는 scoring eligibility만 결정하고 점수에 포함되지 않는다. - -| Gate | -|---| -| generated files | -| static safety | -| local images | -| no external network dependency | -| console safety | -| responsive layout | -| accessibility | - -quality rubric: - -| Category | Max | -|---|---:| -| requirements fidelity | 25 | -| visual completeness | 25 | -| responsive accessibility | 15 | -| image/detail usage | 10 | -| behavior stability | 10 | -| code quality | 10 | -| self verification | 5 | -| Total | 100 | - -## 14. 실패 처리와 재개 원칙 - -| 상황 | 조치 | -|---|---| -| manifest invalid | source/manifest를 수정하고 validate부터 다시 시작한다. run을 만들지 않는다. | -| preflight not ready | blocker를 해결하고 fresh preflight한다. attempt를 할당하지 않는다. | -| caller launch 전 interruption | retained evidence를 보존한다. run tree를 직접 수정하지 않는다. | -| lifecycle/parser failure | exact retained output으로 source 원인을 수정하고 deterministic regression을 추가한다. | -| completed이지만 product/harness/process/artifact gate 실패 또는 timed_out/cancelled | 각 축 evidence로 보존한다. 암묵 retry하지 않는다. | -| state가 `running`이지만 process가 없음 | manual JSON 수정/삭제/reconcile을 하지 않는다. reviewer evidence로 남기고 승인된 새 plan에서만 다음 상태를 결정한다. | -| scoring_failed | 0점 처리하지 않는다. 명시적 retry 권한 없이는 중단한다. | -| report unavailable | run evidence를 수정하거나 report를 수작업 생성하지 않는다. | - -현재 milestone의 원칙은 old incomplete/failed run을 `resume --retry-failed`하지 않고 distinct fresh run을 만드는 것이다. 일반 CLI가 `resume`을 지원한다는 사실이 현재 benchmark에서 사용 권한을 뜻하지 않는다. - -## 15. Secret-safe 기록 규칙 - -다음 값은 tracked docs, task review, run metadata, log, metric label과 command argument에 남기지 않는다. - -- IOP principal token -- raw provider credential -- private key, at-rest key, issuer/recipient private key -- slot alias와 lease id -- credential-bearing URL -- raw prompt/response, tool input/output -- caller/provider session content - -허용되는 내용: - -- secret file의 상대 path와 존재/mode -- route/model/stage 이름 -- safe credential slot reference/revision -- hashed endpoint/config/spec identity -- redacted lifecycle 상태, duration과 usage count -- run id와 attempt identity가 필요한 controller evidence - -incident evidence를 보존하기 전에 repository와 run output에서 secret 원문이 없는지 확인한다. 의심되는 artifact는 내용을 복사하지 말고 path와 redaction failure만 보고한다. - -## 16. 실행 전 체크리스트 - -- [ ] 현재 manifest validation 통과 -- [ ] focused/full deterministic tests fresh PASS -- [ ] benchmark runner와 `../iop-s2` provenance 확인 -- [ ] Claude/agy/Codex command와 current version/help 확인 -- [ ] `token/.iop-bench`, CA와 provider provisioning source 존재/mode 확인; 내용 출력 없음 -- [ ] dev runtime source/build identity, process와 listener 확인 -- [ ] managed projection, provider slot/route와 no-legacy-fallback 확인 -- [ ] config observation이 runtime route/stage와 정확히 일치 -- [ ] direct qualification `ready=5`, fresh attempt 5개, `unresolved=0`, `running=0`, `interrupted=0`, terminal evidence 5개와 infrastructure block 없음 -- [ ] public preflight `ready=9` -- [ ] 현재 plan이 exactly one scored run을 소유하고 사용자 권한이 명확함 -- [ ] old run resume/retry/state edit 계획 없음 -- [ ] run 이후 status, scoring, report의 run id 전달 경로 준비 - -## 17. 관련 구현 - -- controller/state: `scripts/agent_benchmark/attempts.py` -- manifest: `scripts/agent_benchmark/manifest.py` -- lifecycle: `scripts/agent_benchmark/lifecycle.py` -- workspace isolation: `scripts/agent_benchmark/workspace.py` -- live routing/admission: `scripts/agent_benchmark/live_iop.py` -- Claude adapter: `scripts/agent_benchmark/claude_iop.py` -- agy adapter: `scripts/agent_benchmark/agy_iop.py` -- Codex adapter: `scripts/agent_benchmark/codex_iop.py` -- measurement: `scripts/agent_benchmark/measurement.py` -- browser/web gate: `scripts/agent_benchmark/web_validation.py` -- blind scoring: `scripts/agent_benchmark/scoring.py` -- rubric: `scripts/agent_benchmark/rubric.py` -- report: `scripts/agent_benchmark/reporting.py` diff --git a/docs/edge-local-dev-guide.md b/docs/edge-local-dev-guide.md index d2ead4cd..35b5a53b 100644 --- a/docs/edge-local-dev-guide.md +++ b/docs/edge-local-dev-guide.md @@ -280,8 +280,8 @@ The deterministic Messages qualification succeeds alongside Chat: the Control Pl 공식 `agy` 1.1.12 API-key provider는 upstream Gemini key가 아니라 관리형 IOP principal token을 사용한다. dev operator가 이미 발급한 token과 CA 파일을 보호된 `token/` 아래에 둔 경우 값을 명령행에 직접 쓰지 않고 다음처럼 읽는다. ```bash -read -r IOP_BENCH_TOKEN < token/.iop-bench -export GEMINI_API_KEY="$IOP_BENCH_TOKEN" +read -r IOP_AGY_SMOKE_TOKEN < token/.iop-principal +export GEMINI_API_KEY="$IOP_AGY_SMOKE_TOKEN" export SSL_CERT_FILE="$PWD/token/iop-dev-ca.pem" export NODE_EXTRA_CA_CERTS="$PWD/token/iop-dev-ca.pem" @@ -293,12 +293,10 @@ GOOGLE_GEMINI_BASE_URL="https://:/gemini/ bool: - """Match one documented help token, never a prefix or a suffix.""" - return re.search(r"(? ConnectivityIssue: - return ConnectivityIssue(code, ISSUE_RESUME_CODES[code]) - - -def _requested_binding(cell: MatrixCell) -> RequestedEffectiveBinding: - return RequestedEffectiveBinding( - cell.id, - cell.caller, - cell.iop.route_kind, - cell.iop.route_id, - cell.iop.request_model, - cell.iop.requested_effort, - ) - - -def parse_documented_agy_capabilities(help_output: str) -> AgyDocumentedCapabilities: - """Parse only complete documented tokens from a known agy help surface.""" - if not isinstance(help_output, str): - return AgyDocumentedCapabilities((), (), ()) - return AgyDocumentedCapabilities( - tuple(token for token in _DOCUMENTED_OPTIONS if _exact_token_present(help_output, token)), - (), - ("stream-json",) if _exact_token_present(help_output, "stream-json") else (), - ) - - -def inspect_agy_iop_capability(version_output: str, help_output: str) -> AgyCapability: - """Inspect only public, versioned help text for the closed IOP transport. - - The known release is pinned because its Gemini provider environment is not - printed by ``--help``. The public options and stream format still have to - match exactly; a changed release fails closed until re-qualified. - """ - if not isinstance(version_output, str) or not isinstance(help_output, str): - return AgyCapability(None, False, False, False, False, False, (), ()) - matched = _VERSION_RE.fullmatch(version_output.strip()) - version = matched.group(1) if matched else None - known_version = version == AGY_KNOWN_VERSION - documented = parse_documented_agy_capabilities(help_output) - endpoint_supported = known_version - auth_supported = known_version - protocol_supported = known_version and all( - option in documented.options for option in _DOCUMENTED_OPTIONS - ) - stream_supported = "stream-json" in documented.output_formats - supported = endpoint_supported and auth_supported and protocol_supported and stream_supported - if not supported: - return AgyCapability( - version, False, endpoint_supported, auth_supported, protocol_supported, stream_supported, (), () - ) - return AgyCapability( - version, - True, - True, - True, - True, - True, - ("direct", "execution_preset"), - ("high", "low", "medium"), - ) - - -def _runtime_identity(label: str, value: str) -> str: - return "sha256:" + hashlib.sha256( - b"agy-iop-runtime-v1\0" + label.encode("ascii") + b"\0" + value.encode("utf-8") - ).hexdigest() - - -def _validate_config_owner_observation( - cell: MatrixCell, observation: AgyRuntimeObservation -) -> None: - """Validate an opaque observation supplied by the independent config owner.""" - if not isinstance(observation, AgyRuntimeObservation): - raise AgyAdapterError("agy runtime observation is invalid") - if (observation.cell_id, observation.route_kind, observation.route_id) != ( - cell.id, cell.iop.route_kind, cell.iop.route_id, - ): - raise AgyAdapterError("agy IOP runtime observation mismatch") - if not all(_IDENTITY_RE.fullmatch(value) for value in ( - observation.endpoint_identity, - observation.credential_identity, - observation.config_identity, - )): - raise AgyAdapterError("agy IOP runtime observation identity is invalid") - - -def validate_agy_iop_runtime( - cell: MatrixCell, runtime: AgyRuntimeInputs, observation: AgyRuntimeObservation -) -> _ValidatedAgyRuntime: - """Admit runtime-only launch values only when IOP identity proof is exact.""" - if not isinstance(cell, MatrixCell) or cell.caller != AGY_CALLER: - raise AgyAdapterError("agy runtime validation requires an agy matrix cell") - if not isinstance(runtime, AgyRuntimeInputs) or not isinstance(observation, AgyRuntimeObservation): - raise AgyAdapterError("agy runtime inputs are invalid") - if not all(isinstance(value, str) and value for value in (runtime.binary, runtime.endpoint, runtime.credential)): - raise AgyAdapterError("agy runtime values are unavailable") - endpoint = urlsplit(runtime.endpoint) - route_path = f"/gemini/{cell.iop.route_id}" - if ( - endpoint.scheme != "https" - or not endpoint.netloc - or endpoint.query - or endpoint.fragment - or endpoint.path.rstrip("/") != route_path - ): - raise AgyAdapterError("agy IOP endpoint is invalid") - _validate_config_owner_observation(cell, observation) - if observation.endpoint_identity != _runtime_identity("endpoint", runtime.endpoint): - raise AgyAdapterError("agy IOP runtime observation mismatch") - if observation.credential_identity != _runtime_identity("credential", runtime.credential): - raise AgyAdapterError("agy IOP runtime observation mismatch") - return _ValidatedAgyRuntime(runtime.binary, runtime.endpoint, runtime.credential, observation) - - -def preflight_agy_iop( - cell: MatrixCell, - capability: AgyCapability, - runtime: AgyRuntimeInputs, - observation: AgyRuntimeObservation, -) -> AgyPreflightResult: - """Classify only registration and implementation gaps without launching agy.""" - if not isinstance(cell, MatrixCell) or cell.caller != AGY_CALLER: - raise AgyAdapterError("agy preflight requires an agy matrix cell") - if not isinstance(capability, AgyCapability) or not isinstance(runtime, AgyRuntimeInputs): - raise AgyAdapterError("agy preflight inputs are invalid") - - issues: list[ConnectivityIssue] = [] - validated_runtime: _ValidatedAgyRuntime | None = None - if runtime.credential: - try: - validated_runtime = validate_agy_iop_runtime(cell, runtime, observation) - except AgyAdapterError: - issues.append(_issue("endpoint_incompatible")) - if not runtime.credential: - issues.append(_issue("credential_missing")) - if cell.iop.request_model not in AGY_MODEL_LABELS: - issues.append(_issue("model_missing")) - if not runtime.endpoint: - issues.append(_issue("endpoint_incompatible")) - if not capability.endpoint_supported: - issues.append(_issue("endpoint_incompatible")) - if not capability.auth_supported: - issues.append(_issue("auth_incompatible")) - if not capability.protocol_supported: - issues.append(_issue("protocol_incompatible")) - if not capability.stream_supported: - issues.append(_issue("stream_incompatible")) - if capability.iop_transport_supported and cell.iop.route_kind not in capability.route_kinds: - issues.append(_issue("protocol_incompatible")) - elif capability.iop_transport_supported and cell.iop.requested_effort not in capability.efforts: - issues.append(_issue("effort_unsupported")) - - # Preserve connectivity.py's canonical issue order without leaking values. - unique = {item.code: item for item in issues} - ordered = tuple( - unique[code] - for code in ( - "credential_missing", "model_missing", "route_missing", "effort_unsupported", - "endpoint_incompatible", "auth_incompatible", "protocol_incompatible", "stream_incompatible", - ) - if code in unique - ) - status = classify_issues(ordered) - return AgyPreflightResult(capability, _requested_binding(cell), ordered, status, validated_runtime) - - -def build_agy_invocation( - cell: MatrixCell, - prepared: PreparedWorkspace, - task_payload: bytes, - timeout: Timeout, - preflight: AgyPreflightResult, -) -> InvocationSpec: - """Build one isolated official agy print invocation after a ready preflight.""" - if preflight.status != "ready" or not preflight.capability.iop_transport_supported or preflight.runtime is None: - raise AgyAdapterError("agy IOP transport is not proven") - runtime = preflight.runtime - if not runtime.binary or not Path(runtime.binary).is_file(): - raise AgyAdapterError("agy binary is unavailable") - if ( - not isinstance(prepared, PreparedWorkspace) - or not Path(prepared.workspace_dir).is_dir() - or not Path(prepared.session_dir).is_dir() - or not Path(prepared.attempt_root).is_dir() - ): - raise AgyAdapterError("prepared workspace is unavailable") - if not isinstance(task_payload, bytes) or not task_payload: - raise AgyAdapterError("agy task payload is unavailable") - try: - task_text = task_payload.decode("utf-8") - except UnicodeDecodeError as exc: - raise AgyAdapterError("agy task payload must be UTF-8") from exc - - _stage_agy_provider_settings(prepared.session_dir) - - # The child receives a minimal environment and explicit IOP-only provider - # settings. No parent agy/Gemini config or session variable is inherited. - environment = { - "PATH": os.environ.get("PATH", "/usr/bin:/bin"), - "LANG": "C.UTF-8", - "LC_ALL": "C.UTF-8", - "TZ": "UTC", - "HOME": prepared.session_dir, - AGY_ENDPOINT_ENV: runtime.endpoint, - AGY_AUTH_ENV: runtime.credential, - } - environment.update(inherited_tls_ca_environment()) - return InvocationSpec( - argv=( - runtime.binary, - "--dangerously-skip-permissions", - "--output-format", "stream-json", - "--model", AGY_MODEL_LABELS[cell.iop.request_model], - "--print", task_text, - ), - cwd=prepared.workspace_dir, - env=env_pairs(environment), - env_allowlist=(AGY_ENDPOINT_ENV, AGY_AUTH_ENV, *TLS_CA_ENV_KEYS), - submission_mode=SUBMISSION_ARGV_TASK, - completion_mode=COMPLETION_EXIT_AFTER_IDLE, - timeout=timeout, - evidence_dir=prepared.attempt_root, - control_dir=str(Path(prepared.attempt_root) / "agy-control"), - ) - - -def _stage_agy_provider_settings(session_dir: str) -> None: - """Create the exact secret-free provider selector in one isolated HOME.""" - try: - session = Path(session_dir).resolve(strict=True) - if not session.is_dir(): - raise OSError("session is not a directory") - settings = session / AGY_SETTINGS_RELATIVE_PATH - settings.parent.mkdir(mode=0o700, parents=True, exist_ok=True) - if settings.parent.resolve(strict=True) != session / AGY_SETTINGS_RELATIVE_PATH.parent: - raise OSError("settings directory escapes session") - payload = ( - json.dumps(_AGY_ISOLATED_SETTINGS, sort_keys=True, separators=(",", ":")) + "\n" - ).encode("utf-8") - flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL - if hasattr(os, "O_NOFOLLOW"): - flags |= os.O_NOFOLLOW - descriptor = os.open(settings, flags, 0o600) - with os.fdopen(descriptor, "wb") as handle: - handle.write(payload) - except (OSError, ValueError) as exc: - raise AgyAdapterError("agy isolated provider settings are unavailable") from exc - - -def _safe_identifier(value: Any) -> str | None: - return value if isinstance(value, str) and re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.:+-]{0,63}", value) else None - - -def redact_agy_event(raw_line: str, sensitive_values: tuple[str, ...] = ()) -> str: - """Return a canonical allowlisted event projection, never caller content.""" - try: - parsed = json.loads(raw_line) - except (TypeError, json.JSONDecodeError): - return '{"event":"unparseable"}' - if not isinstance(parsed, dict): - return '{"event":"unparseable"}' - event = _safe_identifier(parsed.get("event")) - if event is None or event in sensitive_values: - return '{"event":"unparseable"}' - safe: dict[str, str] = {"event": event} - payload = parsed.get(event) - if not isinstance(payload, dict): - payload = {} - for field in _SAFE_EVENT_FIELDS[1:]: - value = _safe_identifier(payload.get(field)) - if value is not None and value not in sensitive_values: - safe[field] = value - return json.dumps(safe, sort_keys=True, separators=(",", ":"), ensure_ascii=True) - - -class AgyEventParser: - """Parse the official 1.1.12 stream while trusting binding only from config.""" - - def __init__(self, cell: MatrixCell, admitted_binding: RequestedEffectiveBinding) -> None: - if not isinstance(admitted_binding, RequestedEffectiveBinding): - raise AgyAdapterError("agy admitted binding is invalid") - if admitted_binding.cell_id != cell.id or admitted_binding.caller != cell.caller: - raise AgyAdapterError("agy admitted binding mismatch") - self._cell = cell - self._admitted_binding = admitted_binding - self._init_seen = False - self._result_seen = False - self._latest_usage: dict[str, Any] | None = None - - def __call__(self, stream: str, raw_line: str) -> Any: - return self.parse(stream, raw_line) - - def parse(self, stream: str, raw_line: str) -> Any: - if stream != "stdout": - return None - try: - item = json.loads(raw_line) - except (TypeError, json.JSONDecodeError): - raise AgyAdapterError("malformed agy event") - if not isinstance(item, dict): - raise AgyAdapterError("malformed agy event") - event = item.get("event") - payload = item.get(event) if isinstance(event, str) else None - if not isinstance(payload, dict): - raise AgyAdapterError("malformed agy event") - if event == "init": - if self._init_seen or self._result_seen: - raise AgyAdapterError("malformed agy event") - self._init_seen = True - return None - if event == "step_update": - if not self._init_seen or self._result_seen: - raise AgyAdapterError("malformed agy event") - usage = payload.get("usage") - if usage is not None: - if self._usage_metrics(usage) is None: - raise AgyAdapterError("malformed agy event") - self._latest_usage = usage - return None - if event != "result" or not self._init_seen or self._result_seen: - raise AgyAdapterError("malformed agy event") - self._result_seen = True - if payload.get("status") != "SUCCESS": - if payload.get("status") != "ERROR": - raise AgyAdapterError("malformed agy result status") - return ( - CallerTerminal(CALLER_STATUS_FAILED, CALLER_REASON_ERROR), - CallerEvent("finish"), - CallerEvent("idle"), - ) - metrics: list[ParsedMetric] = [] - usage = payload.get("usage", self._latest_usage) - if usage is not None: - parsed_usage = self._usage_metrics(usage) - if parsed_usage is None: - raise AgyAdapterError("malformed agy usage") - metrics.extend(parsed_usage) - try: - if "duration_seconds" in payload: - metrics.append(duration_metric( - "total_duration", payload["duration_seconds"], reported_unit="s", - model=self._cell.iop.request_model, - )) - if "num_turns" in payload: - metrics.append(count_metric( - "model_calls", payload["num_turns"], model=self._cell.iop.request_model, - )) - except LifecycleMetricError: - raise AgyAdapterError("malformed agy metric") - return tuple(metrics) + ( - CallerTerminal(CALLER_STATUS_SUCCEEDED, CALLER_REASON_SUCCESS), - CallerEvent("finish"), - CallerEvent("idle"), - ) - - def _usage_metrics(self, usage: Any) -> tuple[ParsedMetric, ...] | None: - if not isinstance(usage, dict) or not set(usage) <= set(_AGY_USAGE_METRICS): - return None - metrics: list[ParsedMetric] = [] - try: - for wire_name, metric_name in _AGY_USAGE_METRICS.items(): - if wire_name in usage: - metrics.append(count_metric( - metric_name, usage[wire_name], model=self._cell.iop.request_model, - )) - except LifecycleMetricError: - return None - return tuple(metrics) - - def observed_result(self, capability: AgyCapability, lifecycle: InvocationResult) -> ConnectivityResult: - requested = _requested_binding(self._cell) - closed_gap = (_issue("stream_incompatible"),) - caller_capability = CallerCapability(AGY_CALLER, capability.route_kinds, capability.efforts) - if ( - not isinstance(lifecycle, InvocationResult) - or lifecycle.product.status != CALLER_STATUS_SUCCEEDED - or lifecycle.harness.status != "passed" - or not lifecycle.harness.ordered_terminal - or not self._result_seen - ): - return make_result(self._cell, caller_capability, requested, closed_gap) - try: - return make_result(self._cell, caller_capability, self._admitted_binding) - except Exception: - return make_result(self._cell, caller_capability, requested, closed_gap) - - -def run_agy_invocation( - spec: InvocationSpec, - parser: AgyEventParser, - preflight: AgyPreflightResult, - on_started: Callable[[SupervisorLocator], None], -) -> InvocationResult: - """Run a prepared agy call with structural output redaction only.""" - if ( - not isinstance(preflight, AgyPreflightResult) - or preflight.status != "ready" - or not preflight.capability.iop_transport_supported - or preflight.runtime is None - ): - raise AgyAdapterError("agy IOP transport is not proven") - runtime = preflight.runtime - sensitive = (runtime.endpoint, runtime.credential) - structural = lambda line: redact_agy_event(line, sensitive) - # exact replacement is retained as a final defence for non-JSON stderr. - exact = exact_value_redactor(sensitive) - return run_invocation( - spec, - parse_event=parser, - on_started=on_started, - redact=lambda line: structural(exact(line)), - ) diff --git a/scripts/agent_benchmark/agy_iop_test.py b/scripts/agent_benchmark/agy_iop_test.py deleted file mode 100644 index 0f2fae59..00000000 --- a/scripts/agent_benchmark/agy_iop_test.py +++ /dev/null @@ -1,303 +0,0 @@ -"""Credential-free tests for the official agy 1.1.12 IOP adapter.""" - -from __future__ import annotations - -import json -import os -import sys -import tempfile -import unittest -from dataclasses import replace -from pathlib import Path -from unittest.mock import patch - -from scripts.agent_benchmark.agy_iop import ( - AGY_AUTH_ENV, - AGY_ENDPOINT_ENV, - AGY_KNOWN_VERSION, - AGY_SETTINGS_RELATIVE_PATH, - AgyAdapterError, - AgyEventParser, - AgyRuntimeInputs, - AgyRuntimeObservation, - _runtime_identity, - build_agy_invocation, - inspect_agy_iop_capability, - preflight_agy_iop, - redact_agy_event, - run_agy_invocation, -) -from scripts.agent_benchmark.connectivity import EffectiveBinding, RequestedEffectiveBinding -from scripts.agent_benchmark.lifecycle import ( - REASON_MALFORMED_EVENT, - REASON_NONZERO_EXIT, - SUBMISSION_ARGV_TASK, - InvocationSpec, - env_pairs, -) -from scripts.agent_benchmark.manifest import ExpectedBinding, IopCell, MatrixCell, Timeout -from scripts.agent_benchmark.workspace import AttemptIdentity, PreparedWorkspace, TestbedProvenance - - -def _help() -> str: - return "--print --output-format stream-json --dangerously-skip-permissions --model --effort" - - -def _cell() -> MatrixCell: - return MatrixCell( - "agy-direct", "agy", - IopCell("gemini-3.6-flash", "high", "direct", "agy-direct", ( - ExpectedBinding("request", "gemini-3.6-flash", "high"), - )), - ) - - -def _binding() -> RequestedEffectiveBinding: - return RequestedEffectiveBinding( - "agy-direct", "agy", "direct", "agy-direct", "gemini-3.6-flash", "high", - "direct", "agy-direct", "gemini-3.6-flash", "high", - (EffectiveBinding("request", "gemini-3.6-flash", "high"),), - ) - - -class AgyIopTest(unittest.TestCase): - def setUp(self) -> None: - self.temp = tempfile.TemporaryDirectory() - self.root = Path(self.temp.name) - self.workspace = self.root / "workspace" - self.workspace.mkdir() - self.session = self.root / "session" - self.session.mkdir() - self.runtime = AgyRuntimeInputs( - sys.executable, "https://private.invalid/gemini/agy-direct", "iop_secret_123456789" - ) - - def tearDown(self) -> None: - self.temp.cleanup() - - def _observation(self, runtime: AgyRuntimeInputs | None = None) -> AgyRuntimeObservation: - value = runtime or self.runtime - return AgyRuntimeObservation( - "agy-direct", "direct", "agy-direct", - _runtime_identity("endpoint", value.endpoint), - _runtime_identity("credential", value.credential), - "sha256:" + "c" * 64, - ) - - def _prepared(self) -> PreparedWorkspace: - return PreparedWorkspace( - AttemptIdentity("run", "agy-direct", 1, 1), str(self.root), str(self.workspace), - str(self.session), "fresh-session", True, "sha256:" + "0" * 64, - "isolated", TestbedProvenance("/testbed", "main", "0" * 40, "sha256:" + "1" * 64, True), - "2026-01-01T00:00:00+00:00", - ) - - def _preflight(self, runtime: AgyRuntimeInputs | None = None): - value = runtime or self.runtime - return preflight_agy_iop( - _cell(), inspect_agy_iop_capability("agy 1.1.12", _help()), value, - self._observation(value), - ) - - def _run_lines(self, lines: list[str], parser: AgyEventParser, *, exit_code: int = 0): - evidence = self.root / f"evidence-{len(list(self.root.glob('evidence-*')))}" - evidence.mkdir() - source = ( - "import sys; lines=" + repr(lines) - + "; [print(line) for line in lines]; raise SystemExit(" + repr(exit_code) + ")" - ) - spec = InvocationSpec( - argv=(sys.executable, "-u", "-c", source), cwd=str(self.root), - env=env_pairs({"PATH": os.environ.get("PATH", "/usr/bin:/bin")}), - submission_mode="stdin_once", completion_mode="exit_after_idle", - timeout=Timeout(5, 1, 1, 1), evidence_dir=str(evidence), task_payload=b"task", - ) - return run_agy_invocation(spec, parser, self._preflight(), lambda _: None) - - def test_official_public_surface_is_pinned_without_invented_environment(self) -> None: - capability = inspect_agy_iop_capability("1.1.12", _help()) - self.assertEqual(capability.version, AGY_KNOWN_VERSION) - self.assertTrue(capability.iop_transport_supported) - self.assertTrue(capability.endpoint_supported) - self.assertTrue(capability.auth_supported) - self.assertFalse(inspect_agy_iop_capability("1.1.11", _help()).iop_transport_supported) - self.assertFalse(inspect_agy_iop_capability("1.1.12", _help().replace("stream-json", "json")).stream_supported) - missing_permission = inspect_agy_iop_capability( - "1.1.12", _help().replace("--dangerously-skip-permissions", "") - ) - self.assertFalse(missing_permission.protocol_supported) - self.assertFalse(missing_permission.iop_transport_supported) - - def test_build_uses_official_gemini_api_key_transport(self) -> None: - with patch.dict(os.environ, {"SSL_CERT_FILE": "/operator/dev-ca.pem", "NODE_EXTRA_CA_CERTS": "/operator/dev-ca.pem"}): - spec = build_agy_invocation(_cell(), self._prepared(), b"one task", Timeout(5, 1, 1, 1), self._preflight()) - environment = dict(spec.env) - self.assertEqual(environment[AGY_ENDPOINT_ENV], self.runtime.endpoint) - self.assertEqual(environment[AGY_AUTH_ENV], self.runtime.credential) - self.assertEqual(environment["HOME"], str(self.session)) - self.assertNotIn("AGY_PROVIDER", environment) - self.assertNotIn("AGY_OPENAI_BASE_URL", environment) - self.assertNotIn("AGY_OPENAI_API_KEY", environment) - self.assertNotIn("--sandbox", spec.argv) - self.assertEqual(spec.argv.count("--dangerously-skip-permissions"), 1) - self.assertNotIn("--effort", spec.argv) - self.assertEqual(spec.argv[spec.argv.index("--model") + 1], "Gemini 3.6 Flash") - self.assertEqual(spec.argv[-2:], ("--print", "one task")) - self.assertEqual(spec.submission_mode, SUBMISSION_ARGV_TASK) - self.assertEqual(spec.task_payload, b"") - self.assertEqual(environment["SSL_CERT_FILE"], "/operator/dev-ca.pem") - self.assertEqual(environment["NODE_EXTRA_CA_CERTS"], "/operator/dev-ca.pem") - self.assertEqual(spec.env_allowlist, (AGY_ENDPOINT_ENV, AGY_AUTH_ENV, "SSL_CERT_FILE", "NODE_EXTRA_CA_CERTS")) - settings_path = self.session / AGY_SETTINGS_RELATIVE_PATH - self.assertEqual(json.loads(settings_path.read_text(encoding="utf-8")), { - "enableTelemetry": False, - "modelProvider": "gemini", - "toolPermission": "always-proceed", - }) - self.assertEqual(settings_path.stat().st_mode & 0o777, 0o600) - settings_text = settings_path.read_text(encoding="utf-8") - self.assertNotIn(self.runtime.endpoint, settings_text) - self.assertNotIn(self.runtime.credential, settings_text) - - def test_build_rejects_preexisting_isolated_settings(self) -> None: - settings_path = self.session / AGY_SETTINGS_RELATIVE_PATH - settings_path.parent.mkdir(parents=True) - settings_path.write_text('{"modelProvider":"other"}\n', encoding="utf-8") - with self.assertRaisesRegex(AgyAdapterError, "isolated provider settings"): - build_agy_invocation( - _cell(), self._prepared(), b"task", Timeout(5, 1, 1, 1), self._preflight() - ) - - def test_build_rejects_missing_prepared_workspace_or_session(self) -> None: - prepared = self._prepared() - for field in ("workspace_dir", "session_dir", "attempt_root"): - missing = prepared.__class__(**{ - **prepared.__dict__, field: str(self.root / f"missing-{field}"), - }) - with self.subTest(field=field): - with self.assertRaisesRegex(AgyAdapterError, "prepared workspace"): - build_agy_invocation( - _cell(), missing, b"task", Timeout(5, 1, 1, 1), self._preflight() - ) - - def test_build_rejects_non_utf8_task_for_print_argument(self) -> None: - with self.assertRaisesRegex(AgyAdapterError, "must be UTF-8"): - build_agy_invocation( - _cell(), self._prepared(), b"\xff", Timeout(5, 1, 1, 1), self._preflight() - ) - - def test_route_qualified_https_runtime_is_required(self) -> None: - for endpoint in ( - "http://private.invalid/gemini/agy-direct", - "https://private.invalid/v1", - "https://private.invalid/gemini/other", - ): - runtime = replace(self.runtime, endpoint=endpoint) - result = self._preflight(runtime) - self.assertEqual(result.status, "implementation_gap") - self.assertEqual([issue.code for issue in result.issues], ["endpoint_incompatible"]) - missing = replace(self.runtime, credential="") - result = preflight_agy_iop( - _cell(), inspect_agy_iop_capability("1.1.12", _help()), missing, - self._observation(self.runtime), - ) - self.assertEqual(result.status, "registration_required") - self.assertEqual([issue.code for issue in result.issues], ["credential_missing"]) - - def test_unknown_model_and_unvalidated_observation_fail_closed(self) -> None: - unsupported = replace(_cell(), iop=replace(_cell().iop, request_model="gemini-unknown")) - result = preflight_agy_iop( - unsupported, inspect_agy_iop_capability("1.1.12", _help()), self.runtime, - self._observation(), - ) - self.assertIn("model_missing", [issue.code for issue in result.issues]) - mismatched = replace(self._observation(), endpoint_identity="sha256:" + "d" * 64) - result = preflight_agy_iop( - _cell(), inspect_agy_iop_capability("1.1.12", _help()), self.runtime, mismatched, - ) - self.assertIsNone(result.runtime) - with self.assertRaises(AgyAdapterError): - build_agy_invocation(_cell(), self._prepared(), b"task", Timeout(5, 1, 1, 1), result) - - def test_official_fixture_completes_and_preserves_metrics(self) -> None: - parser = AgyEventParser(_cell(), _binding()) - fixture = Path("scripts/fixtures/agent-comparison-benchmark/agy-iop-stream.jsonl") - result = self._run_lines(fixture.read_text(encoding="utf-8").splitlines(), parser) - self.assertTrue(result.product.status == "succeeded") - self.assertTrue(result.harness.ordered_terminal) - metrics = {metric.name: metric for metric in result.metrics} - self.assertEqual( - set(metrics), - {"total_duration", "model_calls", "input_tokens", "cached_input_tokens", "output_tokens", "reasoning_tokens", "total_tokens"}, - ) - self.assertEqual(metrics["total_duration"].value, 12_000_000) - self.assertEqual(metrics["model_calls"].value, 1) - self.assertEqual(metrics["total_tokens"].value, 12) - capability = inspect_agy_iop_capability("1.1.12", _help()) - self.assertEqual(parser.observed_result(capability, result).binding, _binding()) - - def test_result_must_follow_init_be_unique_and_successful(self) -> None: - result = { - "event": "result", "result": { - "status": "SUCCESS", "duration_seconds": 0.1, "num_turns": 1, - "usage": {"input_tokens": 1}, "response": "secret content", - }, - } - for lines, reason in ( - ([json.dumps(result)], REASON_MALFORMED_EVENT), - ([json.dumps({"event": "init", "init": {}}), json.dumps(result), json.dumps(result)], REASON_MALFORMED_EVENT), - ): - parser = AgyEventParser(_cell(), _binding()) - invocation = self._run_lines(lines, parser) - self.assertFalse(invocation.product.status == "succeeded") - self.assertEqual(invocation.harness.reason, "parser_error") - - parser = AgyEventParser(_cell(), _binding()) - error_result = json.dumps({ - "event": "result", "result": {"status": "ERROR", "response": "private"}, - }) - invocation = self._run_lines([ - json.dumps({"event": "init", "init": {}}), error_result, - ], parser, exit_code=1) - self.assertFalse(invocation.product.status == "succeeded") - self.assertEqual(invocation.product.status, "failed") - self.assertEqual(invocation.harness.status, "passed") - self.assertEqual(invocation.process.exit_code, 1) - - def test_latest_step_usage_is_used_only_when_result_omits_usage(self) -> None: - parser = AgyEventParser(_cell(), _binding()) - lines = [ - json.dumps({"event": "init", "conversation_id": "c", "init": {"model": "Gemini 3.6 Flash"}}), - json.dumps({"event": "step_update", "step_update": {"state": "DONE", "step_index": 0, "step_type": "agent_response", "text_delta": "private", "usage": {"input_tokens": 2, "output_tokens": 1, "total_tokens": 3}}}), - json.dumps({"event": "result", "result": {"status": "SUCCESS", "duration_seconds": 0.2, "num_turns": 1, "response": "private"}}), - ] - invocation = self._run_lines(lines, parser) - self.assertTrue(invocation.product.status == "succeeded") - self.assertEqual({m.name: m.value for m in invocation.metrics}["total_tokens"], 3) - - def test_malformed_usage_fails_without_partial_metric(self) -> None: - parser = AgyEventParser(_cell(), _binding()) - invocation = self._run_lines([ - '{"event":"init","init":{}}', - '{"event":"result","result":{"status":"SUCCESS","usage":{"input_tokens":"1"}}}', - ], parser) - self.assertFalse(invocation.product.status == "succeeded") - self.assertEqual(invocation.harness.reason, "parser_error") - self.assertEqual(invocation.metrics, ()) - - def test_structural_redaction_excludes_response_tools_endpoint_and_secret(self) -> None: - raw = json.dumps({ - "event": "result", "result": { - "status": "SUCCESS", "response": "raw prompt", - "tool_input": {"secret": "x"}, "endpoint": self.runtime.endpoint, - "token": self.runtime.credential, - }, - }) - redacted = redact_agy_event(raw, (self.runtime.endpoint, self.runtime.credential)) - self.assertEqual(redacted, '{"event":"result","status":"SUCCESS"}') - for forbidden in ("raw prompt", "tool_input", self.runtime.endpoint, self.runtime.credential): - self.assertNotIn(forbidden, redacted) - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/agent_benchmark/attempts.py b/scripts/agent_benchmark/attempts.py deleted file mode 100644 index a4ddb20f..00000000 --- a/scripts/agent_benchmark/attempts.py +++ /dev/null @@ -1,2376 +0,0 @@ -"""Durable, append-only benchmark run and attempt state. - -Attempt-directory creation is the allocation marker. The directory remains -empty until the workspace layer has prepared it; only then is the identity -bound running record published. This preserves the workspace API's -empty-root contract while retaining a crash-recoverable allocation boundary. -""" - -from __future__ import annotations - -import contextlib -import datetime as _datetime -import fcntl -import hashlib -import json -import os -import re -import secrets -import stat -import tempfile -from dataclasses import dataclass -from pathlib import Path -from types import SimpleNamespace -from typing import Any, Callable, Iterator, Mapping, Protocol - -from scripts.agent_benchmark.connectivity import ( - CallerCapability, - ConnectivityIssue, - ConnectivityResult, - EffectiveBinding, - RequestedEffectiveBinding, - canonical_evidence_bytes, - validate_requested_binding, - validate_result, -) - -from scripts.agent_benchmark.lifecycle import ( - CALLER_REASON_SUCCESS, - COMPLETION_MODES, - EVENT_CALLER_TERMINAL, - EVENT_FINISH, - EVENT_IDLE, - EVENT_QUIET, - EVENT_SUBMITTED, - InvocationResult, - JOURNAL_VERSION, - HARNESS_REASONS, - HARNESS_STATUSES, - HarnessOutcome, - LifecycleValidationError, - PROCESS_STATUSES, - ProcessOutcome, - PRODUCT_REASONS, - PRODUCT_STATUSES, - ProductOutcome, - LifecycleRecoveryError, - RECEIPT_VERSION, - REASON_CONTROLLER_LOST, - REASON_RECOVERED_STOP, - SOURCE_CALLER_OUTPUT, - SOCKET_FILENAME, - SUBMISSION_MODES, - SupervisorLocator, - TERMINAL_REASONS, - recover_invocation, -) -from scripts.agent_benchmark.manifest import ( - Manifest, - MatrixCell, - Timeout, - validate_manifest_bytes, -) -from scripts.agent_benchmark.measurement import ( - MEASUREMENT_FILENAME, - MeasurementError, - WorkspaceWriteObservation, - WorkspaceWriteObserver, - build_measurement, - build_recovery_measurement, - load_measurement, - publish_measurement, - validate_measurement_lifecycle_binding, -) -from scripts.agent_benchmark.web_validation import ( - WEB_VALIDATION_FILENAME, - WebValidationError, - load_web_validation, - publish_web_validation, - validate_web_attempt, -) -from scripts.agent_benchmark.workspace import AttemptIdentity, PreparedWorkspace - -RUN_ID_RE = re.compile(r"^run-[0-9]{8}T[0-9]{6}Z-[0-9a-f]{12}$") -ATTEMPT_RE = re.compile(r"^attempt-([0-9]{6})$") -PREFLIGHT_RE = re.compile(r"^preflight-([0-9]{6})\.json$") -DIGEST_RE = re.compile(r"^sha256:[0-9a-f]{64}$") -PREFLIGHT_SCHEMA_VERSION = "1" -PREFLIGHT_STATUSES = ("ready", "registration_required", "implementation_gap") -TERMINAL_STATES = frozenset(("completed", "timed_out", "cancelled", "interrupted")) -NONTERMINAL_STATE = "running" -ATTEMPT_RESULT_VERSION = 2 -MEASUREMENT_POLICY_REQUIRED_V1 = "required-v1" -MEASUREMENT_POLICIES = frozenset((MEASUREMENT_POLICY_REQUIRED_V1,)) -MEASUREMENT_POLICY_FILENAME = "attempt-measurement-policy.json" -MEASUREMENT_POLICY_RECORD = "attempt-measurement-policy" -MEASUREMENT_POLICY_VERSION = 1 -WEB_VALIDATION_POLICY_REQUIRED_V1 = "required-v1" -WEB_VALIDATION_POLICIES = frozenset((WEB_VALIDATION_POLICY_REQUIRED_V1,)) -WEB_VALIDATION_POLICY_FILENAME = "web-validation-policy.json" -WEB_VALIDATION_POLICY_RECORD = "web-validation-policy" -WEB_VALIDATION_POLICY_VERSION = 1 -SUCCESS_EVIDENCE_KINDS = (EVENT_SUBMITTED, EVENT_FINISH, EVENT_IDLE, EVENT_QUIET) -BOUND_EVIDENCE_KINDS = SUCCESS_EVIDENCE_KINDS + (EVENT_CALLER_TERMINAL,) -CONTROL_ALIAS_PREFIX = "iop-bench-attempt-" -CONTROL_DIRECTORY_NAME = "control" -CONTROL_ALIAS_DIGEST_HEX_LENGTH = 24 -UNIX_SOCKET_PATH_MAX_BYTES = 103 -RECEIPT_FIELDS = { - "receipt_version", "supervisor_pid", "challenge_digest", "reason", "exit_code", - "signal", "caller_launched", "cleanup_complete", "process_group_alive", "completed_at", -} -RECEIPT_ONLY_TERMINAL_REASONS = frozenset( - (REASON_CONTROLLER_LOST, REASON_RECOVERED_STOP) -) - - -def _outcome_records(result: InvocationResult) -> dict[str, dict[str, Any]]: - return { - "product": { - "status": result.product.status, - "reason": result.product.reason, - }, - "harness": { - "status": result.harness.status, - "reason": result.harness.reason, - "ordered_terminal": result.harness.ordered_terminal, - "cleanup_complete": result.harness.cleanup_complete, - }, - "process": { - "status": result.process.status, - "exit_code": result.process.exit_code, - "signal": result.process.signal, - }, - } - - -def _unknown_terminal( - reason: str, *, process_status: str = "not_started", - exit_code: int | None = None, signal: int | None = None, -) -> dict[str, dict[str, Any]]: - return { - "product": {"status": "unknown", "reason": "unavailable"}, - "harness": { - "status": "failed", - "reason": reason, - "ordered_terminal": False, - "cleanup_complete": True, - }, - "process": { - "status": process_status, - "exit_code": exit_code, - "signal": signal, - }, - } - - -def _terminal_reason(terminal: Mapping[str, Any]) -> str: - harness = terminal.get("harness") - return str(harness.get("reason") if isinstance(harness, Mapping) else "") - - -def _terminal_passed(terminal: Mapping[str, Any]) -> bool: - process = terminal.get("process") - return ( - isinstance(terminal.get("product"), Mapping) - and terminal["product"].get("status") == "succeeded" - and isinstance(terminal.get("harness"), Mapping) - and terminal["harness"].get("status") == "passed" - and isinstance(process, Mapping) - and process.get("status") == "exited" - and process.get("exit_code") == 0 - and process.get("signal") is None - ) - - -class AttemptError(Exception): - """Base error whose message is safe to present to a benchmark caller.""" - - -class RunPathError(AttemptError): - pass - - -class RunBusyError(AttemptError): - pass - - -class AttemptStateError(AttemptError): - pass - - -class CapabilityUnavailable(AttemptError): - pass - - -@dataclass(frozen=True) -class RunIdentity: - run_id: str - manifest_digest: str - root: str - - -@dataclass(frozen=True) -class Slot: - cell_id: str - repetition: int - - -@dataclass(frozen=True) -class Attempt: - identity: AttemptIdentity - root: str - state: str - - -@dataclass(frozen=True) -class PreflightObservation: - """One adapter result plus opaque identities safe for durable evidence.""" - - result: ConnectivityResult - endpoint_identity: str - config_identity: str - - -@dataclass(frozen=True) -class AttemptControlLease: - """One deterministic short pathname bound to an exact attempt root.""" - - alias: str - control_dir: str - socket_path: str - - -class PreflightAdapter(Protocol): - """Closed adapter boundary consumed by the public preflight controller.""" - - capability: CallerCapability - - def preflight(self, cell: "MatrixCell") -> PreflightObservation: - """Return one typed observation without exposing raw caller output.""" - - -class ExecutionAdapter(PreflightAdapter, Protocol): - """Typed caller boundary for one preflight-approved scored attempt.""" - - def invoke( - self, - cell: "MatrixCell", - prepared: PreparedWorkspace, - attempt: Attempt, - control_dir: str, - task_payload: bytes, - timeout: Timeout, - on_started: Callable[[SupervisorLocator, str], None], - ) -> InvocationResult: - """Submit the exact task once for the bound cell and prepared identity.""" - - -def _json_bytes(value: Any) -> bytes: - return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("utf-8") + b"\n" - - -def _fsync_dir(path: Path) -> None: - fd = os.open(path, os.O_RDONLY) - try: - os.fsync(fd) - finally: - os.close(fd) - - -def _write_new(path: Path, data: bytes) -> None: - fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) - try: - os.write(fd, data) - os.fsync(fd) - finally: - os.close(fd) - _fsync_dir(path.parent) - - -def _replace(path: Path, data: bytes) -> None: - fd, tmp = tempfile.mkstemp(prefix=".attempt-", dir=path.parent) - try: - with os.fdopen(fd, "wb") as handle: - handle.write(data) - handle.flush() - os.fsync(handle.fileno()) - os.chmod(tmp, 0o600) - os.replace(tmp, path) - _fsync_dir(path.parent) - finally: - try: - os.unlink(tmp) - except FileNotFoundError: - pass - - -def _open_regular(path: Path, label: str, flags: int = os.O_RDONLY) -> int: - """Open one durable file without following links, blocking or trusting its type.""" - try: - fd = os.open(path, flags | os.O_NOFOLLOW | os.O_NONBLOCK) - except OSError as exc: - raise AttemptStateError(f"{label} is unavailable") from exc - try: - opened = os.fstat(fd) - if not stat.S_ISREG(opened.st_mode): - raise AttemptStateError(f"{label} must be a regular file") - except BaseException: - os.close(fd) - raise - return fd - - -def _read_regular_bytes(path: Path, label: str) -> bytes: - """Read only the no-follow descriptor that was verified as regular.""" - fd = _open_regular(path, label) - try: - chunks: list[bytes] = [] - while True: - chunk = os.read(fd, 1 << 20) - if not chunk: - return b"".join(chunks) - chunks.append(chunk) - finally: - os.close(fd) - - -def _utc_timestamp(clock: Callable[[], _datetime.datetime]) -> str: - return clock().astimezone(_datetime.timezone.utc).strftime("%Y%m%dT%H%M%SZ") - - -def _directory(path: Path, label: str) -> None: - try: - mode = os.lstat(path).st_mode - except OSError as exc: - raise RunPathError(f"{label} is unavailable") from exc - if not stat.S_ISDIR(mode): - raise RunPathError(f"{label} is invalid") - - -def _contained(path: Path, root: Path) -> bool: - try: - path.resolve(strict=False).relative_to(root.resolve()) - return True - except ValueError: - return False - - -def _requested_binding(cell: MatrixCell) -> RequestedEffectiveBinding: - return RequestedEffectiveBinding( - cell.id, - cell.caller, - cell.iop.route_kind, - cell.iop.route_id, - cell.iop.request_model, - cell.iop.requested_effort, - ) - - -def _overall_preflight_status(statuses: Iterator[str]) -> str: - found = tuple(statuses) - if not found: - raise AttemptStateError("preflight result set is empty") - if any(status == "implementation_gap" for status in found): - return "implementation_gap" - if any(status == "registration_required" for status in found): - return "registration_required" - if all(status == "ready" for status in found): - return "ready" - raise AttemptStateError("preflight status is invalid") - - -def _preflight_counts(results: list[dict[str, Any]]) -> dict[str, int]: - counts = {status: 0 for status in PREFLIGHT_STATUSES} - for result in results: - status = result.get("status") - if status not in counts: - raise AttemptStateError("preflight result status is invalid") - counts[str(status)] += 1 - return counts - - -def _connectivity_result_from_payload( - payload: dict[str, Any], cell: MatrixCell -) -> tuple[ConnectivityResult, str, str]: - """Rebuild one closed result so durable reads re-run semantic validation.""" - if not isinstance(payload, dict) or set(payload) != { - "schema_version", "cell", "status", "binding", "issues", - "endpoint_identity", "config_identity", - }: - raise AttemptStateError("preflight result schema is invalid") - binding_raw = payload.get("binding") - if not isinstance(binding_raw, dict) or set(binding_raw) != { - "cell_id", "caller", "requested_route_kind", "requested_route_id", - "requested_model", "requested_effort", "effective_route_kind", - "effective_route_id", "effective_model", "effective_effort", - "effective_bindings", - }: - raise AttemptStateError("preflight result binding is invalid") - stages_raw = binding_raw.get("effective_bindings") - if not isinstance(stages_raw, list): - raise AttemptStateError("preflight result binding is invalid") - stages: list[EffectiveBinding] = [] - for stage in stages_raw: - if not isinstance(stage, dict) or set(stage) != {"stage", "model", "effort"}: - raise AttemptStateError("preflight result binding is invalid") - stages.append(EffectiveBinding(stage["stage"], stage["model"], stage["effort"])) - issues_raw = payload.get("issues") - if not isinstance(issues_raw, list): - raise AttemptStateError("preflight result issues are invalid") - issues: list[ConnectivityIssue] = [] - for issue in issues_raw: - if not isinstance(issue, dict) or set(issue) != {"code", "resume_code"}: - raise AttemptStateError("preflight result issues are invalid") - issues.append(ConnectivityIssue(issue["code"], issue["resume_code"])) - try: - binding = RequestedEffectiveBinding( - binding_raw["cell_id"], - binding_raw["caller"], - binding_raw["requested_route_kind"], - binding_raw["requested_route_id"], - binding_raw["requested_model"], - binding_raw["requested_effort"], - binding_raw["effective_route_kind"], - binding_raw["effective_route_id"], - binding_raw["effective_model"], - binding_raw["effective_effort"], - tuple(stages), - ) - # Capability is deliberately not serialized. The persisted proof is - # revalidated against the exact immutable cell and its declared route. - capability = CallerCapability( - cell.caller, (cell.iop.route_kind,), (cell.iop.requested_effort,) - ) - result = ConnectivityResult(capability, binding, tuple(issues), payload["status"]) - validate_result(cell, result) - endpoint_identity = payload["endpoint_identity"] - config_identity = payload["config_identity"] - canonical = json.loads( - canonical_evidence_bytes( - cell, result, endpoint_identity, config_identity - ).decode("ascii") - ) - except Exception as exc: - raise AttemptStateError("preflight result is invalid") from exc - if payload != canonical or payload.get("cell") != {"id": cell.id, "caller": cell.caller}: - raise AttemptStateError("preflight result is non-canonical") - return result, endpoint_identity, config_identity - - -def collect_preflight_observations( - manifest: Manifest, - adapters: Mapping[str, PreflightAdapter], -) -> dict[str, PreflightObservation]: - """Validate the full registry, then probe every cell in manifest order.""" - if not isinstance(adapters, Mapping): - raise CapabilityUnavailable("capability-unavailable: caller-adapter") - required_callers = {cell.caller for cell in manifest.matrix} - if any(caller not in adapters for caller in required_callers): - raise CapabilityUnavailable("capability-unavailable: caller-adapter") - - for cell in manifest.matrix: - adapter = adapters[cell.caller] - capability = getattr(adapter, "capability", None) - validate_requested_binding(cell, capability, _requested_binding(cell)) - - observations: dict[str, PreflightObservation] = {} - for cell in manifest.matrix: - observation = adapters[cell.caller].preflight(cell) - if not isinstance(observation, PreflightObservation): - raise AttemptStateError("preflight observation is invalid") - validate_result(cell, observation.result) - # Materializing the canonical bytes validates both opaque identities and - # proves that no adapter-specific/raw value can enter the durable record. - canonical_evidence_bytes( - cell, - observation.result, - observation.endpoint_identity, - observation.config_identity, - ) - observations[cell.id] = observation - return observations - - -class RunStore: - """Filesystem-backed run store rooted at a validated manifest output root.""" - - def __init__( - self, - repo_root: str | Path, - *, - clock: Callable[[], _datetime.datetime] = lambda: _datetime.datetime.now(_datetime.timezone.utc), - token_hex: Callable[[int], str] = secrets.token_hex, - ) -> None: - self.repo_root = Path(repo_root).resolve() - self._clock = clock - self._token_hex = token_hex - - def _output_root(self, manifest: Manifest, *, create: bool) -> Path: - root = self.repo_root / manifest.output_root - if not _contained(root, self.repo_root): - raise RunPathError("output root is invalid") - if create: - root.mkdir(mode=0o700, parents=True, exist_ok=True) - _directory(root, "output root") - if root.is_symlink(): - raise RunPathError("output root is invalid") - return root.resolve() - - def _run_path(self, manifest: Manifest, run_id: str, *, create_root: bool) -> Path: - if not RUN_ID_RE.fullmatch(run_id): - raise RunPathError("run id is invalid") - root = self._output_root(manifest, create=create_root) - path = root / run_id - if path.parent != root or path.is_symlink(): - raise RunPathError("run path is invalid") - return path - - def create(self, manifest: Manifest, manifest_bytes: bytes) -> RunIdentity: - """Create a run and atomically persist immutable source bytes first.""" - loaded = validate_manifest_bytes(manifest_bytes, repo_root=self.repo_root) - if loaded.digest != manifest.digest: - raise AttemptStateError("manifest snapshot does not match manifest digest") - run_id = f"run-{_utc_timestamp(self._clock)}-{self._token_hex(6)}" - if not RUN_ID_RE.fullmatch(run_id): - raise AttemptStateError("generated run id is invalid") - path = self._run_path(manifest, run_id, create_root=True) - try: - path.mkdir(mode=0o700) - except FileExistsError as exc: - raise AttemptStateError("generated run id already exists") from exc - _write_new(path / "manifest.json", manifest_bytes) - _write_new(path / "run.json", _json_bytes({"run_id": run_id, "manifest_digest": manifest.digest})) - _write_new(path / "run.lock", b"") - _fsync_dir(path) - return RunIdentity(run_id, manifest.digest, str(path)) - - def open(self, manifest: Manifest, run_id: str, manifest_bytes: bytes | None = None) -> RunIdentity: - path = self._run_path(manifest, run_id, create_root=False) - _directory(path, "run") - try: - record = json.loads(_read_regular_bytes(path / "run.json", "run record").decode("utf-8")) - snapshot = _read_regular_bytes(path / "manifest.json", "manifest snapshot") - _read_regular_bytes(path / "run.lock", "run lock") - except (OSError, json.JSONDecodeError) as exc: - raise AttemptStateError("run state is unavailable") from exc - if record != {"run_id": run_id, "manifest_digest": manifest.digest}: - raise AttemptStateError("run identity does not match manifest") - if manifest_bytes is not None and snapshot != manifest_bytes: - raise AttemptStateError("manifest bytes do not match run snapshot") - if validate_manifest_bytes(snapshot, repo_root=self.repo_root).digest != manifest.digest: - raise AttemptStateError("run manifest digest is invalid") - return RunIdentity(run_id, manifest.digest, str(path)) - - @contextlib.contextmanager - def writer(self, run: RunIdentity) -> Iterator[None]: - root = Path(run.root) - _directory(root, "run") - lock_path = root / "run.lock" - fd = _open_regular(lock_path, "run lock", os.O_RDWR) - try: - try: - fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) - except BlockingIOError as exc: - raise RunBusyError("run-busy") from exc - yield - finally: - try: - fcntl.flock(fd, fcntl.LOCK_UN) - finally: - os.close(fd) - - @staticmethod - def _preflight_cells(manifest: Manifest) -> tuple[MatrixCell, ...]: - return tuple(manifest.matrix) - - def _preflight_root(self, run: RunIdentity, *, create: bool) -> Path: - root = Path(run.root) / "preflight" - if root.exists() or root.is_symlink(): - try: - mode = os.lstat(root).st_mode - except OSError as exc: - raise AttemptStateError("preflight state is unavailable") from exc - if not stat.S_ISDIR(mode) or root.is_symlink(): - raise AttemptStateError("preflight state is invalid") - return root - if not create: - return root - try: - root.mkdir(mode=0o700) - _fsync_dir(root.parent) - except OSError as exc: - raise AttemptStateError("preflight state is unavailable") from exc - return root - - def _validate_preflight_record( - self, - raw: bytes, - path: Path, - run: RunIdentity, - manifest: Manifest, - expected_sequence: int, - ) -> dict[str, Any]: - try: - record = json.loads(raw.decode("ascii")) - except (UnicodeDecodeError, json.JSONDecodeError) as exc: - raise AttemptStateError("preflight record is invalid") from exc - if not isinstance(record, dict) or set(record) != { - "schema_version", "run_id", "manifest_digest", "sequence", - "status", "results", - }: - raise AttemptStateError("preflight record schema is invalid") - if ( - record["schema_version"] != PREFLIGHT_SCHEMA_VERSION - or record["run_id"] != run.run_id - or record["manifest_digest"] != run.manifest_digest - or record["sequence"] != expected_sequence - or record["status"] not in PREFLIGHT_STATUSES - or not isinstance(record["results"], list) - ): - raise AttemptStateError("preflight record identity is invalid") - preflight_cells = self._preflight_cells(manifest) - if len(record["results"]) != len(preflight_cells): - raise AttemptStateError("preflight result set is invalid") - statuses: list[str] = [] - for cell, result_payload in zip(preflight_cells, record["results"]): - result, _, _ = _connectivity_result_from_payload(result_payload, cell) - statuses.append(result.status) - if record["status"] != _overall_preflight_status(iter(statuses)): - raise AttemptStateError("preflight aggregate status is invalid") - if raw != _json_bytes(record): - raise AttemptStateError("preflight record is non-canonical") - expected_name = f"preflight-{expected_sequence:06d}.json" - if path.name != expected_name: - raise AttemptStateError("preflight sequence is invalid") - return record - - def _preflight_records( - self, run: RunIdentity, manifest: Manifest - ) -> tuple[dict[str, Any], ...]: - root = self._preflight_root(run, create=False) - if not root.exists() and not root.is_symlink(): - return () - records: list[dict[str, Any]] = [] - for expected_sequence, child in enumerate( - sorted(root.iterdir(), key=lambda item: item.name), start=1 - ): - match = PREFLIGHT_RE.fullmatch(child.name) - if match is None or int(match.group(1)) != expected_sequence: - raise AttemptStateError("preflight sequence is invalid") - raw = _read_regular_bytes(child, "preflight record") - records.append( - self._validate_preflight_record( - raw, child, run, manifest, expected_sequence - ) - ) - return tuple(records) - - def preflights( - self, run: RunIdentity, manifest: Manifest - ) -> tuple[dict[str, Any], ...]: - """Read append-only preflight records without creating or reconciling state.""" - bound_run = self.open(manifest, run.run_id) - if bound_run != run: - raise AttemptStateError("run identity is invalid") - return self._preflight_records(bound_run, manifest) - - def record_preflight( - self, - run: RunIdentity, - manifest: Manifest, - observations: Mapping[str, PreflightObservation], - ) -> dict[str, Any]: - """Append one canonical result set while exclusively owning the run writer.""" - bound_run = self.open(manifest, run.run_id) - if bound_run != run: - raise AttemptStateError("run identity is invalid") - with self.writer(bound_run): - return self._record_preflight_locked(bound_run, manifest, observations) - - def _record_preflight_locked( - self, - run: RunIdentity, - manifest: Manifest, - observations: Mapping[str, PreflightObservation], - ) -> dict[str, Any]: - """Append one preflight while the caller owns the run writer.""" - bound_run = self.open(manifest, run.run_id) - if bound_run != run: - raise AttemptStateError("run identity is invalid") - preflight_cells = self._preflight_cells(manifest) - if set(observations) != {cell.id for cell in preflight_cells}: - raise AttemptStateError("preflight observation set is invalid") - results: list[dict[str, Any]] = [] - for cell in preflight_cells: - observation = observations[cell.id] - if not isinstance(observation, PreflightObservation): - raise AttemptStateError("preflight observation is invalid") - try: - encoded = canonical_evidence_bytes( - cell, - observation.result, - observation.endpoint_identity, - observation.config_identity, - ) - payload = json.loads(encoded.decode("ascii")) - except Exception as exc: - raise AttemptStateError("preflight observation is invalid") from exc - _connectivity_result_from_payload(payload, cell) - results.append(payload) - status = _overall_preflight_status( - iter(str(result["status"]) for result in results) - ) - - previous = self._preflight_records(bound_run, manifest) - sequence = len(previous) + 1 - root = self._preflight_root(bound_run, create=True) - path = root / f"preflight-{sequence:06d}.json" - record = { - "schema_version": PREFLIGHT_SCHEMA_VERSION, - "run_id": bound_run.run_id, - "manifest_digest": bound_run.manifest_digest, - "sequence": sequence, - "status": status, - "results": results, - } - raw = _json_bytes(record) - _write_new(path, raw) - self._validate_preflight_record( - _read_regular_bytes(path, "preflight record"), - path, - bound_run, - manifest, - sequence, - ) - return record - - @staticmethod - def slots(manifest: Manifest) -> tuple[Slot, ...]: - return tuple(Slot(cell.id, repetition) for cell in manifest.matrix for repetition in range(1, manifest.repetitions + 1)) - - def _slot_path(self, run: RunIdentity, slot: Slot) -> Path: - if not re.fullmatch(r"[a-z0-9][a-z0-9_-]{0,63}", slot.cell_id) or slot.repetition < 1: - raise AttemptStateError("slot is invalid") - return Path(run.root) / "cells" / slot.cell_id / f"repetition-{slot.repetition:04d}" - - def _ensure_slot_parent(self, run: RunIdentity, slot: Slot) -> Path: - current = Path(run.root) - for segment in ("cells", slot.cell_id, f"repetition-{slot.repetition:04d}"): - current = current / segment - if current.exists() or current.is_symlink(): - _directory(current, "slot state") - if current.is_symlink(): - raise AttemptStateError("slot state is invalid") - continue - try: - current.mkdir(mode=0o700) - except FileExistsError: - _directory(current, "slot state") - if current.is_symlink(): - raise AttemptStateError("slot state is invalid") - return current - - @staticmethod - def _expected_record(run: RunIdentity, identity: AttemptIdentity, state: str) -> dict[str, Any]: - return { - "attempt_result_version": ATTEMPT_RESULT_VERSION, - "run_id": run.run_id, - "manifest_digest": run.manifest_digest, - "cell_id": identity.cell_id, - "repetition": identity.repetition, - "attempt": identity.attempt, - "state": state, - } - - def _attempt_record(self, root: Path, run: RunIdentity, identity: AttemptIdentity, *, absent_ok: bool = False) -> dict[str, Any] | None: - path = root / "attempt.json" - try: - record = json.loads(_read_regular_bytes(path, "attempt record").decode("utf-8")) - except AttemptStateError: - if absent_ok: - try: - os.lstat(path) - except FileNotFoundError: - if self._measurement_policy_start(root, run, identity) is not None: - raise AttemptStateError( - "attempt measurement policy provenance is invalid" - ) - return None - raise - except (OSError, json.JSONDecodeError) as exc: - raise AttemptStateError("attempt record is unavailable") from exc - if not isinstance(record, dict): - raise AttemptStateError("attempt record is invalid") - expected = self._expected_record(run, identity, str(record.get("state", ""))) - if record.get("state") not in TERMINAL_STATES | {NONTERMINAL_STATE}: - raise AttemptStateError("attempt record is invalid") - for key, value in expected.items(): - if record.get(key) != value: - raise AttemptStateError("attempt record identity is invalid") - allowed = set(expected) | { - "locator", "spec_digest", "lifecycle", "measurement_policy", - "web_validation_policy", - } - if set(record) - allowed: - raise AttemptStateError("attempt record schema is invalid") - policy = record.get("measurement_policy") - if policy is not None and policy not in MEASUREMENT_POLICIES: - raise AttemptStateError("attempt measurement policy is invalid") - self._validate_measurement_policy_start(root, run, identity, policy) - web_policy = record.get("web_validation_policy") - if web_policy is not None and web_policy not in WEB_VALIDATION_POLICIES: - raise AttemptStateError("attempt web validation policy is invalid") - self._validate_web_validation_policy_start(root, run, identity, web_policy) - if record["state"] == NONTERMINAL_STATE and "lifecycle" in record: - raise AttemptStateError("running attempt has terminal evidence") - if ("locator" in record) != ("spec_digest" in record): - raise AttemptStateError("attempt invocation identity is invalid") - if "locator" in record: - locator = self._locator_from_record( - root, record["locator"], state=str(record["state"]) - ) - if not isinstance(record["spec_digest"], str) or not DIGEST_RE.fullmatch(record["spec_digest"]): - raise AttemptStateError("attempt invocation digest is invalid") - if record["state"] in TERMINAL_STATES: - lifecycle = record.get("lifecycle") - expected_receipt_reason = ( - _terminal_reason(lifecycle) - if isinstance(lifecycle, dict) - else None - ) - self._validate_terminal_invocation_identity( - root, - locator, - record["spec_digest"], - run=run, - identity=identity, - terminal_state=str(record["state"]), - expected_receipt_reason=expected_receipt_reason, - measurement_policy=policy, - ) - if "lifecycle" in record: - self._validate_outcomes(record["lifecycle"], "attempt lifecycle") - if record["state"] in TERMINAL_STATES: - pre_registration_interrupted = ( - record["state"] == "interrupted" - and "locator" not in record - and record.get("lifecycle") == _unknown_terminal("interrupted") - ) - self._validate_web_validation( - root, - run, - identity, - web_policy, - allow_pre_registration_absence=pre_registration_interrupted, - ) - elif (root / WEB_VALIDATION_FILENAME).exists() or ( - root / WEB_VALIDATION_FILENAME - ).is_symlink(): - self._validate_web_validation(root, run, identity, web_policy) - return record - - def attempts(self, run: RunIdentity, slot: Slot) -> tuple[Attempt, ...]: - parent = self._slot_path(run, slot) - if not parent.exists() and not parent.is_symlink(): - return () - _directory(parent, "slot state") - found: list[Attempt] = [] - for child in sorted(parent.iterdir(), key=lambda item: item.name): - match = ATTEMPT_RE.fullmatch(child.name) - if not match or child.is_symlink() or not child.is_dir(): - raise AttemptStateError("slot contains invalid state") - number = int(match.group(1)) - identity = AttemptIdentity(run.run_id, slot.cell_id, slot.repetition, number) - record = self._attempt_record(child, run, identity, absent_ok=True) - state = NONTERMINAL_STATE if record is None else str(record["state"]) - found.append(Attempt(identity, str(child), state)) - return tuple(found) - - def execution_attempts( - self, run: RunIdentity, manifest: Manifest - ) -> tuple[Attempt, ...]: - """Enumerate every retained execution attempt in canonical slot order. - - Scoring and later reporting use this read-only projection instead of - reconstructing the private ``cells/`` directory grammar. - """ - bound_run = self.open(manifest, run.run_id) - if bound_run != run: - raise AttemptStateError("run identity is invalid") - return tuple( - attempt - for slot in self.slots(manifest) - for attempt in self.attempts(bound_run, slot) - ) - - def attempt_outcomes(self, attempt: Attempt) -> dict[str, Any]: - """Return the independently validated gates for one terminal attempt.""" - run, root = self._bound_attempt(attempt) - record = self._attempt_record(root, run, attempt.identity) - if record is None or record["state"] not in TERMINAL_STATES: - raise AttemptStateError("attempt outcomes require terminal state") - lifecycle = record.get("lifecycle") - if not isinstance(lifecycle, Mapping): - raise AttemptStateError("attempt lifecycle is unavailable") - try: - artifact = load_web_validation(root, manifest=self.open_manifest_snapshot(run)).status - except WebValidationError: - # The only permitted terminal without a web record is a controller - # interruption before caller registration. It remains unresolved. - artifact = "not_run" - return { - "product": lifecycle["product"]["status"], - "harness": lifecycle["harness"]["status"], - "process": lifecycle["process"]["status"], - "artifact": artifact, - "resolved": artifact != "not_run", - "passed": ( - record["state"] == "completed" - and _terminal_passed(lifecycle) - and artifact == "passed" - ), - } - - def allocate(self, run: RunIdentity, slot: Slot) -> Attempt: - """Create the exclusive, deliberately empty attempt root.""" - existing = self.attempts(run, slot) - if any(item.state not in TERMINAL_STATES for item in existing): - raise AttemptStateError("slot already has a nonterminal attempt") - number = max((item.identity.attempt for item in existing), default=0) + 1 - parent = self._ensure_slot_parent(run, slot) - root = parent / f"attempt-{number:06d}" - try: - root.mkdir(mode=0o700) - _fsync_dir(parent) - except FileExistsError as exc: - raise AttemptStateError("attempt allocation collision") from exc - return Attempt(AttemptIdentity(run.run_id, slot.cell_id, slot.repetition, number), str(root), NONTERMINAL_STATE) - - def _bound_attempt(self, attempt: Attempt) -> tuple[RunIdentity, Path]: - root = Path(attempt.root) - try: - run_root = root.parents[3] - except IndexError as exc: - raise AttemptStateError("attempt path is invalid") from exc - _directory(run_root, "run") - try: - run_record = json.loads(_read_regular_bytes(run_root / "run.json", "run record").decode("utf-8")) - snapshot = _read_regular_bytes(run_root / "manifest.json", "manifest snapshot") - _read_regular_bytes(run_root / "run.lock", "run lock") - manifest = validate_manifest_bytes(snapshot, repo_root=self.repo_root) - except Exception as exc: - raise AttemptStateError("attempt run binding is invalid") from exc - if not isinstance(run_record, dict) or set(run_record) != {"run_id", "manifest_digest"}: - raise AttemptStateError("attempt run binding is invalid") - run = RunIdentity(str(run_record["run_id"]), str(run_record["manifest_digest"]), str(run_root)) - if not RUN_ID_RE.fullmatch(run.run_id) or run.manifest_digest != manifest.digest: - raise AttemptStateError("attempt run binding is invalid") - if run_root.resolve() != self._run_path(manifest, run.run_id, create_root=False).resolve(): - raise AttemptStateError("attempt run binding is invalid") - expected = self._slot_path(run, Slot(attempt.identity.cell_id, attempt.identity.repetition)) / f"attempt-{attempt.identity.attempt:06d}" - if root.resolve() != expected.resolve() or attempt.identity.run_id != run.run_id or root.is_symlink(): - raise AttemptStateError("attempt identity is invalid") - _directory(root, "attempt root") - return run, root - - def _initial_record( - self, run: RunIdentity, attempt: Attempt, state: str, *, - reason: str | None = None, measurement_policy: str | None = None, - web_validation_policy: str | None = None, - ) -> dict[str, Any]: - record = self._expected_record(run, attempt.identity, state) - if measurement_policy is not None: - if measurement_policy not in MEASUREMENT_POLICIES: - raise AttemptStateError("attempt measurement policy is invalid") - record["measurement_policy"] = measurement_policy - if web_validation_policy is not None: - if web_validation_policy not in WEB_VALIDATION_POLICIES: - raise AttemptStateError("attempt web validation policy is invalid") - record["web_validation_policy"] = web_validation_policy - if reason is not None: - record["lifecycle"] = _unknown_terminal(reason) - return record - - @staticmethod - def _measurement_policy_start_record( - run: RunIdentity, identity: AttemptIdentity - ) -> dict[str, Any]: - """Return the immutable production measurement-policy start evidence.""" - return { - "record": MEASUREMENT_POLICY_RECORD, - "measurement_policy_version": MEASUREMENT_POLICY_VERSION, - "measurement_policy": MEASUREMENT_POLICY_REQUIRED_V1, - "run_id": run.run_id, - "manifest_digest": run.manifest_digest, - "cell_id": identity.cell_id, - "repetition": identity.repetition, - "attempt": identity.attempt, - } - - def _measurement_policy_start( - self, root: Path, run: RunIdentity, identity: AttemptIdentity - ) -> dict[str, Any] | None: - """Load one canonical, no-follow policy record, or its explicit absence.""" - path = root / MEASUREMENT_POLICY_FILENAME - try: - raw = _read_regular_bytes(path, "measurement policy") - except AttemptStateError: - try: - os.lstat(path) - except FileNotFoundError: - return None - raise - try: - record = json.loads(raw.decode("ascii")) - except (UnicodeDecodeError, json.JSONDecodeError) as exc: - raise AttemptStateError("measurement policy is invalid") from exc - expected = self._measurement_policy_start_record(run, identity) - if record != expected or raw != _json_bytes(expected): - raise AttemptStateError("measurement policy is invalid") - return record - - def _validate_measurement_policy_start( - self, - root: Path, - run: RunIdentity, - identity: AttemptIdentity, - measurement_policy: str | None, - ) -> None: - """Require both policy sources for production and neither for legacy paths.""" - start = self._measurement_policy_start(root, run, identity) - if measurement_policy is None and start is None: - return - if ( - measurement_policy == MEASUREMENT_POLICY_REQUIRED_V1 - and start is not None - ): - return - raise AttemptStateError("attempt measurement policy provenance is invalid") - - @staticmethod - def _web_validation_policy_start_record( - run: RunIdentity, identity: AttemptIdentity - ) -> dict[str, Any]: - return { - "record": WEB_VALIDATION_POLICY_RECORD, - "web_validation_policy_version": WEB_VALIDATION_POLICY_VERSION, - "web_validation_policy": WEB_VALIDATION_POLICY_REQUIRED_V1, - "run_id": run.run_id, - "manifest_digest": run.manifest_digest, - "cell_id": identity.cell_id, - "repetition": identity.repetition, - "attempt": identity.attempt, - } - - def _web_validation_policy_start( - self, root: Path, run: RunIdentity, identity: AttemptIdentity - ) -> dict[str, Any] | None: - path = root / WEB_VALIDATION_POLICY_FILENAME - try: - raw = _read_regular_bytes(path, "web validation policy") - except AttemptStateError: - try: - os.lstat(path) - except FileNotFoundError: - return None - raise - try: - record = json.loads(raw.decode("ascii")) - except (UnicodeDecodeError, json.JSONDecodeError) as exc: - raise AttemptStateError("web validation policy is invalid") from exc - expected = self._web_validation_policy_start_record(run, identity) - if record != expected or raw != _json_bytes(expected): - raise AttemptStateError("web validation policy is invalid") - return record - - def _validate_web_validation_policy_start( - self, root: Path, run: RunIdentity, identity: AttemptIdentity, - policy: str | None, - ) -> None: - start = self._web_validation_policy_start(root, run, identity) - if policy is None and start is None: - return - if policy == WEB_VALIDATION_POLICY_REQUIRED_V1 and start is not None: - return - raise AttemptStateError("attempt web validation policy provenance is invalid") - - def _validate_web_validation( - self, root: Path, run: RunIdentity, identity: AttemptIdentity, - policy: str | None, - *, allow_pre_registration_absence: bool = False, - ) -> None: - path = root / WEB_VALIDATION_FILENAME - if not path.exists() and not path.is_symlink(): - if allow_pre_registration_absence: - return - if policy == WEB_VALIDATION_POLICY_REQUIRED_V1: - raise AttemptStateError("required web validation is unavailable") - return - try: - manifest = self.open_manifest_snapshot(run) - web = load_web_validation(root, manifest=manifest) - measurement = load_measurement(root) - except (WebValidationError, MeasurementError) as exc: - raise AttemptStateError("attempt web validation is invalid") from exc - record = web.record - if ( - (record["attempt"]["run_id"], record["attempt"]["cell_id"], - record["attempt"]["repetition"], record["attempt"]["attempt"]) - != (run.run_id, identity.cell_id, identity.repetition, identity.attempt) - or record["manifest_digest"] != run.manifest_digest - or record["fixture_checksum"] != manifest.fixture.checksum - ): - raise AttemptStateError("attempt web validation identity is invalid") - digest = "sha256:" + hashlib.sha256( - _read_regular_bytes(root / MEASUREMENT_FILENAME, "measurement")).hexdigest() - if record["measurement_digest"] != digest or measurement.run_id != run.run_id: - raise AttemptStateError("attempt web validation measurement is invalid") - if web.status == "not_run": - raise AttemptStateError("terminal workspace web validation was not run") - - def open_manifest_snapshot(self, run: RunIdentity) -> Manifest: - """Load the exact immutable run manifest used by recovery and evidence.""" - try: - return validate_manifest_bytes( - _read_regular_bytes( - Path(run.root) / "manifest.json", "manifest snapshot" - ), - repo_root=self.repo_root, - ) - except Exception as exc: - raise AttemptStateError( - "attempt web validation manifest is invalid" - ) from exc - - def open_manifest_fixture_checksum(self, run: RunIdentity) -> str: - """Read the run-bound manifest only to bind web evidence to its fixture.""" - return self.open_manifest_snapshot(run).fixture.checksum - - def _ensure_required_web_validation( - self, - root: Path, - run: RunIdentity, - identity: AttemptIdentity, - record: Mapping[str, Any], - terminal: Mapping[str, Any], - ) -> None: - """Validate or reconstruct required S12 evidence before terminal commit. - - An existing invalid/colliding record is never replaced. When the - sidecar is absent, reconstruction uses only the immutable run manifest, - conventional attempt workspace, and strict measurement. Any failure - occurs before ``attempt.json`` is replaced, leaving recovery resumable. - """ - policy = record.get("web_validation_policy") - if ( - policy == WEB_VALIDATION_POLICY_REQUIRED_V1 - and record.get("locator") is None - and _terminal_reason(terminal) == "interrupted" - ): - path = root / WEB_VALIDATION_FILENAME - measurement = root / MEASUREMENT_FILENAME - if not ( - path.exists() - or path.is_symlink() - or measurement.exists() - or measurement.is_symlink() - ): - return - if policy != WEB_VALIDATION_POLICY_REQUIRED_V1: - self._validate_web_validation(root, run, identity, policy) - return - path = root / WEB_VALIDATION_FILENAME - if path.exists() or path.is_symlink(): - self._validate_web_validation(root, run, identity, policy) - return - try: - manifest = self.open_manifest_snapshot(run) - measurement = load_measurement(root) - prepared = SimpleNamespace( - workspace_dir=str(root / "workspace"), - attempt_root=str(root), - ) - web = validate_web_attempt( - manifest, - root, - prepared, - measurement, - terminal, - ) - publish_web_validation(root, web) - except (MeasurementError, WebValidationError, OSError) as exc: - raise AttemptStateError( - "required web validation reconstruction failed" - ) from exc - self._validate_web_validation(root, run, identity, policy) - - def _publish_receipt_only_recovery_measurement( - self, - root: Path, - run: RunIdentity, - identity: AttemptIdentity, - record: Mapping[str, Any], - terminal: Mapping[str, Any], - ) -> None: - """Publish the only honest measurement available after controller loss. - - Receipt-only recovery has no lifecycle result or journal to reconstruct - timing, usage, or observer data from. It may nevertheless need the - required web gate, which consumes a strict measurement sidecar. This - method builds the canonical all-unavailable record exactly once: it - publishes it when absent, or strictly reuses the existing sidecar when - it equals that expected record byte-for-byte, so a crash between - measurement and web/terminal publication can resume. Any foreign, - malformed, symlinked, or mismatched target fails closed and leaves the - prior bytes untouched. - """ - if record.get("measurement_policy") != MEASUREMENT_POLICY_REQUIRED_V1: - return - if any( - (root / name).exists() or (root / name).is_symlink() - for name in ("lifecycle-result.json", "lifecycle-journal.jsonl") - ): - return - expected_digest = record.get("spec_digest") - if not isinstance(expected_digest, str) or not DIGEST_RE.fullmatch(expected_digest): - raise AttemptStateError("recovery measurement identity is invalid") - manifest = self.open_manifest_snapshot(run) - cells = [cell for cell in manifest.matrix if cell.id == identity.cell_id] - if len(cells) != 1: - raise AttemptStateError("recovery measurement caller is invalid") - try: - product = ProductOutcome(**terminal["product"]) - harness = HarnessOutcome(**terminal["harness"]) - process = ProcessOutcome(**terminal["process"]) - expected_measurement = build_recovery_measurement( - run_id=run.run_id, - cell_id=identity.cell_id, - repetition=identity.repetition, - attempt=identity.attempt, - caller=cells[0].caller, - spec_digest=expected_digest, - product=product, - harness=harness, - process=process, - ) - measurement_target = root / MEASUREMENT_FILENAME - if measurement_target.exists() or measurement_target.is_symlink(): - if load_measurement(root) != expected_measurement: - raise AttemptStateError("recovery measurement publication failed") - else: - publish_measurement(root, expected_measurement) - except ( - KeyError, - TypeError, - ValueError, - LifecycleValidationError, - MeasurementError, - ) as exc: - raise AttemptStateError("recovery measurement publication failed") from exc - try: - self._validate_measurement( - root, - run, - identity, - expected_digest, - _terminal_reason(terminal), - record.get("measurement_policy"), - ) - measurement = load_measurement(root) - except MeasurementError as exc: - raise AttemptStateError("recovery measurement is invalid") from exc - if ( - measurement.product != product - or measurement.harness != harness - or measurement.process != process - ): - raise AttemptStateError("recovery measurement terminal is invalid") - - @staticmethod - def _control_lease_for_root(root: Path) -> AttemptControlLease: - """Derive the short public alias without reading secret or caller data.""" - canonical_root = root.resolve() - digest = hashlib.sha256( - b"iop-benchmark-attempt-control-v1\0" - + os.fsencode(str(canonical_root)) - ).hexdigest()[:CONTROL_ALIAS_DIGEST_HEX_LENGTH] - alias = Path(tempfile.gettempdir()).resolve() / f"{CONTROL_ALIAS_PREFIX}{digest}" - control_dir = alias / CONTROL_DIRECTORY_NAME - socket_path = control_dir / SOCKET_FILENAME - if len(os.fsencode(str(socket_path))) > UNIX_SOCKET_PATH_MAX_BYTES: - raise AttemptStateError("control socket path exceeds platform budget") - return AttemptControlLease(str(alias), str(control_dir), str(socket_path)) - - @staticmethod - def _validate_control_alias(root: Path, lease: AttemptControlLease) -> None: - alias = Path(lease.alias) - try: - mode = os.lstat(alias).st_mode - except OSError as exc: - raise AttemptStateError("control lease is unavailable") from exc - if not stat.S_ISLNK(mode): - raise AttemptStateError("control lease collision") - try: - target = os.readlink(alias) - except OSError as exc: - raise AttemptStateError("control lease is unavailable") from exc - canonical_root = root.resolve() - try: - resolved_alias = alias.resolve(strict=True) - except OSError as exc: - raise AttemptStateError("control lease is unavailable") from exc - if ( - target != str(canonical_root) - or not Path(target).is_absolute() - or resolved_alias != canonical_root - ): - raise AttemptStateError("control lease target mismatch") - - def acquire_control_lease(self, attempt: Attempt) -> AttemptControlLease: - """Create or authenticate the active attempt's no-overwrite short alias.""" - run, root = self._bound_attempt(attempt) - record = self._attempt_record(root, run, attempt.identity) - if record is None or record["state"] != NONTERMINAL_STATE: - raise AttemptStateError("control lease requires a running attempt") - lease = self._control_lease_for_root(root) - alias = Path(lease.alias) - try: - os.symlink(str(root.resolve()), alias, target_is_directory=True) - _fsync_dir(alias.parent) - except FileExistsError: - self._validate_control_alias(root, lease) - except OSError as exc: - raise AttemptStateError("control lease is unavailable") from exc - self._validate_control_alias(root, lease) - return lease - - def release_control_lease(self, attempt: Attempt) -> None: - """Remove only this exact owned alias after durable terminal publication.""" - run, root = self._bound_attempt(attempt) - record = self._attempt_record(root, run, attempt.identity) - if record is None or record["state"] not in TERMINAL_STATES: - raise AttemptStateError("control lease release requires terminal state") - lease = self._control_lease_for_root(root) - alias = Path(lease.alias) - try: - os.lstat(alias) - except FileNotFoundError: - return - except OSError as exc: - raise AttemptStateError("control lease is unavailable") from exc - self._validate_control_alias(root, lease) - try: - alias.unlink() - _fsync_dir(alias.parent) - except OSError as exc: - raise AttemptStateError("control lease cleanup failed") from exc - - def publish_terminal(self, attempt: Attempt, state: str, *, result: dict[str, Any] | None = None) -> Attempt: - if state not in TERMINAL_STATES: - raise AttemptStateError("terminal state is invalid") - run, root = self._bound_attempt(attempt) - record = self._attempt_record(root, run, attempt.identity, absent_ok=True) - if record is None: - terminal_result = result or _unknown_terminal(state) - self._validate_outcomes(terminal_result, "attempt terminal") - if (root / WEB_VALIDATION_FILENAME).exists() or ( - root / WEB_VALIDATION_FILENAME - ).is_symlink(): - self._validate_web_validation( - root, run, attempt.identity, policy=None - ) - initial = self._expected_record(run, attempt.identity, state) - initial["lifecycle"] = { - name: dict(terminal_result[name]) - for name in ("product", "harness", "process") - } - _write_new(root / "attempt.json", _json_bytes(initial)) - return Attempt(attempt.identity, attempt.root, state) - if record["state"] in TERMINAL_STATES: - if record["state"] != state: - raise AttemptStateError("terminal attempt is immutable") - return Attempt(attempt.identity, attempt.root, state) - if record["state"] != NONTERMINAL_STATE: - raise AttemptStateError("attempt transition is invalid") - terminal_result = result or _unknown_terminal(state) - self._validate_outcomes(terminal_result, "attempt terminal") - self._ensure_required_web_validation( - root, - run, - attempt.identity, - record, - terminal_result, - ) - record["state"] = state - record["lifecycle"] = { - name: dict(terminal_result[name]) - for name in ("product", "harness", "process") - } - _replace(root / "attempt.json", _json_bytes(record)) - return Attempt(attempt.identity, attempt.root, state) - - def _locator_from_record( - self, root: Path, raw: Any, *, state: str - ) -> SupervisorLocator: - fields = {"supervisor_pid", "start_identity", "socket_path", "challenge", "control_dir", "created_at"} - if not isinstance(raw, dict) or set(raw) != fields: - raise AttemptStateError("locator is invalid") - try: - locator = SupervisorLocator(**raw) - except TypeError as exc: - raise AttemptStateError("locator is invalid") from exc - if not isinstance(locator.supervisor_pid, int) or locator.supervisor_pid < 1 or any(not isinstance(value, str) or not value for value in (locator.start_identity, locator.socket_path, locator.challenge, locator.control_dir, locator.created_at)): - raise AttemptStateError("locator is invalid") - control = Path(locator.control_dir) - socket = Path(locator.socket_path) - lease = self._control_lease_for_root(root) - if control != Path(lease.control_dir) or socket != Path(lease.socket_path): - raise AttemptStateError("locator control binding is invalid") - alias = Path(lease.alias) - try: - os.lstat(alias) - except FileNotFoundError: - if state == NONTERMINAL_STATE: - raise AttemptStateError("running locator control lease is unavailable") - except OSError as exc: - raise AttemptStateError("locator control lease is unavailable") from exc - else: - self._validate_control_alias(root, lease) - if not _contained(control, root) or not _contained(socket, control): - raise AttemptStateError("locator escapes attempt root") - if socket.parent != control: - raise AttemptStateError("locator control binding is invalid") - return locator - - def _validate_measurement( - self, - root: Path, - run: RunIdentity, - identity: AttemptIdentity, - expected_digest: str, - expected_reason: str | None = None, - measurement_policy: str | None = None, - lifecycle: Mapping[str, Any] | None = None, - ) -> None: - """Bind the immutable timing/usage sidecar to this exact invocation. - - The sidecar is optional for historical and lower-level records. A - marked production attempt must have it, and every present sidecar is - bound to the terminal lifecycle evidence without rewriting either. - """ - path = root / MEASUREMENT_FILENAME - if not path.exists() and not path.is_symlink(): - if measurement_policy == MEASUREMENT_POLICY_REQUIRED_V1: - raise AttemptStateError("required attempt measurement is unavailable") - return - try: - measurement = load_measurement(root) - except MeasurementError as exc: - raise AttemptStateError("attempt measurement is invalid") from exc - actual = ( - measurement.run_id, measurement.cell_id, - measurement.repetition, measurement.attempt, - ) - expected = ( - run.run_id, identity.cell_id, identity.repetition, identity.attempt, - ) - if actual != expected or measurement.spec_digest != expected_digest: - raise AttemptStateError("attempt measurement identity is invalid") - if expected_reason is not None and measurement.harness.reason != expected_reason: - raise AttemptStateError("attempt measurement terminal is invalid") - if lifecycle is not None: - try: - validate_measurement_lifecycle_binding(measurement, lifecycle) - except MeasurementError as exc: - raise AttemptStateError("attempt measurement lifecycle is invalid") from exc - - def _validate_terminal_invocation_identity( - self, - root: Path, - locator: SupervisorLocator, - expected_digest: str, - *, - run: RunIdentity, - identity: AttemptIdentity, - terminal_state: str, - expected_receipt_reason: str | None, - measurement_policy: str | None, - ) -> None: - """Rebind cleaned historical records to their durable invocation digest.""" - result_path = root / "lifecycle-result.json" - result_exists = result_path.exists() or result_path.is_symlink() - if result_exists: - result = self._read_bound_lifecycle_terminal( - root, locator, expected_digest - ) - if result is None: - raise AttemptStateError("terminal invocation identity is invalid") - self._validate_measurement( - root, run, identity, expected_digest, expected_receipt_reason, - measurement_policy, result, - ) - journal_path = root / "lifecycle-journal.jsonl" - journal_exists = journal_path.exists() or journal_path.is_symlink() - if journal_exists: - try: - first = _read_regular_bytes( - journal_path, "lifecycle journal" - ).decode("utf-8").splitlines()[0] - header = json.loads(first) - except (IndexError, UnicodeDecodeError, json.JSONDecodeError) as exc: - raise AttemptStateError("terminal invocation identity is invalid") from exc - if not isinstance(header, dict) or header.get("spec_digest") != expected_digest: - raise AttemptStateError("terminal invocation identity is invalid") - if not result_exists and not journal_exists: - # A receipt-only record has no published result or journal, so its only - # authenticated reason is recovery; it may project nothing but interrupted. - if terminal_state != "interrupted" or not isinstance(expected_receipt_reason, str): - raise AttemptStateError("terminal invocation identity is invalid") - self._closed_cleanup_receipt( - root, - locator, - expected_reason=expected_receipt_reason, - required=True, - ) - self._validate_measurement( - root, run, identity, expected_digest, expected_receipt_reason, - measurement_policy, - ) - - def record_locator(self, attempt: Attempt, locator: SupervisorLocator, invocation_digest: str) -> None: - run, root = self._bound_attempt(attempt) - record = self._attempt_record(root, run, attempt.identity) - if record is None or record["state"] != NONTERMINAL_STATE or "locator" in record: - raise AttemptStateError("locator transition is invalid") - if not isinstance(invocation_digest, str) or not DIGEST_RE.fullmatch(invocation_digest): - raise AttemptStateError("invocation digest is invalid") - raw = { - "supervisor_pid": locator.supervisor_pid, - "start_identity": locator.start_identity, - "socket_path": locator.socket_path, - "challenge": locator.challenge, - "control_dir": locator.control_dir, - "created_at": locator.created_at, - } - self._locator_from_record(root, raw, state=NONTERMINAL_STATE) - registered = self._read_json_file(Path(locator.control_dir), "locator.json") - if registered != raw: - raise AttemptStateError("registered locator is invalid") - record["locator"] = raw - record["spec_digest"] = invocation_digest - _replace(root / "attempt.json", _json_bytes(record)) - - @staticmethod - def _state_for_reason(reason: str) -> str: - if reason == "timed_out": - return "timed_out" - if reason == "cancelled": - return "cancelled" - if reason in RECEIPT_ONLY_TERMINAL_REASONS: - return "interrupted" - return "completed" - - @staticmethod - def _state_for_terminal(terminal: Mapping[str, Any]) -> str: - process = terminal.get("process") - status = process.get("status") if isinstance(process, Mapping) else None - if status == "timed_out": - return "timed_out" - if status == "cancelled": - return "cancelled" - reason = _terminal_reason(terminal) - if reason in RECEIPT_ONLY_TERMINAL_REASONS or reason == "interrupted": - return "interrupted" - return "completed" - - def _read_json_file(self, root: Path, name: str) -> dict[str, Any]: - path = root / name - try: - value = json.loads(_read_regular_bytes(path, name).decode("utf-8")) - except (UnicodeDecodeError, json.JSONDecodeError) as exc: - raise AttemptStateError(f"{name} is invalid") from exc - if not isinstance(value, dict): - raise AttemptStateError(f"{name} is invalid") - return value - - @staticmethod - def _exact_fields(value: Any, fields: set[str], label: str) -> dict[str, Any]: - if not isinstance(value, dict) or set(value) != fields: - raise AttemptStateError(f"{label} schema is invalid") - return value - - @staticmethod - def _optional_int(value: Any) -> bool: - return value is None or (isinstance(value, int) and not isinstance(value, bool)) - - def _validate_result_record(self, result: dict[str, Any], locator: SupervisorLocator, expected_digest: str) -> None: - fields = { - "record", "product", "harness", "process", "submitted", - "process_group_alive", - "submission_mode", "completion_mode", "spec_digest", "locator", "started_at", - "ended_at", "duration_ns", "stdout", "stderr", "events", - } - self._exact_fields(result, fields, "lifecycle result") - self._validate_outcomes(result, "lifecycle result") - required_bools = ("submitted", "process_group_alive") - if result["record"] != "result" or any(not isinstance(result[key], bool) for key in required_bools): - raise AttemptStateError("lifecycle result is invalid") - if result["spec_digest"] != expected_digest: - raise AttemptStateError("lifecycle result identity is invalid") - if result["submission_mode"] not in SUBMISSION_MODES or result["completion_mode"] not in COMPLETION_MODES: - raise AttemptStateError("lifecycle result is invalid") - if not all(isinstance(result[key], str) for key in ("started_at", "ended_at")) or not isinstance(result["duration_ns"], int) or isinstance(result["duration_ns"], bool) or result["duration_ns"] < 0: - raise AttemptStateError("lifecycle result is invalid") - expected_public = self._public_locator(locator) - if result["locator"] != expected_public: - raise AttemptStateError("lifecycle terminal locator is invalid") - for name, stream in (("stdout", "stdout"), ("stderr", "stderr")): - capture = self._exact_fields(result[name], {"stream", "text", "line_count", "byte_count", "truncated"}, f"{name} capture") - if capture["stream"] != stream or not isinstance(capture["text"], str) or not isinstance(capture["line_count"], int) or not isinstance(capture["byte_count"], int) or isinstance(capture["line_count"], bool) or isinstance(capture["byte_count"], bool) or capture["line_count"] < 0 or capture["byte_count"] < 0 or not isinstance(capture["truncated"], bool): - raise AttemptStateError("lifecycle capture is invalid") - if not isinstance(result["events"], list): - raise AttemptStateError("lifecycle events are invalid") - for event in result["events"]: - checked = self._exact_fields(event, {"record", "kind", "source", "stream", "monotonic_ns", "source_monotonic_ns", "observed_at", "detail"}, "lifecycle event") - if checked["record"] != "event" or not all(isinstance(checked[key], str) for key in ("kind", "source", "stream", "observed_at", "detail")) or not all(isinstance(checked[key], int) and not isinstance(checked[key], bool) and checked[key] >= 0 for key in ("monotonic_ns", "source_monotonic_ns")): - raise AttemptStateError("lifecycle event is invalid") - if ( - result["process_group_alive"] - or (not result["harness"]["cleanup_complete"]) - != (result["harness"]["reason"] == "cleanup_failed") - ): - raise AttemptStateError("lifecycle terminal outcome is invalid") - if result["harness"]["status"] == "passed" and not result["harness"]["ordered_terminal"]: - raise AttemptStateError("lifecycle terminal outcome is invalid") - - def _validate_outcomes(self, value: Any, label: str) -> None: - if not isinstance(value, Mapping): - raise AttemptStateError(f"{label} schema is invalid") - product = self._exact_fields( - value.get("product"), {"status", "reason"}, f"{label} product" - ) - harness = self._exact_fields( - value.get("harness"), - {"status", "reason", "ordered_terminal", "cleanup_complete"}, - f"{label} harness", - ) - process = self._exact_fields( - value.get("process"), {"status", "exit_code", "signal"}, - f"{label} process", - ) - if ( - product["status"] not in PRODUCT_STATUSES - or product["reason"] not in PRODUCT_REASONS - or (product["status"] == "succeeded") != (product["reason"] == CALLER_REASON_SUCCESS) - or (product["status"] == "failed") != (product["reason"] == "caller_error") - or harness["status"] not in HARNESS_STATUSES - or harness["reason"] not in HARNESS_REASONS - or not isinstance(harness["ordered_terminal"], bool) - or not isinstance(harness["cleanup_complete"], bool) - or (harness["status"] == "passed") != (harness["reason"] == "success") - or (not harness["cleanup_complete"]) - != (harness["reason"] == "cleanup_failed") - or (harness["status"] == "passed" and not harness["ordered_terminal"]) - or process["status"] not in PROCESS_STATUSES - or not self._optional_int(process["exit_code"]) - or not self._optional_int(process["signal"]) - or (process["status"] == "signalled" and process["signal"] is None) - or (process["status"] in {"exited", "not_started"} and process["signal"] is not None) - or (process["status"] == "not_started" and process["exit_code"] is not None) - ): - raise AttemptStateError(f"{label} is invalid") - - @staticmethod - def _instant(value: Any, label: str) -> _datetime.datetime: - """Parse one produced ISO-8601 instant into a comparable UTC value.""" - try: - parsed = _datetime.datetime.fromisoformat(str(value)) - except (TypeError, ValueError) as exc: - raise AttemptStateError(f"{label} timestamp is invalid") from exc - return parsed if parsed.tzinfo is not None else parsed.replace(tzinfo=_datetime.timezone.utc) - - @staticmethod - def _validate_terminal_events(events: list[Any]) -> dict[str, int]: - """Return the unique ordinal of every product/harness evidence event.""" - positions: dict[str, int] = {} - for index, event in enumerate(events): - kind = event["kind"] - if kind not in BOUND_EVIDENCE_KINDS: - continue - if kind in positions: - raise AttemptStateError("lifecycle events are invalid") - positions[kind] = index - return positions - - def _validate_terminal_coherence(self, result: dict[str, Any], receipt: dict[str, Any], positions: dict[str, int]) -> None: - """Bind result, ordered events and cleanup receipt to one terminal projection.""" - submitted, finish, idle, quiet = (positions.get(kind) for kind in SUCCESS_EVIDENCE_KINDS) - caller_terminal = positions.get(EVENT_CALLER_TERMINAL) - ordered = finish is not None and idle is not None and quiet is not None and finish < idle < quiet - if ordered != result["harness"]["ordered_terminal"]: - raise AttemptStateError("lifecycle ordered evidence is invalid") - if result["process"]["exit_code"] != receipt["exit_code"] or result["process"]["signal"] != receipt["signal"]: - raise AttemptStateError("lifecycle terminal outcome is invalid") - if result["submitted"] and not receipt["caller_launched"]: - raise AttemptStateError("lifecycle terminal outcome is invalid") - if result["product"]["status"] != "unknown": - product = result["product"] - expected_detail = f"status={product['status']} reason={product['reason']}" - terminal_event = ( - None if caller_terminal is None else result["events"][caller_terminal] - ) - if ( - not result["submitted"] - or submitted is None - or finish is None - or idle is None - or caller_terminal is None - or submitted > finish - or submitted > caller_terminal - or caller_terminal > idle - or terminal_event["source"] != SOURCE_CALLER_OUTPUT - or terminal_event["detail"] != expected_detail - ): - raise AttemptStateError("lifecycle product evidence is invalid") - started = self._instant(result["started_at"], "lifecycle result") - ended = self._instant(result["ended_at"], "lifecycle result") - completed = self._instant(receipt["completed_at"], "cleanup receipt") - if completed < started or ended < completed: - raise AttemptStateError("lifecycle terminal chronology is invalid") - - def _validate_receipt_record(self, receipt: dict[str, Any], locator: SupervisorLocator) -> dict[str, Any]: - """Validate one cleanup receipt against the registered supervisor identity.""" - self._exact_fields(receipt, RECEIPT_FIELDS, "cleanup receipt") - if receipt["receipt_version"] != RECEIPT_VERSION or receipt["supervisor_pid"] != locator.supervisor_pid or receipt["challenge_digest"] != self._public_locator(locator)["challenge_digest"] or receipt["reason"] not in TERMINAL_REASONS or not self._optional_int(receipt["exit_code"]) or not self._optional_int(receipt["signal"]) or not isinstance(receipt["caller_launched"], bool) or not isinstance(receipt["cleanup_complete"], bool) or (not receipt["cleanup_complete"]) != (receipt["reason"] == "cleanup_failed") or receipt["process_group_alive"] is not False or not isinstance(receipt["completed_at"], str): - raise AttemptStateError("cleanup receipt is invalid") - return receipt - - def _closed_cleanup_receipt( - self, - root: Path, - locator: SupervisorLocator, - *, - expected_reason: str = REASON_CONTROLLER_LOST, - required: bool = False, - ) -> dict[str, Any] | None: - """Read one canonical, authenticated receipt after its socket is closed.""" - if expected_reason not in RECEIPT_ONLY_TERMINAL_REASONS: - raise AttemptStateError("closed cleanup receipt reason is invalid") - control = root / CONTROL_DIRECTORY_NAME - try: - control_mode = os.lstat(control).st_mode - except OSError as exc: - raise AttemptStateError("attempt control directory is unavailable") from exc - if not stat.S_ISDIR(control_mode): - raise AttemptStateError("attempt control directory is invalid") - - registered = self._read_json_file(control, "locator.json") - expected_locator = { - "supervisor_pid": locator.supervisor_pid, - "start_identity": locator.start_identity, - "socket_path": locator.socket_path, - "challenge": locator.challenge, - "control_dir": locator.control_dir, - "created_at": locator.created_at, - } - if registered != expected_locator: - raise AttemptStateError("registered locator is invalid") - - receipt_path = control / "cleanup-receipt.json" - try: - os.lstat(receipt_path) - except FileNotFoundError: - if required: - raise AttemptStateError("closed cleanup receipt is unavailable") - return None - except OSError as exc: - raise AttemptStateError("closed cleanup receipt is unavailable") from exc - receipt = self._validate_receipt_record( - self._read_json_file(control, receipt_path.name), locator - ) - # Result-bound receipts get their instant parsed by terminal coherence; the - # receipt-only path is the sole authority here, so parse it independently. - self._instant(receipt["completed_at"], "cleanup receipt") - if receipt["reason"] != expected_reason: - raise AttemptStateError("closed cleanup receipt is invalid") - - socket_path = control / SOCKET_FILENAME - try: - socket_mode = os.lstat(socket_path).st_mode - except FileNotFoundError: - return receipt - except OSError as exc: - raise AttemptStateError("closed cleanup socket is unavailable") from exc - if not stat.S_ISSOCK(socket_mode): - raise AttemptStateError("closed cleanup socket is invalid") - if expected_reason == REASON_CONTROLLER_LOST: - raise AttemptStateError("closed cleanup socket is still active") - return receipt - - @staticmethod - def _public_locator(locator: SupervisorLocator) -> dict[str, Any]: - return { - "supervisor_pid": locator.supervisor_pid, - "start_identity": locator.start_identity, - "socket_path": locator.socket_path, - "control_dir": locator.control_dir, - "challenge_digest": hashlib.sha256(locator.challenge.encode("utf-8")).hexdigest(), - "created_at": locator.created_at, - } - - def _validate_journal_record(self, root: Path, result: dict[str, Any], expected_digest: str) -> None: - """Validate the append-only journal against the published result record.""" - journal = root / "lifecycle-journal.jsonl" - try: - lines = [json.loads(line) for line in _read_regular_bytes(journal, "lifecycle journal").decode("utf-8").splitlines()] - except (UnicodeDecodeError, json.JSONDecodeError) as exc: - raise AttemptStateError("lifecycle journal is invalid") from exc - if len(lines) < 2: - raise AttemptStateError("lifecycle journal is invalid") - header, terminal = lines[0], lines[-1] - self._exact_fields(header, {"record", "journal_version", "spec_digest", "submission_mode", "completion_mode", "started_at"}, "lifecycle journal header") - self._exact_fields(terminal, {"record", "product", "harness", "process", "process_group_alive", "ended_at"}, "lifecycle journal terminal") - self._validate_outcomes(terminal, "lifecycle journal terminal") - if header["record"] != "header" or header["journal_version"] != JOURNAL_VERSION or header["spec_digest"] != expected_digest or header["submission_mode"] != result["submission_mode"] or header["completion_mode"] != result["completion_mode"] or header["started_at"] != result["started_at"] or not isinstance(header["started_at"], str) or terminal["record"] != "terminal" or terminal["product"] != result["product"] or terminal["harness"] != result["harness"] or terminal["process"] != result["process"] or terminal["process_group_alive"] is not False or terminal["ended_at"] != result["ended_at"] or not isinstance(terminal["ended_at"], str): - raise AttemptStateError("lifecycle journal is invalid") - for event in lines[1:-1]: - self._exact_fields(event, {"record", "kind", "source", "stream", "monotonic_ns", "source_monotonic_ns", "observed_at", "detail"}, "lifecycle journal event") - if lines[1:-1] != result["events"]: - raise AttemptStateError("lifecycle journal events are invalid") - - def _read_bound_lifecycle_terminal(self, root: Path, locator: SupervisorLocator, expected_digest: str) -> dict[str, Any] | None: - result_path = root / "lifecycle-result.json" - if not result_path.exists() and not result_path.is_symlink(): - return None - result = self._read_json_file(root, "lifecycle-result.json") - self._validate_result_record(result, locator, expected_digest) - self._validate_journal_record(root, result, expected_digest) - # ``locator.control_dir`` is the live short lease alias. It is - # intentionally removed after terminal publication, so terminal reads - # must rebind the receipt through the immutable attempt-owned control - # directory rather than resolve a stale alias. - control = root / CONTROL_DIRECTORY_NAME - _directory(control, "attempt control directory") - registered = self._read_json_file(control, "locator.json") - expected_locator = { - "supervisor_pid": locator.supervisor_pid, - "start_identity": locator.start_identity, - "socket_path": locator.socket_path, - "challenge": locator.challenge, - "control_dir": locator.control_dir, - "created_at": locator.created_at, - } - if registered != expected_locator: - raise AttemptStateError("registered locator is invalid") - receipt_data = self._validate_receipt_record( - self._read_json_file(control, "cleanup-receipt.json"), locator - ) - if receipt_data["reason"] != result["harness"]["reason"]: - raise AttemptStateError("cleanup receipt is invalid") - events = self._validate_terminal_events(result["events"]) - self._validate_terminal_coherence(result, receipt_data, events) - return result - - def validate_invocation_terminal(self, attempt: Attempt, invocation: InvocationResult) -> dict[str, Any]: - """Validate direct lifecycle output against the identity committed before launch.""" - if not isinstance(invocation, InvocationResult): - raise AttemptStateError("invocation result is invalid") - run, root = self._bound_attempt(attempt) - record = self._attempt_record(root, run, attempt.identity) - if record is None or record["state"] != NONTERMINAL_STATE: - raise AttemptStateError("invocation transition is invalid") - raw_locator = record.get("locator") - expected_digest = record.get("spec_digest") - if raw_locator is None or not isinstance(expected_digest, str): - raise AttemptStateError("invocation identity was not committed") - locator = self._locator_from_record( - root, raw_locator, state=NONTERMINAL_STATE - ) - expected_result_path = root / "lifecycle-result.json" - expected_journal_path = root / "lifecycle-journal.jsonl" - if invocation.locator != locator or invocation.spec_digest != expected_digest or Path(invocation.result_path).resolve(strict=False) != expected_result_path.resolve() or Path(invocation.journal_path).resolve(strict=False) != expected_journal_path.resolve(): - raise AttemptStateError("invocation result identity is invalid") - terminal = self._read_bound_lifecycle_terminal(root, locator, expected_digest) - if terminal is None: - raise AttemptStateError("lifecycle terminal is unavailable") - self._validate_measurement( - root, run, attempt.identity, expected_digest, - _terminal_reason(terminal), - record.get("measurement_policy"), terminal, - ) - self._validate_web_validation( - root, - run, - attempt.identity, - record.get("web_validation_policy"), - ) - if ( - _outcome_records(invocation)["product"] != terminal["product"] - or _outcome_records(invocation)["harness"] != terminal["harness"] - or _outcome_records(invocation)["process"] != terminal["process"] - ): - raise AttemptStateError("invocation result does not match durable terminal") - fields = ("submitted", "process_group_alive", "spec_digest", "started_at", "ended_at", "duration_ns") - if any(getattr(invocation, field) != terminal[field] for field in fields): - raise AttemptStateError("invocation result does not match durable terminal") - return terminal - - def publish_attempt_measurement( - self, - attempt: Attempt, - caller: str, - result: InvocationResult, - observation: WorkspaceWriteObservation, - ) -> None: - """Publish one immutable timing/usage sidecar before terminal commit. - - Publication happens while the attempt is still running so a collision, - a corrupt projection or an identity mismatch fails closed before any - terminal state is written, and never rewrites bytes it does not own. - """ - if not isinstance(result, InvocationResult): - raise AttemptStateError("invocation result is invalid") - run, root = self._bound_attempt(attempt) - record = self._attempt_record(root, run, attempt.identity) - if record is None or record["state"] != NONTERMINAL_STATE: - raise AttemptStateError("measurement requires a running attempt") - expected_digest = record.get("spec_digest") - if not isinstance(expected_digest, str): - # The invocation identity was never committed, so terminal - # validation owns this failure and there is nothing to bind to. - return - if expected_digest != result.spec_digest: - raise AttemptStateError("measurement invocation identity is invalid") - try: - publish_measurement(root, build_measurement( - run_id=run.run_id, - cell_id=attempt.identity.cell_id, - repetition=attempt.identity.repetition, - attempt=attempt.identity.attempt, - caller=caller, - result=result, - observation=observation, - )) - except MeasurementError as exc: - raise AttemptStateError("attempt measurement is invalid") from exc - self._validate_measurement( - root, run, attempt.identity, expected_digest, result.harness.reason, - record.get("measurement_policy"), - ) - - def publish_attempt_web_validation( - self, attempt: Attempt, web_validation: Any, - ) -> None: - """Publish one web sidecar while the attempt is running. - - The sidecar is deliberately independent from lifecycle success: a page - may fail a quality gate while its caller lifecycle remains successful. - It is nevertheless required provenance for every production attempt. - """ - run, root = self._bound_attempt(attempt) - record = self._attempt_record(root, run, attempt.identity) - if record is None or record["state"] != NONTERMINAL_STATE: - raise AttemptStateError("web validation requires a running attempt") - if record.get("web_validation_policy") != WEB_VALIDATION_POLICY_REQUIRED_V1: - raise AttemptStateError("web validation policy is unavailable") - try: - publish_web_validation(root, web_validation) - except WebValidationError as exc: - raise AttemptStateError("attempt web validation is invalid") from exc - self._validate_web_validation( - root, run, attempt.identity, record.get("web_validation_policy"), - ) - - def reconcile(self, attempt: Attempt) -> Attempt: - """Commit only authenticated terminal evidence before recovery cleanup.""" - run, root = self._bound_attempt(attempt) - record = self._attempt_record(root, run, attempt.identity, absent_ok=True) - if record is None: - terminal = self.publish_terminal( - attempt, "interrupted", result=_unknown_terminal("interrupted") - ) - self.release_control_lease(terminal) - return terminal - if record["state"] in TERMINAL_STATES: - terminal = Attempt(attempt.identity, attempt.root, str(record["state"])) - self.release_control_lease(terminal) - return terminal - raw_locator = record.get("locator") - if raw_locator is None: - terminal = self.publish_terminal( - attempt, "interrupted", result=_unknown_terminal("interrupted") - ) - self.release_control_lease(terminal) - return terminal - locator = self._locator_from_record( - root, raw_locator, state=NONTERMINAL_STATE - ) - expected_digest = record.get("spec_digest") - if not isinstance(expected_digest, str) or not DIGEST_RE.fullmatch(expected_digest): - raise AttemptStateError("recovery identity is invalid") - terminal = self._read_bound_lifecycle_terminal(root, locator, expected_digest) - if terminal is not None: - self._validate_measurement( - root, run, attempt.identity, expected_digest, - _terminal_reason(terminal), - record.get("measurement_policy"), terminal, - ) - self._ensure_required_web_validation( - root, run, attempt.identity, record, terminal - ) - published = self.publish_terminal( - attempt, self._state_for_terminal(terminal), result=terminal - ) - self.release_control_lease(published) - return published - closed_receipt = self._closed_cleanup_receipt(root, locator) - if closed_receipt is not None: - recovery_terminal = _unknown_terminal( - closed_receipt["reason"], - process_status=( - "signalled" if closed_receipt.get("signal") is not None - else "exited" - ), - exit_code=closed_receipt.get("exit_code"), - signal=closed_receipt.get("signal"), - ) - self._publish_receipt_only_recovery_measurement( - root, run, attempt.identity, record, recovery_terminal - ) - self._ensure_required_web_validation( - root, run, attempt.identity, record, recovery_terminal - ) - published = self.publish_terminal( - attempt, - self._state_for_terminal(recovery_terminal), - result=recovery_terminal, - ) - self.release_control_lease(published) - return published - try: - outcome = recover_invocation(locator, stop=True) - except LifecycleRecoveryError as exc: - # The supervisor may complete cleanup and durably publish its - # authenticated receipt immediately before the control reply is - # lost. Re-read that exact receipt instead of treating a missing - # reply as proof that cleanup did not happen. - try: - recovered_receipt = self._closed_cleanup_receipt( - root, - locator, - expected_reason=REASON_RECOVERED_STOP, - ) - except AttemptStateError: - raise AttemptStateError("recovery is unverified") from exc - if recovered_receipt is None: - raise AttemptStateError("recovery is unverified") from exc - recovery_terminal = _unknown_terminal( - recovered_receipt["reason"], - process_status=( - "signalled" if recovered_receipt.get("signal") is not None - else "exited" - ), - exit_code=recovered_receipt.get("exit_code"), - signal=recovered_receipt.get("signal"), - ) - self._publish_receipt_only_recovery_measurement( - root, run, attempt.identity, record, recovery_terminal - ) - self._ensure_required_web_validation( - root, run, attempt.identity, record, recovery_terminal - ) - terminal = self.publish_terminal( - attempt, - self._state_for_terminal(recovery_terminal), - result=recovery_terminal, - ) - self.release_control_lease(terminal) - return terminal - receipt = Path(outcome.receipt_path) - if not outcome.cleanup_complete or outcome.process_group_alive or receipt.parent != Path(locator.control_dir) or not _contained(receipt, root): - raise AttemptStateError("recovery cleanup is unverified") - try: - self._validate_receipt_record(self._read_json_file(receipt.parent, receipt.name), locator) - except AttemptStateError as exc: - raise AttemptStateError("recovery cleanup is unverified") from exc - recovery_terminal = _unknown_terminal( - outcome.reason, - process_status="signalled" if outcome.signal is not None else "exited", - exit_code=outcome.exit_code, - signal=outcome.signal, - ) - self._publish_receipt_only_recovery_measurement( - root, run, attempt.identity, record, recovery_terminal - ) - self._ensure_required_web_validation( - root, run, attempt.identity, record, recovery_terminal - ) - terminal = self.publish_terminal( - attempt, - self._state_for_terminal(recovery_terminal), - result=recovery_terminal, - ) - self.release_control_lease(terminal) - return terminal - - def execute_attempt( - self, - attempt: Attempt, - *, - prepare: Callable[[Attempt], Any], - invoke: Callable[[Attempt, Callable[[SupervisorLocator, str], None]], InvocationResult], - require_measurement: bool = False, - require_web_validation: bool = False, - ) -> Attempt: - """Prepare once, publish running identity, then invoke lifecycle once.""" - run, root = self._bound_attempt(attempt) - if self._attempt_record(root, run, attempt.identity, absent_ok=True) is not None: - raise AttemptStateError("attempt has already been prepared") - try: - prepare(attempt) - except Exception: - _write_new(root / "attempt.json", _json_bytes(self._initial_record(run, attempt, "interrupted", reason="launch_failed"))) - raise - _write_new( - root / "attempt.json", - _json_bytes(self._initial_record( - run, attempt, NONTERMINAL_STATE, - measurement_policy=(MEASUREMENT_POLICY_REQUIRED_V1 - if require_measurement else None), - web_validation_policy=(WEB_VALIDATION_POLICY_REQUIRED_V1 - if require_web_validation else None), - )), - ) - if require_measurement: - _write_new( - root / MEASUREMENT_POLICY_FILENAME, - _json_bytes(self._measurement_policy_start_record(run, attempt.identity)), - ) - if require_web_validation: - _write_new( - root / WEB_VALIDATION_POLICY_FILENAME, - _json_bytes(self._web_validation_policy_start_record(run, attempt.identity)), - ) - result = invoke(attempt, lambda locator, digest: self.record_locator(attempt, locator, digest)) - terminal = self.validate_invocation_terminal(attempt, result) - published = self.publish_terminal( - attempt, self._state_for_terminal(terminal), result=terminal - ) - self.release_control_lease(published) - return published - - def status(self, run: RunIdentity, manifest: Manifest) -> dict[str, Any]: - """Read-only deterministic status; it neither creates nor reconciles.""" - bound_run = self.open(manifest, run.run_id) - if bound_run != run: - raise AttemptStateError("run identity is invalid") - states = {name: 0 for name in sorted(TERMINAL_STATES | {NONTERMINAL_STATE})} - outcomes = { - "product": {name: 0 for name in ("succeeded", "failed", "unknown")}, - "harness": {name: 0 for name in ("passed", "failed")}, - "process": { - name: 0 for name in ( - "exited", "signalled", "timed_out", "cancelled", "not_started" - ) - }, - "artifact": { - name: 0 for name in ("passed", "failed", "blocked", "not_run") - }, - "unresolved": 0, - } - for slot in self.slots(manifest): - retained = self.attempts(bound_run, slot) - for attempt in retained: - states[attempt.state] += 1 - if not retained or retained[-1].state == NONTERMINAL_STATE: - outcomes["unresolved"] += 1 - continue - projection = self.attempt_outcomes(retained[-1]) - for axis in ("product", "harness", "process", "artifact"): - outcomes[axis][projection[axis]] += 1 - if not projection["resolved"]: - outcomes["unresolved"] += 1 - preflights = self._preflight_records(bound_run, manifest) - latest = preflights[-1] if preflights else None - projection = { - "records": len(preflights), - "latest_sequence": 0 if latest is None else latest["sequence"], - "latest_status": "unavailable" if latest is None else latest["status"], - **( - {status: 0 for status in PREFLIGHT_STATUSES} - if latest is None - else _preflight_counts(latest["results"]) - ), - } - return { - "run_id": bound_run.run_id, - "manifest_digest": bound_run.manifest_digest, - "preflight": projection, - "attempts": states, - "outcomes": outcomes, - } - - -def preflight_manifest( - store: RunStore, - manifest: Manifest, - manifest_bytes: bytes, - *, - adapters: Mapping[str, PreflightAdapter], -) -> tuple[RunIdentity, dict[str, Any]]: - """Collect all-cell observations, then create one run and append one record.""" - observations = collect_preflight_observations(manifest, adapters) - run = store.create(manifest, manifest_bytes) - return run, store.record_preflight(run, manifest, observations) - - -def _validate_prepared_binding( - store: RunStore, - manifest: Manifest, - cell: MatrixCell, - attempt: Attempt, - prepared: PreparedWorkspace, -) -> None: - """Reject any caller/cell/workspace identity drift before invocation.""" - if not isinstance(prepared, PreparedWorkspace): - raise AttemptStateError("prepared workspace is invalid") - if cell.id != attempt.identity.cell_id or prepared.identity != attempt.identity: - raise AttemptStateError("prepared workspace identity mismatch") - - attempt_root = Path(attempt.root).resolve() - workspace = Path(prepared.workspace_dir) - session = Path(prepared.session_dir) - if Path(prepared.attempt_root).resolve() != attempt_root: - raise AttemptStateError("prepared attempt root mismatch") - if ( - workspace.is_symlink() - or session.is_symlink() - or not workspace.is_dir() - or not session.is_dir() - or workspace.resolve() != attempt_root / "workspace" - or session.resolve() != attempt_root / "session" - ): - raise AttemptStateError("prepared workspace path mismatch") - if ( - not prepared.session_is_fresh - or not isinstance(prepared.session_id, str) - or not prepared.session_id - or prepared.workspace_checksum != manifest.fixture.checksum - or prepared.setup_cache_policy != manifest.setup_cache_policy - ): - raise AttemptStateError("prepared workspace policy mismatch") - expected_testbed = (store.repo_root / manifest.testbed).resolve() - if ( - not prepared.testbed_provenance.clean - or Path(prepared.testbed_provenance.path).resolve() != expected_testbed - ): - raise AttemptStateError("prepared testbed provenance mismatch") - - -def run_slots( - store: RunStore, - run: RunIdentity, - manifest: Manifest, - *, - adapters: Mapping[str, ExecutionAdapter], - prepare: Callable[[Manifest, Attempt], PreparedWorkspace], - retry_failed: bool = False, -) -> tuple[Attempt, ...]: - """Append fresh preflight, then execute eligible slots under one writer.""" - if not isinstance(adapters, Mapping): - raise CapabilityUnavailable("capability-unavailable: caller-adapter") - required_callers = {cell.caller for cell in manifest.matrix} - if any( - caller not in adapters or not callable(getattr(adapters[caller], "invoke", None)) - for caller in required_callers - ): - raise CapabilityUnavailable("capability-unavailable: caller-adapter") - if not callable(prepare): - raise AttemptStateError("workspace preparer is invalid") - - observations = collect_preflight_observations(manifest, adapters) - bound_run = store.open(manifest, run.run_id) - if bound_run != run: - raise AttemptStateError("run identity is invalid") - cells = {cell.id: cell for cell in manifest.matrix} - if len(cells) != len(manifest.matrix): - raise AttemptStateError("manifest cell identity is invalid") - completed: list[Attempt] = [] - with store.writer(bound_run): - preflight = store._record_preflight_locked( - bound_run, manifest, observations - ) - if preflight["status"] != "ready": - return () - if frozenset(observations) != frozenset(cells): - return () - for slot in store.slots(manifest): - cell = cells.get(slot.cell_id) - if cell is None: - raise AttemptStateError("slot cell identity is invalid") - existing = store.attempts(bound_run, slot) - if existing and existing[-1].state not in TERMINAL_STATES: - store.reconcile(existing[-1]) - existing = store.attempts(bound_run, slot) - if existing: - if store.attempt_outcomes(existing[-1])["passed"]: - continue - if not retry_failed: - continue - attempt = store.allocate(bound_run, slot) - prepared: PreparedWorkspace | None = None - - def prepare_bound(current: Attempt) -> PreparedWorkspace: - nonlocal prepared - candidate = prepare(manifest, current) - _validate_prepared_binding(store, manifest, cell, current, candidate) - prepared = candidate - return candidate - - def invoke_bound( - current: Attempt, - on_started: Callable[[SupervisorLocator, str], None], - ) -> InvocationResult: - if prepared is None: - raise AttemptStateError("prepared workspace is unavailable") - lease = store.acquire_control_lease(current) - # The observer's baseline must be older than any caller write, - # so it starts before the caller can be launched and is joined - # on every success, error, timeout and cancellation path. - observer = WorkspaceWriteObserver(prepared.workspace_dir) - observer.start() - try: - result = adapters[cell.caller].invoke( - cell, - prepared, - current, - lease.control_dir, - manifest.fixture.prompt_content, - manifest.timeout, - on_started, - ) - finally: - observation = observer.stop() - if not observer.stopped: - raise AttemptStateError("workspace observer did not stop") - store.publish_attempt_measurement( - current, cell.caller, result, observation - ) - measurement = load_measurement(current.root) - web = validate_web_attempt( - manifest, current.root, prepared, measurement, result, - ) - store.publish_attempt_web_validation(current, web) - return result - - completed.append( - store.execute_attempt( - attempt, - prepare=prepare_bound, - invoke=invoke_bound, - require_measurement=True, - require_web_validation=True, - ) - ) - return tuple(completed) diff --git a/scripts/agent_benchmark/attempts_test.py b/scripts/agent_benchmark/attempts_test.py deleted file mode 100644 index 17d2057f..00000000 --- a/scripts/agent_benchmark/attempts_test.py +++ /dev/null @@ -1,2450 +0,0 @@ -"""Credential-free production-path tests for durable benchmark attempts.""" - -from __future__ import annotations - -import contextlib -import datetime -import io -import json -import os -import signal -import socket -import stat -import subprocess -import sys -import tempfile -import threading -import time -import unittest -from pathlib import Path -from unittest import mock - -from scripts import agent_comparison_benchmark as benchmark_cli -from scripts.agent_benchmark.attempts import ( - Attempt, - AttemptStateError, - CapabilityUnavailable, - MEASUREMENT_POLICY_FILENAME, - PreflightObservation, - RunBusyError, - RunIdentity, - RunStore, - Slot, - WEB_VALIDATION_POLICY_FILENAME, - _unknown_terminal, - run_slots, -) -from scripts.agent_benchmark.connectivity import ( - ISSUE_RESUME_CODES, - CallerCapability, - ConnectivityIssue, - EffectiveBinding, - RequestedEffectiveBinding, - make_result, -) -from scripts.agent_benchmark.lifecycle import ( - CALLER_REASON_SUCCESS, - CALLER_STATUS_SUCCEEDED, - COMPLETION_EXIT_AFTER_IDLE, - SUBMISSION_ARGV_TASK, - InvocationResult, - InvocationSpec, - CallerEvent, - CallerTerminal, - LifecycleRecoveryError, - REASON_CONTROLLER_LOST, - REASON_RECOVERED_STOP, - SupervisorLocator, - count_metric, - duration_metric, - env_pairs, - recover_invocation, - run_invocation, - spec_digest, -) -from scripts.agent_benchmark.manifest import AssetMapping, Timeout, digest_workspace_inputs, load_manifest -from scripts.agent_benchmark.measurement import ( - MEASUREMENT_FILENAME, - WorkspaceWriteObserver, - load_measurement, - path_digest, -) -from scripts.agent_benchmark.web_validation import ( - WEB_VALIDATION_FILENAME, - WebValidationError, - _digest, - load_web_validation, -) -from scripts.agent_benchmark.workspace import AttemptIdentity, prepare_workspace - - -def _manifest( - root: Path, - repetitions: int = 1, - execution_order_seed: str | None = None, - cell_ids: tuple[str, ...] = ("a",), -): - fixtures = root / "scripts/fixtures" - fixtures.mkdir(parents=True, exist_ok=True) - (fixtures / "prompt.md").write_text("prompt", encoding="utf-8") - (fixtures / "reference.txt").write_text("reference", encoding="utf-8") - fixture = { - "version": "v1", "prompt": "scripts/fixtures/prompt.md", - "assets": [{"source": "scripts/fixtures/reference.txt", "workspace_path": "workspace/reference.txt"}], - "checksum": digest_workspace_inputs((AssetMapping("scripts/fixtures/reference.txt", "workspace/reference.txt", b"reference"),)), - } - data = { - "pipeline_version": "2", "environment": "dev", "testbed": "../iop-s2", - "session_policy": "fresh", "setup_cache_policy": "isolated", - "timeout": {"run_seconds": 1, "idle_seconds": 1, "quiet_seconds": 1, "cleanup_grace_seconds": 1}, - "viewports": [{"id": "desktop", "width": 1, "height": 1}], "rubric_version": "landing-quality-v1", - "evaluator": {"caller": "codex", "iop": {"request_model": "judge", "requested_effort": "high", "route_kind": "direct", "route_id": "judge", "expected_bindings": [{"stage": "request", "model": "judge", "effort": "high"}]}}, - "output_root": "agent-test/runs/a", "fixture": fixture, "repetitions": repetitions, - "matrix": [ - {"id": cell_id, "caller": "claude", "iop": {"request_model": "model", "requested_effort": "high", "route_kind": "direct", "route_id": "route", "expected_bindings": [{"stage": "request", "model": "model", "effort": "high"}]}} - for cell_id in cell_ids - ], - } - if execution_order_seed is not None: - data["execution_order_seed"] = execution_order_seed - path = root / "manifest.json" - raw = json.dumps(data, sort_keys=True).encode("utf-8") - path.write_bytes(raw) - return load_manifest(path, repo_root=root), raw, path - - -def _events(_: str, line: str): - return { - "FINISH": ( - CallerTerminal(CALLER_STATUS_SUCCEEDED, CALLER_REASON_SUCCESS), - CallerEvent("finish"), - ), - "IDLE": CallerEvent("idle"), - }.get(line.strip()) - - -def _preflight_observation(cell, issue_code: str | None = None) -> PreflightObservation: - capability = CallerCapability( - cell.caller, ("direct", "execution_preset"), (cell.iop.requested_effort,) - ) - if issue_code is None: - bindings = tuple( - EffectiveBinding(item.stage, item.model, item.effort) - for item in cell.iop.expected_bindings - ) - binding = RequestedEffectiveBinding( - cell.id, cell.caller, cell.iop.route_kind, cell.iop.route_id, - cell.iop.request_model, cell.iop.requested_effort, - cell.iop.route_kind, cell.iop.route_id, cell.iop.request_model, - cell.iop.requested_effort, bindings, - ) - issues = () - else: - binding = RequestedEffectiveBinding( - cell.id, cell.caller, cell.iop.route_kind, cell.iop.route_id, - cell.iop.request_model, cell.iop.requested_effort, - ) - issues = (ConnectivityIssue(issue_code, ISSUE_RESUME_CODES[issue_code]),) - return PreflightObservation( - make_result(cell, capability, binding, issues), - "sha256:" + "1" * 64, - "sha256:" + "2" * 64, - ) - - -_PROBE_TIMEOUT_SECONDS = 30.0 - - -def _controller_loss_child(payload_json: str) -> None: - """Run one real lifecycle controller that the parent regression will kill.""" - payload = json.loads(payload_json) - store = RunStore(payload["repo"]) - manifest = load_manifest( - Path(payload["manifest"]), repo_root=Path(payload["repo"]) - ) - run = RunIdentity(payload["run_id"], manifest.digest, payload["run_root"]) - attempt = Attempt( - AttemptIdentity( - payload["run_id"], - payload["cell_id"], - payload["repetition"], - payload["attempt_number"], - ), - payload["attempt_root"], - "running", - ) - - def invoke(current, started): - lease = store.acquire_control_lease(current) - spec = InvocationSpec( - argv=( - sys.executable, - "-u", - "-c", - "import time; print('START', flush=True); time.sleep(30)", - ), - cwd=payload["repo"], - env=env_pairs({"PATH": os.environ.get("PATH", "/usr/bin:/bin")}), - submission_mode=SUBMISSION_ARGV_TASK, - completion_mode=COMPLETION_EXIT_AFTER_IDLE, - timeout=Timeout(60, 1, 1, 1), - evidence_dir=current.root, - control_dir=lease.control_dir, - ) - return run_invocation( - spec, - parse_event=_events, - on_started=lambda locator: started(locator, spec_digest(spec)), - ) - - def prepare(current): - if payload.get("prepare_workspace"): - return prepare_workspace( - manifest, - current.root, - current.identity, - repo_root=Path(payload["repo"]), - ) - return None - - with store.writer(run): - store.execute_attempt( - attempt, - prepare=prepare, - invoke=invoke, - require_measurement=bool(payload.get("require_measurement")), - require_web_validation=bool(payload.get("require_web_validation")), - ) - -# Every durable read runs in a bounded child so a blocking special file cannot -# hang the suite; the child reports whether the store fails closed. -_PROBE_SOURCE = """ -import json -import os -import sys -from pathlib import Path - -from scripts.agent_benchmark.attempts import Attempt, AttemptStateError, RunIdentity, RunStore, Slot -from scripts.agent_benchmark.lifecycle import SupervisorLocator -from scripts.agent_benchmark.manifest import load_manifest -from scripts.agent_benchmark.workspace import AttemptIdentity - -payload = json.loads(sys.argv[1]) -store = RunStore(payload["repo"]) -manifest = load_manifest(Path(payload["manifest"]), repo_root=Path(payload["repo"])) -run = RunIdentity(payload["run_id"], manifest.digest, payload["run_root"]) -attempt = Attempt( - AttemptIdentity(payload["run_id"], payload["cell_id"], payload["repetition"], payload["attempt_number"]), - payload["attempt_root"], "running", -) - - -def lease(): - with store.writer(run): - pass - - -def locator(): - store.record_locator( - attempt, - SupervisorLocator(**payload["locator"]), - "sha256:" + "0" * 64, - ) - - -operations = { - "open": lambda: store.open(manifest, run.run_id), - "lease": lease, - "attempts": lambda: store.attempts(run, Slot("a", 1)), - "reconcile": lambda: store.reconcile(attempt), - "locator": locator, -} -try: - operations[payload["operation"]]() -except AttemptStateError: - print("rejected") - sys.exit(0) -print("accepted") -sys.exit(1) -""" - - -def _reordered(events: list[dict]) -> list[dict]: - """Return production events with finish and idle transposed.""" - kinds = [event["kind"] for event in events] - swapped = list(events) - finish, idle = kinds.index("finish"), kinds.index("idle") - swapped[finish], swapped[idle] = swapped[idle], swapped[finish] - return swapped - - -def _without(kind: str): - return lambda events: [event for event in events if event["kind"] != kind] - - -def _duplicated(kind: str): - return lambda events: events + [event for event in events if event["kind"] == kind] - - -class ControllerCrash(RuntimeError): - """Test-only controller loss after lifecycle evidence has been published.""" - - -class FakeExecutionAdapter: - """Typed fake that exercises the production run_slots boundary.""" - - def __init__( - self, - owner: "AttemptBase", - reason: str, - calls: list[str], - issue_code: str | None = None, - ) -> None: - self.owner = owner - self.reason = reason - self.calls = calls - self.issue_code = issue_code - self.capability = CallerCapability( - "claude", ("direct", "execution_preset"), ("high",) - ) - - def preflight(self, cell): - self.calls.append("preflight") - return _preflight_observation(cell, self.issue_code) - - def invoke( - self, - cell, - prepared, - attempt, - control_dir, - task_payload, - timeout, - on_started, - ): - self.calls.append("invoke") - if cell.id != attempt.identity.cell_id or prepared.identity != attempt.identity: - raise AssertionError("typed execution identity drift") - if task_payload != self.owner.manifest.fixture.prompt_content: - raise AssertionError("task payload drift") - source = ( - "print('FINISH'); print('IDLE')" - if self.reason == "success" - else "import sys; print('FAILED'); sys.exit(3)" - ) - if Path(control_dir).resolve(strict=False) != Path(attempt.root).resolve() / "control": - raise AssertionError("controller control binding drift") - spec = self.owner._spec(attempt, source, control_dir=control_dir) - return run_invocation( - spec, - parse_event=_events, - on_started=lambda locator: on_started(locator, spec_digest(spec)), - ) - - def __call__(self, attempt, on_started): - """Retain the lower-level RunStore lifecycle seam for recovery tests.""" - self.calls.append("invoke") - source = ( - "print('FINISH'); print('IDLE')" - if self.reason == "success" - else "import sys; print('FAILED'); sys.exit(3)" - ) - spec = self.owner._spec(attempt, source) - return run_invocation( - spec, - parse_event=_events, - on_started=lambda locator: on_started(locator, spec_digest(spec)), - ) - - -def _measured_events(_stream: str, line: str): - """Parse the fake caller's terminal lines plus one typed usage report.""" - text = line.strip() - if text.startswith("USAGE "): - _, duration, tokens = text.split() - return ( - duration_metric("total_duration", duration, model="model"), - count_metric("input_tokens", int(tokens), model="model"), - ) - return _events(_stream, line) - - -_MEASURING_SOURCES = { - "success": ( - "from pathlib import Path\n" - "Path({workspace!r}).joinpath('answer.txt').write_text('generated')\n" - "print('USAGE 12.5 11')\nprint('FINISH')\nprint('IDLE')\n" - ), - "failed": "import sys\nprint('FAILED')\nsys.exit(3)\n", - "timeout": "import time\ntime.sleep(30)\n", -} - - -class MeasuringExecutionAdapter: - """Typed fake whose caller writes into the prepared workspace and reports usage.""" - - def __init__(self, owner: "AttemptBase", mode: str = "success", *, collide: bool = False) -> None: - self.owner = owner - self.mode = mode - self.collide = collide - self.capability = CallerCapability( - "claude", ("direct", "execution_preset"), ("high",) - ) - - def preflight(self, cell): - return _preflight_observation(cell) - - def invoke(self, cell, prepared, attempt, control_dir, task_payload, timeout, on_started): - spec = InvocationSpec( - argv=( - sys.executable, "-u", "-c", - _MEASURING_SOURCES[self.mode].format(workspace=prepared.workspace_dir), - ), - cwd=prepared.workspace_dir, - env=env_pairs({"PATH": os.environ.get("PATH", "/usr/bin:/bin")}), - submission_mode=SUBMISSION_ARGV_TASK, - completion_mode=COMPLETION_EXIT_AFTER_IDLE, - timeout=Timeout(1, 1, 1, 1) if self.mode == "timeout" else Timeout(5, 1, 1, 1), - evidence_dir=attempt.root, - control_dir=control_dir, - ) - result = run_invocation( - spec, parse_event=_measured_events, - on_started=lambda locator: on_started(locator, spec_digest(spec)), - ) - if self.collide: - # Simulate a concurrent owner that already published this sidecar. - (Path(attempt.root) / MEASUREMENT_FILENAME).write_bytes(b'{"record":"prior"}\n') - return result - - -class AttemptBase(unittest.TestCase): - def setUp(self) -> None: - self.temp = tempfile.TemporaryDirectory(dir="/tmp", prefix="b") - self.root = Path(self.temp.name) / "r" - self.root.mkdir() - self._control_aliases: list[Path] = [] - self.manifest, self.raw, self.manifest_path = _manifest(self.root) - self.store = RunStore( - self.root, - clock=lambda: datetime.datetime(2026, 1, 2, 3, 4, 5, tzinfo=datetime.timezone.utc), - token_hex=lambda _: "abcdef123456", - ) - - def tearDown(self) -> None: - for alias in self._control_aliases: - try: - alias.unlink() - except FileNotFoundError: - pass - self.temp.cleanup() - - def create_run(self): - return self.store.create(self.manifest, self.raw) - - def _control_dir(self, attempt) -> str: - lease = self.store.acquire_control_lease(attempt) - self._control_aliases.append(Path(lease.alias)) - return lease.control_dir - - def _spec( - self, attempt, source: str, *, control_dir: str | None = None - ) -> InvocationSpec: - bound_control_dir = control_dir or self._control_dir(attempt) - return InvocationSpec( - argv=(sys.executable, "-u", "-c", source), - cwd=str(self.root), - env=env_pairs({"PATH": os.environ.get("PATH", "/usr/bin:/bin")}), - submission_mode=SUBMISSION_ARGV_TASK, - completion_mode=COMPLETION_EXIT_AFTER_IDLE, - timeout=Timeout(5, 1, 1, 1), - evidence_dir=attempt.root, - control_dir=bound_control_dir, - ) - - def adapter( - self, - reason: str, - calls: list[str], - issue_code: str | None = None, - ) -> FakeExecutionAdapter: - return FakeExecutionAdapter(self, reason, calls, issue_code) - - def preparer(self, calls: list[str]): - def prepare(manifest, attempt): - calls.append("prepare") - return prepare_workspace( - manifest, attempt.root, attempt.identity, repo_root=self.root - ) - - return prepare - - def _init_testbed(self) -> None: - testbed = self.root.parent / "iop-s2" - testbed.mkdir() - (testbed / "README.md").write_text("testbed", encoding="utf-8") - for command in ( - ("git", "init"), ("git", "config", "user.name", "test"), - ("git", "config", "user.email", "test@example.invalid"), ("git", "add", "."), - ("git", "commit", "-m", "testbed"), - ): - subprocess.run(command, cwd=testbed, check=True, capture_output=True) - - -class AttemptStoreTest(AttemptBase): - def test_slots_follow_seeded_manifest_order_before_repetitions(self): - self.manifest, self.raw, self.manifest_path = _manifest( - self.root, - repetitions=2, - execution_order_seed="seed-a", - cell_ids=("cell-a", "cell-b", "cell-c"), - ) - matrix_order = [cell.id for cell in self.manifest.matrix] - self.assertEqual(matrix_order, ["cell-c", "cell-a", "cell-b"]) - - slots = self.store.slots(self.manifest) - self.assertEqual( - [(slot.cell_id, slot.repetition) for slot in slots], - [ - ("cell-c", 1), - ("cell-c", 2), - ("cell-a", 1), - ("cell-a", 2), - ("cell-b", 1), - ("cell-b", 2), - ], - ) - self.assertEqual( - [slot.cell_id for slot in slots if slot.repetition == 1], - matrix_order, - ) - - def test_slots_and_append_only_terminals(self): - self.manifest, self.raw, self.manifest_path = _manifest(self.root, repetitions=2) - run = self.create_run() - self.assertEqual([slot.repetition for slot in self.store.slots(self.manifest)], [1, 2]) - with self.store.writer(run): - first = self.store.allocate(run, Slot("a", 1)) - self.assertEqual(list(Path(first.root).iterdir()), []) - terminal = self.store.publish_terminal(first, "interrupted") - with self.assertRaises(AttemptStateError): - self.store.publish_terminal(terminal, "completed") - second = self.store.allocate(run, Slot("a", 1)) - self.assertEqual(second.identity.attempt, 2) - - def test_writer_is_fail_fast_and_status_is_read_only(self): - run = self.create_run() - before = (Path(run.root) / "run.json").read_bytes() - with self.store.writer(run): - with self.assertRaises(RunBusyError): - with self.store.writer(run): - pass - self.assertEqual(self.store.status(run, self.manifest)["attempts"]["running"], 0) - self.assertEqual(before, (Path(run.root) / "run.json").read_bytes()) - - def test_open_rejects_changed_snapshot_and_empty_allocation_reconciles(self): - run = self.create_run() - with self.assertRaises(AttemptStateError): - self.store.open(self.manifest, run.run_id, self.raw + b" ") - with self.store.writer(run): - attempt = self.store.allocate(run, Slot("a", 1)) - interrupted = self.store.reconcile(attempt) - self.assertEqual(interrupted.state, "interrupted") - - def test_foreign_record_and_symlink_fail_closed_without_status_mutation(self): - run = self.create_run() - with self.store.writer(run): - attempt = self.store.allocate(run, Slot("a", 1)) - self.store.publish_terminal(attempt, "interrupted") - record = Path(attempt.root) / "attempt.json" - foreign = json.loads(record.read_text(encoding="utf-8")) - foreign["run_id"] = "run-20260102T030405Z-ffffffffffff" - record.write_text(json.dumps(foreign), encoding="utf-8") - before = record.read_bytes() - with self.assertRaises(AttemptStateError): - self.store.status(run, self.manifest) - self.assertEqual(before, record.read_bytes()) - record.unlink() - record.symlink_to(Path(attempt.root) / "other.json") - with self.assertRaises(AttemptStateError): - self.store.attempts(run, Slot("a", 1)) - - def test_preflight_records_append_in_order_and_status_is_read_only(self): - run = self.create_run() - cell = self.manifest.matrix[0] - first = self.store.record_preflight( - run, self.manifest, {cell.id: _preflight_observation(cell)} - ) - second = self.store.record_preflight( - run, - self.manifest, - {cell.id: _preflight_observation(cell, "credential_missing")}, - ) - before = { - path.name: path.read_bytes() - for path in (Path(run.root) / "preflight").iterdir() - } - - self.assertEqual((first["sequence"], second["sequence"]), (1, 2)) - self.assertEqual( - [record["status"] for record in self.store.preflights(run, self.manifest)], - ["ready", "registration_required"], - ) - status = self.store.status(run, self.manifest) - self.assertEqual( - status["preflight"], - { - "records": 2, - "latest_sequence": 2, - "latest_status": "registration_required", - "ready": 0, - "registration_required": 1, - "implementation_gap": 0, - }, - ) - self.assertEqual(status["attempts"]["running"], 0) - self.assertFalse((Path(run.root) / "cells").exists()) - self.assertEqual( - before, - { - path.name: path.read_bytes() - for path in (Path(run.root) / "preflight").iterdir() - }, - ) - - def test_preflight_corruption_and_symlink_fail_closed(self): - run = self.create_run() - cell = self.manifest.matrix[0] - self.store.record_preflight( - run, self.manifest, {cell.id: _preflight_observation(cell)} - ) - record = Path(run.root) / "preflight/preflight-000001.json" - original = record.read_bytes() - record.write_bytes(original + b" ") - with self.assertRaises(AttemptStateError): - self.store.preflights(run, self.manifest) - record.write_bytes(original) - record.unlink() - record.symlink_to(Path(run.root) / "run.json") - with self.assertRaises(AttemptStateError): - self.store.status(run, self.manifest) - - def test_preflight_rejects_sequence_gap_and_foreign_result(self): - run = self.create_run() - cell = self.manifest.matrix[0] - self.store.record_preflight( - run, self.manifest, {cell.id: _preflight_observation(cell)} - ) - first = Path(run.root) / "preflight/preflight-000001.json" - first.rename(first.with_name("preflight-000002.json")) - with self.assertRaises(AttemptStateError): - self.store.preflights(run, self.manifest) - - first.with_name("preflight-000002.json").rename(first) - raw = json.loads(first.read_text(encoding="ascii")) - raw["results"][0]["binding"]["requested_model"] = "fallback" - first.write_bytes(json.dumps(raw, sort_keys=True, separators=(",", ":")).encode("ascii") + b"\n") - with self.assertRaises(AttemptStateError): - self.store.preflights(run, self.manifest) - - def test_preflight_evidence_contains_no_unmodeled_adapter_values(self): - run = self.create_run() - cell = self.manifest.matrix[0] - sentinel = "private_endpoint_or_token_must_not_persist" - observation = _preflight_observation(cell, "stream_incompatible") - # An adapter may retain runtime-only values on itself, but the writer - # accepts only the closed PreflightObservation projection above. - adapter = type("Adapter", (), {"runtime_value": sentinel})() - self.assertEqual(adapter.runtime_value, sentinel) - self.store.record_preflight(run, self.manifest, {cell.id: observation}) - durable = b"".join( - path.read_bytes() - for path in Path(run.root).rglob("*") - if path.is_file() - ) - self.assertNotIn(sentinel.encode("ascii"), durable) - - -class AttemptOrchestrationTest(AttemptBase): - def test_run_slots_prepares_workspace_and_invokes_once(self): - self._init_testbed() - run = self.create_run() - calls: list[str] = [] - - def prepare(manifest, attempt): - self.assertTrue((Path(run.root) / "preflight/preflight-000001.json").is_file()) - calls.append("prepare") - return prepare_workspace(manifest, attempt.root, attempt.identity, repo_root=self.root) - - completed = run_slots(self.store, run, self.manifest, adapters={"claude": self.adapter("success", calls)}, prepare=prepare) - self.assertEqual([item.state for item in completed], ["completed"]) - self.assertEqual(calls, ["preflight", "prepare", "invoke"]) - attempt_root = Path(completed[0].root) - self.assertTrue((attempt_root / "prepared.json").is_file()) - state = json.loads((attempt_root / "attempt.json").read_text(encoding="utf-8")) - self.assertEqual(state["attempt_result_version"], 2) - alias = Path(state["locator"]["control_dir"]).parent - self.assertFalse(os.path.lexists(alias)) - self.assertTrue((attempt_root / "control/locator.json").is_file()) - self.assertTrue((attempt_root / "control/cleanup-receipt.json").is_file()) - self.assertEqual(self.store.status(run, self.manifest)["attempts"]["completed"], 1) - record_path = attempt_root / "attempt.json" - original = record_path.read_bytes() - corruptions = ( - ( - "arbitrary-control-path", - lambda raw: raw["locator"].update( - { - "control_dir": "/tmp/iop-bench-attempt-000000000000000000000000/control", - "socket_path": "/tmp/iop-bench-attempt-000000000000000000000000/control/control.sock", - } - ), - ), - ( - "mismatched-invocation-digest", - lambda raw: raw.__setitem__("spec_digest", "sha256:" + "0" * 64), - ), - ) - for name, corrupt in corruptions: - with self.subTest(name=name): - raw = json.loads(original.decode("utf-8")) - corrupt(raw) - record_path.write_text( - json.dumps(raw, sort_keys=True, separators=(",", ":")) + "\n", - encoding="utf-8", - ) - before = record_path.read_bytes() - with self.assertRaises(AttemptStateError): - self.store.status(run, self.manifest) - self.assertEqual(before, record_path.read_bytes()) - record_path.write_bytes(original) - - def test_control_lease_rejects_collision_and_mismatched_target(self): - run = self.create_run() - with self.store.writer(run): - attempt = self.store.allocate(run, Slot("a", 1)) - with self.assertRaisesRegex(ControllerCrash, "before lease"): - self.store.execute_attempt( - attempt, - prepare=lambda _: None, - invoke=lambda _attempt, _started: (_ for _ in ()).throw( - ControllerCrash("before lease") - ), - ) - expected = self.store._control_lease_for_root(Path(attempt.root)) - alias = Path(expected.alias) - try: - alias.touch(mode=0o600) - with self.assertRaisesRegex(AttemptStateError, "collision"): - self.store.acquire_control_lease(attempt) - alias.unlink() - - alias.symlink_to(self.root, target_is_directory=True) - with self.assertRaisesRegex(AttemptStateError, "target mismatch"): - self.store.acquire_control_lease(attempt) - alias.unlink() - - first = self.store.acquire_control_lease(attempt) - second = self.store.acquire_control_lease(attempt) - self._control_aliases.append(Path(first.alias)) - self.assertEqual(first, second) - self.assertLessEqual( - len(os.fsencode(first.socket_path)), 103 - ) - with self.store.writer(run): - terminal = self.store.reconcile(attempt) - self.assertEqual(terminal.state, "interrupted") - self.assertFalse(os.path.lexists(alias)) - finally: - alias.unlink(missing_ok=True) - - def test_preparation_failure_is_sealed_without_launch(self): - run = self.create_run() - calls: list[str] = [] - - def fail_prepare(_manifest, _attempt): - calls.append("prepare") - raise RuntimeError("prepare failure") - - with self.assertRaisesRegex(RuntimeError, "prepare failure"): - run_slots(self.store, run, self.manifest, adapters={"claude": self.adapter("success", calls)}, prepare=fail_prepare) - self.assertEqual(calls, ["preflight", "prepare"]) - self.assertEqual(self.store.attempts(run, Slot("a", 1))[-1].state, "interrupted") - - def test_retry_and_skip_preserve_prior_terminal_bytes(self): - self._init_testbed() - run = self.create_run() - calls: list[str] = [] - run_slots( - self.store, - run, - self.manifest, - adapters={"claude": self.adapter("failed", calls)}, - prepare=self.preparer(calls), - ) - first = self.store.attempts(run, Slot("a", 1))[0] - prior = (Path(first.root) / "attempt.json").read_bytes() - self.assertEqual( - run_slots( - self.store, - run, - self.manifest, - adapters={"claude": self.adapter("success", calls)}, - prepare=self.preparer(calls), - ), - (), - ) - self.assertEqual(prior, (Path(first.root) / "attempt.json").read_bytes()) - retry = run_slots( - self.store, - run, - self.manifest, - adapters={"claude": self.adapter("success", calls)}, - prepare=self.preparer(calls), - retry_failed=True, - ) - self.assertEqual(retry[0].identity.attempt, 2) - self.assertEqual(prior, (Path(first.root) / "attempt.json").read_bytes()) - self.assertEqual(len(self.store.preflights(run, self.manifest)), 3) - - def test_preflight_blocker_appends_without_attempt_allocation(self): - run = self.create_run() - calls: list[str] = [] - completed = run_slots( - self.store, - run, - self.manifest, - adapters={ - "claude": self.adapter( - "success", calls, issue_code="credential_missing" - ) - }, - prepare=lambda _manifest, _attempt: self.fail("preparer must not run"), - ) - self.assertEqual(completed, ()) - self.assertEqual(calls, ["preflight"]) - self.assertFalse((Path(run.root) / "cells").exists()) - self.assertEqual( - self.store.status(run, self.manifest)["preflight"]["latest_status"], - "registration_required", - ) - - def test_missing_adapter_has_no_output_root_side_effect(self): - fake_run = RunIdentity("run-20260102T030405Z-abcdef123456", self.manifest.digest, str(self.root / "absent")) - output = self.root / self.manifest.output_root - self.assertFalse(output.exists()) - with self.assertRaises(CapabilityUnavailable): - run_slots(self.store, fake_run, self.manifest, adapters={}, prepare=lambda _manifest, _attempt: None) - self.assertFalse(output.exists()) - - -class AttemptMeasurementTest(AttemptBase): - """The immutable timing/usage sidecar around one production invocation.""" - - def _run(self, mode: str = "success", *, collide: bool = False): - self._init_testbed() - run = self.create_run() - completed = run_slots( - self.store, run, self.manifest, - adapters={"claude": MeasuringExecutionAdapter(self, mode, collide=collide)}, - prepare=self.preparer([]), - ) - return run, completed - - def test_successful_attempt_publishes_one_bound_measurement(self): - threads_before = set(threading.enumerate()) - run, completed = self._run() - self.assertEqual([item.state for item in completed], ["completed"]) - attempt_root = Path(completed[0].root) - measurement = load_measurement(attempt_root) - - self.assertEqual( - (measurement.run_id, measurement.cell_id, measurement.repetition, measurement.attempt), - (run.run_id, "a", 1, 1), - ) - self.assertEqual(measurement.caller, "claude") - self.assertEqual(measurement.harness.reason, "success") - state = json.loads((attempt_root / "attempt.json").read_text(encoding="utf-8")) - self.assertEqual(measurement.spec_digest, state["spec_digest"]) - self.assertEqual(state["measurement_policy"], "required-v1") - marker = json.loads( - (attempt_root / MEASUREMENT_POLICY_FILENAME).read_text(encoding="ascii") - ) - self.assertEqual(marker["measurement_policy"], "required-v1") - self.assertEqual(marker["run_id"], run.run_id) - self.assertEqual(marker["manifest_digest"], run.manifest_digest) - self.assertEqual( - (marker["cell_id"], marker["repetition"], marker["attempt"]), - ("a", 1, 1), - ) - - self.assertEqual(measurement.usage["total_duration"].value, 12_500_000) - self.assertEqual(measurement.usage["total_duration"].clock, "caller_reported") - self.assertEqual(measurement.usage["input_tokens"].value, 11) - # The fake caller reports no provider total, so it stays unavailable. - self.assertEqual(measurement.usage["total_tokens"].status, "unavailable") - self.assertIsNone(measurement.usage["total_tokens"].value) - - timeline = measurement.timeline - self.assertEqual(timeline["submitted_at"].clock, "harness_monotonic") - self.assertEqual(timeline["first_output_at"].status, "observed") - self.assertEqual(timeline["first_write_observed_at"].source, "workspace_poll") - self.assertEqual(timeline["first_write_mtime"].clock, "filesystem_mtime") - self.assertEqual(measurement.observer.path_digest, path_digest("answer.txt")) - self.assertGreater(measurement.observer.precision_ns, 0) - # The caller-chosen filename is digested, never persisted verbatim. - self.assertNotIn( - b"answer.txt", (attempt_root / MEASUREMENT_FILENAME).read_bytes() - ) - self.assertEqual(set(threading.enumerate()) - threads_before, set()) - - def _assert_unavailable_measurement(self, mode: str, state: str, reason: str) -> None: - _run, completed = self._run(mode) - self.assertEqual(completed[0].state, state) - measurement = load_measurement(Path(completed[0].root)) - self.assertEqual(measurement.harness.reason, reason) - self.assertEqual(measurement.observations, ()) - for name in ("total_duration", "input_tokens", "model_calls"): - self.assertEqual(measurement.usage[name].status, "unavailable") - self.assertIsNone(measurement.usage[name].value) - self.assertFalse(measurement.observer.observed) - self.assertEqual( - measurement.timeline["first_write_observed_at"].reason, "not_observed" - ) - - def test_failed_attempt_keeps_unavailable_values(self): - self._assert_unavailable_measurement("failed", "completed", "nonzero_exit") - - def test_timed_out_attempt_keeps_unavailable_values(self): - self._assert_unavailable_measurement("timeout", "timed_out", "timed_out") - - def test_measurement_collision_fails_closed_without_touching_prior_bytes(self): - self._init_testbed() - run = self.create_run() - with self.assertRaises(AttemptStateError): - run_slots( - self.store, run, self.manifest, - adapters={"claude": MeasuringExecutionAdapter(self, collide=True)}, - prepare=self.preparer([]), - ) - attempt = self.store.attempts(run, Slot("a", 1))[-1] - sidecar = Path(attempt.root) / MEASUREMENT_FILENAME - self.assertEqual(sidecar.read_bytes(), b'{"record":"prior"}\n') - self.assertEqual(attempt.state, "running") - - def test_tampered_or_unbound_measurement_fails_closed_and_preserves_bytes(self): - run, completed = self._run() - sidecar = Path(completed[0].root) / MEASUREMENT_FILENAME - original = sidecar.read_bytes() - record = json.loads(original.decode("ascii")) - cases = { - "foreign-attempt": {**record, "attempt": {**record["attempt"], "cell_id": "other"}}, - "foreign-digest": {**record, "spec_digest": "sha256:" + "0" * 64}, - "rewritten-terminal": { - **record, - "harness": {**record["harness"], "reason": "timed_out"}, - }, - "rewritten-product": { - **record, - "product": {"status": "failed", "reason": "caller_error"}, - }, - "rewritten-process": { - **record, - "process": {**record["process"], "exit_code": 7}, - }, - "invented-total": { - **record, - "usage": { - **record["usage"], - "total_tokens": { - "status": "observed", "value": 11, "unit": "tokens", - "clock": "none", "source": "caller_output", - }, - }, - }, - } - for name, payload in cases.items(): - with self.subTest(name=name): - sidecar.write_bytes( - json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() + b"\n" - ) - before = sidecar.read_bytes() - with self.assertRaises(AttemptStateError): - self.store.status(run, self.manifest) - self.assertEqual(before, sidecar.read_bytes()) - sidecar.write_bytes(original) - self.assertEqual(self.store.status(run, self.manifest)["attempts"]["completed"], 1) - - def test_marked_measurement_is_required_and_bound_to_lifecycle_events(self): - run, completed = self._run() - attempt = completed[0] - sidecar = Path(attempt.root) / MEASUREMENT_FILENAME - original = sidecar.read_bytes() - sidecar.unlink() - with self.assertRaises(AttemptStateError): - self.store.status(run, self.manifest) - with self.assertRaises(AttemptStateError): - self.store.reconcile(attempt) - sidecar.write_bytes(original) - - record = json.loads(original.decode("ascii")) - rewritten = {**record} - rewritten["observations"] = [ - {**record["observations"][0], "value": 999}, - *record["observations"][1:], - ] - rewritten["usage"] = { - **record["usage"], - "total_duration": { - **record["usage"]["total_duration"], "value": 999, - }, - } - sidecar.write_bytes( - json.dumps(rewritten, sort_keys=True, separators=(",", ":")).encode() + b"\n" - ) - with self.assertRaises(AttemptStateError): - self.store.status(run, self.manifest) - self.assertEqual(sidecar.read_bytes(), json.dumps( - rewritten, sort_keys=True, separators=(",", ":") - ).encode() + b"\n") - - def test_measurement_policy_start_evidence_rejects_downgrade_and_tampering(self): - run, completed = self._run() - attempt = completed[0] - root = Path(attempt.root) - record = root / "attempt.json" - marker = root / MEASUREMENT_POLICY_FILENAME - sidecar = root / MEASUREMENT_FILENAME - saved_record, saved_marker, saved_sidecar = ( - record.read_bytes(), marker.read_bytes(), sidecar.read_bytes() - ) - - def durable_marker() -> bytes | str | None: - try: - mode = os.lstat(marker).st_mode - except FileNotFoundError: - return None - if stat.S_ISREG(mode): - return marker.read_bytes() - return f"nonregular:{stat.S_IFMT(mode)}" - - def reject_on_status_and_reconcile() -> None: - before = (record.read_bytes(), durable_marker()) - with self.assertRaises(AttemptStateError): - self.store.status(run, self.manifest) - with self.assertRaises(AttemptStateError): - self.store.reconcile(attempt) - self.assertEqual(before, (record.read_bytes(), durable_marker())) - - try: - downgraded = json.loads(saved_record.decode("utf-8")) - downgraded.pop("measurement_policy") - record.write_bytes(json.dumps( - downgraded, sort_keys=True, separators=(",", ":") - ).encode("ascii") + b"\n") - sidecar.unlink() - reject_on_status_and_reconcile() - sidecar.write_bytes(saved_sidecar) - record.write_bytes(saved_record) - - marker.unlink() - reject_on_status_and_reconcile() - marker.write_bytes(saved_marker) - - record.unlink() - with self.assertRaises(AttemptStateError): - self.store.status(run, self.manifest) - with self.assertRaises(AttemptStateError): - self.store.reconcile(attempt) - self.assertEqual(marker.read_bytes(), saved_marker) - record.write_bytes(saved_record) - - mismatched = json.loads(saved_marker.decode("ascii")) - mismatched["cell_id"] = "other" - marker.write_bytes(json.dumps( - mismatched, sort_keys=True, separators=(",", ":") - ).encode("ascii") + b"\n") - reject_on_status_and_reconcile() - marker.write_bytes(saved_marker) - - marker.write_bytes(json.dumps(json.loads(saved_marker), indent=2).encode("ascii")) - reject_on_status_and_reconcile() - finally: - record.write_bytes(saved_record) - if marker.exists() or marker.is_symlink(): - if marker.is_dir() and not marker.is_symlink(): - marker.rmdir() - else: - marker.unlink() - marker.write_bytes(saved_marker) - - for kind in ("fifo", "symlink", "directory"): - with self.subTest(kind=kind): - AttemptRecoveryTest._substitute(marker, kind, saved_marker) - try: - reject_on_status_and_reconcile() - finally: - AttemptRecoveryTest._restore(marker, saved_marker) - self.assertEqual(self.store.status(run, self.manifest)["attempts"]["completed"], 1) - - def test_explicitly_unmarked_lower_level_attempt_remains_compatible(self): - run = self.create_run() - with self.store.writer(run): - attempt = self.store.allocate(run, Slot("a", 1)) - terminal = self.store.execute_attempt( - attempt, prepare=lambda _: None, - invoke=self.adapter("success", []), require_measurement=False, - ) - root = Path(terminal.root) - record = json.loads((root / "attempt.json").read_text(encoding="utf-8")) - self.assertNotIn("measurement_policy", record) - self.assertFalse((root / MEASUREMENT_POLICY_FILENAME).exists()) - self.assertFalse((root / MEASUREMENT_FILENAME).exists()) - self.assertEqual(self.store.status(run, self.manifest)["attempts"]["completed"], 1) - - def test_nonregular_measurement_fails_closed_without_blocking(self): - run, completed = self._run() - sidecar = Path(completed[0].root) / MEASUREMENT_FILENAME - saved = sidecar.read_bytes() - for kind in ("fifo", "symlink", "directory"): - with self.subTest(kind=kind): - AttemptRecoveryTest._substitute(sidecar, kind, saved) - with self.assertRaises(AttemptStateError): - self.store.attempts(run, Slot("a", 1)) - AttemptRecoveryTest._restore(sidecar, saved) - self.assertEqual(self.store.attempts(run, Slot("a", 1))[-1].state, "completed") - - def test_recovery_commits_only_a_valid_bound_sidecar(self): - self._init_testbed() - run = self.create_run() - adapter = MeasuringExecutionAdapter(self) - prepared: dict[str, object] = {} - - def prepare(attempt): - prepared["workspace"] = prepare_workspace( - self.manifest, attempt.root, attempt.identity, repo_root=self.root - ) - return prepared["workspace"] - - def invoke(attempt, started): - workspace = prepared["workspace"] - observer = WorkspaceWriteObserver(workspace.workspace_dir) - observer.start() - try: - result = adapter.invoke( - self.manifest.matrix[0], workspace, attempt, - self._control_dir(attempt), b"task", self.manifest.timeout, started, - ) - finally: - observation = observer.stop() - self.store.publish_attempt_measurement(attempt, "claude", result, observation) - raise ControllerCrash("controller crash") - - with self.store.writer(run): - attempt = self.store.allocate(run, Slot("a", 1)) - with self.assertRaisesRegex(RuntimeError, "controller crash"): - self.store.execute_attempt(attempt, prepare=prepare, invoke=invoke) - - sidecar = Path(attempt.root) / MEASUREMENT_FILENAME - original = sidecar.read_bytes() - sidecar.write_bytes( - json.dumps( - {**json.loads(original.decode("ascii")), "caller": ""}, - sort_keys=True, separators=(",", ":"), - ).encode() + b"\n" - ) - with self.store.writer(run): - with self.assertRaises(AttemptStateError): - self.store.reconcile(attempt) - self.assertEqual(self.store.attempts(run, Slot("a", 1))[-1].state, "running") - - sidecar.write_bytes(original) - with self.store.writer(run): - recovered = self.store.reconcile(attempt) - self.assertEqual(recovered.state, "completed") - self.assertEqual(load_measurement(Path(attempt.root)).caller, "claude") - - -class AttemptWebValidationTest(AttemptBase): - """Required S12 policy, lifecycle mapping, and recovery-before-terminal.""" - - def test_pre_registration_interruption_needs_no_impossible_sidecars(self): - run = self.create_run() - with self.store.writer(run): - attempt = self.store.allocate(run, Slot("a", 1)) - with self.assertRaisesRegex(ControllerCrash, "before registration"): - self.store.execute_attempt( - attempt, - prepare=lambda _: None, - invoke=lambda _attempt, _started: (_ for _ in ()).throw( - ControllerCrash("before registration") - ), - require_measurement=True, - require_web_validation=True, - ) - terminal = self.store.reconcile(attempt) - self.assertEqual(terminal.state, "interrupted") - root = Path(terminal.root) - self.assertFalse((root / MEASUREMENT_FILENAME).exists()) - self.assertFalse((root / WEB_VALIDATION_FILENAME).exists()) - self.assertEqual( - self.store.status(run, self.manifest)["attempts"]["interrupted"], 1 - ) - - def _run(self, mode: str = "success"): - self._init_testbed() - run = self.create_run() - completed = run_slots( - self.store, - run, - self.manifest, - adapters={"claude": MeasuringExecutionAdapter(self, mode)}, - prepare=self.preparer([]), - ) - return run, completed - - def _running_required_web( - self, *, generated: bool = False, return_result: bool = False - ): - self._init_testbed() - run = self.create_run() - adapter = MeasuringExecutionAdapter(self) - prepared: dict[str, object] = {} - - def prepare(attempt): - workspace = prepare_workspace( - self.manifest, - attempt.root, - attempt.identity, - repo_root=self.root, - ) - prepared["workspace"] = workspace - return workspace - - def invoke(attempt, started): - workspace = prepared["workspace"] - if generated: - root = Path(workspace.workspace_dir) - (root / "index.html").write_text( - "

ready

go
", - encoding="utf-8", - ) - (root / "styles.css").write_text( - "body{color:#111;background:#fff}a:focus{outline:2px solid #05f}", - encoding="utf-8", - ) - (root / "script.js").write_text("", encoding="utf-8") - observer = WorkspaceWriteObserver(workspace.workspace_dir) - observer.start() - try: - result = adapter.invoke( - self.manifest.matrix[0], - workspace, - attempt, - self._control_dir(attempt), - b"task", - self.manifest.timeout, - started, - ) - finally: - observation = observer.stop() - self.store.publish_attempt_measurement( - attempt, "claude", result, observation - ) - if return_result: - return result - raise ControllerCrash("controller crash before web publication") - - with self.store.writer(run): - attempt = self.store.allocate(run, Slot("a", 1)) - expected = AttemptStateError if return_result else ControllerCrash - with self.assertRaises(expected): - self.store.execute_attempt( - attempt, - prepare=prepare, - invoke=invoke, - require_measurement=True, - require_web_validation=True, - ) - return run, attempt - - def _receipt_only_required_attempt(self, *, generated: bool): - """Create the exact post-supervisor, pre-measurement recovery boundary.""" - self._init_testbed() - run = self.create_run() - with self.store.writer(run): - attempt = self.store.allocate(run, Slot("a", 1)) - attempt_root = Path(attempt.root) - lease = self.store._control_lease_for_root(attempt_root) - alias = Path(lease.alias) - self._control_aliases.append(alias) - payload = json.dumps( - { - "repo": str(self.root), - "manifest": str(self.manifest_path), - "run_id": run.run_id, - "run_root": run.root, - "attempt_root": attempt.root, - "cell_id": attempt.identity.cell_id, - "repetition": attempt.identity.repetition, - "attempt_number": attempt.identity.attempt, - "prepare_workspace": True, - "require_measurement": True, - "require_web_validation": True, - } - ) - child = subprocess.Popen( - [ - sys.executable, - "-c", - ( - "import sys; " - "from scripts.agent_benchmark.attempts_test import " - "_controller_loss_child; " - "_controller_loss_child(sys.argv[1])" - ), - payload, - ], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - env={ - **os.environ, - "PYTHONPATH": str(Path(__file__).resolve().parents[2]), - }, - ) - - def child_failure(label: str) -> None: - if child.poll() is None: - return - stdout, stderr = child.communicate() - self.fail( - f"controller exited before {label}: returncode={child.returncode} " - f"stdout={stdout!r} stderr={stderr!r}" - ) - - try: - locator: SupervisorLocator | None = None - deadline = time.monotonic() + 10 - attempt_record = attempt_root / "attempt.json" - while locator is None: - child_failure("locator commit") - if attempt_record.is_file(): - raw = json.loads(attempt_record.read_text(encoding="utf-8")) - if raw.get("locator") is not None: - locator = SupervisorLocator(**raw["locator"]) - break - if time.monotonic() >= deadline: - self.fail("controller did not commit its locator") - threading.Event().wait(0.01) - - deadline = time.monotonic() + 10 - while True: - child_failure("caller launch") - try: - status = recover_invocation(locator, stop=False) - except LifecycleRecoveryError: - status = None - if status is not None and status.caller_launched: - break - if time.monotonic() >= deadline: - self.fail("controller did not launch its caller") - threading.Event().wait(0.01) - - child.kill() - stdout, stderr = child.communicate(timeout=5) - self.assertEqual(child.returncode, -signal.SIGKILL, (stdout, stderr)) - - receipt_path = attempt_root / "control" / "cleanup-receipt.json" - deadline = time.monotonic() + 10 - while True: - if receipt_path.is_file(): - receipt = json.loads(receipt_path.read_text(encoding="utf-8")) - if ( - receipt.get("reason") == REASON_CONTROLLER_LOST - and receipt.get("cleanup_complete") is True - and receipt.get("process_group_alive") is False - ): - break - if time.monotonic() >= deadline: - self.fail("supervisor did not publish controller-loss receipt") - threading.Event().wait(0.01) - - self.assertFalse((attempt_root / "lifecycle-result.json").exists()) - self.assertFalse((attempt_root / "lifecycle-journal.jsonl").exists()) - self.assertFalse((attempt_root / MEASUREMENT_FILENAME).exists()) - if generated: - workspace = attempt_root / "workspace" - (workspace / "index.html").write_text( - "

ready

go
", - encoding="utf-8", - ) - (workspace / "styles.css").write_text( - "body{color:#111;background:#fff}a:focus{outline:2px solid #05f}", - encoding="utf-8", - ) - (workspace / "script.js").write_text("", encoding="utf-8") - return run, attempt - finally: - if child.poll() is None: - child.kill() - child.communicate(timeout=5) - - def test_receipt_only_required_evidence_reconstructs_measurement_and_web(self): - for generated in (False, True): - with self.subTest(generated=generated): - self.tearDown() - self.setUp() - run, attempt = self._receipt_only_required_attempt(generated=generated) - root = Path(attempt.root) - renderer = ( - mock.patch( - "scripts.agent_benchmark.web_validation.BrowserRenderer.render", - side_effect=FileNotFoundError("missing browser"), - ) - if generated - else contextlib.nullcontext() - ) - with renderer: - with self.store.writer(run): - terminal = self.store.reconcile(attempt) - measurement = load_measurement(root) - self.assertEqual(terminal.state, "interrupted") - self.assertEqual( - (measurement.run_id, measurement.cell_id, measurement.repetition, measurement.attempt), - (run.run_id, "a", 1, 1), - ) - self.assertEqual(measurement.caller, "claude") - self.assertEqual(measurement.product.status, "unknown") - self.assertEqual(measurement.harness.reason, REASON_CONTROLLER_LOST) - self.assertEqual(measurement.observations, ()) - for observation in measurement.timeline.values(): - self.assertEqual(observation.status, "unavailable") - self.assertEqual(observation.reason, "not_observed") - for observation in measurement.usage.values(): - self.assertEqual(observation.status, "unavailable") - self.assertEqual(observation.reason, "not_reported") - self.assertFalse(measurement.observer.observed) - self.assertEqual(measurement.observer.reason, "observer_unavailable") - self.assertEqual(measurement.observer.samples, 0) - self.assertNotEqual(load_web_validation(root).status, "not_run") - counts = self.store.status(run, self.manifest)["attempts"] - self.assertEqual(counts["interrupted"], 1) - self.assertEqual(counts["running"], 0) - - def test_receipt_only_measurement_collision_preserves_running_and_prior_bytes(self): - run, attempt = self._receipt_only_required_attempt(generated=False) - root = Path(attempt.root) - attempt_path = root / "attempt.json" - measurement_path = root / MEASUREMENT_FILENAME - measurement_path.write_bytes(b'{"record":"prior"}\n') - before = (attempt_path.read_bytes(), measurement_path.read_bytes()) - with self.store.writer(run): - with self.assertRaises(AttemptStateError): - self.store.reconcile(attempt) - self.assertEqual(before, (attempt_path.read_bytes(), measurement_path.read_bytes())) - self.assertEqual(json.loads(attempt_path.read_text())["state"], "running") - - def test_receipt_only_recovery_resumes_after_measurement_publication(self): - run, attempt = self._receipt_only_required_attempt(generated=False) - root = Path(attempt.root) - attempt_path = root / "attempt.json" - measurement_path = root / MEASUREMENT_FILENAME - receipt = json.loads( - (root / "control" / "cleanup-receipt.json").read_text(encoding="utf-8") - ) - terminal = _unknown_terminal( - receipt["reason"], - process_status=("signalled" if receipt.get("signal") is not None else "exited"), - exit_code=receipt.get("exit_code"), - signal=receipt.get("signal"), - ) - record = json.loads(attempt_path.read_text(encoding="utf-8")) - with self.store.writer(run): - self.store._publish_receipt_only_recovery_measurement( - root, run, attempt.identity, record, terminal - ) - self.assertTrue(measurement_path.exists()) - before_measurement = measurement_path.read_bytes() - self.assertEqual(json.loads(attempt_path.read_text())["state"], "running") - self.assertFalse((root / WEB_VALIDATION_FILENAME).exists()) - with self.store.writer(run): - published = self.store.reconcile(attempt) - self.assertEqual(measurement_path.read_bytes(), before_measurement) - self.assertEqual(published.state, "interrupted") - self.assertEqual( - json.loads(attempt_path.read_text())["state"], "interrupted" - ) - self.assertNotEqual(load_web_validation(root).status, "not_run") - counts = self.store.status(run, self.manifest)["attempts"] - self.assertEqual(counts["running"], 0) - self.assertEqual(counts["interrupted"], 1) - - def test_lifecycle_status_matrix_validates_every_terminal_workspace(self): - cases = ( - ("success", "completed", "failed"), - ("failed", "completed", "failed"), - ("timeout", "timed_out", "failed"), - ) - for mode, terminal, web_status in cases: - with self.subTest(mode=mode): - self.tearDown() - self.setUp() - _run, completed = self._run(mode) - self.assertEqual(completed[0].state, terminal) - web = load_web_validation(Path(completed[0].root)) - self.assertEqual(web.status, web_status) - self.assertNotEqual(web.status, "not_run") - - def test_normal_terminal_requires_web_sidecar_before_commit(self): - _run, attempt = self._running_required_web(return_result=True) - record = Path(attempt.root) / "attempt.json" - self.assertEqual(json.loads(record.read_text())["state"], "running") - self.assertFalse((Path(attempt.root) / WEB_VALIDATION_FILENAME).exists()) - - def test_recovery_reconstructs_web_before_terminal_commit(self): - run, attempt = self._running_required_web() - root = Path(attempt.root) - attempt_record = root / "attempt.json" - before = attempt_record.read_bytes() - with self.store.writer(run): - recovered = self.store.reconcile(attempt) - self.assertEqual(recovered.state, "completed") - self.assertNotEqual(attempt_record.read_bytes(), before) - self.assertEqual(load_web_validation(root).status, "failed") - self.assertEqual(self.store.status(run, self.manifest)["attempts"]["completed"], 1) - - def test_recovery_browser_start_failure_publishes_blocked(self): - run, attempt = self._running_required_web(generated=True) - with mock.patch( - "scripts.agent_benchmark.web_validation.BrowserRenderer.render", - side_effect=FileNotFoundError("missing browser"), - ): - with self.store.writer(run): - recovered = self.store.reconcile(attempt) - self.assertEqual(recovered.state, "completed") - web = load_web_validation(Path(attempt.root)) - self.assertEqual(web.status, "blocked") - self.assertFalse(web.record["screenshots"]) - - def test_recovery_collision_preserves_running_and_prior_bytes(self): - run, attempt = self._running_required_web() - root = Path(attempt.root) - web_path = root / WEB_VALIDATION_FILENAME - web_path.write_bytes(b'{"record":"prior"}\n') - attempt_path = root / "attempt.json" - before = (attempt_path.read_bytes(), web_path.read_bytes()) - with self.store.writer(run): - with self.assertRaises(AttemptStateError): - self.store.reconcile(attempt) - self.assertEqual(before, (attempt_path.read_bytes(), web_path.read_bytes())) - self.assertEqual(json.loads(attempt_path.read_text())["state"], "running") - - def test_policy_record_and_artifact_faults_fail_closed(self): - run, completed = self._run() - attempt = completed[0] - root = Path(attempt.root) - attempt_path = root / "attempt.json" - web_path = root / WEB_VALIDATION_FILENAME - marker_path = root / WEB_VALIDATION_POLICY_FILENAME - saved = { - "attempt": attempt_path.read_bytes(), - "web": web_path.read_bytes(), - "marker": marker_path.read_bytes(), - } - - def rejected() -> None: - before = attempt_path.read_bytes() - with self.assertRaises(AttemptStateError): - self.store.status(run, self.manifest) - with self.assertRaises(AttemptStateError): - self.store.reconcile(attempt) - self.assertEqual(attempt_path.read_bytes(), before) - - downgraded = json.loads(saved["attempt"]) - downgraded.pop("web_validation_policy") - attempt_path.write_bytes( - json.dumps(downgraded, sort_keys=True, separators=(",", ":")).encode() - + b"\n" - ) - rejected() - attempt_path.write_bytes(saved["attempt"]) - - marker_path.unlink() - rejected() - marker_path.write_bytes(saved["marker"]) - - for mutation in ("identity", "measurement", "schema"): - with self.subTest(mutation=mutation): - record = json.loads(saved["web"]) - if mutation == "identity": - record["attempt"]["cell_id"] = "other" - elif mutation == "measurement": - record["measurement_digest"] = "sha256:" + "0" * 64 - else: - record["browser"]["unknown"] = True - web_path.write_bytes( - json.dumps(record, sort_keys=True, separators=(",", ":")).encode() - + b"\n" - ) - rejected() - web_path.write_bytes(saved["web"]) - - web_path.unlink() - web_path.mkdir() - rejected() - web_path.rmdir() - web_path.write_bytes(saved["web"]) - self.assertEqual(self.store.status(run, self.manifest)["attempts"]["completed"], 1) - - -class AttemptRecoveryTest(AttemptBase): - def _running_with_terminal(self): - run = self.create_run() - with self.store.writer(run): - attempt = self.store.allocate(run, Slot("a", 1)) - with self.assertRaisesRegex(RuntimeError, "controller crash"): - self.store.execute_attempt( - attempt, prepare=lambda _: None, - invoke=self._invoke_then_crash, - ) - return run, attempt - - def _invoke_then_crash(self, attempt, started): - result = self.adapter("success", [])(attempt, started) - self.assertTrue((Path(attempt.root) / "lifecycle-result.json").is_file()) - raise ControllerCrash("controller crash") - - def test_real_terminal_first_recovery_commits_once(self): - run, attempt = self._running_with_terminal() - running = json.loads( - (Path(attempt.root) / "attempt.json").read_text(encoding="utf-8") - ) - alias = Path(running["locator"]["control_dir"]).parent - self.assertTrue(alias.is_symlink()) - with self.store.writer(run): - recovered = self.store.reconcile(attempt) - self.assertEqual(recovered.state, "completed") - self.assertFalse(os.path.lexists(alias)) - self.assertEqual(self.store.status(run, self.manifest)["attempts"]["completed"], 1) - self.assertEqual(self.store.reconcile(recovered).state, "completed") - - def test_corrupt_terminal_variants_fail_closed_and_preserve_bytes(self): - run, attempt = self._running_with_terminal() - cases = ( - ("contradictory-success", "lifecycle-result.json", lambda raw: raw.__setitem__("success", False)), - ("extra-result-field", "lifecycle-result.json", lambda raw: raw.__setitem__("unexpected", True)), - ("mismatched-digest", "lifecycle-result.json", lambda raw: raw.__setitem__("spec_digest", "sha256:" + "0" * 64)), - ("receipt-cleanup", "cleanup-receipt.json", lambda raw: raw.__setitem__("cleanup_complete", False)), - ) - for name, filename, corrupt in cases: - with self.subTest(name=name): - target = Path(attempt.root) / "control" / filename if filename == "cleanup-receipt.json" else Path(attempt.root) / filename - original = target.read_bytes() - raw = json.loads(target.read_text(encoding="utf-8")) - corrupt(raw) - target.write_text(json.dumps(raw), encoding="utf-8") - evidence_before = {path: path.read_bytes() for path in (Path(attempt.root) / "attempt.json", Path(attempt.root) / "lifecycle-result.json", Path(attempt.root) / "lifecycle-journal.jsonl", Path(attempt.root) / "control" / "cleanup-receipt.json")} - with self.store.writer(run): - with self.assertRaises(AttemptStateError): - self.store.reconcile(attempt) - self.assertEqual(evidence_before, {path: path.read_bytes() for path in evidence_before}) - target.write_bytes(original) - - @staticmethod - def _evidence_bytes(run, attempt) -> dict[str, bytes]: - """Snapshot every durable run and attempt record published so far.""" - run_root, attempt_root = Path(run.root), Path(attempt.root) - paths = ( - run_root / "run.json", run_root / "manifest.json", run_root / "run.lock", - attempt_root / "attempt.json", attempt_root / "lifecycle-result.json", - attempt_root / "lifecycle-journal.jsonl", attempt_root / "control" / "locator.json", - attempt_root / "control" / "cleanup-receipt.json", - ) - return {str(path): path.read_bytes() for path in paths} - - @staticmethod - def _substitute(target: Path, kind: str, saved: bytes) -> None: - """Replace one durable file with a non-regular object of the given kind.""" - target.unlink() - if kind == "fifo": - os.mkfifo(target, 0o600) - elif kind == "directory": - target.mkdir(mode=0o700) - elif kind == "socket": - short = Path(tempfile.mkdtemp(dir="/tmp", prefix="s")) / "s" - with contextlib.closing(socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)) as endpoint: - endpoint.bind(str(short)) # bound in a short path, then moved into place - os.replace(short, target) - short.parent.rmdir() - else: - copy = target.with_name(target.name + ".copy") - copy.write_bytes(saved) - target.symlink_to(copy) - - @staticmethod - def _restore(target: Path, saved: bytes) -> None: - """Discard the substituted object and republish the original bytes.""" - if target.is_symlink() or not target.is_dir(): - target.unlink() - else: - target.rmdir() - target.with_name(target.name + ".copy").unlink(missing_ok=True) - target.write_bytes(saved) - os.chmod(target, 0o600) - - def _assert_probe_rejected(self, run, attempt, operation: str, locator: dict | None = None) -> None: - """Run one store operation in a bounded child and require a closed failure.""" - payload = json.dumps({ - "repo": str(self.root), "manifest": str(self.manifest_path), "run_id": run.run_id, - "run_root": run.root, "attempt_root": attempt.root, "operation": operation, - "cell_id": attempt.identity.cell_id, "repetition": attempt.identity.repetition, - "attempt_number": attempt.identity.attempt, "locator": locator, - }) - child = subprocess.Popen( - [sys.executable, "-c", _PROBE_SOURCE, payload], - stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, - env={**os.environ, "PYTHONPATH": str(Path(__file__).resolve().parents[2])}, - ) - try: - out, err = child.communicate(timeout=_PROBE_TIMEOUT_SECONDS) - except subprocess.TimeoutExpired: - child.kill() - child.communicate() - self.fail(f"{operation} did not return within {_PROBE_TIMEOUT_SECONDS:.0f}s") - self.assertEqual((child.returncode, out.strip()), (0, "rejected"), err) - self.assertIsNotNone(child.poll()) - - def test_nonregular_durable_files_fail_closed_without_blocking(self): - run, attempt = self._running_with_terminal() - run_root, attempt_root = Path(run.root), Path(attempt.root) - surfaces = ( - ("run record", run_root / "run.json", "open"), - ("manifest snapshot", run_root / "manifest.json", "open"), - ("run lock read", run_root / "run.lock", "open"), - ("run lock lease", run_root / "run.lock", "lease"), - ("attempt record", attempt_root / "attempt.json", "attempts"), - ("lifecycle result", attempt_root / "lifecycle-result.json", "reconcile"), - ("lifecycle journal", attempt_root / "lifecycle-journal.jsonl", "reconcile"), - ("cleanup receipt", attempt_root / "control" / "cleanup-receipt.json", "reconcile"), - ) - kinds = ("fifo", "directory", "socket", "symlink") - covered: list[tuple[str, str]] = [] - for label, target, operation in surfaces: - for kind in kinds: - with self.subTest(surface=label, kind=kind): - before = self._evidence_bytes(run, attempt) - saved = before[str(target)] - self._substitute(target, kind, saved) - try: - self._assert_probe_rejected(run, attempt, operation) - self.assertFalse(stat.S_ISREG(os.lstat(target).st_mode)) - finally: - self._restore(target, saved) - self.assertEqual(before, self._evidence_bytes(run, attempt)) - covered.append((label, kind)) - with self.store.writer(run): - locator_attempt = self.store.allocate(run, Slot("a", 2)) - locator_root = Path(locator_attempt.root) - - def stop_before_locator(_attempt, _started): - raise ControllerCrash("locator setup") - - with self.assertRaisesRegex(ControllerCrash, "locator setup"): - self.store.execute_attempt( - locator_attempt, prepare=lambda _: None, invoke=stop_before_locator, - ) - lease = self.store.acquire_control_lease(locator_attempt) - self._control_aliases.append(Path(lease.alias)) - control = Path(lease.control_dir) - control.mkdir(mode=0o700) - locator = { - "supervisor_pid": os.getpid(), "start_identity": "probe", - "socket_path": lease.socket_path, "challenge": "challenge", - "control_dir": lease.control_dir, "created_at": "created", - } - target = control / "locator.json" - target.write_text(json.dumps(locator), encoding="utf-8") - running = (locator_root / "attempt.json").read_bytes() - - for kind in kinds: - with self.subTest(surface="registered locator", kind=kind): - saved = target.read_bytes() - self._substitute(target, kind, saved) - try: - self._assert_probe_rejected(run, locator_attempt, "locator", locator) - self.assertFalse(stat.S_ISREG(os.lstat(target).st_mode)) - finally: - self._restore(target, saved) - self.assertEqual(saved, target.read_bytes()) - self.assertTrue(stat.S_ISREG(os.lstat(target).st_mode)) - self.assertEqual(running, (locator_root / "attempt.json").read_bytes()) - covered.append(("registered locator", kind)) - digest = "sha256:" + "0" * 64 - self.store.record_locator( - locator_attempt, SupervisorLocator(**locator), digest, - ) - record = json.loads((locator_root / "attempt.json").read_text(encoding="utf-8")) - self.assertEqual(record.get("locator"), locator) - self.assertEqual(record.get("spec_digest"), digest) - located = self.store.attempts(run, Slot("a", 2)) - self.assertEqual([(item.identity, item.state) for item in located], [(locator_attempt.identity, "running")]) - self.assertFalse((locator_root / "lifecycle-result.json").exists()) - self.assertFalse((locator_root / "lifecycle-journal.jsonl").exists()) - self.assertEqual(self.store.attempts(run, Slot("a", 3)), ()) - # Passing subtests are silent, so bind the executed matrix explicitly. - self.assertEqual(len(covered), (len(surfaces) + 1) * len(kinds)) - - @staticmethod - def _mutate_terminal(paths: dict[str, Path], record: str, mutate) -> None: - """Apply one contradiction to a single record or to the ordered event evidence.""" - if record != "events": - raw = json.loads(paths[record].read_text(encoding="utf-8")) - mutate(raw) - paths[record].write_text(json.dumps(raw), encoding="utf-8") - return - result = json.loads(paths["result"].read_text(encoding="utf-8")) - result["events"] = mutate(result["events"]) - paths["result"].write_text(json.dumps(result), encoding="utf-8") - lines = [json.loads(line) for line in paths["journal"].read_text(encoding="utf-8").splitlines()] - rewritten = [lines[0], *result["events"], lines[-1]] - paths["journal"].write_text("".join(json.dumps(line) + "\n" for line in rewritten), encoding="utf-8") - - def test_cross_record_terminal_corruption_fails_closed_and_preserves_bytes(self): - run, attempt = self._running_with_terminal() - root = Path(attempt.root) - paths = { - "result": root / "lifecycle-result.json", - "journal": root / "lifecycle-journal.jsonl", - "receipt": root / "control" / "cleanup-receipt.json", - } - saved = {key: path.read_bytes() for key, path in paths.items()} - cases = ( - ("receipt-exit-code", "receipt", lambda raw: raw.__setitem__("exit_code", 9)), - ("receipt-signal", "receipt", lambda raw: raw.__setitem__("signal", 9)), - ("receipt-reason", "receipt", lambda raw: raw.__setitem__("reason", "failed")), - ("receipt-caller-launched", "receipt", lambda raw: raw.__setitem__("caller_launched", False)), - ("receipt-completed-before-start", "receipt", lambda raw: raw.__setitem__("completed_at", "2000-01-01T00:00:00+00:00")), - ("receipt-completed-after-end", "receipt", lambda raw: raw.__setitem__("completed_at", "2100-01-01T00:00:00+00:00")), - ("receipt-completed-unparseable", "receipt", lambda raw: raw.__setitem__("completed_at", "not-a-timestamp")), - ("result-submitted", "result", lambda raw: raw.__setitem__("submitted", False)), - ( - "result-exit-code", - "result", - lambda raw: raw["process"].__setitem__("exit_code", 7), - ), - ("events-cleared", "events", lambda events: []), - ("events-missing-submitted", "events", _without("submitted")), - ("events-missing-caller-terminal", "events", _without("caller_terminal")), - ("events-missing-finish", "events", _without("finish")), - ("events-missing-idle", "events", _without("idle")), - ("events-missing-quiet", "events", _without("quiet")), - ("events-out-of-order", "events", _reordered), - ("events-duplicate-finish", "events", _duplicated("finish")), - ( - "events-caller-terminal-detail", - "events", - lambda events: [ - {**event, "detail": "status=failed reason=caller_error"} - if event["kind"] == "caller_terminal" - else event - for event in events - ], - ), - ) - for name, record, mutate in cases: - with self.subTest(case=name): - self._mutate_terminal(paths, record, mutate) - try: - before = self._evidence_bytes(run, attempt) - with self.store.writer(run): - with self.assertRaises(AttemptStateError): - self.store.reconcile(attempt) - self.assertEqual(before, self._evidence_bytes(run, attempt)) - self.assertEqual([item.state for item in self.store.attempts(run, Slot("a", 1))], ["running"]) - finally: - for key, path in paths.items(): - path.write_bytes(saved[key]) - record = root / "attempt.json" - running = record.read_bytes() - with self.store.writer(run): - recovered = self.store.reconcile(attempt) - published = record.read_bytes() - self.assertEqual(recovered.state, "completed") - self.assertNotEqual(running, published) - with self.store.writer(run): - self.assertEqual(self.store.reconcile(recovered).state, "completed") - self.assertEqual(published, record.read_bytes()) - self.assertEqual(len(self.store.attempts(run, Slot("a", 1))), 1) - - def test_coherent_cleanup_failure_remains_a_terminal_independent_axis(self): - run, attempt = self._running_with_terminal() - root = Path(attempt.root) - result_path = root / "lifecycle-result.json" - journal_path = root / "lifecycle-journal.jsonl" - receipt_path = root / "control" / "cleanup-receipt.json" - - result = json.loads(result_path.read_text(encoding="utf-8")) - result["harness"].update({ - "status": "failed", - "reason": "cleanup_failed", - "cleanup_complete": False, - }) - result_path.write_text(json.dumps(result), encoding="utf-8") - - journal = [ - json.loads(line) - for line in journal_path.read_text(encoding="utf-8").splitlines() - ] - journal[-1]["harness"] = result["harness"] - journal_path.write_text( - "".join(json.dumps(line) + "\n" for line in journal), - encoding="utf-8", - ) - - receipt = json.loads(receipt_path.read_text(encoding="utf-8")) - receipt.update({"reason": "cleanup_failed", "cleanup_complete": False}) - receipt_path.write_text(json.dumps(receipt), encoding="utf-8") - - with self.store.writer(run): - terminal = self.store.reconcile(attempt) - projection = self.store.attempt_outcomes(terminal) - self.assertEqual(terminal.state, "completed") - self.assertEqual( - (projection["product"], projection["harness"], projection["process"]), - ("succeeded", "failed", "exited"), - ) - - def test_symlink_lifecycle_evidence_fails_closed(self): - run, attempt = self._running_with_terminal() - journal = Path(attempt.root) / "lifecycle-journal.jsonl" - saved = journal.read_bytes() - target = Path(attempt.root) / "journal-copy.jsonl" - target.write_bytes(saved) - journal.unlink() - journal.symlink_to(target) - record = Path(attempt.root) / "attempt.json" - before = record.read_bytes() - with self.store.writer(run): - with self.assertRaises(AttemptStateError): - self.store.reconcile(attempt) - self.assertEqual(before, record.read_bytes()) - - def test_direct_result_requires_bound_production_evidence(self): - run = self.create_run() - with self.store.writer(run): - attempt = self.store.allocate(run, Slot("a", 1)) - - def contradictory(current, started): - result = self.adapter("success", [])(current, started) - path = Path(current.root) / "lifecycle-result.json" - raw = json.loads(path.read_text(encoding="utf-8")) - raw["product"]["status"] = "unknown" - path.write_text(json.dumps(raw), encoding="utf-8") - return result - - with self.assertRaises(AttemptStateError): - self.store.execute_attempt(attempt, prepare=lambda _: None, invoke=contradictory) - record = Path(attempt.root) / "attempt.json" - self.assertEqual(json.loads(record.read_text(encoding="utf-8"))["state"], "running") - - def test_controller_process_loss_reconciles_durable_receipt(self): - run = self.create_run() - with self.store.writer(run): - attempt = self.store.allocate(run, Slot("a", 1)) - attempt_root = Path(attempt.root) - expected_lease = self.store._control_lease_for_root(attempt_root) - alias = Path(expected_lease.alias) - self._control_aliases.append(alias) - payload = json.dumps( - { - "repo": str(self.root), - "manifest": str(self.manifest_path), - "run_id": run.run_id, - "run_root": run.root, - "attempt_root": attempt.root, - "cell_id": attempt.identity.cell_id, - "repetition": attempt.identity.repetition, - "attempt_number": attempt.identity.attempt, - } - ) - child = subprocess.Popen( - [ - sys.executable, - "-c", - ( - "import sys; " - "from scripts.agent_benchmark.attempts_test import " - "_controller_loss_child; " - "_controller_loss_child(sys.argv[1])" - ), - payload, - ], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - env={ - **os.environ, - "PYTHONPATH": str(Path(__file__).resolve().parents[2]), - }, - ) - locator: SupervisorLocator | None = None - - def child_failure(label: str) -> None: - if child.poll() is None: - return - stdout, stderr = child.communicate() - self.fail( - f"controller exited before {label}: returncode={child.returncode} " - f"stdout={stdout!r} stderr={stderr!r}" - ) - - try: - deadline = time.monotonic() + 10 - attempt_record = attempt_root / "attempt.json" - registered_path = attempt_root / "control" / "locator.json" - while locator is None: - child_failure("locator commit") - if attempt_record.is_file() and registered_path.is_file(): - record = json.loads(attempt_record.read_text(encoding="utf-8")) - raw_locator = record.get("locator") - if raw_locator is not None: - registered = json.loads( - registered_path.read_text(encoding="utf-8") - ) - if registered == raw_locator: - locator = SupervisorLocator(**raw_locator) - break - if time.monotonic() >= deadline: - self.fail("controller did not commit its locator") - threading.Event().wait(0.01) - - deadline = time.monotonic() + 10 - while True: - child_failure("caller launch") - try: - status = recover_invocation(locator, stop=False) - except LifecycleRecoveryError: - status = None - if status is not None and status.caller_launched: - break - if time.monotonic() >= deadline: - self.fail("controller did not launch its caller") - threading.Event().wait(0.01) - - child.kill() - stdout, stderr = child.communicate(timeout=5) - self.assertEqual(child.returncode, -signal.SIGKILL, (stdout, stderr)) - - receipt_path = attempt_root / "control" / "cleanup-receipt.json" - canonical_socket = attempt_root / "control" / "control.sock" - deadline = time.monotonic() + 10 - receipt: dict[str, object] | None = None - while receipt is None: - if receipt_path.is_file(): - candidate = json.loads(receipt_path.read_text(encoding="utf-8")) - if ( - candidate.get("reason") == REASON_CONTROLLER_LOST - and candidate.get("caller_launched") is True - and candidate.get("cleanup_complete") is True - and candidate.get("process_group_alive") is False - and not os.path.lexists(canonical_socket) - and not os.path.lexists(locator.socket_path) - ): - receipt = candidate - break - if time.monotonic() >= deadline: - self.fail("supervisor did not publish closed controller-loss receipt") - threading.Event().wait(0.01) - - self.assertTrue(alias.is_symlink()) - self.assertFalse((attempt_root / "lifecycle-result.json").exists()) - self.assertFalse((attempt_root / "lifecycle-journal.jsonl").exists()) - - def durable_bytes() -> dict[str, bytes]: - paths = [attempt_record] - paths.extend( - path - for path in sorted((attempt_root / "control").iterdir()) - if stat.S_ISREG(os.lstat(path).st_mode) - ) - return { - str(path.relative_to(attempt_root)): path.read_bytes() - for path in paths - } - - clean_running = durable_bytes() - tamper_cases = ( - ( - "registered-locator", - registered_path, - lambda raw: raw.__setitem__( - "challenge", str(raw["challenge"]) + "-tampered" - ), - ), - ( - "receipt-identity", - receipt_path, - lambda raw: raw.__setitem__("challenge_digest", "0" * 64), - ), - ( - "receipt-reason", - receipt_path, - lambda raw: raw.__setitem__("reason", "recovered_stop"), - ), - ( - "receipt-incomplete-cleanup", - receipt_path, - lambda raw: raw.__setitem__("cleanup_complete", False), - ), - ( - "receipt-live-process-group", - receipt_path, - lambda raw: raw.__setitem__("process_group_alive", True), - ), - ( - "receipt-schema", - receipt_path, - lambda raw: raw.__setitem__("unexpected", True), - ), - ( - "receipt-completed-at", - receipt_path, - lambda raw: raw.__setitem__("completed_at", "not-a-timestamp"), - ), - ) - for name, target, tamper in tamper_cases: - with self.subTest(phase="running", case=name): - original = target.read_bytes() - raw = json.loads(original.decode("utf-8")) - tamper(raw) - target.write_text(json.dumps(raw), encoding="utf-8") - before = durable_bytes() - with self.store.writer(run): - with self.assertRaises(AttemptStateError): - self.store.reconcile(attempt) - self.assertEqual(before, durable_bytes()) - self.assertTrue(alias.is_symlink()) - target.write_bytes(original) - self.assertEqual(clean_running, durable_bytes()) - - original_release = self.store.release_control_lease - release_observations: list[str] = [] - - def release_after_publication(current) -> None: - published = json.loads(attempt_record.read_text(encoding="utf-8")) - self.assertEqual(published["state"], "interrupted") - self.assertEqual( - published["lifecycle"]["harness"]["reason"], - REASON_CONTROLLER_LOST, - ) - self.assertTrue(alias.is_symlink()) - release_observations.append(published["state"]) - original_release(current) - - with mock.patch.object( - self.store, - "release_control_lease", - side_effect=release_after_publication, - ): - with self.store.writer(run): - recovered = self.store.reconcile(attempt) - successor = self.store.allocate(run, Slot("a", 1)) - - self.assertEqual(recovered.state, "interrupted") - self.assertEqual(successor.identity.attempt, 2) - self.assertEqual(release_observations, ["interrupted"]) - self.assertFalse(os.path.lexists(alias)) - - clean_terminal = durable_bytes() - terminal_tamper_cases = ( - *tamper_cases, - ( - "terminal-record-reason", - attempt_record, - lambda raw: raw["lifecycle"]["harness"].__setitem__( - "reason", "success" - ), - ), - ( - "terminal-record-state-success", - attempt_record, - lambda raw: raw.__setitem__("state", "completed"), - ), - ( - "terminal-record-state-failed", - attempt_record, - lambda raw: raw.__setitem__("state", "cancelled"), - ), - ) - for name, target, tamper in terminal_tamper_cases: - with self.subTest(phase="terminal-status", case=name): - original = target.read_bytes() - raw = json.loads(original.decode("utf-8")) - tamper(raw) - target.write_text(json.dumps(raw), encoding="utf-8") - before = durable_bytes() - with self.assertRaises(AttemptStateError): - self.store.status(run, self.manifest) - self.assertEqual(before, durable_bytes()) - target.write_bytes(original) - self.assertEqual(clean_terminal, durable_bytes()) - - projected = self.store.status(run, self.manifest)["attempts"] - self.assertEqual(projected["interrupted"], 1) - self.assertEqual(projected["running"], 1) - self.assertFalse(receipt["process_group_alive"]) - self.assertFalse(os.path.lexists(alias)) - finally: - if child.poll() is None: - child.kill() - child.communicate(timeout=5) - - def test_live_survivor_cleanup_precedes_successor(self): - run = self.create_run() - with self.store.writer(run): - attempt = self.store.allocate(run, Slot("a", 1)) - locator_ready = threading.Event() - worker_outcomes: list[BaseException | Attempt] = [] - reconciler_outcomes: list[BaseException | Attempt] = [] - - def long_running(current, started): - spec = self._spec(current, "import time; print('START', flush=True); time.sleep(30)") - - def commit(locator): - started(locator, spec_digest(spec)) - locator_ready.set() - - return run_invocation(spec, parse_event=_events, on_started=commit) - - def invoke() -> None: - try: - worker_outcomes.append( - self.store.execute_attempt( - attempt, prepare=lambda _: None, invoke=long_running - ) - ) - except BaseException as exc: # concurrent reconciliation seals this attempt first - worker_outcomes.append(exc) - - worker = threading.Thread(target=invoke) - worker.start() - self.assertTrue(locator_ready.wait(5)) - locator = SupervisorLocator( - **json.loads( - (Path(attempt.root) / "attempt.json").read_text(encoding="utf-8") - )["locator"] - ) - deadline = time.monotonic() + 5 - while not recover_invocation(locator, stop=False).caller_launched: - if time.monotonic() >= deadline: - self.fail("caller did not launch before recovery") - threading.Event().wait(0.01) - actual_recover = recover_invocation - - def lose_cleanup_reply(current, stop=True): - outcome = actual_recover(current, stop=stop) - if stop: - raise LifecycleRecoveryError("simulated lost cleanup reply") - return outcome - - with mock.patch( - "scripts.agent_benchmark.attempts.recover_invocation", - side_effect=lose_cleanup_reply, - ): - try: - with self.store.writer(run): - reconciler_outcomes.append(self.store.reconcile(attempt)) - except BaseException as exc: - reconciler_outcomes.append(exc) - worker.join(10) - self.assertFalse(worker.is_alive()) - self.assertEqual(len(worker_outcomes), 1) - self.assertEqual(len(reconciler_outcomes), 1) - outcomes = (*worker_outcomes, *reconciler_outcomes) - for outcome in outcomes: - if isinstance(outcome, BaseException): - self.assertIsInstance(outcome, AttemptStateError) - else: - self.assertEqual(outcome.state, "interrupted") - self.assertTrue(any(isinstance(outcome, Attempt) for outcome in outcomes)) - - attempt_root = Path(attempt.root) - record = json.loads( - (attempt_root / "attempt.json").read_text(encoding="utf-8") - ) - result = json.loads( - (attempt_root / "lifecycle-result.json").read_text(encoding="utf-8") - ) - receipt = json.loads( - (attempt_root / "control/cleanup-receipt.json").read_text( - encoding="utf-8" - ) - ) - self.assertEqual(record["state"], "interrupted") - self.assertEqual(record["lifecycle"]["harness"]["reason"], REASON_RECOVERED_STOP) - self.assertEqual(result["harness"]["reason"], REASON_RECOVERED_STOP) - self.assertEqual(receipt["reason"], REASON_RECOVERED_STOP) - self.assertTrue(receipt["cleanup_complete"]) - self.assertFalse(receipt["process_group_alive"]) - - terminal = Attempt(attempt.identity, attempt.root, "interrupted") - self.store.release_control_lease(terminal) - self.store.release_control_lease(terminal) - alias = Path(self.store._control_lease_for_root(attempt_root).alias) - self.assertFalse(os.path.lexists(alias)) - - with self.store.writer(run): - successor = self.store.allocate(run, Slot("a", 1)) - self.assertEqual(successor.identity.attempt, 2) - - def test_cross_process_lease_contention_and_crash_release(self): - run = self.create_run() - script = "import fcntl, os, sys, time; f=os.open(sys.argv[1], os.O_RDWR); fcntl.flock(f, fcntl.LOCK_EX); print('locked', flush=True); time.sleep(30)" - child = subprocess.Popen([sys.executable, "-c", script, str(Path(run.root) / "run.lock")], stdout=subprocess.PIPE, text=True) - self.assertEqual(child.stdout.readline().strip(), "locked") - with self.assertRaises(RunBusyError): - with self.store.writer(run): - pass - child.kill() - child.wait(timeout=5) - child.stdout.close() - with self.store.writer(run): - pass - - -class AttemptCliContractTest(AttemptBase): - def test_cli_status_is_read_only_and_run_resume_block_before_attempts(self): - run = self.create_run() - run_before = { - path.relative_to(run.root): path.read_bytes() - for path in Path(run.root).rglob("*") - if path.is_file() - } - output = io.StringIO() - with mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root), contextlib.redirect_stdout(output): - self.assertEqual(benchmark_cli.main(["status", "--manifest", str(self.manifest_path), "--run-id", run.run_id]), 0) - self.assertIn("running=0", output.getvalue()) - self.assertIn("product_succeeded=0", output.getvalue()) - self.assertIn("artifact_passed=0", output.getvalue()) - self.assertIn("unresolved=1", output.getvalue()) - self.assertEqual( - run_before, - { - path.relative_to(run.root): path.read_bytes() - for path in Path(run.root).rglob("*") - if path.is_file() - }, - ) - with mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root), contextlib.redirect_stderr(io.StringIO()): - self.assertEqual(benchmark_cli.main(["resume", "--manifest", str(self.manifest_path), "--run-id", run.run_id]), 69) - self.assertFalse((Path(run.root) / "cells").exists()) - self.assertEqual(len(self.store.preflights(run, self.manifest)), 1) - absent_root = self.root / "agent-test/runs/absent" - raw = json.loads(self.raw) - raw["output_root"] = "agent-test/runs/absent" - absent = self.root / "absent.json" - absent.write_text(json.dumps(raw), encoding="utf-8") - with mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root), contextlib.redirect_stderr(io.StringIO()): - self.assertEqual(benchmark_cli.main(["run", "--manifest", str(absent)]), 69) - created = list(absent_root.glob("run-*")) - self.assertEqual(len(created), 1) - self.assertTrue((created[0] / "preflight/preflight-000001.json").is_file()) - self.assertFalse((created[0] / "cells").exists()) - - def test_terminal_failed_attempt_resolves_without_passing(self): - """A terminal failed attempt with complete web evidence resolves (unresolved=0) but does not pass.""" - self._init_testbed() - run = self.create_run() - completed = run_slots( - self.store, - run, - self.manifest, - adapters={"claude": MeasuringExecutionAdapter(self, "failed")}, - prepare=self.preparer([]), - ) - self.assertEqual([item.state for item in completed], ["completed"]) - outcomes = self.store.attempt_outcomes(completed[0]) - # The MeasuringExecutionAdapter produces web-validation.json with status=failed. - self.assertEqual(outcomes["artifact"], "failed") - self.assertEqual(outcomes["resolved"], True) - self.assertEqual(outcomes["passed"], False) - status = self.store.status(run, self.manifest) - self.assertEqual(status["outcomes"]["unresolved"], 0) - self.assertEqual(status["outcomes"]["artifact"]["failed"], 1) - - def test_terminal_timed_out_attempt_resolves_without_passing(self): - """A terminal timed-out attempt with complete web evidence resolves (unresolved=0) but does not pass.""" - self._init_testbed() - run = self.create_run() - completed = run_slots( - self.store, - run, - self.manifest, - adapters={"claude": MeasuringExecutionAdapter(self, "timeout")}, - prepare=self.preparer([]), - ) - self.assertEqual([item.state for item in completed], ["timed_out"]) - outcomes = self.store.attempt_outcomes(completed[0]) - self.assertEqual(outcomes["artifact"], "failed") - self.assertEqual(outcomes["resolved"], True) - self.assertEqual(outcomes["passed"], False) - status = self.store.status(run, self.manifest) - self.assertEqual(status["outcomes"]["unresolved"], 0) - - def test_absent_slot_remains_unresolved(self): - """A slot with no attempt record remains unresolved.""" - run = self.create_run() - status = self.store.status(run, self.manifest) - self.assertEqual(status["outcomes"]["unresolved"], 1) - - def test_running_slot_remains_unresolved(self): - """An attempt still in running state remains unresolved.""" - run = self.create_run() - with self.store.writer(run): - attempt = self.store.allocate(run, Slot("a", 1)) - self.assertEqual(attempt.state, "running") - status = self.store.status(run, self.manifest) - self.assertEqual(status["outcomes"]["unresolved"], 1) - - def test_interruption_without_web_evidence_remains_unresolved(self): - """A controller interruption before caller registration has no web record and stays unresolved.""" - run = self.create_run() - with self.store.writer(run): - attempt = self.store.allocate(run, Slot("a", 1)) - interrupted = self.store.reconcile(attempt) - self.assertEqual(interrupted.state, "interrupted") - outcomes = self.store.attempt_outcomes(interrupted) - self.assertEqual(outcomes["artifact"], "not_run") - self.assertEqual(outcomes["resolved"], False) - status = self.store.status(run, self.manifest) - self.assertEqual(status["outcomes"]["unresolved"], 1) - - def test_mixed_terminal_outcomes_resolve_with_independent_failure_counts(self): - """A run with resolved terminals preserves independent axis counts even when all fail.""" - self._init_testbed() - run = self.create_run() - completed = run_slots( - self.store, - run, - self.manifest, - adapters={"claude": MeasuringExecutionAdapter(self, "failed")}, - prepare=self.preparer([]), - ) - self.assertEqual([item.state for item in completed], ["completed"]) - status = self.store.status(run, self.manifest) - self.assertEqual(status["outcomes"]["unresolved"], 0) - self.assertEqual(status["attempts"]["completed"], 1) - # The fake caller's non-terminal output yields product=unknown, harness=failed. - self.assertEqual(status["outcomes"]["product"]["unknown"], 1) - self.assertEqual(status["outcomes"]["harness"]["failed"], 1) - self.assertEqual(status["outcomes"]["artifact"]["failed"], 1) - output = io.StringIO() - with mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root), contextlib.redirect_stdout(output): - self.assertEqual(benchmark_cli.main(["status", "--manifest", str(self.manifest_path), "--run-id", run.run_id]), 0) - self.assertIn("unresolved=0", output.getvalue()) - self.assertIn("artifact_failed=1", output.getvalue()) - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/agent_benchmark/browser_cdp.py b/scripts/agent_benchmark/browser_cdp.py deleted file mode 100644 index 469e0c1d..00000000 --- a/scripts/agent_benchmark/browser_cdp.py +++ /dev/null @@ -1,870 +0,0 @@ -"""Contained loopback Chromium rendering for benchmark web evidence. - -Only regular files beneath one workspace are served. The CDP client is a -small, bounded RFC6455 implementation and every browser is launched in an -owned process group which is reaped before its temporary profile is removed. -""" -from __future__ import annotations - -import base64 -import hashlib -import ipaddress -import json -import os -import secrets -import shutil -import signal -import socket -import stat -import struct -import subprocess -import tempfile -import threading -import time -from dataclasses import dataclass -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from pathlib import Path -from urllib.parse import unquote_to_bytes, urlparse -from urllib.request import urlopen - -from scripts.agent_benchmark.manifest import VIEWPORT_ID_RE - - -MAX_HANDSHAKE_BYTES = 16 * 1024 -MAX_MESSAGE_BYTES = 8 * 1024 * 1024 -MAX_STATIC_BYTES = 32 * 1024 * 1024 -MAX_PENDING_FIRE_COMMANDS = 4096 -WEBSOCKET_GUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11" - - -class BrowserError(Exception): - """A redacted renderer/protocol failure safe for durable evidence.""" - - -@dataclass(frozen=True) -class ViewportObservation: - id: str - width: int - height: int - screenshot: str - screenshot_digest: str - screenshot_size: int - image_facts: tuple[dict, ...] - layout: dict - accessibility: dict - - -@dataclass(frozen=True) -class RenderObservation: - browser: str - origin: str - requests: tuple[dict, ...] - console: tuple[dict, ...] - viewports: tuple[ViewportObservation, ...] - - -def _digest(data: bytes) -> str: - return "sha256:" + hashlib.sha256(data).hexdigest() - - -def _decode_request_path(raw: str) -> tuple[tuple[str, ...], str]: - try: - decoded = unquote_to_bytes(raw).decode("utf-8", "strict") - except (UnicodeDecodeError, ValueError) as exc: - raise FileNotFoundError from exc - if not decoded.startswith("/") or "\x00" in decoded or "\\" in decoded: - raise FileNotFoundError - parts = tuple(decoded[1:].split("/")) - if not parts or any(part in ("", ".", "..") for part in parts): - raise FileNotFoundError - return parts, decoded - - -def _read_contained_regular_no_follow(root: Path, raw: str) -> tuple[bytes, str]: - """Open every path component relative to ``root`` without following links.""" - parts, decoded = _decode_request_path(raw) - directory_flags = os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC - if hasattr(os, "O_NOFOLLOW"): - directory_flags |= os.O_NOFOLLOW - file_flags = os.O_RDONLY | os.O_CLOEXEC | os.O_NONBLOCK - if hasattr(os, "O_NOFOLLOW"): - file_flags |= os.O_NOFOLLOW - descriptors: list[int] = [] - try: - current = os.open(root, directory_flags) - descriptors.append(current) - for component in parts[:-1]: - current = os.open(component, directory_flags, dir_fd=current) - descriptors.append(current) - fd = os.open(parts[-1], file_flags, dir_fd=current) - descriptors.append(fd) - info = os.fstat(fd) - if not stat.S_ISREG(info.st_mode) or info.st_size > MAX_STATIC_BYTES: - raise FileNotFoundError - chunks: list[bytes] = [] - remaining = info.st_size - while remaining: - chunk = os.read(fd, min(remaining, 1024 * 1024)) - if not chunk: - raise FileNotFoundError - chunks.append(chunk) - remaining -= len(chunk) - # Refuse a file that grew beyond the bounded snapshot while being read. - if os.read(fd, 1): - raise FileNotFoundError - return b"".join(chunks), Path(decoded).suffix.lower() - except (OSError, ValueError) as exc: - raise FileNotFoundError from exc - finally: - for descriptor in reversed(descriptors): - try: - os.close(descriptor) - except OSError: - pass - - -class _StaticServer(ThreadingHTTPServer): - daemon_threads = True - allow_reuse_address = True - - def __init__(self, root: Path): - if root.is_symlink() or not root.is_dir(): - raise BrowserError("workspace_unavailable") - self.root = root.resolve() - self.requests: list[dict] = [] - self._requests_lock = threading.Lock() - super().__init__(("127.0.0.1", 0), _StaticHandler) - - def record_request(self, record: dict) -> None: - with self._requests_lock: - self.requests.append(record) - - def request_snapshot(self) -> tuple[dict, ...]: - with self._requests_lock: - return tuple(dict(item) for item in self.requests) - - -class _StaticHandler(BaseHTTPRequestHandler): - def log_message(self, *_args): - return - - def do_GET(self): - server: _StaticServer = self.server # type: ignore[assignment] - raw = self.path.split("?", 1)[0].split("#", 1)[0] - if raw == "/favicon.ico": - server.record_request( - {"kind": "local", "path": raw, "allowed": True, "status": 204} - ) - self.send_response(204) - self.end_headers() - return - if raw == "/": - raw = "/index.html" - try: - data, suffix = _read_contained_regular_no_follow(server.root, raw) - except FileNotFoundError: - server.record_request( - {"kind": "local", "path": raw, "allowed": False, "status": 404} - ) - self.send_error(404) - return - server.record_request( - {"kind": "local", "path": raw, "allowed": True, "status": 200} - ) - content_type = { - ".html": "text/html; charset=utf-8", - ".css": "text/css; charset=utf-8", - ".js": "application/javascript; charset=utf-8", - ".svg": "image/svg+xml", - ".png": "image/png", - ".jpg": "image/jpeg", - ".jpeg": "image/jpeg", - }.get(suffix, "application/octet-stream") - self.send_response(200) - self.send_header("Content-Type", content_type) - self.send_header("Content-Length", str(len(data))) - self.end_headers() - self.wfile.write(data) - - -class _CDP: - """Bounded RFC6455 client with strict response/event correlation.""" - - def __init__(self, url: str, deadline: float): - parsed = urlparse(url) - try: - address = ipaddress.ip_address(parsed.hostname or "") - except ValueError as exc: - raise BrowserError("CDP endpoint is not loopback") from exc - if parsed.scheme != "ws" or not address.is_loopback or parsed.port is None: - raise BrowserError("CDP endpoint is not loopback") - self.deadline = deadline - self.seq = 0 - self.events: list[dict] = [] - self.event_handler = None - self._fire_ids: set[int] = set() - self._buffer = bytearray() - self._closed = False - timeout = self._remaining() - try: - self.sock = socket.create_connection( - (parsed.hostname, parsed.port), timeout - ) - key = base64.b64encode(secrets.token_bytes(16)).decode("ascii") - request = ( - f"GET {parsed.path or '/'} HTTP/1.1\r\n" - f"Host: {parsed.hostname}:{parsed.port}\r\n" - "Upgrade: websocket\r\nConnection: Upgrade\r\n" - f"Sec-WebSocket-Key: {key}\r\n" - "Sec-WebSocket-Version: 13\r\n\r\n" - ).encode("ascii") - self._sendall(request) - self._read_http(key) - except Exception: - sock = getattr(self, "sock", None) - if sock is not None: - try: - sock.close() - except OSError: - pass - raise - - def _remaining(self) -> float: - remaining = self.deadline - time.monotonic() - if remaining <= 0: - raise BrowserError("CDP deadline expired") - return remaining - - def _read_http(self, key: str) -> None: - data = bytearray() - while b"\r\n\r\n" not in data: - if len(data) >= MAX_HANDSHAKE_BYTES: - raise BrowserError("oversized CDP handshake") - try: - self.sock.settimeout(self._remaining()) - chunk = self.sock.recv(4096) - except (OSError, socket.timeout) as exc: - raise BrowserError("CDP handshake failed") from exc - if not chunk: - raise BrowserError("CDP handshake closed") - data.extend(chunk) - if len(data) > MAX_HANDSHAKE_BYTES: - raise BrowserError("oversized CDP handshake") - header, tail = bytes(data).split(b"\r\n\r\n", 1) - lines = header.split(b"\r\n") - status_parts = lines[0].split(b" ", 2) - if len(status_parts) < 2 or status_parts[0] != b"HTTP/1.1" or status_parts[1] != b"101": - raise BrowserError("CDP WebSocket upgrade failed") - headers: dict[str, str] = {} - for line in lines[1:]: - if b":" not in line: - raise BrowserError("CDP WebSocket headers are invalid") - name, value = line.split(b":", 1) - try: - headers[name.decode("ascii").strip().lower()] = value.decode( - "ascii" - ).strip() - except UnicodeDecodeError as exc: - raise BrowserError("CDP WebSocket headers are invalid") from exc - expected = base64.b64encode( - hashlib.sha1((key + WEBSOCKET_GUID).encode("ascii")).digest() - ).decode("ascii") - if ( - headers.get("upgrade", "").lower() != "websocket" - or "upgrade" - not in {part.strip().lower() for part in headers.get("connection", "").split(",")} - or headers.get("sec-websocket-accept") != expected - ): - raise BrowserError("CDP WebSocket handshake is invalid") - self._buffer.extend(tail) - - def _exact(self, size: int) -> bytes: - data = bytearray() - if self._buffer: - take = min(size, len(self._buffer)) - data.extend(self._buffer[:take]) - del self._buffer[:take] - while len(data) < size: - try: - self.sock.settimeout(self._remaining()) - chunk = self.sock.recv(size - len(data)) - except socket.timeout as exc: - raise BrowserError("CDP deadline expired") from exc - except OSError as exc: - raise BrowserError("CDP socket read failed") from exc - if not chunk: - raise BrowserError("CDP socket closed") - data.extend(chunk) - return bytes(data) - - def _frame(self) -> tuple[bool, int, bytes]: - first, second = self._exact(2) - fin = bool(first & 0x80) - if first & 0x70: - raise BrowserError("reserved CDP frame bits") - opcode = first & 0x0F - masked = bool(second & 0x80) - if masked: - raise BrowserError("masked CDP server frame") - length = second & 0x7F - if length == 126: - length = struct.unpack("!H", self._exact(2))[0] - if length < 126: - raise BrowserError("non-canonical CDP frame length") - elif length == 127: - encoded = self._exact(8) - if encoded[0] & 0x80: - raise BrowserError("invalid CDP frame length") - length = struct.unpack("!Q", encoded)[0] - if length <= 65535: - raise BrowserError("non-canonical CDP frame length") - if length > MAX_MESSAGE_BYTES: - raise BrowserError("oversized CDP frame") - if opcode >= 8 and (not fin or length > 125): - raise BrowserError("invalid CDP control frame") - return fin, opcode, self._exact(length) - - def receive_message(self) -> dict: - fragments = bytearray() - fragmented = False - while True: - fin, opcode, payload = self._frame() - if opcode == 8: - if len(payload) == 1: - raise BrowserError("invalid CDP close frame") - try: - self._send(payload, opcode=8) - except BrowserError: - pass - raise BrowserError("CDP socket closed") - if opcode == 9: - self._send(payload, opcode=10) - continue - if opcode == 10: - continue - if opcode == 2: - raise BrowserError("binary CDP message") - if opcode == 1: - if fragmented: - raise BrowserError("interleaved CDP data frame") - fragments.extend(payload) - fragmented = not fin - elif opcode == 0: - if not fragmented: - raise BrowserError("unexpected CDP continuation") - fragments.extend(payload) - fragmented = not fin - else: - raise BrowserError("unsupported CDP frame") - if len(fragments) > MAX_MESSAGE_BYTES: - raise BrowserError("oversized CDP message") - if fragmented: - continue - try: - message = json.loads(fragments.decode("utf-8")) - except (UnicodeDecodeError, json.JSONDecodeError) as exc: - raise BrowserError("malformed CDP message") from exc - if not isinstance(message, dict): - raise BrowserError("malformed CDP message") - return message - - def _sendall(self, data: bytes) -> None: - try: - self.sock.settimeout(self._remaining()) - self.sock.sendall(data) - except socket.timeout as exc: - raise BrowserError("CDP deadline expired") from exc - except OSError as exc: - raise BrowserError("CDP socket write failed") from exc - - def _send(self, data: bytes, *, opcode: int = 1) -> None: - if len(data) > MAX_MESSAGE_BYTES: - raise BrowserError("oversized CDP client message") - mask = secrets.token_bytes(4) - size = len(data) - if size < 126: - header = bytes((0x80 | opcode, 0x80 | size)) - elif size <= 65535: - header = bytes((0x80 | opcode, 0x80 | 126)) + struct.pack("!H", size) - else: - header = bytes((0x80 | opcode, 0x80 | 127)) + struct.pack("!Q", size) - masked = bytes(value ^ mask[index % 4] for index, value in enumerate(data)) - self._sendall(header + mask + masked) - - @staticmethod - def _command(ident: int, method: str, params: dict | None) -> bytes: - return json.dumps( - {"id": ident, "method": method, "params": params or {}}, - separators=(",", ":"), - ).encode("utf-8") - - def call(self, method: str, params: dict | None = None) -> dict: - self.seq += 1 - ident = self.seq - self._send(self._command(ident, method, params)) - while True: - message = self.receive_message() - response_id = message.get("id") - if response_id is not None: - if isinstance(response_id, bool) or not isinstance(response_id, int): - raise BrowserError("invalid CDP response id") - if response_id in self._fire_ids: - self._fire_ids.remove(response_id) - continue - if response_id != ident: - raise BrowserError("uncorrelated CDP response") - if "error" in message: - raise BrowserError(f"CDP {method} failed") - result = message.get("result", {}) - if not isinstance(result, dict): - raise BrowserError("invalid CDP result") - return result - if not isinstance(message.get("method"), str) or not isinstance( - message.get("params", {}), dict - ): - raise BrowserError("invalid CDP event") - self.events.append(message) - if self.event_handler is not None: - self.event_handler(message) - - def fire(self, method: str, params: dict | None = None) -> None: - """Send an event-callback command without entering a nested receive loop.""" - if len(self._fire_ids) >= MAX_PENDING_FIRE_COMMANDS: - raise BrowserError("too many pending CDP commands") - self.seq += 1 - self._fire_ids.add(self.seq) - self._send(self._command(self.seq, method, params)) - - def close(self) -> None: - if self._closed: - return - self._closed = True - try: - self.sock.shutdown(socket.SHUT_RDWR) - except OSError: - pass - try: - self.sock.close() - except OSError: - pass - - -_OBSERVATION_SCRIPT = r"""(() => { -const rect = e => { const r=e.getBoundingClientRect(); return {x:r.x,y:r.y,width:r.width,height:r.height,right:r.right,bottom:r.bottom}; }; -const visible = e => { const r=e.getBoundingClientRect(),s=getComputedStyle(e); return r.width>0&&r.height>0&&s.display!=='none'&&s.visibility!=='hidden'&&Number(s.opacity)>0; }; -const controls=[...document.querySelectorAll('a[href],button,input,select,textarea,[tabindex]')].filter(e=>!e.disabled&&visible(e)); -const clipped=[...document.querySelectorAll('img,a[href],button,input,select,textarea,[tabindex]')].filter(visible).filter(e=>{const r=e.getBoundingClientRect();return r.left<0||r.right>innerWidth;}).length; -let overlaps=0; for(let i=0;iMath.max(a.left,b.left)&&Math.min(a.bottom,b.bottom)>Math.max(a.top,b.top))overlaps++;} -const name=e=>(e.getAttribute('aria-label')||e.getAttribute('alt')||e.getAttribute('title')||e.value||e.textContent||'').trim(); -const rgba=s=>{let m=s.match(/[\d.]+/g)||[];return m.slice(0,4).map(Number);}; -const lum=c=>{c=c/255;return c<=.03928?c/12.92:Math.pow((c+.055)/1.055,2.4);}; -const contrast=e=>{let s=getComputedStyle(e),f=rgba(s.color),b=rgba(s.backgroundColor),p=e.parentElement;while((b.length<3||(b.length>=4&&b[3]===0))&&p){b=rgba(getComputedStyle(p).backgroundColor);p=p.parentElement;}if(f.length<3||b.length<3)return 0;let a=.2126*lum(f[0])+.7152*lum(f[1])+.0722*lum(f[2]),z=.2126*lum(b[0])+.7152*lum(b[1])+.0722*lum(b[2]);return (Math.max(a,z)+.05)/(Math.min(a,z)+.05);}; -const focusStyle=e=>{let s=getComputedStyle(e);return {outline_style:s.outlineStyle,outline_width:s.outlineWidth,outline_color:s.outlineColor,outline_offset:s.outlineOffset,box_shadow:s.boxShadow,background_color:s.backgroundColor,background_image:s.backgroundImage,border_top:[s.borderTopWidth,s.borderTopStyle,s.borderTopColor],border_right:[s.borderRightWidth,s.borderRightStyle,s.borderRightColor],border_bottom:[s.borderBottomWidth,s.borderBottomStyle,s.borderBottomColor],border_left:[s.borderLeftWidth,s.borderLeftStyle,s.borderLeftColor]};}; -const changed=(before,after,keys)=>keys.some(key=>before[key]!==after[key]); -const colorProbe=document.createElement('canvas').getContext('2d'); -const paintAlpha=value=>{if(!colorProbe)return 0;colorProbe.fillStyle='rgba(0,0,0,0)';colorProbe.fillStyle=value;let normalized=colorProbe.fillStyle,m=normalized.match(/^rgba?\(([^)]+)\)$/i);if(m){let parts=m[1].replace(/\//g,' ').split(/[,\s]+/).filter(Boolean);return normalized.toLowerCase().startsWith('rgba')?Number(parts[3]||0):1;}if(/^#[0-9a-f]{6}$/i.test(normalized))return 1;if(/^#[0-9a-f]{8}$/i.test(normalized))return parseInt(normalized.slice(7,9),16)/255;return 0;}; -const splitShadows=value=>{let result=[],start=0,depth=0;for(let i=0;i(parseFloat(after.outline_width)||0)>0&&after.outline_style!=='none'&&after.outline_style!=='hidden'&&paintAlpha(after.outline_color)>0&&changed(before,after,['outline_style','outline_width','outline_color','outline_offset']); -const paintedShadowChanged=(before,after)=>before.box_shadow!==after.box_shadow&&after.box_shadow!=='none'&&splitShadows(after.box_shadow).some(shadow=>{let color=(shadow.match(/rgba?\([^)]*\)|#[0-9a-f]{3,8}\b/ig)||[])[0]||'transparent';return paintAlpha(color)>0&&(shadow.match(/-?\d+(?:\.\d+)?px/g)||[]).some(value=>Math.abs(parseFloat(value))>0);}); -const paintedBorderChanged=(before,after)=>['border_top','border_right','border_bottom','border_left'].some(key=>{let current=after[key],prior=before[key];return (parseFloat(current[0])||0)>0&¤t[1]!=='none'&¤t[1]!=='hidden'&&paintAlpha(current[2])>0&¤t.some((value,index)=>value!==prior[index]);}); -const paintedBackgroundColorChanged=(before,after)=>before.background_color!==after.background_color&&paintAlpha(after.background_color)>0; -const unfocus=()=>{let active=document.activeElement;if(active&&typeof active.blur==='function')active.blur();}; -const focus=controls.map(e=>{unfocus();e.blur();let before=focusStyle(e);e.focus();let after=focusStyle(e),focused=document.activeElement===e;let indicator=paintedOutlineChanged(before,after)||paintedShadowChanged(before,after)||paintedBorderChanged(before,after)||paintedBackgroundColorChanged(before,after);return {name:!!name(e),tab_index:e.tabIndex,focused,focus_visible:focused&&indicator,contrast:contrast(e)};}); -const headings=[...document.querySelectorAll('h1,h2,h3,h4,h5,h6')].map(e=>Number(e.tagName.slice(1))); -let heading_progression=true; for(let i=1;iheadings[i-1]+1)heading_progression=false; -return { - images:[...document.images].map(e=>({src:e.getAttribute('src')||'',alt:e.alt||'',complete:e.complete,natural_width:e.naturalWidth,natural_height:e.naturalHeight,visible:visible(e),rect:rect(e)})), - layout:{scroll_width:document.documentElement.scrollWidth,client_width:document.documentElement.clientWidth,clipped,overlaps}, - accessibility:{h1_count:document.querySelectorAll('h1').length,headings,heading_progression,main_count:document.querySelectorAll('main').length,landmarks:document.querySelectorAll('main,nav,header,footer,[role="main"],[role="navigation"],[role="banner"],[role="contentinfo"]').length,controls:focus} -}; })()""" - - -def _write_new(path: Path, data: bytes) -> None: - flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_CLOEXEC - if hasattr(os, "O_NOFOLLOW"): - flags |= os.O_NOFOLLOW - try: - fd = os.open(path, flags, 0o600) - except OSError as exc: - raise BrowserError("screenshot_collision") from exc - try: - with os.fdopen(fd, "wb", closefd=True) as handle: - handle.write(data) - handle.flush() - os.fsync(handle.fileno()) - except Exception: - try: - path.unlink() - except OSError: - pass - raise - - -def _terminate_owned_process_group(process: subprocess.Popen, grace: float = 3.0) -> None: - """Terminate then kill the session led by ``process`` and reap its leader.""" - pgid = process.pid - if pgid == os.getpgrp(): - raise BrowserError("browser process group ownership is invalid") - try: - os.killpg(pgid, signal.SIGTERM) - except ProcessLookupError: - pass - except OSError as exc: - raise BrowserError("browser process cleanup failed") from exc - try: - process.wait(timeout=grace) - except subprocess.TimeoutExpired: - try: - os.killpg(pgid, signal.SIGKILL) - except ProcessLookupError: - pass - except OSError as exc: - raise BrowserError("browser process cleanup failed") from exc - try: - process.wait(timeout=grace) - except subprocess.TimeoutExpired as exc: - raise BrowserError("browser process cleanup failed") from exc - # The leader can exit before a descendant. Kill any surviving member and - # give the kernel a bounded moment to reap it under its own parent. - try: - os.killpg(pgid, 0) - except ProcessLookupError: - return - except OSError: - return - try: - os.killpg(pgid, signal.SIGKILL) - except ProcessLookupError: - return - end = time.monotonic() + grace - while time.monotonic() < end: - try: - os.killpg(pgid, 0) - except ProcessLookupError: - return - except OSError: - return - time.sleep(0.02) - raise BrowserError("browser process cleanup failed") - - -def _console_projection(events: list[dict]) -> tuple[dict, ...]: - projected: list[dict] = [] - for event in events: - method = event.get("method") - params = event.get("params", {}) - if method == "Runtime.consoleAPICalled": - projected.append({"kind": "console", "level": str(params.get("type", "unknown"))[:32]}) - elif method == "Runtime.exceptionThrown": - projected.append({"kind": "exception", "level": "error"}) - elif method == "Log.entryAdded": - entry = params.get("entry", {}) - projected.append({"kind": "log", "level": str(entry.get("level", "unknown"))[:32]}) - return tuple(projected) - - -class BrowserRenderer: - def __init__(self, browser_binary: str = "chromium"): - self.browser_binary = browser_binary - - def render( - self, - *, - workspace_root: str | Path, - output_root: str | Path, - viewports, - timeout_seconds: int, - ) -> RenderObservation: - raw_root, raw_out = Path(workspace_root), Path(output_root) - if raw_root.is_symlink() or not raw_root.is_dir(): - raise BrowserError("workspace_unavailable") - if raw_out.is_symlink() or not raw_out.is_dir(): - raise BrowserError("output_unavailable") - root, out = raw_root.resolve(), raw_out.resolve() - viewport_list = tuple(viewports) - if not viewport_list: - raise BrowserError("viewport_unavailable") - for viewport in viewport_list: - if ( - not isinstance(getattr(viewport, "id", None), str) - or not VIEWPORT_ID_RE.fullmatch(viewport.id) - or isinstance(getattr(viewport, "width", None), bool) - or not isinstance(viewport.width, int) - or not 1 <= viewport.width <= 10000 - or isinstance(getattr(viewport, "height", None), bool) - or not isinstance(viewport.height, int) - or not 1 <= viewport.height <= 10000 - ): - raise BrowserError("viewport_unavailable") - targets = [out / f"screenshot-{viewport.id}.png" for viewport in viewport_list] - if len(set(targets)) != len(targets) or any( - target.exists() or target.is_symlink() for target in targets - ): - raise BrowserError("screenshot_collision") - binary = self.browser_binary - if os.sep not in binary: - binary = shutil.which(binary) or "" - if not binary: - raise BrowserError("browser_unavailable") - transient = { - "browser_start_failed", - "browser_cdp_unavailable", - "CDP handshake failed", - "CDP handshake closed", - "CDP socket read failed", - "CDP socket closed", - "CDP socket write failed", - } - for attempt in range(3): - try: - return self._render_once( - root=root, - out=out, - viewport_list=viewport_list, - targets=targets, - binary=binary, - timeout_seconds=timeout_seconds, - ) - except BrowserError as exc: - if str(exc) not in transient or attempt == 2: - raise - self._remove_partial_screenshots(out, viewport_list) - raise BrowserError("browser_restart_exhausted") - - @staticmethod - def _remove_partial_screenshots(output_root: Path, viewports) -> None: - for viewport in viewports: - target = output_root / f"screenshot-{getattr(viewport, 'id', '')}.png" - try: - if target.is_file() and not target.is_symlink(): - target.unlink() - elif target.exists() or target.is_symlink(): - raise BrowserError("screenshot_cleanup_failed") - except OSError as exc: - raise BrowserError("screenshot_cleanup_failed") from exc - - def _render_once( - self, - *, - root: Path, - out: Path, - viewport_list, - targets: list[Path], - binary: str, - timeout_seconds: int, - ) -> RenderObservation: - deadline = time.monotonic() + max(3, timeout_seconds) - server = _StaticServer(root) - thread = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - profile = tempfile.TemporaryDirectory( - prefix="iop-browser-", ignore_cleanup_errors=True - ) - process: subprocess.Popen | None = None - cdp: _CDP | None = None - cleanup_error: Exception | None = None - completed = False - try: - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as port_socket: - port_socket.bind(("127.0.0.1", 0)) - port = int(port_socket.getsockname()[1]) - try: - process = subprocess.Popen( - [ - binary, - "--headless", - "--no-sandbox", - "--disable-gpu", - "--disable-extensions", - "--disable-background-networking", - "--no-first-run", - "--remote-allow-origins=*", - "--remote-debugging-address=127.0.0.1", - f"--remote-debugging-port={port}", - f"--user-data-dir={profile.name}", - "about:blank", - ], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - start_new_session=True, - ) - except OSError as exc: - raise BrowserError("browser_start_failed") from exc - target = None - while time.monotonic() < deadline: - if process.poll() is not None: - raise BrowserError("browser_start_failed") - try: - with urlopen( - f"http://127.0.0.1:{port}/json", timeout=0.5 - ) as response: - candidates = json.load(response) - if not isinstance(candidates, list): - raise ValueError("invalid target list") - target = next( - ( - item - for item in candidates - if isinstance(item, dict) and item.get("type") == "page" - ), - None, - ) - if target: - break - except (OSError, ValueError, json.JSONDecodeError): - time.sleep(0.05) - if not target or not isinstance(target.get("webSocketDebuggerUrl"), str): - raise BrowserError("browser_cdp_unavailable") - cdp = _CDP(target["webSocketDebuggerUrl"], deadline) - for method in ( - "Page.enable", - "Runtime.enable", - "Log.enable", - "Accessibility.enable", - ): - cdp.call(method) - cdp.call("Page.bringToFront") - version = str(cdp.call("Browser.getVersion").get("product", "unknown"))[:128] - origin = f"http://127.0.0.1:{server.server_address[1]}" - denied: list[dict] = [] - - def intercept(event): - if event.get("method") != "Fetch.requestPaused": - return - params = event.get("params", {}) - request = params.get("request", {}) - url = request.get("url", "") - request_id = params.get("requestId") - if not isinstance(request_id, str) or not request_id: - raise BrowserError("browser request identity is invalid") - if isinstance(url, str) and url.startswith(origin + "/"): - cdp.fire( - "Fetch.continueRequest", {"requestId": request_id} - ) - else: - encoded = str(url).encode("utf-8", "replace") - denied.append( - { - "kind": "external", - "url_digest": _digest(encoded), - "allowed": False, - "status": 0, - } - ) - cdp.fire( - "Fetch.failRequest", - { - "requestId": request_id, - "errorReason": "BlockedByClient", - }, - ) - - cdp.event_handler = intercept - cdp.call( - "Fetch.enable", - {"patterns": [{"urlPattern": "*", "requestStage": "Request"}]}, - ) - results: list[ViewportObservation] = [] - for viewport, screenshot in zip(viewport_list, targets): - cdp.call( - "Emulation.setDeviceMetricsOverride", - { - "width": viewport.width, - "height": viewport.height, - "deviceScaleFactor": 1, - "mobile": viewport.width < 600, - }, - ) - cdp.call("Page.navigate", {"url": origin + "/index.html"}) - cdp.call( - "Runtime.evaluate", - { - "expression": "new Promise((resolve,reject)=>{const done=()=>requestAnimationFrame(()=>requestAnimationFrame(resolve));if(document.readyState==='complete')done();else window.addEventListener('load',done,{once:true});setTimeout(()=>reject(new Error('load timeout')),5000);})", - "awaitPromise": True, - "returnByValue": True, - }, - ) - facts = cdp.call( - "Runtime.evaluate", - {"expression": _OBSERVATION_SCRIPT, "returnByValue": True}, - ) - parsed = facts.get("result", {}).get("value", {}) - if not isinstance(parsed, dict) or set(parsed) != { - "images", - "layout", - "accessibility", - }: - raise BrowserError("browser_observation_invalid") - ax_nodes = cdp.call("Accessibility.getFullAXTree").get("nodes", []) - if not isinstance(ax_nodes, list): - raise BrowserError("browser_accessibility_invalid") - ax = { - "nodes": len(ax_nodes), - "non_ignored": sum( - 1 for node in ax_nodes if isinstance(node, dict) and not node.get("ignored", False) - ), - "named": sum( - 1 - for node in ax_nodes - if isinstance(node, dict) - and isinstance(node.get("name"), dict) - and bool(node["name"].get("value")) - ), - } - accessibility = dict(parsed["accessibility"]) - accessibility["ax"] = ax - try: - png = base64.b64decode( - cdp.call("Page.captureScreenshot", {"format": "png"})["data"], - validate=True, - ) - except (KeyError, ValueError, TypeError) as exc: - raise BrowserError("browser_screenshot_invalid") from exc - if not png.startswith(b"\x89PNG\r\n\x1a\n"): - raise BrowserError("browser_screenshot_invalid") - _write_new(screenshot, png) - results.append( - ViewportObservation( - viewport.id, - viewport.width, - viewport.height, - screenshot.name, - _digest(png), - len(png), - tuple(parsed["images"]), - parsed["layout"], - accessibility, - ) - ) - observation = RenderObservation( - version, - origin, - tuple([*server.request_snapshot(), *denied]), - _console_projection(cdp.events), - tuple(results), - ) - completed = True - return observation - finally: - if cdp is not None: - cdp.close() - server.shutdown() - server.server_close() - thread.join(2) - if process is not None: - try: - _terminate_owned_process_group(process) - except Exception as exc: # preserve cleanup failure after body success - cleanup_error = exc - if not completed: - for target in targets: - try: - if target.is_file() and not target.is_symlink(): - target.unlink() - except OSError: - cleanup_error = cleanup_error or BrowserError( - "screenshot_cleanup_failed" - ) - profile.cleanup() - if cleanup_error is not None: - raise cleanup_error diff --git a/scripts/agent_benchmark/browser_cdp_test.py b/scripts/agent_benchmark/browser_cdp_test.py deleted file mode 100644 index 2b0068d4..00000000 --- a/scripts/agent_benchmark/browser_cdp_test.py +++ /dev/null @@ -1,591 +0,0 @@ -from __future__ import annotations - -import base64 -import hashlib -import json -import os -import signal -import socket -import struct -import subprocess -import sys -import tempfile -import threading -import time -import unittest -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from pathlib import Path -from types import SimpleNamespace -from urllib.error import HTTPError -from urllib.request import urlopen - -from scripts.agent_benchmark.browser_cdp import ( - MAX_MESSAGE_BYTES, - WEBSOCKET_GUID, - BrowserError, - BrowserRenderer, - _CDP, - _StaticServer, - _terminate_owned_process_group, -) -from scripts.agent_benchmark.web_validation import _runtime_gates - - -def _frame(payload: bytes, *, opcode: int = 1, fin: bool = True, masked: bool = False) -> bytes: - first = (0x80 if fin else 0) | opcode - size = len(payload) - if size < 126: - header = bytes((first, (0x80 if masked else 0) | size)) - elif size <= 65535: - header = bytes((first, (0x80 if masked else 0) | 126)) + struct.pack("!H", size) - else: - header = bytes((first, (0x80 if masked else 0) | 127)) + struct.pack("!Q", size) - if not masked: - return header + payload - mask = b"mask" - return header + mask + bytes(value ^ mask[index % 4] for index, value in enumerate(payload)) - - -def _exact(connection: socket.socket, size: int) -> bytes: - chunks = bytearray() - while len(chunks) < size: - chunk = connection.recv(size - len(chunks)) - if not chunk: - raise EOFError - chunks.extend(chunk) - return bytes(chunks) - - -def _client_frame(connection: socket.socket) -> tuple[int, bytes]: - first, second = _exact(connection, 2) - size = second & 0x7F - if size == 126: - size = struct.unpack("!H", _exact(connection, 2))[0] - elif size == 127: - size = struct.unpack("!Q", _exact(connection, 8))[0] - if not second & 0x80: - raise AssertionError("client frame was not masked") - mask = _exact(connection, 4) - payload = _exact(connection, size) - return first & 0x0F, bytes( - value ^ mask[index % 4] for index, value in enumerate(payload) - ) - - -class _WebSocketFixture: - def __init__(self, script, *, valid_accept: bool = True): - self.script = script - self.valid_accept = valid_accept - self.error: BaseException | None = None - self.listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - self.listener.bind(("127.0.0.1", 0)) - self.listener.listen(1) - self.thread = threading.Thread(target=self._serve, daemon=True) - self.thread.start() - - @property - def url(self) -> str: - return f"ws://127.0.0.1:{self.listener.getsockname()[1]}/devtools/page/1" - - def _serve(self) -> None: - try: - connection, _ = self.listener.accept() - with connection: - request = bytearray() - while b"\r\n\r\n" not in request: - request.extend(connection.recv(4096)) - key = "" - for line in bytes(request).split(b"\r\n"): - if line.lower().startswith(b"sec-websocket-key:"): - key = line.split(b":", 1)[1].strip().decode("ascii") - accept = base64.b64encode( - hashlib.sha1((key + WEBSOCKET_GUID).encode("ascii")).digest() - ).decode("ascii") - if not self.valid_accept: - accept = "invalid" - connection.sendall( - ( - "HTTP/1.1 101 Switching Protocols\r\n" - "Upgrade: websocket\r\nConnection: keep-alive, Upgrade\r\n" - f"Sec-WebSocket-Accept: {accept}\r\n\r\n" - ).encode("ascii") - ) - if self.valid_accept: - self.script(connection) - except BaseException as exc: # surfaced by close() - self.error = exc - finally: - self.listener.close() - - def close(self) -> None: - self.thread.join(2) - if self.thread.is_alive(): - self.listener.close() - self.thread.join(2) - if self.error is not None: - raise self.error - - -class BrowserProtocolTest(unittest.TestCase): - def test_loopback_handler_rejects_escape_and_all_symlinks(self): - with tempfile.TemporaryDirectory() as raw: - root = Path(raw) - (root / "nested").mkdir() - (root / "index.html").write_text("ok", encoding="utf-8") - (root / "nested" / "ok.txt").write_text("nested", encoding="utf-8") - outside = root.parent / f"browser-cdp-outside-{root.name}.txt" - outside.write_text("no", encoding="utf-8") - (root / "internal-link").symlink_to(root / "index.html") - (root / "outside-link").symlink_to(outside) - (root / "nested-link").symlink_to(root / "nested", target_is_directory=True) - server = _StaticServer(root) - thread = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - try: - base = f"http://127.0.0.1:{server.server_address[1]}" - with urlopen(base + "/index.html") as response: - self.assertEqual(response.read(), b"ok") - for path in ( - "/internal-link", - "/outside-link", - "/nested-link/ok.txt", - "/%2e%2e/index.html", - "/nested/%2e%2e/index.html", - ): - with self.subTest(path=path), self.assertRaises(HTTPError): - urlopen(base + path) - finally: - server.shutdown() - server.server_close() - thread.join(2) - outside.unlink(missing_ok=True) - - def test_fragment_ping_event_and_response_correlation(self): - def script(connection: socket.socket) -> None: - opcode, payload = _client_frame(connection) - command = json.loads(payload) - self.assertEqual(opcode, 1) - connection.sendall(_frame(b'{"method":"Page.ready","params":{"ok":true}}')) - connection.sendall(_frame(b"ping", opcode=9)) - response = json.dumps({"id": command["id"], "result": {"ok": True}}).encode() - split = len(response) // 2 - connection.sendall(_frame(response[:split], fin=False)) - connection.sendall(_frame(response[split:], opcode=0)) - pong, pong_payload = _client_frame(connection) - self.assertEqual((pong, pong_payload), (10, b"ping")) - - server = _WebSocketFixture(script) - client = _CDP(server.url, time.monotonic() + 2) - try: - self.assertEqual(client.call("Page.enable"), {"ok": True}) - self.assertEqual(client.events[0]["method"], "Page.ready") - finally: - client.close() - server.close() - - def test_fire_and_call_responses_can_be_interleaved(self): - def script(connection: socket.socket) -> None: - _, first = _client_frame(connection) - _, second = _client_frame(connection) - fire, call = json.loads(first), json.loads(second) - connection.sendall(_frame(json.dumps({"id": fire["id"], "result": {}}).encode())) - connection.sendall(_frame(b'{"method":"Fetch.paused","params":{}}')) - connection.sendall(_frame(json.dumps({"id": call["id"], "result": {"done": 1}}).encode())) - - server = _WebSocketFixture(script) - client = _CDP(server.url, time.monotonic() + 2) - try: - client.fire("Fetch.continueRequest", {"requestId": "request"}) - self.assertEqual(client.call("Runtime.evaluate"), {"done": 1}) - self.assertEqual(client.events[0]["method"], "Fetch.paused") - finally: - client.close() - server.close() - - def test_extended_frame_lengths_are_canonical_and_bounded(self): - for size in (200, 70_000): - with self.subTest(size=size): - value = "x" * size - - def script(connection: socket.socket, value=value) -> None: - _, payload = _client_frame(connection) - command = json.loads(payload) - response = json.dumps( - {"id": command["id"], "result": {"value": value}}, - separators=(",", ":"), - ).encode() - connection.sendall(_frame(response)) - - server = _WebSocketFixture(script) - client = _CDP(server.url, time.monotonic() + 2) - try: - self.assertEqual( - client.call("Runtime.evaluate")["value"], value - ) - finally: - client.close() - server.close() - - def test_handshake_and_malformed_frame_matrix_fail_closed(self): - bad_accept = _WebSocketFixture(lambda _connection: None, valid_accept=False) - with self.assertRaises(BrowserError): - _CDP(bad_accept.url, time.monotonic() + 1) - bad_accept.close() - - cases = { - "masked": _frame(b"{}", masked=True), - "binary": _frame(b"{}", opcode=2), - "continuation": _frame(b"{}", opcode=0), - "malformed-json": _frame(b"{"), - "close": _frame(b"", opcode=8), - "oversized": bytes((0x81, 127)) + struct.pack("!Q", MAX_MESSAGE_BYTES + 1), - "noncanonical": bytes((0x81, 126)) + struct.pack("!H", 1) + b"x", - } - for name, response in cases.items(): - with self.subTest(name=name): - def script(connection: socket.socket, response=response) -> None: - _client_frame(connection) - connection.sendall(response) - - server = _WebSocketFixture(script) - client = _CDP(server.url, time.monotonic() + 1) - try: - with self.assertRaises(BrowserError): - client.call("Runtime.evaluate") - finally: - client.close() - server.close() - - def test_uncorrelated_response_and_deadline_fail_closed(self): - def wrong_id(connection: socket.socket) -> None: - _, payload = _client_frame(connection) - ident = json.loads(payload)["id"] - connection.sendall(_frame(json.dumps({"id": ident + 1, "result": {}}).encode())) - - server = _WebSocketFixture(wrong_id) - client = _CDP(server.url, time.monotonic() + 1) - try: - with self.assertRaises(BrowserError): - client.call("Runtime.evaluate") - finally: - client.close() - server.close() - - def no_response(connection: socket.socket) -> None: - _client_frame(connection) - time.sleep(0.2) - - server = _WebSocketFixture(no_response) - client = _CDP(server.url, time.monotonic() + 0.05) - try: - with self.assertRaises(BrowserError): - client.call("Runtime.evaluate") - finally: - client.close() - server.close() - - def test_owned_process_group_is_terminated_and_reaped(self): - source = ( - "import signal,subprocess,sys,time;" - "signal.signal(signal.SIGTERM,signal.SIG_IGN);" - "p=subprocess.Popen([sys.executable,'-c'," - "'import signal,time;signal.signal(signal.SIGTERM,signal.SIG_IGN);time.sleep(60)']);" - "print(p.pid,flush=True);time.sleep(60)" - ) - process = subprocess.Popen( - [sys.executable, "-c", source], - stdout=subprocess.PIPE, - text=True, - start_new_session=True, - ) - assert process.stdout is not None - child_pid = int(process.stdout.readline().strip()) - try: - _terminate_owned_process_group(process, grace=1) - self.assertIsNotNone(process.poll()) - with self.assertRaises(ProcessLookupError): - os.killpg(process.pid, 0) - with self.assertRaises(ProcessLookupError): - os.kill(child_pid, 0) - finally: - try: - os.killpg(process.pid, signal.SIGKILL) - except ProcessLookupError: - pass - process.wait(timeout=2) - process.stdout.close() - - def test_renderer_error_reaps_its_owned_process_group(self): - with tempfile.TemporaryDirectory(dir=Path.cwd()) as raw: - root = Path(raw) - (root / "index.html").write_text("
unused
", encoding="utf-8") - pid_path = root / "browser-pids" - binary = root / "fake-browser" - binary.write_text( - "#!/usr/bin/env python3\n" - "import os, signal, subprocess, sys, time\n" - "signal.signal(signal.SIGTERM, signal.SIG_IGN)\n" - "child = subprocess.Popen([sys.executable, '-c', " - "'import signal,time;signal.signal(signal.SIGTERM,signal.SIG_IGN);time.sleep(60)'])\n" - f"with open({str(pid_path)!r}, 'a', encoding='ascii') as handle:\n" - " handle.write(f'{os.getpid()} {child.pid}\\n')\n" - " handle.flush()\n" - " os.fsync(handle.fileno())\n" - "time.sleep(60)\n", - encoding="utf-8", - ) - binary.chmod(0o700) - pids: list[int] = [] - try: - with self.assertRaisesRegex(BrowserError, "browser_cdp_unavailable"): - BrowserRenderer(str(binary)).render( - workspace_root=root, - output_root=root, - viewports=( - SimpleNamespace( - id="mobile.small+wide", width=375, height=700 - ), - ), - timeout_seconds=1, - ) - pids = [int(value) for value in pid_path.read_text().split()] - self.assertEqual(len(pids), 6) - for pid in pids: - with self.subTest(pid=pid), self.assertRaises(ProcessLookupError): - os.kill(pid, 0) - finally: - for pid in pids: - try: - os.kill(pid, signal.SIGKILL) - except ProcessLookupError: - pass - - def test_renderer_restarts_only_closed_transient_failures(self): - with tempfile.TemporaryDirectory() as raw: - root = Path(raw) - (root / "index.html").write_text("
unused
", encoding="utf-8") - viewport = SimpleNamespace(id="desktop", width=800, height=600) - renderer = BrowserRenderer(sys.executable) - calls = 0 - expected = object() - - def transient_then_success(**_kwargs): - nonlocal calls - calls += 1 - screenshot = root / "screenshot-desktop.png" - if calls < 3: - screenshot.write_bytes(b"partial") - raise BrowserError("CDP socket closed") - self.assertFalse(screenshot.exists()) - return expected - - renderer._render_once = transient_then_success - self.assertIs( - renderer.render( - workspace_root=root, - output_root=root, - viewports=(viewport,), - timeout_seconds=1, - ), - expected, - ) - self.assertEqual(calls, 3) - - for error in ("CDP socket closed", "screenshot_collision"): - with self.subTest(error=error): - calls = 0 - - def fail(**_kwargs): - nonlocal calls - calls += 1 - raise BrowserError(error) - - renderer._render_once = fail - with self.assertRaisesRegex(BrowserError, error): - renderer.render( - workspace_root=root, - output_root=root, - viewports=(viewport,), - timeout_seconds=1, - ) - self.assertEqual(calls, 3 if error == "CDP socket closed" else 1) - - -class _Counter(BaseHTTPRequestHandler): - count = 0 - - def do_GET(self): - type(self).count += 1 - self.send_response(200) - self.end_headers() - - def log_message(self, *_args): - pass - - -class BrowserIntegrationTest(unittest.TestCase): - _VIEWPORTS = ( - SimpleNamespace(id="desktop", width=900, height=700), - SimpleNamespace(id="mobile.small+wide", width=375, height=700), - ) - - @staticmethod - def _valid_page( - root: Path, - extra_image: str = "", - *, - focus_css: str = "", - autofocus: bool = False, - ) -> None: - (root / "assets").mkdir() - for name in ("a.svg", "b.svg"): - (root / "assets" / name).write_text( - "", - encoding="utf-8", - ) - focus_attribute = " autofocus" if autofocus else "" - (root / "index.html").write_text( - "" - "

Ready

A" - "B" - f"{extra_image}go
" - "", - encoding="utf-8", - ) - (root / "styles.css").write_text( - f"body{{color:#111;background:#fff}}{focus_css}", encoding="utf-8" - ) - (root / "script.js").write_text("document.body.dataset.ready='1'", encoding="utf-8") - - def test_valid_page_emits_complete_two_viewport_observations(self): - with tempfile.TemporaryDirectory() as raw: - root = Path(raw) - self._valid_page(root) - render = BrowserRenderer().render( - workspace_root=root, - output_root=root, - viewports=self._VIEWPORTS, - timeout_seconds=20, - ) - self.assertEqual( - [view.id for view in render.viewports], - ["desktop", "mobile.small+wide"], - ) - self.assertTrue(all((root / view.screenshot).stat().st_size > 0 for view in render.viewports)) - self.assertTrue(all(len(view.image_facts) == 2 for view in render.viewports)) - self.assertFalse([item for item in render.requests if item["kind"] == "external"]) - - def test_denied_page_dispatches_no_external_request(self): - _Counter.count = 0 - with tempfile.TemporaryDirectory() as raw: - root = Path(raw) - counter = ThreadingHTTPServer(("127.0.0.1", 0), _Counter) - thread = threading.Thread(target=counter.serve_forever, daemon=True) - thread.start() - try: - self._valid_page( - root, - f"blocked", - ) - render = BrowserRenderer().render( - workspace_root=root, - output_root=root, - viewports=self._VIEWPORTS, - timeout_seconds=20, - ) - self.assertEqual(_Counter.count, 0) - self.assertTrue(any(item["kind"] == "external" for item in render.requests)) - finally: - counter.shutdown() - counter.server_close() - thread.join(2) - - def test_focus_visibility_uses_computed_indicator(self): - manifest = SimpleNamespace( - fixture=SimpleNamespace( - assets=tuple( - SimpleNamespace(workspace_path=f"assets/{name}.svg") - for name in ("a", "b") - ) - ), - viewports=self._VIEWPORTS, - ) - cases = ( - ("suppressed", "a:focus{outline:none;box-shadow:none}", False, False), - ( - "transparent-outline", - "a:focus{outline:4px solid rgba(0,85,255,0);box-shadow:none}", - False, - False, - ), - ( - "transparent-shadow", - "a:focus{outline:none;box-shadow:0 0 0 4px rgba(0,85,255,0)}", - False, - False, - ), - ( - "transparent-border", - "a:focus{outline:none;box-shadow:none;border:4px solid rgba(0,85,255,0)}", - False, - False, - ), - ( - "transparent-gradient", - "a:focus{outline:none;box-shadow:none;background-image:linear-gradient(rgba(0,85,255,0),rgba(0,85,255,0))}", - False, - False, - ), - ( - "autofocus-visible-outline", - "a:focus{outline:4px solid #05f;box-shadow:none}", - True, - True, - ), - ( - "visible-outline", - "a:focus{outline:4px solid #05f;box-shadow:none}", - True, - False, - ), - ( - "visible-shadow", - "a:focus{outline:none;box-shadow:0 0 0 4px #05f}", - True, - False, - ), - ( - "visible-border", - "a:focus{outline:none;box-shadow:none;border:4px solid #05f}", - True, - False, - ), - ( - "visible-background", - "a:focus{outline:none;box-shadow:none;background-color:#8cf}", - True, - False, - ), - ) - for label, focus_css, expected, autofocus in cases: - with self.subTest(case=label), tempfile.TemporaryDirectory() as raw: - root = Path(raw) - self._valid_page( - root, focus_css=focus_css, autofocus=autofocus - ) - render = BrowserRenderer().render( - workspace_root=root, - output_root=root, - viewports=self._VIEWPORTS, - timeout_seconds=20, - ) - observed = [ - control["focus_visible"] - for viewport in render.viewports - for control in viewport.accessibility["controls"] - ] - self.assertEqual(observed, [expected, expected]) - if not expected: - self.assertFalse( - _runtime_gates(manifest, render)["accessibility"]["passed"] - ) diff --git a/scripts/agent_benchmark/claude_iop.py b/scripts/agent_benchmark/claude_iop.py deleted file mode 100644 index 3208e01b..00000000 --- a/scripts/agent_benchmark/claude_iop.py +++ /dev/null @@ -1,535 +0,0 @@ -"""Secret-safe Claude Code adapter for the IOP benchmark lifecycle. - -The generic lifecycle deliberately does not know Claude's command line or its -JSONL protocol. This module converts one immutable benchmark cell and one -prepared workspace into that closed boundary. It has no network dependency; -tests use a local fake executable. -""" - -from __future__ import annotations - -import json -import os -import shutil -from dataclasses import dataclass -from pathlib import Path -from typing import Any, Callable - -from scripts.agent_benchmark.connectivity import ( - CallerCapability, - ConnectivityResult, - EffectiveBinding, - RequestedEffectiveBinding, - make_result, -) -from scripts.agent_benchmark.lifecycle import ( - CALLER_REASON_ERROR, - CALLER_REASON_SUCCESS, - CALLER_STATUS_FAILED, - CALLER_STATUS_SUCCEEDED, - COMPLETION_EXIT_AFTER_IDLE, - SUBMISSION_STDIN_ONCE, - CallerEvent, - CallerTerminal, - InvocationSpec, - LifecycleMetricError, - ParsedMetric, - TLS_CA_ENV_KEYS, - count_metric, - duration_metric, - exact_value_redactor, - inherited_tls_ca_environment, - is_reported_number, -) -from scripts.agent_benchmark.manifest import MatrixCell, Timeout -from scripts.agent_benchmark.workspace import PreparedWorkspace - - -REDACTED = "[redacted]" -CLAUDE_ROUTE_KINDS = ("direct", "execution_preset") -# This is lexical order, as required by the closed connectivity capability -# tuple. The cell's requested effort is still passed through unchanged. -CLAUDE_EFFORTS = ("high", "low", "max", "medium", "xhigh") -_STRUCTURAL_SECRET_KEYS = frozenset( - { - "content", "text", "input", "arguments", "tool_input", "prompt", "query", - "result", "error", "errors", "error_message", - } -) -# The exact Claude result fields this adapter is allowed to observe. Anything -# outside these allowlists never becomes timing or usage evidence, and -# ``duration_api_ms`` is marked as overlapping because it is reported inside -# the same wall-clock window as ``duration_ms``. -_CLAUDE_DURATION_FIELDS = ( - ("duration_ms", "total_duration", False), - ("duration_api_ms", "model_duration", True), -) -_CLAUDE_USAGE_FIELDS = { - "input_tokens": "input_tokens", - "output_tokens": "output_tokens", - "cache_read_input_tokens": "cached_input_tokens", - "cache_creation_input_tokens": "cache_write_tokens", -} -_CLAUDE_USAGE_BOOKKEEPING_FIELDS = { - "cache_creation", "inference_geo", "iterations", "output_tokens_details", - "server_tool_use", "service_tier", "speed", -} -_CLAUDE_USAGE_NESTED_COUNTS = { - "cache_creation": {"ephemeral_1h_input_tokens", "ephemeral_5m_input_tokens"}, - "output_tokens_details": {"thinking_tokens"}, - "server_tool_use": {"web_fetch_requests", "web_search_requests"}, -} - - -class ClaudeIopError(Exception): - """Base exception for configuration and protocol failures.""" - - -class ClaudeIopValidationError(ClaudeIopError): - """Raised when a runtime input or preflight shape is inadmissible.""" - - -class ClaudeIopProtocolError(ClaudeIopError): - """Raised for a malformed or contradictory claimed Claude terminal.""" - - -@dataclass(frozen=True) -class ClaudeIopRuntime: - """Runtime-only IOP inputs. Values are never written to durable evidence.""" - - binary: str - base_url: str - api_key: str - - -def _require_string(value: Any, name: str) -> str: - if not isinstance(value, str) or not value: - raise ClaudeIopValidationError(f"invalid {name}") - return value - - -def _exact_object(raw_line: str) -> dict[str, Any]: - if not isinstance(raw_line, str): - raise ClaudeIopProtocolError("invalid Claude JSONL line") - try: - value = json.loads(raw_line) - except (TypeError, ValueError) as exc: - raise ClaudeIopProtocolError("invalid Claude JSONL line") from exc - if not isinstance(value, dict): - raise ClaudeIopProtocolError("invalid Claude JSONL object") - return value - - -def _required_string(data: dict[str, Any], name: str) -> str: - value = data.get(name) - if not isinstance(value, str) or not value: - raise ClaudeIopProtocolError(f"missing Claude {name}") - return value - - -def _structural_redact( - value: Any, - sensitive_values: tuple[str, ...], - key: str = "", - *, - redact_terminal_message: bool = False, -) -> Any: - if key in _STRUCTURAL_SECRET_KEYS: - return REDACTED - if redact_terminal_message and key == "message": - return REDACTED - if isinstance(value, str): - for sensitive in sensitive_values: - if sensitive: - value = value.replace(sensitive, REDACTED) - return value - if isinstance(value, list): - return [ - _structural_redact( - item, sensitive_values, redact_terminal_message=redact_terminal_message - ) - for item in value - ] - if isinstance(value, dict): - return { - str(name): _structural_redact( - item, - sensitive_values, - str(name), - redact_terminal_message=redact_terminal_message, - ) - for name, item in value.items() - } - return value - - -def redact_claude_event(raw_line: str, sensitive_values: tuple[str, ...]) -> str: - """Return canonical JSON without task/tool content or runtime secrets. - - A malformed line is represented by a fixed marker so error reporting cannot - accidentally retain the raw malformed payload. - """ - try: - event = _exact_object(raw_line) - except ClaudeIopProtocolError: - return '{"type":"invalid_claude_json"}' - redacted = _structural_redact( - event, - sensitive_values, - redact_terminal_message=event.get("type") in ("error", "result"), - ) - return json.dumps(redacted, sort_keys=True, separators=(",", ":"), ensure_ascii=True) - - -def parse_preflight_binding(raw_line: str, cell: MatrixCell) -> RequestedEffectiveBinding: - """Parse an explicit, secret-free IOP binding observation from JSONL. - - The adapter never derives effective values from the requested cell. A - caller must provide all requested/effective scalars and every stage, or the - connectivity contract rejects the observation as incomplete. - """ - event = _exact_object(raw_line) - if event.get("type") != "system" or event.get("subtype") != "iop_binding": - raise ClaudeIopProtocolError("invalid Claude preflight event") - binding = event.get("binding") - if not isinstance(binding, dict) or set(binding) != { - "cell_id", "caller", "requested_route_kind", "requested_route_id", - "requested_model", "requested_effort", "effective_route_kind", - "effective_route_id", "effective_model", "effective_effort", "effective_bindings", - }: - raise ClaudeIopProtocolError("invalid Claude preflight binding") - raw_stages = binding["effective_bindings"] - if not isinstance(raw_stages, list): - raise ClaudeIopProtocolError("invalid Claude preflight stages") - stages: list[EffectiveBinding] = [] - for item in raw_stages: - if not isinstance(item, dict) or set(item) != {"stage", "model", "effort"}: - raise ClaudeIopProtocolError("invalid Claude preflight stage") - stages.append(EffectiveBinding(item["stage"], item["model"], item["effort"])) - try: - result = RequestedEffectiveBinding( - binding["cell_id"], binding["caller"], binding["requested_route_kind"], - binding["requested_route_id"], binding["requested_model"], - binding["requested_effort"], binding["effective_route_kind"], - binding["effective_route_id"], binding["effective_model"], - binding["effective_effort"], tuple(stages), - ) - # Validation is intentionally delegated to the one shared contract. - make_result(cell, claude_capability(), result) - except Exception as exc: - if isinstance(exc, ClaudeIopProtocolError): - raise - raise ClaudeIopProtocolError("invalid Claude preflight binding") from exc - return result - - -class ClaudeStreamParser: - """Parse one fresh Claude Code stream without inventing terminal bindings.""" - - def __init__(self, cell: MatrixCell, session_id: str) -> None: - if not isinstance(cell, MatrixCell) or cell.caller != "claude": - raise ClaudeIopValidationError("Claude adapter requires a Claude matrix cell") - self.cell = cell - # The prepared workspace identity proves that this is a fresh caller - # invocation. Claude Code emits its own UUID in system/init, so it - # must be derived from that event rather than compared to this local - # opaque label. - self.prepared_session_id = _require_string(session_id, "session_id") - self.claude_session_id: str | None = None - self._phase = "await_init" - self._assistant_messages = 0 - self._active_message_id: str | None = None - self._completed_message_ids: set[str] = set() - self._continuation_pending = False - self._api_error_seen = False - - def _require_bound_session(self, event: dict[str, Any]) -> None: - if self.claude_session_id is None: - raise ClaudeIopProtocolError("missing Claude init") - if _required_string(event, "session_id") != self.claude_session_id: - raise ClaudeIopProtocolError("Claude session binding mismatch") - - def _consume_init(self, event: dict[str, Any]) -> None: - if self._phase != "await_init": - raise ClaudeIopProtocolError("duplicate or out-of-order Claude init") - if _required_string(event, "model") != self.cell.iop.request_model: - raise ClaudeIopProtocolError("Claude model binding mismatch") - self.claude_session_id = _required_string(event, "session_id") - self._phase = "await_assistant" - - def _consume_assistant(self, event: dict[str, Any]) -> CallerEvent | None: - if self._phase != "await_assistant": - raise ClaudeIopProtocolError("duplicate or out-of-order Claude assistant") - self._require_bound_session(event) - message = event.get("message") - if not isinstance(message, dict): - raise ClaudeIopProtocolError("invalid Claude assistant event") - if event.get("is_api_error_message") is True: - if message.get("model") != "" or message.get("stop_reason") != "stop_sequence": - raise ClaudeIopProtocolError("invalid Claude API error event") - self._complete_active_message() - self._api_error_seen = True - self._phase = "await_error_result" - return None - if _required_string(message, "model") != self.cell.iop.request_model: - raise ClaudeIopProtocolError("Claude model binding mismatch") - stop_reason = message.get("stop_reason") - if stop_reason not in (None, "tool_use", "end_turn"): - raise ClaudeIopProtocolError("invalid Claude assistant stop reason") - message_id = message.get("id") - if message_id is not None and (not isinstance(message_id, str) or not message_id): - raise ClaudeIopProtocolError("invalid Claude assistant message id") - if self._continuation_pending: - if message_id != self._active_message_id: - self._complete_active_message() - self._continuation_pending = False - if stop_reason is None: - if message_id is None: - raise ClaudeIopProtocolError("unbound Claude assistant snapshot") - if message_id in self._completed_message_ids: - raise ClaudeIopProtocolError("duplicate Claude assistant message") - if self._active_message_id not in (None, message_id): - raise ClaudeIopProtocolError("overlapping Claude assistant messages") - self._active_message_id = message_id - return None - - if message_id is None: - # Older fixture-shaped output did not include a message id. It is - # admissible only for the single final assistant event. - if stop_reason != "end_turn" or self._active_message_id is not None: - raise ClaudeIopProtocolError("unbound Claude assistant terminal") - self._assistant_messages += 1 - else: - if message_id in self._completed_message_ids: - raise ClaudeIopProtocolError("duplicate Claude assistant message") - if self._active_message_id not in (None, message_id): - raise ClaudeIopProtocolError("overlapping Claude assistant messages") - self._active_message_id = message_id - self._complete_active_message() - if stop_reason == "tool_use": - self._phase = "await_tool_result" - return None - self._phase = "await_result" - return CallerEvent("finish") - - def _consume_user(self, event: dict[str, Any]) -> None: - if self._phase not in ("await_assistant", "await_tool_result"): - raise ClaudeIopProtocolError("out-of-order Claude user event") - self._require_bound_session(event) - if self._active_message_id is not None: - # Claude Code may publish cumulative snapshots with the same - # assistant message id on both sides of one or more tool results. - # The next assistant snapshot (or the result terminal) determines - # whether this message continues or a new model call begins. - self._continuation_pending = True - elif self._phase != "await_tool_result": - raise ClaudeIopProtocolError("unexpected Claude user event") - self._phase = "await_assistant" - - def _complete_active_message(self) -> None: - if self._active_message_id is None: - return - if self._active_message_id in self._completed_message_ids: - raise ClaudeIopProtocolError("duplicate Claude assistant message") - self._completed_message_ids.add(self._active_message_id) - self._active_message_id = None - self._continuation_pending = False - self._assistant_messages += 1 - - def _consume_result(self, event: dict[str, Any]) -> tuple[Any, ...] | None: - if self._phase == "await_error_result": - self._require_bound_session(event) - if ( - not self._api_error_seen - or event.get("subtype") != "success" - or event.get("is_error") is not True - or event.get("terminal_reason") != "api_error" - ): - raise ClaudeIopProtocolError("invalid Claude API error terminal") - self._phase = "complete" - return ( - CallerTerminal(CALLER_STATUS_FAILED, CALLER_REASON_ERROR), - CallerEvent("finish"), - CallerEvent("idle"), - ) - result_completes_active_message = False - if self._phase == "await_assistant" and self._active_message_id is not None: - if self._continuation_pending: - raise ClaudeIopProtocolError("Claude result followed an unresolved tool result") - self._complete_active_message() - result_completes_active_message = True - elif self._phase != "await_result": - raise ClaudeIopProtocolError("duplicate or out-of-order Claude result") - self._require_bound_session(event) - if event.get("subtype") != "success" or event.get("is_error") is True: - raise ClaudeIopProtocolError("invalid Claude result terminal") - self._phase = "complete" - terminal = ( - (CallerEvent("finish"), CallerEvent("idle")) - if result_completes_active_message - else (CallerEvent("idle"),) - ) - return ( - *self._observations(event), - CallerTerminal(CALLER_STATUS_SUCCEEDED, CALLER_REASON_SUCCESS), - *terminal, - ) - - def _observations(self, event: dict[str, Any]) -> tuple[ParsedMetric, ...]: - """Convert only allowlisted reported Claude values into observations.""" - model = self.cell.iop.request_model - observations: list[ParsedMetric] = [] - try: - for field, name, overlap in _CLAUDE_DURATION_FIELDS: - if field in event: - if not is_reported_number(event[field]): - raise ClaudeIopProtocolError("invalid Claude duration observation") - observations.append(duration_metric( - name, event[field], reported_unit="ms", - model=model, overlap=overlap, - )) - observations.append(count_metric( - "model_calls", self._assistant_messages, model=model - )) - observations.extend(self._usage_observations(event, model)) - except LifecycleMetricError as exc: - raise ClaudeIopProtocolError("invalid Claude usage observation") from exc - return tuple(observations) - - @staticmethod - def _usage_observations(event: dict[str, Any], model: str) -> list[ParsedMetric]: - usage = event.get("usage") - if usage is None: - return [] - allowed = set(_CLAUDE_USAGE_FIELDS) | _CLAUDE_USAGE_BOOKKEEPING_FIELDS - if not isinstance(usage, dict) or not set(usage) <= allowed: - raise ClaudeIopProtocolError("invalid Claude usage observation") - ClaudeStreamParser._validate_usage_bookkeeping(usage) - return [ - count_metric(_CLAUDE_USAGE_FIELDS[field], value, model=model) - for field, value in sorted(usage.items()) if field in _CLAUDE_USAGE_FIELDS - ] - - @staticmethod - def _validate_usage_bookkeeping(usage: dict[str, Any]) -> None: - for field, nested_fields in _CLAUDE_USAGE_NESTED_COUNTS.items(): - if field not in usage: - continue - value = usage[field] - if not isinstance(value, dict) or not set(value) <= nested_fields: - raise ClaudeIopProtocolError("invalid Claude usage observation") - if any(not is_reported_number(item) for item in value.values()): - raise ClaudeIopProtocolError("invalid Claude usage observation") - if "iterations" in usage and usage["iterations"] != []: - raise ClaudeIopProtocolError("invalid Claude usage observation") - for field in ("inference_geo", "service_tier", "speed"): - if field in usage and ( - not isinstance(usage[field], str) or len(usage[field]) > 64 - ): - raise ClaudeIopProtocolError("invalid Claude usage observation") - - def __call__(self, stream: str, raw_line: str) -> Any: - if stream != "stdout": - return None - event = _exact_object(raw_line) - event_type = event.get("type") - if event_type == "system" and event.get("subtype") == "init": - self._consume_init(event) - return None - if event_type == "assistant": - return self._consume_assistant(event) - if event_type == "user": - self._consume_user(event) - return None - if event_type == "result": - return self._consume_result(event) - # Informational events are deliberately ignored only after they have - # passed exact JSON-object decoding above. - return None - - -def claude_capability() -> CallerCapability: - return CallerCapability("claude", CLAUDE_ROUTE_KINDS, CLAUDE_EFFORTS) - - -def resolve_claude_binary(binary: str) -> str: - """Resolve one executable before lifecycle process creation.""" - candidate = _require_string(binary, "Claude binary") - path = Path(candidate) - resolved = str(path.resolve()) if path.parent != Path(".") else shutil.which(candidate) - if not resolved or not Path(resolved).is_file() or not os.access(resolved, os.X_OK): - raise ClaudeIopValidationError("Claude binary is unavailable") - return str(Path(resolved).resolve()) - - -class ClaudeIopAdapter: - """Build safe Claude invocations and parse their IOP-bound stream evidence.""" - - capability = claude_capability() - - def __init__(self, cell: MatrixCell, workspace: PreparedWorkspace, runtime: ClaudeIopRuntime) -> None: - if not isinstance(cell, MatrixCell) or cell.caller != "claude": - raise ClaudeIopValidationError("Claude adapter requires a Claude matrix cell") - if not isinstance(workspace, PreparedWorkspace) or not workspace.session_is_fresh: - raise ClaudeIopValidationError("Claude adapter requires a fresh prepared workspace") - if not isinstance(runtime, ClaudeIopRuntime): - raise ClaudeIopValidationError("invalid Claude runtime") - self.cell = cell - self.workspace = workspace - self.runtime = runtime - self.binary = resolve_claude_binary(runtime.binary) - self.base_url = _require_string(runtime.base_url, "IOP base URL") - self.api_key = _require_string(runtime.api_key, "IOP API key") - - def preflight(self, raw_line: str) -> ConnectivityResult: - return make_result(self.cell, self.capability, parse_preflight_binding(raw_line, self.cell)) - - def parser(self) -> ClaudeStreamParser: - return ClaudeStreamParser(self.cell, self.workspace.session_id) - - def redactor(self, task: str) -> Callable[[str], str]: - _require_string(task, "task") - structural_values = (task, self.base_url, self.api_key) - exact = exact_value_redactor(structural_values) - - def _redact(line: str) -> str: - return exact(redact_claude_event(line, structural_values)) - - return _redact - - def invocation(self, task: str, evidence_dir: str | Path, timeout: Timeout) -> InvocationSpec: - _require_string(task, "task") - if not isinstance(timeout, Timeout): - raise ClaudeIopValidationError("invalid invocation timeout") - evidence_path = _require_string(str(evidence_dir), "evidence directory") - cwd = Path(self.workspace.workspace_dir) - if not cwd.is_dir(): - raise ClaudeIopValidationError("prepared workspace is unavailable") - env = ( - ("PATH", os.environ.get("PATH", "/usr/bin:/bin")), - ("ANTHROPIC_BASE_URL", self.base_url), - ("ANTHROPIC_API_KEY", self.api_key), - ("CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC", "1"), - ("CLAUDE_CODE_DISABLE_AUTOUPDATER", "1"), - ) + inherited_tls_ca_environment() - return InvocationSpec( - argv=( - self.binary, "--bare", "--print", "--verbose", - "--input-format", "text", "--output-format", "stream-json", - "--model", self.cell.iop.request_model, "--effort", self.cell.iop.requested_effort, - "--no-session-persistence", "--permission-mode", "dontAsk", - "--tools", "Read,Write,Edit", "--allowedTools", "Read,Write,Edit", - ), - cwd=str(cwd), - env=env, - env_allowlist=( - "ANTHROPIC_BASE_URL", "ANTHROPIC_API_KEY", - "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC", "CLAUDE_CODE_DISABLE_AUTOUPDATER", - *TLS_CA_ENV_KEYS, - ), - submission_mode=SUBMISSION_STDIN_ONCE, - completion_mode=COMPLETION_EXIT_AFTER_IDLE, - timeout=timeout, - evidence_dir=evidence_path, - task_payload=task.encode("utf-8"), - ) diff --git a/scripts/agent_benchmark/claude_iop_test.py b/scripts/agent_benchmark/claude_iop_test.py deleted file mode 100644 index d1ea5028..00000000 --- a/scripts/agent_benchmark/claude_iop_test.py +++ /dev/null @@ -1,484 +0,0 @@ -"""Hermetic contract tests for the Claude Code IOP benchmark adapter.""" - -from __future__ import annotations - -import json -import os -import tempfile -import textwrap -import unittest -from pathlib import Path -from unittest.mock import patch - -from scripts.agent_benchmark.claude_iop import ( - ClaudeIopAdapter, - ClaudeIopProtocolError, - ClaudeIopRuntime, - ClaudeIopValidationError, - ClaudeStreamParser, - parse_preflight_binding, - redact_claude_event, -) -from scripts.agent_benchmark.lifecycle import ( - CallerEvent, CallerTerminal, ParsedMetric, - REASON_PARSER_ERROR, REASON_SUCCESS, run_invocation, -) -from scripts.agent_benchmark.manifest import ExpectedBinding, IopCell, MatrixCell, Timeout -from scripts.agent_benchmark.workspace import ( - AttemptIdentity, - PreparedWorkspace, - TestbedProvenance, -) - - -SENTINELS = ("prompt-secret-sentinel", "https://private.iop.invalid", "api-secret-sentinel") -ARBITRARY_SENTINELS = ( - "tool-secret-sentinel", "result-secret-sentinel", "error-secret-sentinel", -) - - -def _cell(route_kind: str = "direct") -> MatrixCell: - bindings = (ExpectedBinding("request", "claude-sonnet", "high"),) - if route_kind == "execution_preset": - bindings = ( - ExpectedBinding("selector", "claude-sonnet", "high"), - ExpectedBinding("plan", "claude-sonnet", "high"), - ExpectedBinding("work", "claude-sonnet", "high"), - ExpectedBinding("review", "claude-sonnet", "high"), - ) - return MatrixCell("claude-direct", "claude", IopCell( - "claude-sonnet", "high", route_kind, "iop-route", bindings, - )) - - -def _workspace(root: Path, session_id: str = "session-fixture") -> PreparedWorkspace: - workspace = root / "workspace" - workspace.mkdir() - return PreparedWorkspace( - AttemptIdentity("run-20260102T030405Z-abcdef123456", "claude-direct", 1, 1), - str(root), str(workspace), str(root / "session"), session_id, True, - "sha256:" + "0" * 64, "isolated", - TestbedProvenance("/testbed", "main", "0" * 40, "sha256:" + "1" * 64, True), - "2026-01-02T03:04:05Z", - ) - - -def _binding_event(cell: MatrixCell) -> str: - return json.dumps({ - "type": "system", "subtype": "iop_binding", "binding": { - "cell_id": cell.id, "caller": cell.caller, - "requested_route_kind": cell.iop.route_kind, "requested_route_id": cell.iop.route_id, - "requested_model": cell.iop.request_model, "requested_effort": cell.iop.requested_effort, - "effective_route_kind": cell.iop.route_kind, "effective_route_id": cell.iop.route_id, - "effective_model": cell.iop.request_model, "effective_effort": cell.iop.requested_effort, - "effective_bindings": [ - {"stage": item.stage, "model": item.model, "effort": item.effort} - for item in cell.iop.expected_bindings - ], - }, - }) - - -class ClaudeIopTest(unittest.TestCase): - def setUp(self) -> None: - # The lifecycle intentionally requires an executable fake CLI. Some - # CI hosts mount /tmp noexec, so keep this short-lived directory under - # the repository worktree instead. - self.temp = tempfile.TemporaryDirectory(dir=Path.cwd(), prefix=".claude-iop-test-") - self.root = Path(self.temp.name) - self.cell = _cell() - self.workspace = _workspace(self.root) - self.binary = self.root / "claude" - self.binary.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") - self.binary.chmod(0o700) - self.runtime = ClaudeIopRuntime(str(self.binary), SENTINELS[1], SENTINELS[2]) - - def tearDown(self) -> None: - self.temp.cleanup() - - def _adapter(self, *, cell: MatrixCell | None = None) -> ClaudeIopAdapter: - return ClaudeIopAdapter(cell or self.cell, self.workspace, self.runtime) - - def _fixture_lines(self) -> list[str]: - fixture = Path("scripts/fixtures/agent-comparison-benchmark/claude-iop-stream.jsonl") - return fixture.read_text(encoding="utf-8").splitlines() - - def _run_fake(self, lines: list[str], *, exit_code: int = 0): - self.binary.write_text(textwrap.dedent(f"""\ - #!/usr/bin/env python3 - import os, sys - assert sys.stdin.read() == {SENTINELS[0]!r} - assert os.environ["ANTHROPIC_BASE_URL"] == {SENTINELS[1]!r} - assert os.environ["ANTHROPIC_API_KEY"] == {SENTINELS[2]!r} - for line in {lines!r}: - print(line) - raise SystemExit({exit_code!r}) - """), encoding="utf-8") - evidence = self.root / f"evidence-{len(tuple(self.root.glob('evidence-*')))}" - evidence.mkdir() - adapter = self._adapter() - result = run_invocation( - adapter.invocation(SENTINELS[0], evidence, Timeout(5, 1, 1, 1)), - parse_event=adapter.parser(), redact=adapter.redactor(SENTINELS[0]), - on_started=lambda _: None, - ) - durable = "\n".join( - path.read_text(encoding="utf-8") for path in sorted(evidence.iterdir()) - ) - return result, durable - - def test_exact_iop_only_invocation_and_fresh_workspace(self) -> None: - adapter = self._adapter() - with patch.dict(os.environ, {"SSL_CERT_FILE": "/operator/dev-ca.pem", "NODE_EXTRA_CA_CERTS": "/operator/dev-ca.pem"}): - spec = adapter.invocation(SENTINELS[0], self.root / "evidence", Timeout(5, 1, 1, 1)) - self.assertEqual(spec.cwd, self.workspace.workspace_dir) - self.assertEqual(spec.submission_mode, "stdin_once") - self.assertEqual(spec.task_payload, SENTINELS[0].encode()) - self.assertEqual(spec.argv[1:], ( - "--bare", "--print", "--verbose", "--input-format", "text", - "--output-format", "stream-json", "--model", "claude-sonnet", "--effort", "high", - "--no-session-persistence", "--permission-mode", "dontAsk", - "--tools", "Read,Write,Edit", "--allowedTools", "Read,Write,Edit", - )) - env = dict(spec.env) - self.assertEqual(env["ANTHROPIC_BASE_URL"], SENTINELS[1]) - self.assertEqual(env["ANTHROPIC_API_KEY"], SENTINELS[2]) - self.assertEqual(env["SSL_CERT_FILE"], "/operator/dev-ca.pem") - self.assertEqual(env["NODE_EXTRA_CA_CERTS"], "/operator/dev-ca.pem") - self.assertNotIn("ANTHROPIC_AUTH_TOKEN", env) - self.assertNotIn("CLAUDE_CONFIG_DIR", env) - (self.root / "other").mkdir() - not_fresh = _workspace(self.root / "other", "session-other") - object.__setattr__(not_fresh, "session_is_fresh", False) - with self.assertRaises(ClaudeIopValidationError): - ClaudeIopAdapter(self.cell, not_fresh, self.runtime) - - def test_direct_and_preset_preflight_are_exact_without_substitution(self) -> None: - for route_kind in ("direct", "execution_preset"): - cell = _cell(route_kind) - event = _binding_event(cell) - adapter = self._adapter(cell=cell) - self.assertEqual(adapter.preflight(event).status, "ready") - mutated = json.loads(event) - mutated["binding"]["effective_model"] = "fallback" - with self.assertRaises(ClaudeIopProtocolError): - parse_preflight_binding(json.dumps(mutated), cell) - incomplete = json.loads(_binding_event(self.cell)) - del incomplete["binding"]["effective_effort"] - with self.assertRaises(ClaudeIopProtocolError): - self._adapter().preflight(json.dumps(incomplete)) - - def test_runtime_requires_available_binary_and_complete_iop_config(self) -> None: - missing_binary = ClaudeIopRuntime( - str(self.root / "missing-claude"), SENTINELS[1], SENTINELS[2] - ) - with self.assertRaises(ClaudeIopValidationError): - ClaudeIopAdapter(self.cell, self.workspace, missing_binary) - for base_url, api_key in (("", SENTINELS[2]), (SENTINELS[1], "")): - with self.assertRaises(ClaudeIopValidationError): - ClaudeIopAdapter( - self.cell, self.workspace, - ClaudeIopRuntime(str(self.binary), base_url, api_key), - ) - - def test_fixture_uses_production_shaped_ordered_terminal_evidence(self) -> None: - lines = self._fixture_lines() - parser = ClaudeStreamParser(self.cell, "session-fixture") - parsed = [parser("stdout", line) for line in lines] - self.assertEqual(parsed[:2], [None, CallerEvent("finish")]) - self.assertEqual(parsed[2][-1], CallerEvent("idle")) - - malformed = json.loads(lines[1]) - del malformed["message"]["model"] - with self.assertRaises(ClaudeIopProtocolError): - parser = ClaudeStreamParser(self.cell, "session-fixture") - parser("stdout", lines[0]) - parser("stdout", json.dumps(malformed)) - with self.assertRaises(ClaudeIopProtocolError): - parser("stdout", "not-json") - - def test_parser_rejects_missing_duplicate_mismatched_and_out_of_order_evidence(self) -> None: - init, assistant, result = self._fixture_lines() - wrong_session = json.loads(assistant) - wrong_session["session_id"] = "other-claude-session" - wrong_model = json.loads(assistant) - wrong_model["message"]["model"] = "fallback-model" - missing_session = json.loads(result) - del missing_session["session_id"] - missing_result = [init, assistant] - cases = { - "assistant-before-init": [assistant], - "result-before-assistant": [init, result], - "duplicate-init": [init, init], - "duplicate-assistant": [init, assistant, assistant], - "session-mismatch": [init, json.dumps(wrong_session)], - "nested-model-mismatch": [init, json.dumps(wrong_model)], - "missing-result-session": [init, assistant, json.dumps(missing_session)], - } - for name, events in cases.items(): - with self.subTest(name=name): - parser = ClaudeStreamParser(self.cell, "session-fixture") - with self.assertRaises(ClaudeIopProtocolError): - for event in events: - parser("stdout", event) - parser = ClaudeStreamParser(self.cell, "session-fixture") - self.assertEqual([parser("stdout", event) for event in missing_result], [None, CallerEvent("finish")]) - - def test_parser_accepts_partial_snapshots_and_tool_result_cycles(self) -> None: - init, _, result = self._fixture_lines() - parser = ClaudeStreamParser(self.cell, "session-fixture") - partial = json.dumps({ - "type": "assistant", "session_id": "claude-session-fixture", - "message": {"id": "msg-tool-1", "model": "claude-sonnet", "stop_reason": None, - "content": [{"type": "tool_use", "id": "tool-1"}]}, - }) - user = json.dumps({ - "type": "user", "session_id": "claude-session-fixture", - "message": {"content": [{"type": "tool_result", "tool_use_id": "tool-1"}]}, - }) - final = json.dumps({ - "type": "assistant", "session_id": "claude-session-fixture", - "message": {"id": "msg-final", "model": "claude-sonnet", - "stop_reason": "end_turn", "content": []}, - }) - self.assertEqual( - [parser("stdout", line) for line in (init, partial, partial, user, final)], - [None, None, None, None, CallerEvent("finish")], - ) - observations = parser("stdout", result) - metrics = { - metric.name: metric.value for metric in observations - if isinstance(metric, ParsedMetric) - } - self.assertEqual(metrics["model_calls"], 2) - - def test_parser_accepts_cumulative_message_ids_and_direct_result(self) -> None: - init, _, result = self._fixture_lines() - parser = ClaudeStreamParser(self.cell, "session-fixture") - partial = json.dumps({ - "type": "assistant", "session_id": "claude-session-fixture", - "message": {"id": "msg-cumulative", "model": "claude-sonnet", - "stop_reason": None, "content": []}, - }) - user = json.dumps({ - "type": "user", "session_id": "claude-session-fixture", - "message": {"content": []}, - }) - terminal = json.loads(result) - terminal["usage"].update({ - "cache_creation": {"ephemeral_1h_input_tokens": 0, "ephemeral_5m_input_tokens": 0}, - "inference_geo": "", "iterations": [], - "output_tokens_details": {"thinking_tokens": 0}, - "server_tool_use": {"web_fetch_requests": 0, "web_search_requests": 0}, - "service_tier": "standard", "speed": "standard", - }) - parsed = [ - parser("stdout", line) - for line in (init, partial, partial, user, partial, user, user, partial) - ] - self.assertEqual(parsed, [None] * len(parsed)) - observations = parser("stdout", json.dumps(terminal)) - self.assertEqual(observations[-2:], (CallerEvent("finish"), CallerEvent("idle"))) - self.assertEqual( - {metric.name: metric.value for metric in observations if isinstance(metric, ParsedMetric)}["model_calls"], 1 - ) - - def test_parser_classifies_synthetic_api_error_without_parser_failure(self) -> None: - init, _, _ = self._fixture_lines() - parser = ClaudeStreamParser(self.cell, "session-fixture") - synthetic = json.dumps({ - "type": "assistant", "session_id": "claude-session-fixture", - "is_api_error_message": True, - "message": {"id": "synthetic", "model": "", - "stop_reason": "stop_sequence", "content": []}, - }) - terminal = json.dumps({ - "type": "result", "session_id": "claude-session-fixture", - "subtype": "success", "is_error": True, "terminal_reason": "api_error", - }) - self.assertEqual( - [parser("stdout", line) for line in (init, synthetic, terminal)], - [None, None, ( - CallerTerminal("failed", "caller_error"), - CallerEvent("finish"), CallerEvent("idle"), - )], - ) - - outcome, _ = self._run_fake([init, synthetic, terminal], exit_code=1) - self.assertFalse(outcome.product.status == "succeeded") - self.assertEqual(outcome.product.status, "failed") - self.assertEqual(outcome.harness.status, "passed") - - def test_lifecycle_accepts_result_direct_active_snapshot(self) -> None: - init, _, result = self._fixture_lines() - partial = json.dumps({ - "type": "assistant", "session_id": "claude-session-fixture", - "message": {"id": "msg-direct", "model": "claude-sonnet", - "stop_reason": None, "content": []}, - }) - outcome, _ = self._run_fake([init, partial, result]) - self.assertTrue(outcome.product.status == "succeeded", outcome) - kinds = [event.kind for event in outcome.events] - self.assertLess(kinds.index("finish"), kinds.index("idle")) - - def test_parser_accepts_explicit_tool_use_terminal_once(self) -> None: - init, _, result = self._fixture_lines() - parser = ClaudeStreamParser(self.cell, "session-fixture") - tool_use = json.dumps({ - "type": "assistant", "session_id": "claude-session-fixture", - "message": {"id": "msg-tool-1", "model": "claude-sonnet", - "stop_reason": "tool_use", "content": []}, - }) - user = json.dumps({ - "type": "user", "session_id": "claude-session-fixture", "message": {"content": []}, - }) - final = json.dumps({ - "type": "assistant", "session_id": "claude-session-fixture", - "message": {"id": "msg-final", "model": "claude-sonnet", - "stop_reason": "end_turn", "content": []}, - }) - self.assertEqual( - [parser("stdout", line) for line in (init, tool_use, user, final)], - [None, None, None, CallerEvent("finish")], - ) - self.assertEqual( - {metric.name: metric.value for metric in parser("stdout", result) if isinstance(metric, ParsedMetric)}["model_calls"], - 2, - ) - duplicate = ClaudeStreamParser(self.cell, "session-fixture") - duplicate("stdout", init) - duplicate("stdout", tool_use) - duplicate("stdout", user) - with self.assertRaises(ClaudeIopProtocolError): - duplicate("stdout", tool_use) - - def test_reported_result_values_become_bound_normalized_observations(self) -> None: - init, assistant, result = self._fixture_lines() - parser = ClaudeStreamParser(self.cell, "session-fixture") - parser("stdout", init) - parser("stdout", assistant) - parsed = parser("stdout", result) - self.assertEqual(parsed[-1], CallerEvent("idle")) - observed = {metric.name: metric for metric in parsed if isinstance(metric, ParsedMetric)} - self.assertEqual(observed["total_duration"].value, 1234 * 10 ** 6) - self.assertFalse(observed["total_duration"].overlap) - # The reported API duration is inside the reported total, so it is - # published as an overlapping interval and never subtracted from it. - self.assertEqual(observed["model_duration"].value, 1000 * 10 ** 6) - self.assertTrue(observed["model_duration"].overlap) - self.assertEqual(observed["model_calls"].value, 1) - self.assertEqual(observed["input_tokens"].value, 11) - self.assertEqual(observed["output_tokens"].value, 22) - self.assertEqual(observed["cached_input_tokens"].value, 5) - # The fixture omits cache creation, so that category stays unreported - # rather than being reported as zero. - self.assertNotIn("cache_write_tokens", observed) - for metric in parsed: - if not isinstance(metric, ParsedMetric): - continue - self.assertEqual(metric.model, "claude-sonnet") - self.assertEqual(metric.source, "caller_output") - self.assertEqual(observed["input_tokens"].clock, "none") - self.assertEqual(observed["total_duration"].clock, "caller_reported") - - def test_unknown_negative_boolean_and_fractional_usage_fails_closed(self) -> None: - init, assistant, result = self._fixture_lines() - base = json.loads(result) - cases = { - "unknown-usage-key": {"usage": {"input_tokens": 1, "web_search_requests": 2}}, - "fractional-token": {"usage": {"input_tokens": 1.5}}, - "boolean-token": {"usage": {"input_tokens": True}}, - "negative-token": {"usage": {"input_tokens": -1}}, - "string-duration": {"duration_ms": "1234"}, - "boolean-duration": {"duration_ms": True}, - "negative-duration": {"duration_ms": -5}, - "non-object-usage": {"usage": [1, 2]}, - } - for name, override in cases.items(): - with self.subTest(name=name): - parser = ClaudeStreamParser(self.cell, "session-fixture") - parser("stdout", init) - parser("stdout", assistant) - with self.assertRaises(ClaudeIopProtocolError): - parser("stdout", json.dumps({**base, **override})) - - def test_structural_redaction_never_retains_sensitive_content(self) -> None: - raw = json.dumps({ - "type": "result", "result": "result-secret-sentinel", - "message": "error-secret-sentinel", - "content": "prompt-secret-sentinel", - "tool_input": {"arguments": "tool-secret-sentinel"}, - "diagnostic": "https://private.iop.invalid api-secret-sentinel", - }) - redacted = redact_claude_event(raw, SENTINELS) - error_redacted = redact_claude_event(json.dumps({ - "type": "error", "message": "error-secret-sentinel", - "error": {"detail": "tool-secret-sentinel"}, - }), SENTINELS) - for sentinel in (*SENTINELS, *ARBITRARY_SENTINELS): - self.assertNotIn(sentinel, redacted) - self.assertNotIn(sentinel, error_redacted) - self.assertIn("[redacted]", redacted) - self.assertEqual(redact_claude_event("raw prompt-secret-sentinel", SENTINELS), - '{"type":"invalid_claude_json"}') - - def test_fake_cli_runs_once_and_durable_evidence_is_redacted(self) -> None: - init, assistant, result = (json.loads(line) for line in self._fixture_lines()) - assistant["message"]["content"] = SENTINELS[0] - assistant["tool_input"] = {"arguments": ARBITRARY_SENTINELS[0]} - result["result"] = ARBITRARY_SENTINELS[1] - diagnostic = {"type": "system", "subtype": "notice", "error": ARBITRARY_SENTINELS[2]} - outcome, durable = self._run_fake([ - json.dumps(init), json.dumps(diagnostic), json.dumps(assistant), json.dumps(result), - ]) - result = outcome - self.assertTrue(result.product.status == "succeeded", result) - self.assertEqual(result.harness.reason, REASON_SUCCESS) - self.assertTrue(result.submitted) - self.assertTrue(result.harness.ordered_terminal) - for sentinel in (*SENTINELS, *ARBITRARY_SENTINELS): - self.assertNotIn(sentinel, durable) - self.assertEqual( - [metric.name for metric in result.metrics], - [ - "total_duration", "model_duration", "model_calls", - "cached_input_tokens", "input_tokens", "output_tokens", - ], - ) - # Numeric observations survive the content redactors that removed every - # sentinel above, and the durable journal keeps them in one closed shape. - self.assertIn('"kind": "metric:total_duration"', durable) - self.assertIn('\\"value\\":1234000000', durable) - - def test_lifecycle_rejects_boundary_violations(self) -> None: - init, assistant, result = self._fixture_lines() - bad_model = json.loads(assistant) - bad_model["message"]["model"] = "fallback-model" - cases = { - "missing-init": [assistant, result], - "missing-result": [init, assistant], - "duplicate-init": [init, init, assistant, result], - "out-of-order-result": [init, result, assistant], - "mismatched-model": [init, json.dumps(bad_model), result], - "malformed-json": [init, "not-json", assistant, result], - } - for name, lines in cases.items(): - with self.subTest(name=name): - outcome, durable = self._run_fake(lines) - self.assertFalse(outcome.product.status == "succeeded", outcome) - for sentinel in (*SENTINELS, *ARBITRARY_SENTINELS): - self.assertNotIn(sentinel, durable) - - def test_metric_prefixed_malformed_output_is_redacted_before_durable_capture(self) -> None: - metric_sentinel = "metric:" + SENTINELS[0] - outcome, durable = self._run_fake([metric_sentinel]) - self.assertFalse(outcome.product.status == "succeeded", outcome) - self.assertEqual(outcome.harness.reason, REASON_PARSER_ERROR) - self.assertTrue(outcome.submitted) - self.assertIn("invalid_claude_json", durable) - for sentinel in (*SENTINELS, metric_sentinel, SENTINELS[0]): - self.assertNotIn(sentinel, durable) - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/agent_benchmark/codex_iop.py b/scripts/agent_benchmark/codex_iop.py deleted file mode 100644 index 18277634..00000000 --- a/scripts/agent_benchmark/codex_iop.py +++ /dev/null @@ -1,504 +0,0 @@ -"""Isolated, secret-safe Codex ``exec`` adapter for IOP benchmark cells. - -The adapter deliberately owns no public runner command. It converts a frozen -manifest cell and a caller-supplied runtime into one bounded lifecycle -invocation. The only executable it starts is a small bridge in the lifecycle -owned process group; that bridge starts ``codex exec``, forwards its JSONL, and -emits an idle marker only after the child has exited and both output streams -reached EOF. -""" - -from __future__ import annotations - -import json -import os -import re -import secrets -import subprocess -import sys -import threading -from dataclasses import dataclass -from pathlib import Path -from typing import Any, Callable, Mapping -from urllib.parse import urlsplit - -# The bridge is intentionally invoked by absolute script path from an isolated -# workspace. Make the repository package importable without inheriting an -# ambient PYTHONPATH. -if __package__ in (None, ""): - sys.path.insert(0, str(Path(__file__).resolve().parents[2])) - -from scripts.agent_benchmark.lifecycle import ( - CALLER_REASON_ERROR, - CALLER_REASON_SUCCESS, - CALLER_STATUS_FAILED, - CALLER_STATUS_SUCCEEDED, - COMPLETION_EXIT_AFTER_IDLE, - SUBMISSION_STDIN_ONCE, - CallerEvent, - CallerTerminal, - InvocationResult, - InvocationSpec, - LifecycleMetricError, - LifecycleValidationError, - ParsedMetric, - SupervisorLocator, - TLS_CA_ENV_KEYS, - count_metric, - duration_metric, - env_pairs, - exact_value_redactor, - inherited_tls_ca_environment, - is_reported_number, - run_invocation, -) -from scripts.agent_benchmark.connectivity import ( - ISSUE_RESUME_CODES, - CallerCapability, - ConnectivityIssue, - ConnectivityResult, - RequestedEffectiveBinding, - make_result, -) -from scripts.agent_benchmark.manifest import MatrixCell, Timeout, TOKEN_RE -from scripts.agent_benchmark.workspace import PreparedWorkspace - - -PROVIDER_ID = "iop_benchmark" -SECRET_ENV_KEY = "IOP_BENCHMARK_API_KEY" -BASE_URL_ENV_KEY = "IOP_BENCHMARK_BASE_URL" -SUPPORTED_EFFORTS = ("xhigh",) -_BRIDGE_IDLE_TYPE = "adapter.idle" -_BRIDGE_ID = "codex_iop" -_SENSITIVE_KEYS = frozenset({ - "api_key", "authorization", "base_url", "command", "content", "endpoint", - "input", "instructions", "message", "output", "prompt", "secret", "text", - "token", "tool_input", "tool_output", "url", -}) -_SAFE_STRING_KEYS = frozenset({ - "adapter", "effort", "model", "nonce", "reasoning_effort", "route_id", - "route_kind", "stage", "status", "type", -}) -# The exact reported Codex usage keys this adapter observes. A tool interval is -# consumed only when the completed item explicitly pairs one bounded duration -# with one call identifier; nothing is derived from event arrival order. -_CODEX_USAGE_FIELDS = { - "input_tokens": "input_tokens", - "cached_input_tokens": "cached_input_tokens", - "cache_write_input_tokens": "cache_write_tokens", - "output_tokens": "output_tokens", - "reasoning_output_tokens": "reasoning_tokens", - "total_tokens": "total_tokens", -} -_CODEX_ITEM_COMPLETED = "item.completed" -_CODEX_CALL_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.:-]{0,63}$") -_CODEX_TOOL_ITEM_TYPES = frozenset({ - "command_execution", "file_change", "mcp_tool_call", "web_search", -}) - - -class CodexIOPError(Exception): - """Base class for closed Codex adapter validation failures.""" - - -class CodexRuntimeError(CodexIOPError): - """The caller-supplied runtime is absent, malformed, or unsafe.""" - - -class CodexJSONLError(CodexIOPError): - """A Codex JSONL event cannot safely satisfy the adapter protocol.""" - - -@dataclass(frozen=True) -class CodexRuntime: - """The two runtime-only values needed to reach the IOP Responses surface.""" - - base_url: str - api_key: str - path: str - - -@dataclass(frozen=True) -class CodexInvocation: - """One immutable spec plus its parser and capture redactor.""" - - spec: InvocationSpec - parser: "CodexJSONLParser" - redact: Callable[[str], str] - - -@dataclass(frozen=True) -class CodexInvocationResult: - """Lifecycle result and independently observed effective binding, if any.""" - - lifecycle: InvocationResult - effective_binding: tuple[str, str, str, str] | None - - -def runtime_from_environment(environment: Mapping[str, str]) -> CodexRuntime: - """Read exactly the named IOP runtime values without consulting ambient env.""" - if not isinstance(environment, Mapping): - raise CodexRuntimeError("invalid runtime environment") - required = {BASE_URL_ENV_KEY, SECRET_ENV_KEY, "PATH"} - if set(environment) != required or not all(isinstance(key, str) for key in environment): - raise CodexRuntimeError("runtime environment has unsupported keys") - values = {key: environment[key] for key in required} - if not all(isinstance(value, str) and value for value in values.values()): - raise CodexRuntimeError("runtime environment is incomplete") - parsed = urlsplit(values[BASE_URL_ENV_KEY]) - if parsed.scheme not in ("http", "https") or not parsed.netloc or parsed.query or parsed.fragment: - raise CodexRuntimeError("invalid IOP base URL") - return CodexRuntime(values[BASE_URL_ENV_KEY], values[SECRET_ENV_KEY], values["PATH"]) - - -def _require_cell(cell: MatrixCell) -> None: - if not isinstance(cell, MatrixCell) or cell.caller != "codex": - raise CodexRuntimeError("Codex adapter requires a Codex matrix cell") - if cell.iop.requested_effort not in SUPPORTED_EFFORTS: - raise CodexRuntimeError("unsupported Codex reasoning effort") - if not TOKEN_RE.fullmatch(cell.iop.request_model): - raise CodexRuntimeError("invalid requested model") - - -def codex_capability() -> CallerCapability: - """The closed local capability claimed by this adapter implementation.""" - return CallerCapability("codex", ("direct", "execution_preset"), SUPPORTED_EFFORTS) - - -def _require_prepared(prepared: PreparedWorkspace) -> None: - if not isinstance(prepared, PreparedWorkspace) or not prepared.session_is_fresh: - raise CodexRuntimeError("Codex invocation requires a fresh prepared workspace") - if not Path(prepared.workspace_dir).is_dir() or not Path(prepared.session_dir).is_dir(): - raise CodexRuntimeError("prepared workspace is unavailable") - if not Path(prepared.attempt_root).is_dir(): - raise CodexRuntimeError("prepared attempt root is unavailable") - - -def _toml_string(value: str) -> str: - if not isinstance(value, str) or any(ord(char) < 0x20 for char in value): - raise CodexRuntimeError("invalid provider configuration value") - # JSON strings are valid TOML basic strings and avoid hand-built quoting. - return json.dumps(value, ensure_ascii=True) - - -def _provider_overrides(cell: MatrixCell, runtime: CodexRuntime) -> tuple[str, ...]: - return ( - f"model_provider={_toml_string(PROVIDER_ID)}", - f"model_providers.{PROVIDER_ID}.name={_toml_string('IOP Benchmark')}", - f"model_providers.{PROVIDER_ID}.base_url={_toml_string(runtime.base_url)}", - f"model_providers.{PROVIDER_ID}.env_key={_toml_string(SECRET_ENV_KEY)}", - f"shell_environment_policy.filters.{SECRET_ENV_KEY}={_toml_string('exclude')}", - f"model_providers.{PROVIDER_ID}.wire_api={_toml_string('responses')}", - f"model_reasoning_effort={_toml_string(cell.iop.requested_effort)}", - ) - - -def build_codex_spec( - cell: MatrixCell, - prepared: PreparedWorkspace, - runtime: CodexRuntime, - task_payload: bytes, - timeout: Timeout, - *, - codex_executable: str | tuple[str, ...] = "codex", -) -> InvocationSpec: - """Build one isolated Codex invocation without reading user configuration. - - The secret remains only in the child environment for provider - authentication. ``shell_environment_policy.filters`` explicitly excludes - the secret from shell tool environments and snapshots so it never - serializes into durable evidence. The base URL is an ephemeral process - argument required by Codex's provider override; neither is serialized - into fixture, capture, lifecycle result, or durable config. - """ - _require_cell(cell) - _require_prepared(prepared) - if not isinstance(runtime, CodexRuntime): - raise CodexRuntimeError("invalid Codex runtime") - if not isinstance(task_payload, bytes) or not task_payload: - raise CodexRuntimeError("Codex task payload is required") - try: - task_payload.decode("utf-8") - except UnicodeDecodeError as exc: - raise CodexRuntimeError("Codex task payload must be UTF-8") from exc - if not isinstance(timeout, Timeout): - raise CodexRuntimeError("invalid timeout") - if isinstance(codex_executable, str): - executable_argv = (codex_executable,) - elif isinstance(codex_executable, tuple) and codex_executable and all( - isinstance(item, str) and item for item in codex_executable - ): - # Test-only seam for a non-executable fixture script; normal callers - # always receive the exact single-token ``codex`` command above. - executable_argv = codex_executable - else: - raise CodexRuntimeError("invalid Codex executable") - - codex_argv: list[str] = [ - *executable_argv, "exec", "--sandbox", "workspace-write", "--json", "--ephemeral", "--ignore-user-config", - "--strict-config", "--skip-git-repo-check", "-C", prepared.workspace_dir, - "-m", cell.iop.request_model, - ] - for override in _provider_overrides(cell, runtime): - codex_argv.extend(("-c", override)) - codex_argv.append("-") - return InvocationSpec( - # The lifecycle changes cwd to the isolated workspace, so use this - # module's absolute script path rather than relying on repository - # import resolution in the child bridge. - argv=(sys.executable, str(Path(__file__).resolve()), "--bridge", "--", *codex_argv), - cwd=prepared.workspace_dir, - env=env_pairs({ - "PATH": runtime.path, - "HOME": prepared.session_dir, - SECRET_ENV_KEY: runtime.api_key, - **dict(inherited_tls_ca_environment()), - }), - env_allowlist=(SECRET_ENV_KEY, *TLS_CA_ENV_KEYS), - submission_mode=SUBMISSION_STDIN_ONCE, - completion_mode=COMPLETION_EXIT_AFTER_IDLE, - timeout=timeout, - evidence_dir=prepared.attempt_root, - task_payload=task_payload, - control_dir=str(Path(prepared.attempt_root) / "codex-control"), - ) - - -def redact_codex_jsonl(text: str, secrets: tuple[str, ...]) -> str: - """Remove JSON content and runtime values before any line becomes evidence.""" - exact = exact_value_redactor(secrets) - try: - value = json.loads(text) - except (TypeError, json.JSONDecodeError): - return "[redacted]" - - def redact(value: Any, key: str | None = None) -> Any: - if key is not None and key.lower() in _SENSITIVE_KEYS: - return "[redacted]" - if isinstance(value, dict): - return {str(item_key): redact(item_value, str(item_key)) for item_key, item_value in value.items()} - if isinstance(value, list): - return [redact(item) for item in value] - if isinstance(value, str): - return exact(value) if key in _SAFE_STRING_KEYS else "[redacted]" - return value - - return json.dumps(redact(value), ensure_ascii=True, separators=(",", ":")) - - -class CodexJSONLParser: - """Parse only terminal evidence and optional explicit IOP effective binding.""" - - def __init__(self, cell: MatrixCell, idle_nonce: str) -> None: - _require_cell(cell) - if not isinstance(idle_nonce, str) or len(idle_nonce) < 16: - raise CodexRuntimeError("invalid bridge idle nonce") - self._cell = cell - self._idle_nonce = idle_nonce - self._effective_binding: tuple[str, str, str, str] | None = None - self._turns = 0 - self._tool_calls: set[str] = set() - - @property - def effective_binding(self) -> tuple[str, str, str, str] | None: - return self._effective_binding - - def connectivity_result(self) -> ConnectivityResult: - """Classify absent evidence without synthesizing unreported stage bindings.""" - iop = self._cell.iop - if self._effective_binding is not None: - # Codex's optional scalar observation proves neither the complete - # stage list nor its exact order. Never manufacture that missing - # contract evidence from the manifest just to produce ``ready``. - raise CodexJSONLError("effective stage binding is unavailable") - binding = RequestedEffectiveBinding( - self._cell.id, self._cell.caller, iop.route_kind, iop.route_id, - iop.request_model, iop.requested_effort, - None, None, None, None, (), - ) - issues: tuple[ConnectivityIssue, ...] = ( - ConnectivityIssue("stream_incompatible", ISSUE_RESUME_CODES["stream_incompatible"]), - ) - return make_result(self._cell, codex_capability(), binding, issues) - - def parse(self, stream: str, line: str) -> Any: - if stream != "stdout": - return None - try: - record = json.loads(line) - except (TypeError, json.JSONDecodeError) as exc: - raise CodexJSONLError("malformed Codex JSONL") from exc - if not isinstance(record, dict) or not isinstance(record.get("type"), str): - raise CodexJSONLError("malformed Codex JSONL") - self._observe_effective_binding(record) - if record["type"] == "turn.completed": - status = record.get("status") - if status is not None and status not in ("completed", "success"): - return ( - CallerTerminal(CALLER_STATUS_FAILED, CALLER_REASON_ERROR), - CallerEvent("finish"), - ) - self._turns += 1 - return ( - *self._turn_observations(record), - CallerTerminal(CALLER_STATUS_SUCCEEDED, CALLER_REASON_SUCCESS), - CallerEvent("finish"), - ) - if record["type"] == _CODEX_ITEM_COMPLETED: - return self._tool_interval(record) - if record["type"] == _BRIDGE_IDLE_TYPE: - if record != {"type": _BRIDGE_IDLE_TYPE, "adapter": _BRIDGE_ID, "nonce": self._idle_nonce, "child_exit": 0}: - raise CodexJSONLError("unverified bridge idle marker") - return CallerEvent("idle") - return None - - def _turn_observations(self, record: dict[str, Any]) -> tuple[ParsedMetric, ...]: - """Observe reported turn usage plus the turn and tool call counts.""" - model = self._cell.iop.request_model - usage = record.get("usage") - if usage is not None and ( - not isinstance(usage, dict) or not set(usage) <= set(_CODEX_USAGE_FIELDS) - ): - raise CodexJSONLError("invalid Codex usage observation") - try: - observations = [ - count_metric(_CODEX_USAGE_FIELDS[field], value, model=model) - for field, value in sorted((usage or {}).items()) - ] - observations.append(count_metric("model_calls", self._turns, model=model)) - observations.append( - count_metric("tool_calls", len(self._tool_calls), model=model) - ) - except LifecycleMetricError as exc: - raise CodexJSONLError("invalid Codex usage observation") from exc - return tuple(observations) - - def _tool_interval(self, record: dict[str, Any]) -> ParsedMetric | None: - """Count a completed tool and optionally observe its reported interval.""" - item = record.get("item") - if not isinstance(item, dict) or item.get("type") not in _CODEX_TOOL_ITEM_TYPES: - return None - call_id = item.get("id") - if not isinstance(call_id, str) or _CODEX_CALL_ID_RE.fullmatch(call_id) is None: - raise CodexJSONLError("unpaired Codex tool completion") - if call_id in self._tool_calls: - raise CodexJSONLError("duplicate Codex tool completion") - if "duration_ms" not in item: - self._tool_calls.add(call_id) - return None - if not is_reported_number(item["duration_ms"]): - raise CodexJSONLError("invalid Codex tool interval") - try: - observation = duration_metric( - "tool_duration", item["duration_ms"], reported_unit="ms", - model=self._cell.iop.request_model, call_id=call_id, overlap=True, - ) - except LifecycleMetricError as exc: - raise CodexJSONLError("invalid Codex tool interval") from exc - self._tool_calls.add(call_id) - return observation - - def _observe_effective_binding(self, record: dict[str, Any]) -> None: - observed = record.get("iop_effective_binding") - if observed is None: - return - if not isinstance(observed, dict) or set(observed) != {"route_kind", "route_id", "model", "effort"}: - raise CodexJSONLError("invalid effective binding observation") - values = tuple(observed[key] for key in ("route_kind", "route_id", "model", "effort")) - if not all(isinstance(value, str) and TOKEN_RE.fullmatch(value) for value in values): - raise CodexJSONLError("invalid effective binding observation") - expected = ( - self._cell.iop.route_kind, - self._cell.iop.route_id, - self._cell.iop.request_model, - self._cell.iop.requested_effort, - ) - if values != expected: - raise CodexJSONLError("effective binding substitution") - if self._effective_binding is not None: - raise CodexJSONLError("duplicate effective binding observation") - self._effective_binding = values # type: ignore[assignment] - - -def build_codex_invocation( - cell: MatrixCell, - prepared: PreparedWorkspace, - runtime: CodexRuntime, - task_payload: bytes, - timeout: Timeout, - *, - codex_executable: str | tuple[str, ...] = "codex", -) -> CodexInvocation: - """Pair the closed invocation spec with its parser and structural redactor.""" - nonce = secrets.token_hex(16) - spec = build_codex_spec(cell, prepared, runtime, task_payload, timeout, codex_executable=codex_executable) - parser = CodexJSONLParser(cell, nonce) - # The nonce must reach the bridge but not Codex. Put it before the bridge - # delimiter so the bridge removes it before starting the child. - argv = (*spec.argv[:3], f"--idle-nonce={nonce}", *spec.argv[3:]) - task_text = task_payload.decode("utf-8") - return CodexInvocation(spec=InvocationSpec(**{**spec.__dict__, "argv": argv}), parser=parser, - redact=lambda line: redact_codex_jsonl(line, (runtime.base_url, runtime.api_key, task_text))) - - -def run_codex_invocation( - invocation: CodexInvocation, - on_started: Callable[[SupervisorLocator], None], -) -> CodexInvocationResult: - """Run exactly one prepared Codex invocation through the generic lifecycle.""" - if not isinstance(invocation, CodexInvocation) or not callable(on_started): - raise CodexRuntimeError("invalid Codex invocation") - result = run_invocation(invocation.spec, parse_event=invocation.parser.parse, - on_started=on_started, redact=invocation.redact) - return CodexInvocationResult(result, invocation.parser.effective_binding) - - -def _forward_stream(stream: Any, destination: Any) -> None: - try: - for line in iter(stream.readline, b""): - destination.buffer.write(line) - destination.buffer.flush() - finally: - stream.close() - - -def _bridge(argv: list[str]) -> int: - """Own a child Codex process and emit idle only after exit plus both EOFs.""" - if not argv or not argv[0].startswith("--idle-nonce="): - return 64 - nonce = argv.pop(0).partition("=")[2] - if not nonce or not argv or argv.pop(0) != "--": - return 64 - try: - child = subprocess.Popen(argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True) - except OSError: - return 127 - assert child.stdin is not None and child.stdout is not None and child.stderr is not None - payload = sys.stdin.buffer.read() - try: - child.stdin.write(payload) - child.stdin.close() - except OSError: - child.terminate() - stdout_thread = threading.Thread(target=_forward_stream, args=(child.stdout, sys.stdout), daemon=True) - stderr_thread = threading.Thread(target=_forward_stream, args=(child.stderr, sys.stderr), daemon=True) - stdout_thread.start() - stderr_thread.start() - exit_code = child.wait() - stdout_thread.join() - stderr_thread.join() - if exit_code == 0: - sys.stdout.write(json.dumps({"type": _BRIDGE_IDLE_TYPE, "adapter": _BRIDGE_ID, "nonce": nonce, "child_exit": 0}, separators=(",", ":")) + "\n") - sys.stdout.flush() - return exit_code - - -def main(argv: list[str] | None = None) -> int: - values = list(sys.argv[1:] if argv is None else argv) - if not values or values.pop(0) != "--bridge": - return 64 - return _bridge(values) - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/agent_benchmark/codex_iop_test.py b/scripts/agent_benchmark/codex_iop_test.py deleted file mode 100644 index 3283605e..00000000 --- a/scripts/agent_benchmark/codex_iop_test.py +++ /dev/null @@ -1,294 +0,0 @@ -"""Credential-free tests for the isolated Codex IOP benchmark adapter.""" - -from __future__ import annotations - -import json -import os -import sys -import tempfile -import unittest -from pathlib import Path -from unittest.mock import patch - -from scripts.agent_benchmark.codex_iop import ( - BASE_URL_ENV_KEY, - SECRET_ENV_KEY, - CodexJSONLError, - CodexJSONLParser, - CodexRuntimeError, - build_codex_invocation, - build_codex_spec, - redact_codex_jsonl, - run_codex_invocation, - runtime_from_environment, -) -from scripts.agent_benchmark.lifecycle import ( - CallerEvent, - ParsedMetric, - REASON_DUPLICATE_EVENT, - REASON_NONZERO_EXIT, - REASON_PARSER_ERROR, -) -from scripts.agent_benchmark.manifest import ExpectedBinding, IopCell, MatrixCell, Timeout -from scripts.agent_benchmark.workspace import AttemptIdentity, PreparedWorkspace, TestbedProvenance - - -_ENDPOINT = "https://iop.private.example/v1" -_SECRET = "iop_test_secret_123456789" -_PROMPT = b"private benchmark prompt must not persist" - - -def _cell(*, effort: str = "xhigh") -> MatrixCell: - return MatrixCell( - "codex-gpt", "codex", - IopCell("gpt-5.6-luna", effort, "direct", "iop-gpt", ( - ExpectedBinding("request", "gpt-5.6-luna", effort), - )), - ) - - -class CodexIOPTest(unittest.TestCase): - def setUp(self) -> None: - self.tmp = tempfile.TemporaryDirectory() - self.root = Path(self.tmp.name) - self.workspace = self.root / "workspace" - self.workspace.mkdir() - self.session = self.root / "session" - self.session.mkdir() - self.evidence = self.root / "attempt" - self.evidence.mkdir() - - def tearDown(self) -> None: - self.tmp.cleanup() - - def _runtime(self): - return runtime_from_environment({ - BASE_URL_ENV_KEY: _ENDPOINT, - SECRET_ENV_KEY: _SECRET, - "PATH": os.environ.get("PATH", "/usr/bin:/bin"), - }) - - def _prepared(self) -> PreparedWorkspace: - return PreparedWorkspace( - identity=AttemptIdentity("run-0001", "codex-gpt", 1, 1), - attempt_root=str(self.evidence), workspace_dir=str(self.workspace), - session_dir=str(self.session), session_id="fresh-session", session_is_fresh=True, - workspace_checksum="sha256:" + "0" * 64, setup_cache_policy="isolated", - testbed_provenance=TestbedProvenance("../iop-s2", "main", "0" * 40, "0" * 64, True), - prepared_at="2026-08-10T00:00:00+00:00", - ) - - def _timeout(self) -> Timeout: - return Timeout(5, 2, 1, 1) - - def _fake_codex(self, records: list[object], exit_code: int = 0) -> str: - path = self.root / f"fake-codex-{len(list(self.root.glob('fake-codex-*')))}.py" - lines = [ - "#!/usr/bin/env python3", "import json, sys", "task = sys.stdin.read()", - ] - for record in records: - if record == "TASK": - lines.append("print(json.dumps({'type': 'turn.completed', 'status': 'completed', 'content': task}), flush=True)") - elif isinstance(record, str): - lines.append(f"print({record!r}, flush=True)") - else: - lines.append(f"print(json.dumps({record!r}), flush=True)") - lines.append(f"raise SystemExit({exit_code})") - path.write_text("\n".join(lines) + "\n", encoding="utf-8") - return str(path) - - def test_exact_isolated_responses_spec_uses_one_stdin_submission(self) -> None: - with patch.dict(os.environ, {"SSL_CERT_FILE": "/operator/dev-ca.pem", "NODE_EXTRA_CA_CERTS": "/operator/dev-ca.pem"}): - spec = build_codex_spec(_cell(), self._prepared(), self._runtime(), _PROMPT, self._timeout()) - argv = list(spec.argv) - codex = argv[argv.index("--") + 1:] - self.assertFalse((self.workspace / ".git").exists()) - self.assertEqual(codex.count("--skip-git-repo-check"), 1) - self.assertEqual(codex[:14], [ - "codex", "exec", "--sandbox", "workspace-write", "--json", "--ephemeral", "--ignore-user-config", - "--strict-config", "--skip-git-repo-check", "-C", str(self.workspace), - "-m", "gpt-5.6-luna", "-c", - ]) - self.assertEqual(codex[-1], "-") - overrides = [codex[index + 1] for index, item in enumerate(codex[:-1]) if item == "-c"] - self.assertEqual(overrides, [ - 'model_provider="iop_benchmark"', - 'model_providers.iop_benchmark.name="IOP Benchmark"', - f'model_providers.iop_benchmark.base_url="{_ENDPOINT}"', - 'model_providers.iop_benchmark.env_key="IOP_BENCHMARK_API_KEY"', - 'shell_environment_policy.filters.IOP_BENCHMARK_API_KEY="exclude"', - 'model_providers.iop_benchmark.wire_api="responses"', - 'model_reasoning_effort="xhigh"', - ]) - self.assertEqual(spec.submission_mode, "stdin_once") - self.assertEqual(spec.completion_mode, "exit_after_idle") - self.assertEqual(dict(spec.env)[SECRET_ENV_KEY], _SECRET) - self.assertEqual(dict(spec.env)["SSL_CERT_FILE"], "/operator/dev-ca.pem") - self.assertEqual(dict(spec.env)["NODE_EXTRA_CA_CERTS"], "/operator/dev-ca.pem") - self.assertNotIn(BASE_URL_ENV_KEY, dict(spec.env)) - self.assertNotIn("OPENAI_API_KEY", dict(spec.env)) - - def test_provider_secret_is_explicitly_excluded_from_shell_environment(self) -> None: - """Regression: the provider env_key injects the secret into the child - environment for authentication, but the same secret is explicitly - excluded from shell tool environments and snapshots. - - Provider authentication, shell exclusion, and the exact invocation - spec are one invariant; both must hold together. - """ - with patch.dict(os.environ, {"SSL_CERT_FILE": "/operator/dev-ca.pem", "NODE_EXTRA_CA_CERTS": "/operator/dev-ca.pem"}): - spec = build_codex_spec(_cell(), self._prepared(), self._runtime(), _PROMPT, self._timeout()) - env = dict(spec.env) - # Provider authentication: the secret must still reach the child process. - self.assertEqual(env[SECRET_ENV_KEY], _SECRET) - # Shell exclusion: the exact override must appear in the invocation. - argv = list(spec.argv) - codex = argv[argv.index("--") + 1:] - overrides = {codex[index + 1] for index, item in enumerate(codex[:-1]) if item == "-c"} - self.assertIn( - f'shell_environment_policy.filters.IOP_BENCHMARK_API_KEY="exclude"', - overrides, - "the benchmark secret must be explicitly excluded from shell environments", - ) - # TLS variables, isolated HOME, and no leaking caller keys. - self.assertEqual(env["SSL_CERT_FILE"], "/operator/dev-ca.pem") - self.assertEqual(env["NODE_EXTRA_CA_CERTS"], "/operator/dev-ca.pem") - self.assertNotIn(BASE_URL_ENV_KEY, env) - self.assertNotIn("OPENAI_API_KEY", env) - - def test_runtime_and_effort_are_closed(self) -> None: - with self.assertRaises(CodexRuntimeError): - runtime_from_environment({BASE_URL_ENV_KEY: _ENDPOINT, SECRET_ENV_KEY: _SECRET}) - with self.assertRaises(CodexRuntimeError): - runtime_from_environment({BASE_URL_ENV_KEY: _ENDPOINT, SECRET_ENV_KEY: _SECRET, "PATH": "/bin", "EXTRA": "x"}) - with self.assertRaises(CodexRuntimeError): - build_codex_spec(_cell(effort="high"), self._prepared(), self._runtime(), _PROMPT, self._timeout()) - - def test_fixture_finish_then_verified_idle_and_structural_redaction(self) -> None: - parser = CodexJSONLParser(_cell(), "fixture-nonce-0001") - fixture = Path("scripts/fixtures/agent-comparison-benchmark/codex-iop-stream.jsonl") - events = [parser.parse("stdout", line) for line in fixture.read_text(encoding="utf-8").splitlines()] - self.assertEqual([events[0], events[-1]], [None, CallerEvent("idle")]) - tool = events[1] - self.assertEqual((tool.name, tool.value, tool.call_id), ("tool_duration", 7_250_000, "call-1")) - # A tool interval is reported inside the turn, so it is published as an - # overlapping interval instead of a subtractable slice. - self.assertTrue(tool.overlap) - self.assertEqual(events[2][-1], CallerEvent("finish")) - turn = {metric.name: metric.value for metric in events[2] if isinstance(metric, ParsedMetric)} - self.assertEqual(turn, { - "cache_write_tokens": 6, "cached_input_tokens": 8, - "input_tokens": 31, "output_tokens": 12, "reasoning_tokens": 4, - "model_calls": 1, "tool_calls": 1, - }) - # The fixture omits the provider total, so it is never reconstructed - # from the reported categories. - self.assertNotIn("total_tokens", turn) - redacted = redact_codex_jsonl( - json.dumps({"type": "turn.completed", "content": _PROMPT.decode(), "endpoint": _ENDPOINT, "token": _SECRET}), - (_ENDPOINT, _SECRET, _PROMPT.decode()), - ) - self.assertNotIn(_PROMPT.decode(), redacted) - self.assertNotIn(_ENDPOINT, redacted) - self.assertNotIn(_SECRET, redacted) - self.assertIn("[redacted]", redacted) - - def test_bridge_proves_finish_then_idle_after_child_exit(self) -> None: - fake = self._fake_codex(["TASK"]) - invocation = build_codex_invocation(_cell(), self._prepared(), self._runtime(), _PROMPT, self._timeout(), codex_executable=(sys.executable, fake)) - result = run_codex_invocation(invocation, lambda _: None) - self.assertTrue(result.lifecycle.product.status == "succeeded") - self.assertEqual([event.kind for event in result.lifecycle.events], [ - "submitted", "first_output", "metric:model_calls", "metric:tool_calls", - "caller_terminal", "finish", "idle", "exited", "quiet", - ]) - capture = result.lifecycle.stdout.text - self.assertNotIn(_PROMPT.decode(), capture) - self.assertNotIn(_ENDPOINT, capture) - self.assertNotIn(_SECRET, capture) - self.assertIsNone(result.effective_binding) - - def test_child_failure_never_synthesizes_idle(self) -> None: - fake = self._fake_codex([{"type": "turn.completed", "status": "completed"}], exit_code=7) - result = run_codex_invocation(build_codex_invocation(_cell(), self._prepared(), self._runtime(), _PROMPT, self._timeout(), codex_executable=(sys.executable, fake)), lambda _: None) - self.assertFalse(result.lifecycle.product.status == "succeeded") - self.assertEqual(result.lifecycle.harness.reason, REASON_NONZERO_EXIT) - self.assertNotIn("idle", [event.kind for event in result.lifecycle.events]) - - def test_duplicate_malformed_and_unverified_idle_fail_closed(self) -> None: - cases = ( - ([{"type": "turn.completed", "status": "completed"}, {"type": "turn.completed", "status": "completed"}], REASON_DUPLICATE_EVENT), - (["not-json"], REASON_PARSER_ERROR), - ([{"type": "adapter.idle", "adapter": "codex_iop", "nonce": "not-the-bridge-nonce", "child_exit": 0}], REASON_PARSER_ERROR), - ) - for index, (records, reason) in enumerate(cases): - with self.subTest(reason=reason): - evidence = self.root / f"attempt-{reason}-{index}" - evidence.mkdir() - prepared = self._prepared().__class__(**{**self._prepared().__dict__, "attempt_root": str(evidence)}) - fake = self._fake_codex(records) - result = run_codex_invocation(build_codex_invocation(_cell(), prepared, self._runtime(), _PROMPT, self._timeout(), codex_executable=(sys.executable, fake)), lambda _: None) - self.assertFalse(result.lifecycle.product.status == "succeeded") - self.assertEqual(result.lifecycle.harness.reason, reason) - - def test_tool_intervals_require_one_explicit_unique_pairing(self) -> None: - parser = CodexJSONLParser(_cell(), "0123456789abcdef") - # A non-tool item carries neither a tool count nor an interval. - self.assertIsNone(parser.parse("stdout", json.dumps( - {"type": "item.completed", "item": {"type": "agent_message", "text": "x"}} - ))) - # Real Codex command completions omit duration; they must still count. - self.assertIsNone(parser.parse("stdout", json.dumps( - {"type": "item.completed", "item": {"id": "call-0", "type": "command_execution"}} - ))) - paired = json.dumps({"type": "item.completed", "item": { - "id": "call-1", "type": "command_execution", "duration_ms": 3, - }}) - self.assertEqual(parser.parse("stdout", paired).call_id, "call-1") - for name, item in { - "duplicate-call": {"id": "call-1", "type": "command_execution", "duration_ms": 4}, - "unpaired-duration": {"type": "command_execution", "duration_ms": 4}, - "unsafe-call-id": {"id": "call 1", "type": "command_execution", "duration_ms": 4}, - "string-duration": {"id": "call-2", "type": "command_execution", "duration_ms": "4"}, - "negative-duration": {"id": "call-3", "type": "command_execution", "duration_ms": -4}, - }.items(): - with self.subTest(name=name): - with self.assertRaises(CodexJSONLError): - parser.parse("stdout", json.dumps({"type": "item.completed", "item": item})) - turn = parser.parse("stdout", json.dumps({"type": "turn.completed"})) - counts = {metric.name: metric.value for metric in turn if isinstance(metric, ParsedMetric)} - self.assertEqual(counts, {"model_calls": 1, "tool_calls": 2}) - - def test_unknown_or_fractional_turn_usage_fails_closed(self) -> None: - for usage in ( - {"unknown_tokens": 1}, - {"input_tokens": 1.5}, - {"input_tokens": True}, - {"input_tokens": -1}, - ): - with self.subTest(usage=usage): - parser = CodexJSONLParser(_cell(), "0123456789abcdef") - with self.assertRaises(CodexJSONLError): - parser.parse("stdout", json.dumps({ - "type": "turn.completed", "status": "completed", "usage": usage, - })) - - def test_effective_binding_is_optional_but_any_observation_is_exact(self) -> None: - parser = CodexJSONLParser(_cell(), "0123456789abcdef") - self.assertIsNone(parser.effective_binding) - self.assertEqual(parser.connectivity_result().status, "implementation_gap") - parser.parse("stdout", json.dumps({"type": "thread.started", "iop_effective_binding": { - "route_kind": "direct", "route_id": "iop-gpt", "model": "gpt-5.6-luna", "effort": "xhigh", - }})) - self.assertEqual(parser.effective_binding, ("direct", "iop-gpt", "gpt-5.6-luna", "xhigh")) - with self.assertRaises(CodexJSONLError): - parser.connectivity_result() - mismatch = CodexJSONLParser(_cell(), "0123456789abcdef") - with self.assertRaises(CodexJSONLError): - mismatch.parse("stdout", json.dumps({"type": "thread.started", "iop_effective_binding": { - "route_kind": "direct", "route_id": "iop-gpt", "model": "gpt-alias", "effort": "xhigh", - }})) - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/agent_benchmark/connectivity.py b/scripts/agent_benchmark/connectivity.py deleted file mode 100644 index cd4e355e..00000000 --- a/scripts/agent_benchmark/connectivity.py +++ /dev/null @@ -1,679 +0,0 @@ -"""Closed, secret-safe connectivity preflight contracts for benchmark cells. - -This module deliberately has no caller, provider, or network dependency. Caller -adapters provide typed observations, while this boundary proves that those -observations exactly match one immutable manifest cell before they can be stored. -A blocked caller may report that it observed no effective binding at all, but it -can never report a partial or manifest-derived synthetic one. -""" - -from __future__ import annotations - -import json -import os -import re -import stat -from dataclasses import dataclass -from pathlib import Path -from typing import Any, Protocol - -from scripts.agent_benchmark.manifest import ( - CALLER_ENUM, - ROUTE_KIND_ENUM, - STAGE_ENUM, - STAGE_RANK, - ExpectedBinding, - MatrixCell, - TOKEN_RE, -) - - -SCHEMA_VERSION = "1" -IDENTITY_RE = re.compile(r"^sha256:[0-9a-f]{64}$") -MAX_EVIDENCE_BYTES = 65536 - -# Canonical issue order: registration blockers first, then implementation gaps. -ISSUE_CODE_ORDER = ( - "credential_missing", - "model_missing", - "route_missing", - "effort_unsupported", - "endpoint_incompatible", - "auth_incompatible", - "protocol_incompatible", - "stream_incompatible", -) -REGISTRATION_ISSUE_CODES = frozenset(ISSUE_CODE_ORDER[:4]) -IMPLEMENTATION_ISSUE_CODES = frozenset(ISSUE_CODE_ORDER[4:]) -ISSUE_CODES = REGISTRATION_ISSUE_CODES | IMPLEMENTATION_ISSUE_CODES -ISSUE_RANK: dict[str, int] = {code: rank for rank, code in enumerate(ISSUE_CODE_ORDER)} -# The only resume vocabulary; callers can never attach their own text. -ISSUE_RESUME_CODES: dict[str, str] = { - "credential_missing": "register_credential", - "model_missing": "register_model", - "route_missing": "register_route", - "effort_unsupported": "register_effort_support", - "endpoint_incompatible": "implement_endpoint_adapter", - "auth_incompatible": "implement_auth_adapter", - "protocol_incompatible": "implement_protocol_adapter", - "stream_incompatible": "implement_stream_adapter", -} -RESUME_CODES = frozenset(ISSUE_RESUME_CODES.values()) -RESULT_STATUSES = ("ready", "registration_required", "implementation_gap") -EFFECTIVE_FIELDS = ( - "effective_route_kind", - "effective_route_id", - "effective_model", - "effective_effort", -) - - -class ConnectivityError(Exception): - """Base class for closed connectivity contract failures.""" - - -class ConnectivityValidationError(ConnectivityError): - """Raised when a caller observation or evidence value is not admissible.""" - - -class ConnectivityEvidenceError(ConnectivityError): - """Raised when durable evidence cannot be safely written or read.""" - - -@dataclass(frozen=True) -class CallerCapability: - """Static public capability claimed by one benchmark caller adapter.""" - - caller: str - route_kinds: tuple[str, ...] - efforts: tuple[str, ...] - - -@dataclass(frozen=True) -class EffectiveBinding: - """One observed stage binding, including its exact effective effort.""" - - stage: str - model: str - effort: str | None = None - - -@dataclass(frozen=True) -class RequestedEffectiveBinding: - """Requested identity plus one all-or-none observed effective binding. - - The requested fields are always required. The effective group is either - fully absent, which only a blocked result may report, or complete and exact. - """ - - cell_id: str - caller: str - requested_route_kind: str - requested_route_id: str - requested_model: str - requested_effort: str - effective_route_kind: str | None = None - effective_route_id: str | None = None - effective_model: str | None = None - effective_effort: str | None = None - effective_bindings: tuple[EffectiveBinding, ...] = () - - -@dataclass(frozen=True) -class ConnectivityIssue: - """Closed blocker code paired with its one permitted resume code.""" - - code: str - resume_code: str - - -@dataclass(frozen=True) -class ConnectivityResult: - """Validated immutable result for exactly one caller/cell preflight.""" - - capability: CallerCapability - binding: RequestedEffectiveBinding - issues: tuple[ConnectivityIssue, ...] - status: str - - -class CallerPreflight(Protocol): - """Adapter boundary; implementations must not return raw caller output.""" - - capability: CallerCapability - - def preflight( - self, cell: MatrixCell - ) -> tuple[RequestedEffectiveBinding, tuple[ConnectivityIssue, ...]]: - """Return typed, secret-free observation for ``cell``.""" - - -def _fail(message: str) -> None: - # Messages contain only fixed contract field names, never input values. - raise ConnectivityValidationError(message) - - -def _require_identifier(value: Any, field_name: str) -> str: - if not isinstance(value, str) or not TOKEN_RE.fullmatch(value): - _fail(f"invalid {field_name}") - return value - - -def _require_identity(value: Any, field_name: str) -> str: - if not isinstance(value, str) or not IDENTITY_RE.fullmatch(value): - raise ConnectivityEvidenceError(f"invalid {field_name}") - return value - - -def _validate_closed_tuple( - values: Any, - field_name: str, - *, - allowed: tuple[str, ...] | None = None, -) -> None: - """Check shape, item type, membership and uniqueness before canonical order. - - Ordering is checked last so unknown or unhashable entries can never reach - ``set`` or an enum index lookup and escape as a built-in exception. - """ - if not isinstance(values, tuple) or not values: - _fail(f"invalid {field_name}") - for item in values: - if not isinstance(item, str): - _fail(f"invalid {field_name} item") - if allowed is None: - _require_identifier(item, f"{field_name} item") - elif item not in allowed: - _fail(f"invalid {field_name} item") - if len(set(values)) != len(values): - _fail(f"duplicate {field_name} item") - if tuple(sorted(values, key=allowed.index if allowed is not None else None)) != values: - _fail(f"non-canonical {field_name}") - - -def _validate_capability(capability: CallerCapability) -> None: - if not isinstance(capability, CallerCapability): - _fail("invalid caller capability") - if capability.caller not in CALLER_ENUM: - _fail("invalid capability caller") - _validate_closed_tuple( - capability.route_kinds, "capability route_kinds", allowed=ROUTE_KIND_ENUM - ) - _validate_closed_tuple(capability.efforts, "capability efforts") - - -def _validate_binding_shape(binding: RequestedEffectiveBinding) -> bool: - """Validate requested fields, then the all-or-none effective group. - - Returns whether the caller reported an effective observation at all. - """ - if not isinstance(binding, RequestedEffectiveBinding): - _fail("invalid requested/effective binding") - _require_identifier(binding.cell_id, "binding cell_id") - if binding.caller not in CALLER_ENUM: - _fail("invalid binding caller") - if binding.requested_route_kind not in ROUTE_KIND_ENUM: - _fail("invalid requested route_kind") - for name in ("requested_route_id", "requested_model", "requested_effort"): - _require_identifier(getattr(binding, name), f"binding {name}") - - if not isinstance(binding.effective_bindings, tuple): - _fail("invalid effective_bindings") - observed = tuple(getattr(binding, name) is not None for name in EFFECTIVE_FIELDS) - if any(observed) != all(observed): - _fail("partial effective observation") - if not any(observed): - if binding.effective_bindings: - _fail("partial effective observation") - return False - - if binding.effective_route_kind not in ROUTE_KIND_ENUM: - _fail("invalid effective route_kind") - for name in ("effective_route_id", "effective_model", "effective_effort"): - _require_identifier(getattr(binding, name), f"binding {name}") - if not binding.effective_bindings: - _fail("partial effective observation") - stages: list[str] = [] - for item in binding.effective_bindings: - if not isinstance(item, EffectiveBinding): - _fail("invalid effective binding item") - if item.stage not in STAGE_ENUM: - _fail("invalid effective binding stage") - _require_identifier(item.model, "effective binding model") - if item.effort is not None: - _require_identifier(item.effort, "effective binding effort") - stages.append(item.stage) - if len(set(stages)) != len(stages): - _fail("duplicate effective binding stage") - if stages != sorted(stages, key=STAGE_RANK.__getitem__): - _fail("non-canonical effective binding order") - return True - - -def _expected_as_effective(expected: ExpectedBinding) -> EffectiveBinding: - return EffectiveBinding(expected.stage, expected.model, expected.effort) - - -def _cell_binding(cell: MatrixCell) -> tuple[str, str, str, str]: - return ( - cell.iop.route_kind, - cell.iop.route_id, - cell.iop.request_model, - cell.iop.requested_effort, - ) - - -def validate_requested_binding( - cell: MatrixCell, - capability: CallerCapability, - binding: RequestedEffectiveBinding, -) -> None: - """Fail closed unless caller identity and requested route/model/effort are exact.""" - if not isinstance(cell, MatrixCell): - _fail("invalid matrix cell") - _validate_capability(capability) - _validate_binding_shape(binding) - - if capability.caller != cell.caller or binding.caller != cell.caller: - _fail("caller identity mismatch") - if binding.cell_id != cell.id: - _fail("cell identity mismatch") - if cell.iop.route_kind not in capability.route_kinds: - _fail("unsupported route_kind") - if cell.iop.requested_effort not in capability.efforts: - _fail("unsupported effort") - - requested = ( - binding.requested_route_kind, - binding.requested_route_id, - binding.requested_model, - binding.requested_effort, - ) - if requested != _cell_binding(cell): - _fail("requested binding mismatch") - - -def validate_effective_binding( - cell: MatrixCell, - binding: RequestedEffectiveBinding, - *, - required: bool, -) -> None: - """Require the observation only for ``ready``; any present group must be exact.""" - if not isinstance(cell, MatrixCell): - _fail("invalid matrix cell") - if not _validate_binding_shape(binding): - if required: - _fail("missing effective observation") - return - - effective = ( - binding.effective_route_kind, - binding.effective_route_id, - binding.effective_model, - binding.effective_effort, - ) - if effective != _cell_binding(cell): - _fail("effective binding substitution") - expected_stages = tuple(_expected_as_effective(item) for item in cell.iop.expected_bindings) - if binding.effective_bindings != expected_stages: - _fail("effective stage binding mismatch") - - -def validate_binding( - cell: MatrixCell, - capability: CallerCapability, - binding: RequestedEffectiveBinding, - *, - require_effective: bool = True, -) -> None: - """Validate the requested phase and then the effective phase of one binding.""" - validate_requested_binding(cell, capability, binding) - validate_effective_binding(cell, binding, required=require_effective) - - -def _validate_issue(issue: ConnectivityIssue) -> None: - if not isinstance(issue, ConnectivityIssue): - _fail("invalid issue") - if not isinstance(issue.code, str) or issue.code not in ISSUE_CODES: - _fail("invalid issue code") - if not isinstance(issue.resume_code, str) or issue.resume_code != ISSUE_RESUME_CODES[issue.code]: - _fail("invalid issue resume_code") - - -def classify_issues(issues: tuple[ConnectivityIssue, ...]) -> str: - """Return the only permitted status, with implementation gaps taking precedence.""" - if not isinstance(issues, tuple): - _fail("issues must be a tuple") - codes: list[str] = [] - for issue in issues: - _validate_issue(issue) - codes.append(issue.code) - if len(set(codes)) != len(codes): - _fail("duplicate issue code") - if codes != sorted(codes, key=ISSUE_RANK.__getitem__): - _fail("non-canonical issue order") - if any(code in IMPLEMENTATION_ISSUE_CODES for code in codes): - return "implementation_gap" - if any(code in REGISTRATION_ISSUE_CODES for code in codes): - return "registration_required" - return "ready" - - -def make_result( - cell: MatrixCell, - capability: CallerCapability, - binding: RequestedEffectiveBinding, - issues: tuple[ConnectivityIssue, ...] = (), -) -> ConnectivityResult: - """Construct a result only after phase-exact no-substitution validation.""" - status = classify_issues(issues) - validate_requested_binding(cell, capability, binding) - validate_effective_binding(cell, binding, required=status == "ready") - return ConnectivityResult(capability, binding, issues, status) - - -def validate_result(cell: MatrixCell, result: ConnectivityResult) -> None: - """Revalidate a received result before it is consumed or persisted.""" - if not isinstance(result, ConnectivityResult): - _fail("invalid connectivity result") - status = classify_issues(result.issues) - if result.status not in RESULT_STATUSES or result.status != status: - _fail("result status mismatch") - validate_requested_binding(cell, result.capability, result.binding) - validate_effective_binding(cell, result.binding, required=status == "ready") - - -def _binding_dict(binding: RequestedEffectiveBinding) -> dict[str, Any]: - return { - "cell_id": binding.cell_id, - "caller": binding.caller, - "requested_route_kind": binding.requested_route_kind, - "requested_route_id": binding.requested_route_id, - "requested_model": binding.requested_model, - "requested_effort": binding.requested_effort, - "effective_route_kind": binding.effective_route_kind, - "effective_route_id": binding.effective_route_id, - "effective_model": binding.effective_model, - "effective_effort": binding.effective_effort, - "effective_bindings": [ - {"stage": item.stage, "model": item.model, "effort": item.effort} - for item in binding.effective_bindings - ], - } - - -def canonical_evidence_bytes( - cell: MatrixCell, - result: ConnectivityResult, - endpoint_identity: str, - config_identity: str, -) -> bytes: - """Return deterministic schema-closed evidence containing no raw endpoint data.""" - validate_result(cell, result) - _require_identity(endpoint_identity, "endpoint_identity") - _require_identity(config_identity, "config_identity") - payload = { - "schema_version": SCHEMA_VERSION, - "cell": {"id": cell.id, "caller": cell.caller}, - "status": result.status, - "binding": _binding_dict(result.binding), - "issues": [ - {"code": issue.code, "resume_code": issue.resume_code} - for issue in result.issues - ], - "endpoint_identity": endpoint_identity, - "config_identity": config_identity, - } - return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("ascii") + b"\n" - - -def _safe_relative_path(relative_path: str | Path) -> Path: - if not isinstance(relative_path, (str, Path)): - raise ConnectivityEvidenceError("invalid evidence relative path") - path = Path(relative_path) - if ( - path.is_absolute() - or not path.parts - or any(part in ("", ".", "..") for part in path.parts) - or path.suffix != ".json" - ): - raise ConnectivityEvidenceError("invalid evidence relative path") - return path - - -def _require_nofollow_support() -> None: - if ( - not hasattr(os, "O_NOFOLLOW") - or not hasattr(os, "O_DIRECTORY") - or os.open not in os.supports_dir_fd - or os.mkdir not in os.supports_dir_fd - ): - raise ConnectivityEvidenceError("evidence no-follow traversal unsupported") - - -def _open_directory(name: str, dir_fd: int | None, message: str) -> int: - flags = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW - try: - return os.open(name, flags, dir_fd=dir_fd) - except OSError as exc: - raise ConnectivityEvidenceError(message) from exc - - -def _open_root(root: str | Path) -> int: - """Open the evidence root by descending one no-follow component at a time.""" - if not isinstance(root, (str, Path)): - raise ConnectivityEvidenceError("invalid evidence root") - path = Path(root) - parts = list(path.parts) - if path.is_absolute(): - descriptor = _open_directory(parts[0], None, "invalid evidence root") - parts = parts[1:] - else: - descriptor = _open_directory(".", None, "invalid evidence root") - try: - for part in parts: - if part in ("", ".", ".."): - raise ConnectivityEvidenceError("invalid evidence root") - child = _open_directory(part, descriptor, "invalid evidence root") - os.close(descriptor) - descriptor = child - except BaseException: - os.close(descriptor) - raise - return descriptor - - -def _open_evidence_parent( - root: str | Path, relative_path: str | Path, *, create: bool -) -> tuple[int, str]: - """Return a descriptor for the verified parent directory and the final name.""" - relative = _safe_relative_path(relative_path) - _require_nofollow_support() - descriptor = _open_root(root) - try: - for part in relative.parts[:-1]: - if create: - try: - os.mkdir(part, 0o700, dir_fd=descriptor) - except FileExistsError: - pass - except OSError as exc: - raise ConnectivityEvidenceError("unsafe evidence parent") from exc - child = _open_directory(part, descriptor, "unsafe evidence parent") - os.close(descriptor) - descriptor = child - except BaseException: - os.close(descriptor) - raise - return descriptor, relative.parts[-1] - - -def write_evidence( - root: str | Path, - relative_path: str | Path, - cell: MatrixCell, - result: ConnectivityResult, - endpoint_identity: str, - config_identity: str, -) -> None: - """Atomically create evidence once; any existing or symlinked target is rejected.""" - payload = canonical_evidence_bytes(cell, result, endpoint_identity, config_identity) - parent, name = _open_evidence_parent(root, relative_path, create=True) - try: - flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW - try: - descriptor = os.open(name, flags, 0o600, dir_fd=parent) - except FileExistsError as exc: - raise ConnectivityEvidenceError("evidence target already exists") from exc - except OSError as exc: - raise ConnectivityEvidenceError("evidence write rejected") from exc - try: - with os.fdopen(descriptor, "wb") as stream: - stream.write(payload) - stream.flush() - os.fsync(stream.fileno()) - except OSError as exc: - raise ConnectivityEvidenceError("evidence write rejected") from exc - finally: - os.close(parent) - - -def _read_bounded(descriptor: int) -> bytes: - """Read at most one byte past the cap so oversized input fails closed.""" - chunks: list[bytes] = [] - remaining = MAX_EVIDENCE_BYTES + 1 - try: - while remaining > 0: - chunk = os.read(descriptor, remaining) - if not chunk: - break - chunks.append(chunk) - remaining -= len(chunk) - except OSError as exc: - raise ConnectivityEvidenceError("invalid evidence bytes") from exc - raw = b"".join(chunks) - if len(raw) > MAX_EVIDENCE_BYTES: - raise ConnectivityEvidenceError("evidence target too large") - return raw - - -def _read_evidence_bytes(root: str | Path, relative_path: str | Path) -> bytes: - parent, name = _open_evidence_parent(root, relative_path, create=False) - try: - try: - descriptor = os.open( - name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=parent - ) - except OSError as exc: - raise ConnectivityEvidenceError("unsafe evidence target") from exc - try: - info = os.fstat(descriptor) - if not stat.S_ISREG(info.st_mode): - raise ConnectivityEvidenceError("unsafe evidence target") - if info.st_size > MAX_EVIDENCE_BYTES: - raise ConnectivityEvidenceError("evidence target too large") - return _read_bounded(descriptor) - finally: - os.close(descriptor) - finally: - os.close(parent) - - -def _no_duplicate_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]: - result: dict[str, Any] = {} - for key, value in pairs: - if key in result: - raise ConnectivityEvidenceError("duplicate evidence field") - result[key] = value - return result - - -def _read_binding(data: Any) -> RequestedEffectiveBinding: - if not isinstance(data, dict) or set(data) != { - "cell_id", "caller", "requested_route_kind", "requested_route_id", - "requested_model", "requested_effort", "effective_route_kind", - "effective_route_id", "effective_model", "effective_effort", "effective_bindings", - }: - raise ConnectivityEvidenceError("invalid evidence binding") - stages = data["effective_bindings"] - if not isinstance(stages, list): - raise ConnectivityEvidenceError("invalid evidence binding") - bindings: list[EffectiveBinding] = [] - for item in stages: - if not isinstance(item, dict) or set(item) != {"stage", "model", "effort"}: - raise ConnectivityEvidenceError("invalid evidence binding") - bindings.append(EffectiveBinding(item["stage"], item["model"], item["effort"])) - try: - binding = RequestedEffectiveBinding( - data["cell_id"], data["caller"], data["requested_route_kind"], - data["requested_route_id"], data["requested_model"], data["requested_effort"], - data["effective_route_kind"], data["effective_route_id"], - data["effective_model"], data["effective_effort"], tuple(bindings), - ) - _validate_binding_shape(binding) - except ConnectivityValidationError as exc: - raise ConnectivityEvidenceError("invalid evidence binding") from exc - return binding - - -def _validate_evidence_payload(payload: dict[str, Any], cell: MatrixCell) -> None: - if not isinstance(cell, MatrixCell): - raise ConnectivityEvidenceError("invalid matrix cell") - if not isinstance(payload.get("cell"), dict) or set(payload["cell"]) != {"id", "caller"}: - raise ConnectivityEvidenceError("invalid evidence cell") - binding = _read_binding(payload["binding"]) - if payload["cell"] != {"id": binding.cell_id, "caller": binding.caller}: - raise ConnectivityEvidenceError("evidence cell mismatch") - if binding.cell_id != cell.id or binding.caller != cell.caller: - raise ConnectivityEvidenceError("evidence cell mismatch") - requested = ( - binding.requested_route_kind, - binding.requested_route_id, - binding.requested_model, - binding.requested_effort, - ) - if requested != _cell_binding(cell): - raise ConnectivityEvidenceError("requested binding mismatch") - try: - validate_effective_binding(cell, binding, required=payload["status"] == "ready") - except ConnectivityValidationError as exc: - raise ConnectivityEvidenceError("invalid evidence binding") from exc - issues_raw = payload["issues"] - if not isinstance(issues_raw, list): - raise ConnectivityEvidenceError("invalid evidence issues") - try: - issues = tuple( - ConnectivityIssue(item["code"], item["resume_code"]) - for item in issues_raw - if isinstance(item, dict) and set(item) == {"code", "resume_code"} - ) - if len(issues) != len(issues_raw) or classify_issues(issues) != payload["status"]: - raise ConnectivityValidationError("result status mismatch") - except (ConnectivityValidationError, KeyError, TypeError) as exc: - raise ConnectivityEvidenceError("invalid evidence issues") from exc - - -def read_evidence( - root: str | Path, relative_path: str | Path, cell: MatrixCell -) -> dict[str, Any]: - """Read only canonical, schema-closed evidence; corruption fails closed.""" - raw = _read_evidence_bytes(root, relative_path) - try: - parsed = json.loads(raw.decode("ascii"), object_pairs_hook=_no_duplicate_object) - except (UnicodeDecodeError, json.JSONDecodeError) as exc: - raise ConnectivityEvidenceError("invalid evidence bytes") from exc - if not isinstance(parsed, dict) or set(parsed) != { - "schema_version", "cell", "status", "binding", "issues", - "endpoint_identity", "config_identity", - }: - raise ConnectivityEvidenceError("invalid evidence schema") - if parsed.get("schema_version") != SCHEMA_VERSION or parsed.get("status") not in RESULT_STATUSES: - raise ConnectivityEvidenceError("invalid evidence schema") - _require_identity(parsed.get("endpoint_identity"), "endpoint_identity") - _require_identity(parsed.get("config_identity"), "config_identity") - _validate_evidence_payload(parsed, cell) - canonical = json.dumps(parsed, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("ascii") + b"\n" - if raw != canonical: - raise ConnectivityEvidenceError("non-canonical evidence") - return parsed diff --git a/scripts/agent_benchmark/connectivity_integration_test.py b/scripts/agent_benchmark/connectivity_integration_test.py deleted file mode 100644 index 556c8a14..00000000 --- a/scripts/agent_benchmark/connectivity_integration_test.py +++ /dev/null @@ -1,2873 +0,0 @@ -"""Network-free integration tests for public benchmark preflight.""" - -from __future__ import annotations - -import contextlib -import datetime -import hashlib -import io -import json -import os -import re -import stat -import subprocess -import sys -import tempfile -import threading -import time -import unittest -from dataclasses import replace -from pathlib import Path -from unittest import mock -from urllib.error import HTTPError - -from scripts import agent_comparison_benchmark as benchmark_cli -from scripts.agent_benchmark import live_iop -from scripts.agent_benchmark.browser_cdp import RenderObservation, ViewportObservation -from scripts.agent_benchmark.attempts import ( - AttemptStateError, - CapabilityUnavailable, - PreflightObservation, - RunBusyError, - RunStore, - Slot, - collect_preflight_observations, - preflight_manifest, -) -from scripts.agent_benchmark.connectivity import ( - ISSUE_RESUME_CODES, - CallerCapability, - ConnectivityIssue, - EffectiveBinding, - RequestedEffectiveBinding, - canonical_evidence_bytes, - make_result, -) -from scripts.agent_benchmark.codex_iop import CodexInvocation, CodexInvocationResult -from scripts.agent_benchmark.manifest import ( - AssetMapping, - ExpectedBinding, - MatrixCell, - digest_workspace_inputs, - load_manifest, -) -from scripts.agent_benchmark.lifecycle import ( - CALLER_REASON_ERROR, - CALLER_REASON_SUCCESS, - CALLER_STATUS_SUCCEEDED, - CLOCK_HARNESS_MONOTONIC, - COMPLETION_EXIT_AFTER_IDLE, - METRIC_NAMES, - SOURCE_HARNESS, - SOURCE_WORKSPACE_POLL, - SUBMISSION_STDIN_ONCE, - UNIT_NANOSECONDS, - CaptureStream, - CallerEvent, - CallerTerminal, - InvocationSpec, - InvocationResult, - HarnessOutcome, - LifecycleRecoveryError, - ParsedMetric, - ProcessOutcome, - ProductOutcome, - env_pairs, - recover_invocation, - run_invocation, - spec_digest, -) -from scripts.agent_benchmark.measurement import ( - AttemptMeasurement, - REASON_NOT_OBSERVED, - REASON_NOT_REPORTED, - WorkspaceWriteObservation, - load_measurement, - observed, - path_digest, - publish_measurement, - unavailable, -) -from scripts.agent_benchmark.scoring import BlindWorkspace, ScoringSummary, score_run -from scripts.agent_benchmark.web_validation import ( - WEB_GATES, - build_web_validation, - load_web_validation, - publish_web_validation, -) - - -def _invocation_result( - *, - product_status: str = "succeeded", - harness_reason: str = "success", - exit_code: int = 0, - metrics: tuple[ParsedMetric, ...] = (), - spec_digest_value: str = "sha256:" + "a" * 64, - started_at: str = "2026-08-11T00:00:00+00:00", - ended_at: str = "2026-08-11T00:00:01+00:00", -) -> InvocationResult: - """Build an independent-axis lifecycle value for integration seams.""" - stream = CaptureStream("stdout", "", 0, 0, False) - product_reason = { - "succeeded": CALLER_REASON_SUCCESS, - "failed": CALLER_REASON_ERROR, - "unknown": "unavailable", - }[product_status] - harness_passed = harness_reason == "success" - return InvocationResult( - ProductOutcome(product_status, product_reason), - HarnessOutcome( - "passed" if harness_passed else "failed", - harness_reason, - harness_passed, - True, - ), - ProcessOutcome("exited", exit_code, None), - True, - False, - (), - stream, - replace(stream, stream="stderr"), - "", - "", - None, - spec_digest_value, - started_at, - ended_at, - 1, - metrics, - ) - - -def _cell(cell_id: str, caller: str, model: str, effort: str) -> dict: - return { - "id": cell_id, - "caller": caller, - "iop": { - "request_model": model, - "requested_effort": effort, - "route_kind": "direct", - "route_id": cell_id, - "expected_bindings": [ - {"stage": "request", "model": model, "effort": effort} - ], - }, - } - - -def _preset(cell_id: str, caller: str, model: str, effort: str) -> dict: - return { - "id": cell_id, - "caller": caller, - "iop": { - "request_model": model, - "requested_effort": effort, - "route_kind": "execution_preset", - "route_id": cell_id, - "expected_bindings": [ - {"stage": stage, "model": model} - for stage in ("selector", "plan", "work", "review") - ], - }, - } - - -_SENTINEL_CLASSES = ("task", "secret", "endpoint", "config", "provider") -_LIVE_BRANCHES = ("claude", "agy", "codex") - - -def _branch_sentinels() -> dict[str, dict[str, str]]: - """One distinct sentinel value per caller branch and leak class.""" - return { - caller: {kind: f"branch-{kind}-sentinel-{caller}" for kind in _SENTINEL_CLASSES} - for caller in _LIVE_BRANCHES - } - - -# One production-shaped caller executable. It reads only its own argv, its own -# environment, the harness-submitted stdin task and its private caller config, -# then emits that caller's real stream shape. Every value it can observe is -# echoed back through the exact fields a real caller uses for content, so the -# published evidence proves the production redactors - not the test - removed -# them. The route is taken from the attempt's ``../prepared.json`` because this -# suite's matrix binds ``route_id`` to the cell id. -_CALLER_FIXTURE_BODY = r'''"""Production-shaped benchmark caller fixture.""" -import json -import os -import sys -import uuid -from pathlib import Path - -name = Path(__file__).name -config = json.loads( - Path(__file__).with_name(name + ".config.json").read_text(encoding="utf-8") -) -argv = sys.argv[1:] -task = sys.stdin.buffer.read().decode("utf-8", "replace") -prepared = json.loads((Path.cwd().parent / "prepared.json").read_text(encoding="utf-8")) -route_id = prepared["identity"]["cell_id"] -with open(Path(__file__).with_name(name + ".invocations"), "a", encoding="utf-8") as log: - log.write(prepared["identity"]["run_id"] + " " + route_id + "\n") - - -def option(flag): - return argv[argv.index(flag) + 1] if flag in argv else "" - - -def override(prefix): - for item in argv: - if item.startswith(prefix): - return json.loads(item[len(prefix):]) - return "" - - -leak = json.dumps( - {"argv": argv, "env": dict(os.environ), "task": task, "config": config}, - sort_keys=True, -) -sys.stderr.write(name + ": diagnostic " + leak + "\n") -sys.stderr.flush() - -# Production-shaped benchmark output: exactly the three generated root files, -# using both fixture images. ``script.js`` is written last so the workspace -# observer still has a real caller-produced first-write observation. -Path("index.html").write_text("""Orbit

Orbit landing page

Accessible responsive fixture.

Aurora grid artworkOrbit rings artwork
""", encoding="utf-8") -Path("styles.css").write_text("""*{box-sizing:border-box}body{margin:0;background:#fff;color:#111;font:18px sans-serif}main{max-width:960px;margin:auto;padding:24px}img{display:block;max-width:100%;width:320px;height:auto;margin:16px 0}a:focus,button:focus{outline:3px solid #05f}@media(max-width:600px){main{padding:16px}img{width:100%}}""", encoding="utf-8") -Path("script.js").write_text("document.querySelector('button').addEventListener('click', () => {});", encoding="utf-8") - -if name == "claude": - model = option("--model") - session = str(uuid.uuid4()) - events = [ - {"type": "system", "subtype": "init", "model": model, - "session_id": session, "tools": []}, - {"type": "assistant", "session_id": session, - "message": {"model": model, "stop_reason": "end_turn", - "content": [{"type": "text", "text": leak}]}}, - {"type": "result", "subtype": "success", "session_id": session, - "is_error": False, "duration_ms": 1234, "duration_api_ms": 1000, - "usage": {"input_tokens": 11, "output_tokens": 22, - "cache_read_input_tokens": 5}, - "result": leak}, - ] -elif name == "agy": - events = [ - {"event": "init", "conversation_id": "fixture", - "init": {"model": "Gemini 3.6 Flash", "permission_mode": "sandbox", "tools": []}}, - {"event": "step_update", "step_update": { - "state": "DONE", "step_index": 0, "step_type": "agent_response", - "text_delta": leak, - "usage": {"input_tokens": 11, "output_tokens": 22, "total_tokens": 33}}}, - {"event": "result", "result": { - "conversation_id": "fixture", "status": "SUCCESS", - "duration_seconds": 0.0125, "num_turns": 1, "response": leak, - "usage": {"input_tokens": 11, "output_tokens": 22, "total_tokens": 33}}}, - ] -else: - events = [ - {"type": "item.completed", "item": {"type": "agent_message", "text": leak}}, - {"type": "item.completed", - "item": {"id": "call-1", "type": "command_execution", - "duration_ms": 7.25, "output": leak}}, - {"type": "turn.completed", "status": "completed", - "usage": {"input_tokens": 31, "cached_input_tokens": 8, - "cache_write_input_tokens": 6, "output_tokens": 12}}, - ] - -for event in events: - sys.stdout.write(json.dumps(event) + "\n") - sys.stdout.flush() -''' - - -def _write_manifest( - root: Path, matrix: list[dict], output_id: str = "integration", *, - output_root: str | None = None, prompt: str = "public prompt fixture", -): - fixture_root = root / "scripts/fixtures" - fixture_root.mkdir(parents=True, exist_ok=True) - (fixture_root / "prompt.md").write_text(prompt, encoding="utf-8") - (fixture_root / "reference.txt").write_text("public reference", encoding="utf-8") - for image in ("aurora.svg", "orbit.svg"): - (fixture_root / image).write_text( - "", - encoding="utf-8", - ) - assets = ( - AssetMapping( - "scripts/fixtures/reference.txt", - "brief/reference.txt", - b"public reference", - ), - AssetMapping("scripts/fixtures/aurora.svg", "assets/aurora-grid.svg", (fixture_root / "aurora.svg").read_bytes()), - AssetMapping("scripts/fixtures/orbit.svg", "assets/orbit-rings.svg", (fixture_root / "orbit.svg").read_bytes()), - ) - payload = { - "pipeline_version": "2", - "environment": "dev", - "testbed": "../iop-s2", - "repetitions": 1, - "session_policy": "fresh", - "setup_cache_policy": "isolated", - "timeout": { - "run_seconds": 5, - "idle_seconds": 1, - "quiet_seconds": 1, - "cleanup_grace_seconds": 1, - }, - "viewports": [{"id": "desktop", "width": 900, "height": 700}, {"id": "mobile", "width": 375, "height": 700}], - "rubric_version": "landing-quality-v1", - "evaluator": {"caller": "codex", "iop": {"request_model": "judge", "requested_effort": "high", "route_kind": "direct", "route_id": "judge", "expected_bindings": [{"stage": "request", "model": "judge", "effort": "high"}]}}, - "output_root": output_root or f"agent-test/runs/{output_id}", - "fixture": { - "version": "v1", - "prompt": "scripts/fixtures/prompt.md", - "assets": [ - { - "source": "scripts/fixtures/reference.txt", - "workspace_path": "brief/reference.txt", - } - ,{"source": "scripts/fixtures/aurora.svg", "workspace_path": "assets/aurora-grid.svg"} - ,{"source": "scripts/fixtures/orbit.svg", "workspace_path": "assets/orbit-rings.svg"} - ], - "checksum": digest_workspace_inputs(assets), - }, - "matrix": matrix, - } - path = root / "manifest.json" - raw = json.dumps(payload, sort_keys=True).encode("utf-8") - path.write_bytes(raw) - return load_manifest(path, repo_root=root), raw, path - - -class FakeAdapter: - def __init__( - self, - caller: str, - efforts: tuple[str, ...], - issues_by_cell: dict[str, tuple[str, ...]] | None = None, - *, - sentinel: str = "", - ) -> None: - self.capability = CallerCapability( - caller, ("direct", "execution_preset"), efforts - ) - self.issues_by_cell = issues_by_cell or {} - self.sentinel = sentinel - self.calls: list[str] = [] - self.invocations: list[tuple[str, str, str, bytes]] = [] - self.fail_invocation = False - - def preflight(self, cell: MatrixCell) -> PreflightObservation: - self.calls.append(cell.id) - issue_codes = self.issues_by_cell.get(cell.id, ()) - if issue_codes: - binding = RequestedEffectiveBinding( - cell.id, - cell.caller, - cell.iop.route_kind, - cell.iop.route_id, - cell.iop.request_model, - cell.iop.requested_effort, - ) - else: - binding = RequestedEffectiveBinding( - cell.id, - cell.caller, - cell.iop.route_kind, - cell.iop.route_id, - cell.iop.request_model, - cell.iop.requested_effort, - cell.iop.route_kind, - cell.iop.route_id, - cell.iop.request_model, - cell.iop.requested_effort, - tuple( - EffectiveBinding(item.stage, item.model, item.effort) - for item in cell.iop.expected_bindings - ), - ) - issues = tuple( - ConnectivityIssue(code, ISSUE_RESUME_CODES[code]) - for code in issue_codes - ) - return PreflightObservation( - make_result(cell, self.capability, binding, issues), - "sha256:" + "a" * 64, - "sha256:" + "b" * 64, - ) - - def invoke( - self, - cell, - prepared, - attempt, - control_dir, - task_payload, - timeout, - on_started, - ): - run_root = Path(attempt.root).parents[3] - if not (run_root / "preflight/preflight-000001.json").is_file(): - raise AssertionError("attempt allocated before preflight publication") - if cell.id != attempt.identity.cell_id or prepared.identity != attempt.identity: - raise AssertionError("execution identity drift") - self.invocations.append( - (cell.id, prepared.workspace_dir, prepared.session_id, task_payload) - ) - - if Path(control_dir).resolve(strict=False) != Path(attempt.root).resolve() / "control": - raise AssertionError("controller control binding drift") - source = ( - "import sys; sys.stdin.buffer.read(); print('FAILED'); sys.exit(3)" - if self.fail_invocation - else "import sys; sys.stdin.buffer.read(); print('FINISH'); print('IDLE')" - ) - spec = InvocationSpec( - argv=(sys.executable, "-u", "-c", source), - cwd=prepared.workspace_dir, - env=env_pairs({"PATH": os.environ.get("PATH", "/usr/bin:/bin")}), - submission_mode=SUBMISSION_STDIN_ONCE, - completion_mode=COMPLETION_EXIT_AFTER_IDLE, - timeout=timeout, - evidence_dir=attempt.root, - task_payload=task_payload, - control_dir=control_dir, - ) - return run_invocation( - spec, - parse_event=lambda _stream, line: { - "FINISH": ( - CallerTerminal(CALLER_STATUS_SUCCEEDED, CALLER_REASON_SUCCESS), - CallerEvent("finish"), - ), - "IDLE": CallerEvent("idle"), - }.get(line.strip()), - on_started=lambda locator: on_started(locator, spec_digest(spec)), - ) - - def cleanup(self) -> None: - pass - - -class ConnectivityIntegrationTest(unittest.TestCase): - def setUp(self) -> None: - self.temp = tempfile.TemporaryDirectory(dir="/tmp", prefix="benchmark-preflight-") - self.root = Path(self.temp.name) / "repo" - self.root.mkdir() - self.matrix = [ - _cell("claude-sonnet-direct", "claude", "claude-sonnet-5", "max"), - _cell("claude-gemini-direct", "claude", "gemini-3.6-flash", "high"), - _cell("claude-gpt-direct", "claude", "gpt-5.6-luna", "xhigh"), - _cell("agy-gemini-direct", "agy", "gemini-3.6-flash", "high"), - _cell("codex-gpt-direct", "codex", "gpt-5.6-luna", "xhigh"), - ] - self.manifest, self.raw, self.path = _write_manifest(self.root, self.matrix) - self.store = RunStore( - self.root, - clock=lambda: datetime.datetime( - 2026, 8, 10, 1, 2, 3, tzinfo=datetime.timezone.utc - ), - token_hex=lambda _: "123456abcdef", - ) - - def tearDown(self) -> None: - self.temp.cleanup() - - def _init_testbed(self) -> None: - testbed = self.root.parent / "iop-s2" - testbed.mkdir() - (testbed / "README.md").write_text("testbed", encoding="utf-8") - for command in ( - ("git", "init"), - ("git", "config", "user.name", "test"), - ("git", "config", "user.email", "test@example.invalid"), - ("git", "add", "."), - ("git", "commit", "-m", "testbed"), - ): - subprocess.run(command, cwd=testbed, check=True, capture_output=True) - - def _live_environment(self, *, token: str = "live-token-must-not-persist", manifest=None) -> dict[str, str]: - manifest = self.manifest if manifest is None else manifest - routes = [] - seen: set[tuple[str, str]] = set() - evaluator = MatrixCell("evaluator", manifest.evaluator.caller, manifest.evaluator.iop) - for cell in (*manifest.matrix, evaluator): - key = (cell.iop.route_kind, cell.iop.route_id) - if key not in seen: - seen.add(key) - routes.append( - { - "route_kind": key[0], - "route_id": key[1], - "model": cell.iop.request_model, - "bindings": [ - { - "stage": binding.stage, - "model": binding.model, - "effort": binding.effort, - } - for binding in cell.iop.expected_bindings - ], - } - ) - environment = { - "IOP_BENCH_CONFIG_OBSERVATION_ENV": "BENCH_CONFIG", - "BENCH_CONFIG": json.dumps({"schema_version": "1", "routes": routes}, sort_keys=True), - "BENCH_TOKEN": token, - } - for caller in ("CLAUDE", "AGY", "CODEX"): - environment[f"IOP_BENCH_{caller}_BASE_URL"] = "http://127.0.0.1:18083/v1" - environment[f"IOP_BENCH_{caller}_SECRET_ENV"] = "BENCH_TOKEN" - environment["IOP_BENCH_AGY_BASE_URL"] = "https://127.0.0.1:18083" - return environment - - @staticmethod - def _score_measurement(attempt, caller: str) -> AttemptMeasurement: - timeline = { - "submitted_at": unavailable(REASON_NOT_OBSERVED, SOURCE_HARNESS), - "first_output_at": unavailable(REASON_NOT_OBSERVED, SOURCE_HARNESS), - "first_write_observed_at": unavailable( - REASON_NOT_OBSERVED, SOURCE_WORKSPACE_POLL - ), - "first_write_mtime": unavailable( - REASON_NOT_OBSERVED, SOURCE_WORKSPACE_POLL - ), - "total_duration": observed( - 1, UNIT_NANOSECONDS, CLOCK_HARNESS_MONOTONIC, SOURCE_HARNESS - ), - } - usage = { - name: unavailable(REASON_NOT_REPORTED, SOURCE_HARNESS) - for name in METRIC_NAMES - } - return AttemptMeasurement( - attempt.identity.run_id, - attempt.identity.cell_id, - attempt.identity.repetition, - attempt.identity.attempt, - caller, - "sha256:" + "3" * 64, - ProductOutcome("succeeded", CALLER_REASON_SUCCESS), - HarnessOutcome("passed", "success", True, True), - ProcessOutcome("exited", 0, None), - timeline, - usage, - WorkspaceWriteObservation( - False, None, None, "", 1, 0, REASON_NOT_OBSERVED - ), - (), - ) - - @staticmethod - def _score_view( - attempt_root: Path, - ident: str, - width: int, - height: int, - image_paths: tuple[str, str], - ) -> ViewportObservation: - screenshot = f"screenshot-{ident}.png" - png = b"\x89PNG\r\n\x1a\n" + ident.encode("ascii") - (attempt_root / screenshot).write_bytes(png) - images = tuple( - { - "src": path, - "alt": path, - "complete": True, - "natural_width": 80, - "natural_height": 60, - "visible": True, - "rect": { - "x": 0, - "y": 0, - "width": 80, - "height": 60, - "right": 80, - "bottom": 60, - }, - } - for path in image_paths - ) - return ViewportObservation( - ident, - width, - height, - screenshot, - "sha256:" + hashlib.sha256(png).hexdigest(), - len(png), - images, - { - "scroll_width": width, - "client_width": width, - "clipped": 0, - "overlaps": 0, - }, - { - "h1_count": 1, - "headings": [1], - "heading_progression": True, - "main_count": 1, - "landmarks": 1, - "controls": [ - { - "name": True, - "tab_index": 0, - "focused": True, - "focus_visible": True, - "contrast": 7.0, - } - ], - "ax": {"nodes": 4, "non_ignored": 3, "named": 2}, - }, - ) - - def _successful_score_attempt(self, manifest, run): - cell = manifest.matrix[0] - with self.store.writer(run): - attempt = self.store.allocate(run, Slot(cell.id, 1)) - workspace = Path(attempt.root) / "workspace" - for asset in manifest.fixture.assets: - target = workspace / asset.workspace_path - target.parent.mkdir(parents=True, exist_ok=True) - target.write_bytes(asset.content) - image_paths = tuple( - asset.workspace_path - for asset in manifest.fixture.assets - if Path(asset.workspace_path).suffix.lower() in (".png", ".svg") - ) - if len(image_paths) != 2: - raise AssertionError("scoring fixture requires two images") - (workspace / "index.html").write_text( - "

Ready

" - + "".join( - f"{path}" for path in image_paths - ) - + "" - "
", - encoding="utf-8", - ) - (workspace / "styles.css").write_text( - "body{color:#111;background:#fff}img{width:80px}" - "button:focus{outline:2px solid #05f}", - encoding="utf-8", - ) - (workspace / "script.js").write_text( - "document.body.dataset.ready='1';", encoding="utf-8" - ) - measurement = self._score_measurement(attempt, cell.caller) - publish_measurement(attempt.root, measurement) - render = RenderObservation( - "Chromium/Test", - "http://127.0.0.1:12345", - tuple( - {"kind": "local", "path": "/" + path, "allowed": True, "status": 200} - for path in ("index.html", *image_paths) - ), - (), - tuple( - self._score_view( - Path(attempt.root), - viewport.id, - viewport.width, - viewport.height, - image_paths, - ) - for viewport in manifest.viewports - ), - ) - publish_web_validation( - attempt.root, - build_web_validation(manifest, workspace, measurement, render), - ) - return self.store.publish_terminal( - attempt, - "completed", - result={ - "product": { - "status": "succeeded", - "reason": CALLER_REASON_SUCCESS, - }, - "harness": { - "status": "passed", - "reason": "success", - "ordered_terminal": True, - "cleanup_complete": True, - }, - "process": { - "status": "exited", - "exit_code": 0, - "signal": None, - }, - }, - ) - - def _run_live_scoring_mutation(self, case, mutate): - secret = f"live-{case}-secret-exact-value" - raw = json.loads(self.raw) - raw["evaluator"]["iop"]["requested_effort"] = "xhigh" - raw["evaluator"]["iop"]["expected_bindings"] = [ - {"stage": "request", "model": "judge", "effort": "xhigh"} - ] - raw["output_root"] = f"agent-test/runs/{case}" - manifest_raw = json.dumps(raw, sort_keys=True).encode("utf-8") - manifest_path = self.root / f"{case}.json" - manifest_path.write_bytes(manifest_raw) - manifest = load_manifest(manifest_path, repo_root=self.root) - evaluator = MatrixCell( - "evaluator", manifest.evaluator.caller, manifest.evaluator.iop - ) - run = self.store.create(manifest, manifest_raw) - attempt = self._successful_score_attempt(manifest, run) - environment = self._live_environment(token=secret, manifest=manifest) - base_url = environment["IOP_BENCH_CODEX_BASE_URL"] - - def invoke(invocation, _on_started): - mutate(invocation, secret, base_url) - lifecycle = _invocation_result() - binding = ( - evaluator.iop.route_kind, - evaluator.iop.route_id, - evaluator.iop.request_model, - evaluator.iop.requested_effort, - ) - return CodexInvocationResult(lifecycle, binding) - - adapter = live_iop.build_live_scoring_adapter( - environment, - observer=lambda _runtime: live_iop._Observation( - (evaluator.iop.request_model,), "sha256:" + "b" * 64, True - ), - invoker=invoke, - ) - summary = score_run(self.store, run, manifest, adapter=adapter) - score_root = Path(attempt.root) / "scoring" / "score-000001" - result = json.loads((score_root / "result.json").read_text()) - allocation = json.loads((score_root / "allocation.json").read_text()) - blind_root = Path(run.root) / allocation["blind_path"] - return summary, result, blind_root, secret, base_url - - def _sentinel_live_environment( - self, manifest, sentinels: dict[str, dict[str, str]] - ) -> dict[str, str]: - """Give every branch its own endpoint and credential sentinel value.""" - environment = self._live_environment(manifest=manifest) - del environment["BENCH_TOKEN"] - for caller, branch in sentinels.items(): - prefix = f"IOP_BENCH_{caller.upper()}_" - secret_env = f"BENCH_TOKEN_{caller.upper()}" - environment[secret_env] = branch["secret"] - environment[prefix + "SECRET_ENV"] = secret_env - environment[prefix + "BASE_URL"] = f"http://{branch['endpoint']}.invalid:18083/v1" - if caller == "agy": - environment[prefix + "BASE_URL"] = f"https://{branch['endpoint']}.invalid:18083" - return environment - - @staticmethod - def _registry(issues: dict[str, tuple[str, ...]] | None = None, sentinel: str = ""): - issues = issues or {} - return { - "claude": FakeAdapter( - "claude", ("high", "max", "xhigh"), issues, sentinel=sentinel - ), - "agy": FakeAdapter("agy", ("high",), issues, sentinel=sentinel), - "codex": FakeAdapter("codex", ("xhigh",), issues, sentinel=sentinel), - } - - def test_all_three_callers_append_exact_ready_results_without_attempts(self) -> None: - registry = self._registry() - run, record = preflight_manifest( - self.store, self.manifest, self.raw, adapters=registry - ) - self.assertEqual(record["status"], "ready") - self.assertEqual( - [result["cell"]["id"] for result in record["results"]], - [cell.id for cell in self.manifest.matrix], - ) - self.assertEqual(len(self.store.preflights(run, self.manifest)), 1) - self.assertFalse((Path(run.root) / "cells").exists()) - self.assertEqual( - {caller: adapter.calls for caller, adapter in registry.items()}, - { - "claude": [ - "claude-gemini-direct", - "claude-gpt-direct", - "claude-sonnet-direct", - ], - "agy": ["agy-gemini-direct"], - "codex": ["codex-gpt-direct"], - }, - ) - - def test_registration_and_implementation_blockers_are_distinct_and_no_attempt_allocates(self) -> None: - registry = self._registry( - { - "claude-sonnet-direct": ("credential_missing",), - "agy-gemini-direct": ("stream_incompatible",), - } - ) - run, record = preflight_manifest( - self.store, self.manifest, self.raw, adapters=registry - ) - statuses = [result["status"] for result in record["results"]] - self.assertEqual(statuses.count("registration_required"), 1) - self.assertEqual(statuses.count("implementation_gap"), 1) - self.assertEqual(record["status"], "implementation_gap") - self.assertFalse((Path(run.root) / "cells").exists()) - self.assertEqual(self.store.status(run, self.manifest)["attempts"]["running"], 0) - - @contextlib.contextmanager - def _production_shaped_callers(self, sentinels: dict[str, dict[str, str]]): - """Publish one executable per caller before any invocation is built.""" - # This checkout mounts /tmp with noexec. Keep the fixtures temporary - # and test-owned, but place their executable directory on the current - # executable test filesystem so the real caller adapters can launch - # them through their normal subprocess path. - bin_dir = tempfile.TemporaryDirectory(dir=Path.cwd(), prefix=".bc-") - self.addCleanup(bin_dir.cleanup) - fixture_bin = Path(bin_dir.name) - for caller, branch in sentinels.items(): - executable = fixture_bin / caller - executable.write_text( - f"#!{sys.executable}\n" + _CALLER_FIXTURE_BODY, encoding="utf-8" - ) - executable.chmod(0o700) - # The caller's private configuration is the only source of its - # config/provider sentinels, exactly as a real client config file. - (fixture_bin / f"{caller}.config.json").write_text( - json.dumps( - { - "caller": caller, - "config_identity": branch["config"], - "provider_id": branch["provider"], - }, - sort_keys=True, - ), - encoding="utf-8", - ) - yield fixture_bin - - @staticmethod - def _snapshot_run_root(run_root: Path) -> dict[str, bytes]: - return { - str(item.relative_to(run_root)): item.read_bytes() - for item in sorted(run_root.rglob("*")) - if item.is_file() - } - - def _assert_one_published_spec_digest(self, attempt_root: Path) -> None: - """Prove the admitted, journalled and published digests are one value.""" - state = json.loads((attempt_root / "attempt.json").read_text(encoding="utf-8")) - result = json.loads( - (attempt_root / "lifecycle-result.json").read_text(encoding="utf-8") - ) - header = json.loads( - (attempt_root / "lifecycle-journal.jsonl") - .read_text(encoding="utf-8") - .splitlines()[0] - ) - self.assertIn("locator", state) - digests = {state["spec_digest"], result["spec_digest"], header["spec_digest"]} - self.assertEqual(len(digests), 1, attempt_root) - self.assertRegex(digests.pop(), r"^sha256:[0-9a-f]{64}$") - self.assertEqual(state["state"], "completed") - self.assertEqual(result["product"]["status"], "succeeded") - self.assertEqual(result["harness"]["status"], "passed") - self.assertIs(result["harness"]["ordered_terminal"], True) - self.assertIs(result["harness"]["cleanup_complete"], True) - self.assertEqual(result["process"]["status"], "exited") - self.assertIs(result["process_group_alive"], False) - control_dir = Path(state["locator"]["control_dir"]) - alias = control_dir.parent - self.assertEqual(control_dir.name, "control") - self.assertEqual( - state["locator"]["socket_path"], str(control_dir / "control.sock") - ) - self.assertTrue(alias.name.startswith("iop-bench-attempt-")) - self.assertFalse(os.path.lexists(alias)) - self.assertTrue((attempt_root / "control/locator.json").is_file()) - self.assertTrue((attempt_root / "control/cleanup-receipt.json").is_file()) - - # The exact whole-total categories each caller reports through its own - # allowlist. Everything else must remain explicitly unavailable. - _EXPECTED_TOTALS = { - "claude": { - "total_duration", "model_duration", "model_calls", "input_tokens", - "output_tokens", "cached_input_tokens", - }, - "agy": { - "total_duration", "model_calls", "input_tokens", "output_tokens", - "total_tokens", - }, - "codex": { - "model_calls", "tool_calls", "input_tokens", "output_tokens", - "cached_input_tokens", "cache_write_tokens", - }, - } - - def _assert_measurement_evidence( - self, attempt_roots: list[Path], sentinels: dict[str, dict[str, str]] - ) -> None: - """Every attempt publishes one strict, source-aware, digested sidecar.""" - callers = set() - for attempt_root in attempt_roots: - measurement = load_measurement(attempt_root) - caller = measurement.caller - callers.add(caller) - # This matrix binds the cell id to the caller name. - self.assertEqual(measurement.cell_id, caller) - self.assertEqual(measurement.harness.reason, "success") - observed = { - name for name, item in measurement.usage.items() - if item.status == "observed" - } - self.assertEqual(observed, self._EXPECTED_TOTALS[caller], attempt_root) - # Official agy reports total_tokens; other callers in this fixture - # do not, and no total is reconstructed for them. - if caller == "agy": - self.assertEqual(measurement.usage["total_tokens"].value, 33) - else: - self.assertEqual(measurement.usage["total_tokens"].status, "unavailable") - self.assertIsNone(measurement.usage["total_tokens"].value) - self.assertEqual(measurement.timeline["first_output_at"].status, "observed") - self.assertEqual( - measurement.timeline["first_write_mtime"].clock, "filesystem_mtime" - ) - self.assertTrue(measurement.observer.observed) - self.assertEqual( - measurement.observer.path_digest, - path_digest("index.html"), - ) - web = load_web_validation(attempt_root) - self.assertEqual(web.status, "passed", web.record) - self.assertEqual( - [item["id"] for item in web.record["gates"]], list(WEB_GATES) - ) - self.assertTrue(all(item["passed"] for item in web.record["gates"])) - expected_images = { - item["path"] - for item in web.record["workspace"]["inputs"] - if item["path"].startswith("assets/") - } - self.assertEqual(len(expected_images), 2) - self.assertEqual( - [item["id"] for item in web.record["viewports"]], - ["desktop", "mobile"], - ) - self.assertEqual(len(web.record["screenshots"]), 2) - for viewport, screenshot in zip( - web.record["viewports"], web.record["screenshots"] - ): - self.assertEqual( - {item["src"] for item in viewport["images"]}, expected_images - ) - self.assertEqual( - screenshot, {"id": viewport["id"], **viewport["screenshot"]} - ) - data = (attempt_root / screenshot["file"]).read_bytes() - self.assertEqual(screenshot["size"], len(data)) - self.assertEqual( - screenshot["digest"], - "sha256:" + hashlib.sha256(data).hexdigest(), - ) - self.assertEqual(callers, set(_LIVE_BRANCHES)) - - def _assert_sentinels_absent( - self, published: dict[str, bytes], sentinels: dict[str, dict[str, str]] - ) -> None: - for caller, branch in sentinels.items(): - for kind, value in branch.items(): - encoded = value.encode("ascii") - for relative, data in published.items(): - self.assertNotIn(encoded, data, f"{caller}/{kind} in {relative}") - - def test_cli_live_run_invokes_each_direct_cell_once(self) -> None: - short = tempfile.TemporaryDirectory(dir="/tmp", prefix="b") - self.addCleanup(short.cleanup) - root = Path(short.name) / "r" - root.mkdir() - matrix = [ - _cell("claude", "claude", "sonnet", "max"), - _cell("agy", "agy", "gemini-3.6-flash", "high"), - _cell("codex", "codex", "gpt", "xhigh"), - ] - sentinels = _branch_sentinels() - manifest, _raw, path = _write_manifest( - root, matrix, output_root="agent-test/runs/r", - prompt="public prompt fixture\n" - + "\n".join(sentinels[caller]["task"] for caller in _LIVE_BRANCHES), - ) - testbed = root.parent / "iop-s2" - testbed.mkdir() - (testbed / "README.md").write_text("testbed", encoding="utf-8") - for command in (("git", "init"), ("git", "config", "user.name", "test"), ("git", "config", "user.email", "test@example.invalid"), ("git", "add", "."), ("git", "commit", "-m", "testbed")): - subprocess.run(command, cwd=testbed, check=True, capture_output=True) - environment = self._sentinel_live_environment(manifest, sentinels) - - def observed(_runtime): - return live_iop._Observation( - tuple(sorted(cell.iop.request_model for cell in manifest.matrix)), - "sha256:" + "f" * 64, - True, - "agy 1.1.12", - "--print --output-format --dangerously-skip-permissions --model --effort AGY_PROVIDER AGY_OPENAI_BASE_URL AGY_OPENAI_API_KEY stream-json", - ) - - stdout = io.StringIO() - stderr = io.StringIO() - with self._production_shaped_callers(sentinels) as fixture_bin: - with ( - mock.patch.dict( - os.environ, {"PATH": f"{fixture_bin}:{os.environ['PATH']}"} - ), - mock.patch.object(benchmark_cli, "_REPO_ROOT", root), - contextlib.redirect_stdout(stdout), - contextlib.redirect_stderr(stderr), - ): - registry = live_iop.build_live_adapter_registry( - environment, - observer=observed, - binary_resolver=lambda name: str(fixture_bin / name), - ) - with mock.patch.object( - benchmark_cli, "build_adapter_registry", return_value=registry - ): - exit_code = benchmark_cli.main(["run", "--manifest", str(path)]) - - self.assertEqual(exit_code, 0, stderr.getvalue()) - self.assertIn("ok: run run_id=", stdout.getvalue()) - self.assertEqual(stderr.getvalue(), "") - calls = { - caller: (fixture_bin / f"{caller}.invocations") - .read_text(encoding="utf-8") - .splitlines() - for caller in _LIVE_BRANCHES - } - self.assertEqual({caller: len(item) for caller, item in calls.items()}, - {"claude": 1, "agy": 1, "codex": 1}) - - run_roots = list((root / manifest.output_root).glob("run-*")) - self.assertEqual(len(run_roots), 1) - run_root = run_roots[0] - self.assertTrue((run_root / "preflight/preflight-000001.json").is_file()) - published = self._snapshot_run_root(run_root) - attempt_roots = sorted(run_root.glob("cells/*/repetition-*/attempt-*")) - self.assertEqual(len(attempt_roots), len(manifest.matrix)) - for attempt_root in attempt_roots: - self._assert_one_published_spec_digest(attempt_root) - self._assert_measurement_evidence(attempt_roots, sentinels) - self._assert_sentinels_absent(published, sentinels) - self.assertEqual(published, self._snapshot_run_root(run_root)) - - def test_cli_run_blocker_persists_preflight_and_allocates_zero_attempts(self) -> None: - registry = self._registry( - {"claude-sonnet-direct": ("credential_missing",)} - ) - stdout = io.StringIO() - stderr = io.StringIO() - with ( - mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root), - mock.patch.object( - benchmark_cli, "build_adapter_registry", return_value=registry - ), - contextlib.redirect_stdout(stdout), - contextlib.redirect_stderr(stderr), - ): - exit_code = benchmark_cli.main(["run", "--manifest", str(self.path)]) - - self.assertEqual(exit_code, 69) - self.assertEqual(stdout.getvalue(), "") - self.assertIn("error: preflight blocked", stderr.getvalue()) - run_roots = list((self.root / self.manifest.output_root).glob("run-*")) - self.assertEqual(len(run_roots), 1) - self.assertTrue((run_roots[0] / "preflight/preflight-000001.json").is_file()) - self.assertFalse((run_roots[0] / "cells").exists()) - self.assertTrue(all(adapter.invocations == [] for adapter in registry.values())) - - def test_cli_mixed_manifest_preflights_and_invokes_every_cell(self) -> None: - self._init_testbed() - manifest, _, path = _write_manifest( - self.root, - [ - _cell("direct-ready", "claude", "claude-sonnet-5", "max"), - _preset("preset-ready", "claude", "claude-sonnet-5", "max"), - ], - output_id="mixed-ready", - ) - registry = self._registry() - stdout = io.StringIO() - stderr = io.StringIO() - with ( - mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root), - mock.patch.object( - benchmark_cli, "build_adapter_registry", return_value=registry - ), - contextlib.redirect_stdout(stdout), - contextlib.redirect_stderr(stderr), - ): - exit_code = benchmark_cli.main(["run", "--manifest", str(path)]) - - # All slots resolve (web evidence present), so exit 0 even with independent failures. - self.assertEqual(exit_code, 0) - self.assertIn("unresolved=0", stdout.getvalue()) - self.assertIn("artifact_failed=2", stdout.getvalue()) - run_roots = list((self.root / manifest.output_root).glob("run-*")) - self.assertEqual(len(run_roots), 1) - preflight = json.loads( - (run_roots[0] / "preflight/preflight-000001.json").read_text( - encoding="ascii" - ) - ) - self.assertEqual(preflight["status"], "ready") - self.assertEqual( - [result["cell"]["id"] for result in preflight["results"]], - [cell.id for cell in manifest.matrix], - ) - self.assertEqual( - registry["claude"].calls, - [cell.id for cell in manifest.matrix], - ) - self.assertEqual( - [item[0] for item in registry["claude"].invocations], - [cell.id for cell in manifest.matrix], - ) - self.assertEqual( - len(list(run_roots[0].glob("cells/*/repetition-*/attempt-*"))), - len(manifest.matrix), - ) - - def test_cli_failed_attempt_does_not_prevent_later_slot(self) -> None: - self._init_testbed() - manifest, _, path = _write_manifest( - self.root, - [ - _cell("agy-first", "agy", "gemini-3.6-flash", "high"), - _cell("codex-later", "codex", "gpt-5.6-luna", "xhigh"), - ], - output_id="terminal-failure-continues", - ) - agy = FakeAdapter("agy", ("high",)) - agy.fail_invocation = True - codex = FakeAdapter("codex", ("xhigh",)) - self.addCleanup(agy.cleanup) - self.addCleanup(codex.cleanup) - stdout = io.StringIO() - stderr = io.StringIO() - with ( - mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root), - mock.patch.object( - benchmark_cli, - "build_adapter_registry", - return_value={"agy": agy, "codex": codex}, - ), - contextlib.redirect_stdout(stdout), - contextlib.redirect_stderr(stderr), - ): - exit_code = benchmark_cli.main(["run", "--manifest", str(path)]) - - # All slots resolve (web evidence present), so exit 0 even with independent failures. - self.assertEqual(exit_code, 0) - match = re.search(r"run_id=(run-[0-9A-Za-z-]+)", stdout.getvalue()) - self.assertIsNotNone(match) - self.assertEqual([item[0] for item in agy.invocations], ["agy-first"]) - self.assertEqual([item[0] for item in codex.invocations], ["codex-later"]) - status = self.store.status(manifest=manifest, run=self.store.open( - manifest, match.group(1) # type: ignore[union-attr] - )) - self.assertEqual(status["attempts"]["completed"], 2) - self.assertEqual(status["outcomes"]["product"]["unknown"], 1) - self.assertEqual(status["outcomes"]["product"]["succeeded"], 1) - self.assertEqual(status["outcomes"]["artifact"]["failed"], 2) - self.assertEqual(status["attempts"]["running"], 0) - - def test_cli_resume_retries_append_only_and_status_is_read_only(self) -> None: - self._init_testbed() - manifest, _, path = _write_manifest( - self.root, - [_cell("claude-only", "claude", "claude-sonnet-5", "max")], - output_id="retry", - ) - failed = FakeAdapter("claude", ("max",)) - failed.fail_invocation = True - self.addCleanup(failed.cleanup) - first_stdout = io.StringIO() - first_stderr = io.StringIO() - with ( - mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root), - mock.patch.object( - benchmark_cli, - "build_adapter_registry", - return_value={"claude": failed}, - ), - contextlib.redirect_stdout(first_stdout), - contextlib.redirect_stderr(first_stderr), - ): - first_exit = benchmark_cli.main( - ["run", "--manifest", str(path)] - ) - # All slots resolve (web evidence present), so exit 0 even with independent failures. - self.assertEqual(first_exit, 0) - matched = re.search(r"run_id=(run-[0-9A-Za-z-]+)", first_stdout.getvalue()) - self.assertIsNotNone(matched) - run_id = matched.group(1) # type: ignore[union-attr] - run_root = self.root / manifest.output_root / run_id - first_attempt = next(run_root.glob("cells/*/repetition-*/attempt-000001")) - old_bytes = { - item.relative_to(first_attempt): item.read_bytes() - for item in first_attempt.rglob("*") - if item.is_file() - } - - ready = FakeAdapter("claude", ("max",)) - self.addCleanup(ready.cleanup) - resume_stdout = io.StringIO() - resume_stderr = io.StringIO() - with ( - mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root), - mock.patch.object( - benchmark_cli, - "build_adapter_registry", - return_value={"claude": ready}, - ), - contextlib.redirect_stdout(resume_stdout), - contextlib.redirect_stderr(resume_stderr), - ): - resume_exit = benchmark_cli.main( - [ - "resume", - "--manifest", - str(path), - "--run-id", - run_id, - "--retry-failed", - ] - ) - - # All slots resolve after retry, so exit 0 even with independent failures. - self.assertEqual(resume_exit, 0) - self.assertIn("unresolved=0", resume_stdout.getvalue()) - self.assertIn("product_succeeded=1", resume_stdout.getvalue()) - self.assertIn("artifact_failed=1", resume_stdout.getvalue()) - self.assertEqual( - old_bytes, - { - relative: (first_attempt / relative).read_bytes() - for relative in old_bytes - }, - ) - self.assertTrue( - next(run_root.glob("cells/*/repetition-*/attempt-000002/attempt.json")) - .read_text(encoding="utf-8") - .find('"state":"completed"') - >= 0 - ) - self.assertEqual(len(list((run_root / "preflight").glob("*.json"))), 2) - - before_status = { - item.relative_to(run_root): item.read_bytes() - for item in run_root.rglob("*") - if item.is_file() - } - status_stdout = io.StringIO() - status_stderr = io.StringIO() - with ( - mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root), - contextlib.redirect_stdout(status_stdout), - contextlib.redirect_stderr(status_stderr), - ): - status_exit = benchmark_cli.main( - ["status", "--manifest", str(path), "--run-id", run_id] - ) - self.assertEqual(status_exit, 0, status_stderr.getvalue()) - self.assertIn("product_succeeded=1", status_stdout.getvalue()) - self.assertIn("artifact_failed=1", status_stdout.getvalue()) - self.assertEqual( - before_status, - { - item.relative_to(run_root): item.read_bytes() - for item in run_root.rglob("*") - if item.is_file() - }, - ) - - def test_missing_adapter_is_rejected_before_output_root_mutation(self) -> None: - registry = self._registry() - del registry["codex"] - output_root = self.root / self.manifest.output_root - with self.assertRaises(CapabilityUnavailable): - preflight_manifest( - self.store, self.manifest, self.raw, adapters=registry - ) - self.assertFalse(output_root.exists()) - - def test_preset_only_public_preflight_appends_exact_results(self) -> None: - generic, _, path = _write_manifest( - self.root, - [ - _preset("claude-generic", "claude", "claude-sonnet-5", "high"), - _preset("agy-generic", "agy", "gemini-3.6-flash", "high"), - _preset("codex-generic", "codex", "gpt-5.6-luna", "xhigh"), - ], - output_id="generic", - ) - registry = self._registry() - stdout = io.StringIO() - stderr = io.StringIO() - with ( - mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root), - mock.patch.object(benchmark_cli, "build_adapter_registry", return_value=registry), - contextlib.redirect_stdout(stdout), - contextlib.redirect_stderr(stderr), - ): - exit_code = benchmark_cli.main(["preflight", "--manifest", str(path)]) - - self.assertEqual(exit_code, 0, stderr.getvalue()) - self.assertIn("status=ready", stdout.getvalue()) - self.assertEqual(stderr.getvalue(), "") - run_roots = list((self.root / generic.output_root).glob("run-*")) - self.assertEqual(len(run_roots), 1) - record = json.loads( - (run_roots[0] / "preflight/preflight-000001.json").read_text( - encoding="ascii" - ) - ) - self.assertEqual( - [result["cell"]["id"] for result in record["results"]], - [cell.id for cell in generic.matrix], - ) - self.assertFalse((run_roots[0] / "cells").exists()) - self.assertEqual( - {caller: adapter.calls for caller, adapter in registry.items()}, - { - "claude": ["claude-generic"], - "agy": ["agy-generic"], - "codex": ["codex-generic"], - }, - ) - - def test_preset_blocker_appends_without_attempt_allocation(self) -> None: - generic, _, path = _write_manifest( - self.root, - [ - _preset("claude-generic", "claude", "claude-sonnet-5", "high"), - _preset("agy-generic", "agy", "gemini-3.6-flash", "high"), - _preset("codex-generic", "codex", "gpt-5.6-luna", "xhigh"), - ], - output_id="generic-blocked", - ) - registry = self._registry( - {"agy-generic": ("credential_missing",)} - ) - stdout = io.StringIO() - stderr = io.StringIO() - with ( - mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root), - mock.patch.object(benchmark_cli, "build_adapter_registry", return_value=registry), - contextlib.redirect_stdout(stdout), - contextlib.redirect_stderr(stderr), - ): - exit_code = benchmark_cli.main(["run", "--manifest", str(path)]) - self.assertEqual(exit_code, 69) - self.assertEqual(stdout.getvalue(), "") - self.assertIn("error: preflight blocked", stderr.getvalue()) - run_roots = list((self.root / generic.output_root).glob("run-*")) - self.assertEqual(len(run_roots), 1) - record = json.loads( - (run_roots[0] / "preflight/preflight-000001.json").read_text( - encoding="ascii" - ) - ) - self.assertEqual(record["status"], "registration_required") - self.assertEqual( - [result["cell"]["id"] for result in record["results"]], - [cell.id for cell in generic.matrix], - ) - self.assertFalse((run_roots[0] / "cells").exists()) - self.assertTrue(all(adapter.invocations == [] for adapter in registry.values())) - - def test_all_cell_preflight_rejects_missing_extra_and_reordered_results(self) -> None: - manifest, raw, _ = _write_manifest( - self.root, - [ - _cell("direct", "claude", "claude-sonnet-5", "max"), - _preset("preset", "claude", "claude-sonnet-5", "max"), - ], - output_id="all-cell-corruption", - ) - run, record = preflight_manifest( - self.store, manifest, raw, adapters=self._registry() - ) - path = Path(run.root) / "preflight/preflight-000001.json" - original = path.read_bytes() - mutations = { - "missing": record["results"][:-1], - "extra": [*record["results"], record["results"][0]], - "reordered": list(reversed(record["results"])), - } - for label, results in mutations.items(): - with self.subTest(label=label): - mutated = {**record, "results": results} - path.write_bytes( - json.dumps( - mutated, sort_keys=True, separators=(",", ":") - ).encode("ascii") - + b"\n" - ) - with self.assertRaises(AttemptStateError): - self.store.preflights(run, manifest) - path.write_bytes(original) - self.assertEqual(self.store.preflights(run, manifest), (record,)) - - def test_live_registry_dereferences_secret_names_without_persisting_values(self) -> None: - sentinel = "live-token-must-not-persist" - environment = self._live_environment(token=sentinel) - environment["ANTHROPIC_BASE_URL"] = "must-not-be-read" - - def observed(_runtime): - return live_iop._Observation( - tuple(sorted(cell.iop.request_model for cell in self.manifest.matrix)), - "sha256:" + "c" * 64, - True, - "agy 1.1.12", - "--print --output-format --dangerously-skip-permissions --model --effort AGY_PROVIDER AGY_OPENAI_BASE_URL AGY_OPENAI_API_KEY stream-json", - ) - - registry = live_iop.build_live_adapter_registry( - environment, observer=observed, binary_resolver=lambda _name: "/bin/true" - ) - self.assertEqual(tuple(registry), ("claude", "agy", "codex")) - observations = collect_preflight_observations(self.manifest, registry) - self.assertEqual(set(observations), {cell.id for cell in self.manifest.matrix}) - for observation in observations.values(): - self.assertEqual(observation.result.status, "ready") - self.assertEqual(observation.result.binding.effective_model, observation.result.binding.requested_model) - durable = json.dumps( - {cell_id: item.result.status for cell_id, item in observations.items()}, - sort_keys=True, - ) - self.assertNotIn(sentinel, durable) - self.assertNotIn("must-not-be-read", durable) - - def test_live_registry_ready_binding_requires_catalog_observation(self) -> None: - environment = self._live_environment(token="private-token") - registry = live_iop.build_live_adapter_registry( - environment, - observer=lambda _runtime: live_iop._Observation( - (), "sha256:" + "d" * 64, True - ), - ) - claude_cell = next(cell for cell in self.manifest.matrix if cell.caller == "claude") - observation = registry["claude"].preflight(claude_cell) - self.assertEqual(observation.result.status, "registration_required") - self.assertEqual([issue.code for issue in observation.result.issues], ["model_missing"]) - self.assertIsNone(observation.result.binding.effective_model) - - def test_live_scoring_adapter_preserves_manifest_route_and_ephemeral_secret(self) -> None: - secret = "evaluator-secret-must-not-persist" - source_identities = ( - "source-cell-sentinel", - "source-route-sentinel", - "source-model-sentinel", - "source-effort-sentinel", - ) - - for route_kind in ("direct", "execution_preset"): - with self.subTest(route_kind=route_kind): - expected_bindings = ( - (ExpectedBinding("request", "judge", "xhigh"),) - if route_kind == "direct" - else tuple( - ExpectedBinding(stage, "judge") - for stage in ("selector", "plan", "work", "review") - ) - ) - evaluator = MatrixCell( - "evaluator", self.manifest.evaluator.caller, - replace( - self.manifest.evaluator.iop, - requested_effort="xhigh", - route_kind=route_kind, - expected_bindings=expected_bindings, - ), - ) - scoring_manifest = replace( - self.manifest, - evaluator=replace(self.manifest.evaluator, iop=evaluator.iop), - ) - environment = self._live_environment( - token=secret, manifest=scoring_manifest - ) - captured = {} - - caller_binding = {"value": None} - - def invoke(invocation, _on_started): - captured["spec"] = invocation.spec - lifecycle = _invocation_result() - return CodexInvocationResult( - lifecycle, caller_binding["value"] - ) - - adapter = live_iop.build_live_scoring_adapter( - environment, - observer=lambda _runtime: live_iop._Observation( - (evaluator.iop.request_model,), - "sha256:" + "b" * 64, - True, - ), - invoker=invoke, - ) - observation = adapter.preflight(evaluator) - self.assertEqual(observation.result.status, "ready") - self.assertEqual( - observation.result.binding.effective_bindings, - tuple( - EffectiveBinding(item.stage, item.model, item.effort) - for item in evaluator.iop.expected_bindings - ), - ) - - blind_root = self.root / "runs" / f"blind-{route_kind}" - for name in ("input", "session", "output"): - (blind_root / name).mkdir(parents=True, exist_ok=True) - blind = BlindWorkspace( - f"blind-{route_kind}", str(blind_root), - str(blind_root / "input"), str(blind_root / "session"), - str(blind_root / "output"), "sha256:" + "c" * 64, - "sha256:" + "d" * 64, - ) - prompt = b"Evaluate only anonymous files under input/." - result = adapter.invoke( - evaluator, - blind, - prompt, - self.manifest.timeout, - lambda *_args: None, - ) - self.assertEqual( - (result.product, result.harness, result.process), - ("succeeded", "passed", "exited"), - ) - expected_binding = ( - evaluator.iop.route_kind, - evaluator.iop.route_id, - evaluator.iop.request_model, - evaluator.iop.requested_effort, - ) - self.assertEqual(result.effective_binding, expected_binding) - finalized = adapter.finalize_evidence(blind) - self.assertTrue(finalized.safe) - - caller_binding["value"] = ( - evaluator.iop.route_kind, - "contradictory-route", - evaluator.iop.request_model, - evaluator.iop.requested_effort, - ) - mismatch_root = self.root / "runs" / f"blind-{route_kind}-mismatch" - for name in ("input", "session", "output"): - (mismatch_root / name).mkdir(parents=True, exist_ok=True) - mismatch_blind = BlindWorkspace( - f"blind-{route_kind}-mismatch", str(mismatch_root), - str(mismatch_root / "input"), str(mismatch_root / "session"), - str(mismatch_root / "output"), "sha256:" + "e" * 64, - "sha256:" + "f" * 64, - ) - mismatch = adapter.invoke( - evaluator, - mismatch_blind, - prompt, - self.manifest.timeout, - lambda *_args: None, - ) - self.assertEqual(mismatch.harness, "failed") - self.assertEqual(mismatch.reason, "binding_mismatch") - self.assertEqual( - mismatch.effective_binding, caller_binding["value"] - ) - self.assertTrue(adapter.finalize_evidence(mismatch_blind).safe) - - spec = captured["spec"] - visible = "\n".join( - (*spec.argv, spec.cwd, *(value for pair in spec.env for value in pair)) - ).casefold() - for identity in source_identities: - self.assertNotIn(identity, visible) - durable = b"".join( - path.read_bytes() - for path in blind_root.rglob("*") - if path.is_file() - ) - self.assertNotIn(secret.encode("ascii"), durable) - evidence = canonical_evidence_bytes( - evaluator, - observation.result, - observation.endpoint_identity, - observation.config_identity, - ) - self.assertNotIn(secret.encode("ascii"), evidence) - - def test_live_scoring_alias_is_control_only(self) -> None: - evaluator = MatrixCell( - "evaluator", - self.manifest.evaluator.caller, - replace( - self.manifest.evaluator.iop, - requested_effort="xhigh", - expected_bindings=(ExpectedBinding("request", "judge", "xhigh"),), - ), - ) - scoring_manifest = replace( - self.manifest, - evaluator=replace(self.manifest.evaluator, iop=evaluator.iop), - ) - captured: dict[str, object] = {} - - def invoke(invocation, _on_started): - captured["spec"] = invocation.spec - evidence = Path(invocation.spec.evidence_dir) - (evidence / "lifecycle-journal.jsonl").write_text( - '{"record":"header"}\n{"record":"terminal"}\n', - encoding="utf-8", - ) - (evidence / "lifecycle-result.json").write_text( - '{"record":"result"}\n', encoding="utf-8" - ) - lifecycle = _invocation_result() - binding = ( - evaluator.iop.route_kind, - evaluator.iop.route_id, - evaluator.iop.request_model, - evaluator.iop.requested_effort, - ) - return CodexInvocationResult(lifecycle, binding) - - adapter = live_iop.build_live_scoring_adapter( - self._live_environment(manifest=scoring_manifest), - observer=lambda _runtime: live_iop._Observation( - (evaluator.iop.request_model,), "sha256:" + "b" * 64, True - ), - invoker=invoke, - ) - self.assertEqual(adapter.preflight(evaluator).result.status, "ready") - blind_root = self.root / "alias-run" / "blind" / "blind-alias" - for name in ("input", "session", "output"): - (blind_root / name).mkdir(parents=True, exist_ok=True) - blind = BlindWorkspace( - "blind-alias", - str(blind_root), - str(blind_root / "input"), - str(blind_root / "session"), - str(blind_root / "output"), - "sha256:" + "c" * 64, - "sha256:" + "d" * 64, - ) - result = adapter.invoke( - evaluator, - blind, - b"Evaluate anonymous output.", - self.manifest.timeout, - lambda *_args: None, - ) - self.assertEqual( - (result.product, result.harness, result.process), - ("succeeded", "passed", "exited"), - ) - spec = captured["spec"] - self.assertIsInstance(spec, InvocationSpec) - assert isinstance(spec, InvocationSpec) - self.assertEqual(Path(spec.evidence_dir), blind_root / "output") - alias = Path(spec.control_dir).parent - self.assertTrue(alias.is_symlink()) - self.assertEqual(alias.resolve(strict=True), blind_root / "output") - sidecars = { - path.name: path.read_bytes() - for path in ( - blind_root / "output" / "lifecycle-journal.jsonl", - blind_root / "output" / "lifecycle-result.json", - ) - } - self.assertTrue(adapter.finalize_evidence(blind).safe) - self.assertFalse(alias.exists() or alias.is_symlink()) - for name, data in sidecars.items(): - self.assertEqual((blind_root / "output" / name).read_bytes(), data) - - def test_live_scoring_scrubs_evaluator_secret_output(self) -> None: - secret = "live-evaluator-secret-exact-value" - base_url = "http://127.0.0.1:18083/v1" - evaluator = MatrixCell( - "evaluator", - self.manifest.evaluator.caller, - replace( - self.manifest.evaluator.iop, - requested_effort="xhigh", - expected_bindings=( - ExpectedBinding("request", "judge", "xhigh"), - ), - ), - ) - scoring_manifest = replace( - self.manifest, - evaluator=replace(self.manifest.evaluator, iop=evaluator.iop), - ) - environment = self._live_environment( - token=secret, manifest=scoring_manifest - ) - - def invoke(invocation, _on_started): - output = Path(invocation.spec.evidence_dir) - worksheet = _worksheet_payload = { - "rubric_version": "landing-quality-v1", - "categories": [ - { - "id": ident, - "max_score": maximum, - "score": maximum, - "evidence": secret if index == 0 else "safe evidence", - } - for index, (ident, maximum) in enumerate( - ( - ("task_fidelity", 25), - ("visual_hierarchy", 25), - ("responsive_composition", 20), - ("typography_readability", 15), - ("polish_consistency", 15), - ) - ) - ], - "total": 100, - } - (output / "worksheet.json").write_text( - json.dumps(worksheet), encoding="utf-8" - ) - (output / "diagnostic.txt").write_text( - f"{base_url}\n{secret}\n", encoding="utf-8" - ) - lifecycle = _invocation_result() - binding = ( - evaluator.iop.route_kind, - evaluator.iop.route_id, - evaluator.iop.request_model, - evaluator.iop.requested_effort, - ) - return CodexInvocationResult(lifecycle, binding) - - adapter = live_iop.build_live_scoring_adapter( - environment, - observer=lambda _runtime: live_iop._Observation( - (evaluator.iop.request_model,), - "sha256:" + "b" * 64, - True, - ), - invoker=invoke, - ) - self.assertEqual(adapter.preflight(evaluator).result.status, "ready") - blind_root = self.root / "secret-run" / "blind" / "blind-secret" - for name in ("input", "session", "output"): - (blind_root / name).mkdir(parents=True, exist_ok=True) - blind = BlindWorkspace( - "blind-secret", - str(blind_root), - str(blind_root / "input"), - str(blind_root / "session"), - str(blind_root / "output"), - "sha256:" + "c" * 64, - "sha256:" + "d" * 64, - ) - result = adapter.invoke( - evaluator, - blind, - b"Evaluate anonymous output.", - self.manifest.timeout, - lambda *_args: None, - ) - self.assertEqual( - (result.product, result.harness, result.process), - ("succeeded", "passed", "exited"), - ) - finalized = adapter.finalize_evidence(blind) - self.assertEqual( - (finalized.safe, finalized.reason), - (False, "runtime_secret_leak"), - ) - durable = b"".join( - path.read_bytes() - for path in (self.root / "secret-run").rglob("*") - if path.is_file() - ) - self.assertNotIn(secret.encode("utf-8"), durable) - self.assertNotIn(base_url.encode("utf-8"), durable) - self.assertFalse((blind_root / "output" / "worksheet.json").exists()) - - def test_live_scoring_scrubs_permission_denied_secret_paths(self) -> None: - retained_modes: dict[str, int] = {} - removed_paths: list[Path] = [] - - def mutate(invocation, secret, base_url): - blind_root = Path(invocation.spec.cwd) - input_root = blind_root / "input" - output_root = Path(invocation.spec.evidence_dir) - os.chmod(input_root, 0o700, follow_symlinks=False) - locked = input_root / ("locked-" + secret) - locked.mkdir() - payload = locked / "payload.bin" - payload.write_bytes(secret.encode("utf-8")) - os.chmod(payload, 0o000, follow_symlinks=False) - os.chmod(locked, 0o000, follow_symlinks=False) - removed_paths.append(locked) - - denied = output_root / ("denied-" + secret + ".bin") - denied.write_bytes(base_url.encode("utf-8")) - os.chmod(denied, 0o000, follow_symlinks=False) - removed_paths.append(denied) - safe_dir = output_root / "safe-retained" - safe_dir.mkdir() - safe_file = output_root / "safe-retained.txt" - safe_file.write_text("safe evidence", encoding="utf-8") - os.chmod(safe_file, 0o000, follow_symlinks=False) - os.chmod(safe_dir, 0o000, follow_symlinks=False) - retained_modes["directory"] = 0o000 - retained_modes["file"] = 0o000 - - summary, result, blind_root, secret, base_url = ( - self._run_live_scoring_mutation("permission-denied-secret", mutate) - ) - self.assertEqual((summary.scored, summary.scoring_failed), (0, 1)) - self.assertEqual(result["reason"], "runtime_secret_leak") - self.assertNotIn("worksheet", result) - for path in removed_paths: - self.assertFalse(path.exists() or path.is_symlink()) - safe_dir = blind_root / "output" / "safe-retained" - safe_file = blind_root / "output" / "safe-retained.txt" - self.assertEqual(stat.S_IMODE(os.lstat(safe_dir).st_mode), retained_modes["directory"]) - self.assertEqual(stat.S_IMODE(os.lstat(safe_file).st_mode), retained_modes["file"]) - self.assertNotIn(secret, result["reason"]) - self.assertNotIn(base_url, result["reason"]) - - def test_live_scoring_classifies_safe_invalid_links_without_secret_claim(self) -> None: - cases = ( - ("input", "input_mutated"), - ("output", "evaluator_output_leak"), - ) - for root_kind, expected_reason in cases: - with self.subTest(root_kind=root_kind): - link_path: list[Path] = [] - target_path: list[Path] = [] - - def mutate(invocation, _secret, _base_url): - blind_root = Path(invocation.spec.cwd) - selected = ( - blind_root / "input" - if root_kind == "input" - else Path(invocation.spec.evidence_dir) - ) - if root_kind == "input": - os.chmod(selected, 0o700, follow_symlinks=False) - target = selected / "index.html" - else: - target = selected / "safe-target.txt" - target.write_text("safe evidence", encoding="utf-8") - link = selected / "safe-invalid-link" - link.symlink_to(target.name) - link_path.append(link) - target_path.append(target) - - summary, result, _blind_root, _secret, _base_url = ( - self._run_live_scoring_mutation( - f"safe-link-{root_kind}", mutate - ) - ) - self.assertEqual( - (summary.scored, summary.scoring_failed), (0, 1) - ) - self.assertEqual(result["reason"], expected_reason) - self.assertNotEqual(result["reason"], "runtime_secret_leak") - self.assertFalse(link_path[0].exists() or link_path[0].is_symlink()) - self.assertTrue(target_path[0].is_file()) - self.assertNotIn("worksheet", result) - - def test_live_scoring_scrubs_secret_from_mutated_input_before_failure(self) -> None: - secret = "live-mutated-input-secret-exact-value" - evaluator = MatrixCell( - "evaluator", - self.manifest.evaluator.caller, - replace( - self.manifest.evaluator.iop, - requested_effort="xhigh", - expected_bindings=(ExpectedBinding("request", "judge", "xhigh"),), - ), - ) - manifest_payload = { - **json.loads(self.raw), - "evaluator": { - "caller": evaluator.caller, - "iop": { - "request_model": evaluator.iop.request_model, - "requested_effort": evaluator.iop.requested_effort, - "route_kind": evaluator.iop.route_kind, - "route_id": evaluator.iop.route_id, - "expected_bindings": [ - { - "stage": item.stage, - "model": item.model, - "effort": item.effort, - } - for item in evaluator.iop.expected_bindings - ], - }, - }, - "output_root": "agent-test/runs/mutated-input-secret", - } - manifest_raw = json.dumps(manifest_payload, sort_keys=True).encode("utf-8") - manifest_path = self.root / "mutated-input-secret.json" - manifest_path.write_bytes(manifest_raw) - scoring_manifest = load_manifest(manifest_path, repo_root=self.root) - evaluator = MatrixCell( - "evaluator", - scoring_manifest.evaluator.caller, - scoring_manifest.evaluator.iop, - ) - run = self.store.create(scoring_manifest, manifest_raw) - attempt = self._successful_score_attempt(scoring_manifest, run) - environment = self._live_environment( - token=secret, manifest=scoring_manifest - ) - base_url = environment["IOP_BENCH_CODEX_BASE_URL"] - invocation_count = 0 - - def invoke(invocation, _on_started): - nonlocal invocation_count - invocation_count += 1 - if invocation_count == 1: - input_root = Path(invocation.spec.cwd) / "input" - os.chmod(input_root, 0o700, follow_symlinks=False) - target = input_root / "index.html" - os.chmod(target, 0o600, follow_symlinks=False) - target.write_bytes(target.read_bytes() + secret.encode("utf-8")) - sensitive_dir = input_root / ("copied-" + secret) - sensitive_dir.mkdir() - (sensitive_dir / "runtime.txt").write_text( - secret + "\n" + base_url, encoding="utf-8" - ) - lifecycle = _invocation_result() - binding = ( - evaluator.iop.route_kind, - evaluator.iop.route_id, - evaluator.iop.request_model, - evaluator.iop.requested_effort, - ) - return CodexInvocationResult(lifecycle, binding) - - adapter = live_iop.build_live_scoring_adapter( - environment, - observer=lambda _runtime: live_iop._Observation( - (evaluator.iop.request_model,), "sha256:" + "b" * 64, True - ), - invoker=invoke, - ) - first = score_run( - self.store, run, scoring_manifest, adapter=adapter - ) - self.assertEqual((first.scored, first.scoring_failed), (0, 1)) - first_score = Path(attempt.root) / "scoring" / "score-000001" - first_result = json.loads((first_score / "result.json").read_text()) - self.assertEqual(first_result["reason"], "runtime_secret_leak") - self.assertNotIn("worksheet", first_result) - first_allocation = json.loads((first_score / "allocation.json").read_text()) - first_blind = Path(run.root) / first_allocation["blind_path"] - self.assertFalse((first_blind / "output" / "worksheet.json").exists()) - for path in (first_blind / "input", *(first_blind / "input").rglob("*")): - self.assertEqual(os.lstat(path).st_mode & 0o222, 0) - - prior = { - path: path.read_bytes() - for path in Path(run.root).rglob("*") - if path.is_file() - } - retry = score_run( - self.store, - run, - scoring_manifest, - adapter=adapter, - retry_scoring_failed=True, - ) - self.assertEqual((retry.scored, retry.scoring_failed), (0, 1)) - self.assertEqual(invocation_count, 2) - for path, data in prior.items(): - self.assertEqual(path.read_bytes(), data) - second_score = Path(attempt.root) / "scoring" / "score-000002" - second_allocation = json.loads( - (second_score / "allocation.json").read_text() - ) - self.assertNotEqual( - first_allocation["blind_id"], second_allocation["blind_id"] - ) - self.assertNotEqual( - first_allocation["session_identity"], - second_allocation["session_identity"], - ) - for path in Path(run.root).rglob("*"): - relative = path.relative_to(run.root).as_posix().encode("utf-8") - self.assertNotIn(secret.encode("utf-8"), relative) - if path.is_file(): - data = path.read_bytes() - self.assertNotIn(secret.encode("utf-8"), data) - self.assertNotIn(base_url.encode("utf-8"), data) - - def test_live_scoring_survivor_cleanup_precedes_retry(self) -> None: - evaluator = MatrixCell( - "evaluator", - self.manifest.evaluator.caller, - replace( - self.manifest.evaluator.iop, - requested_effort="xhigh", - expected_bindings=( - ExpectedBinding("request", "judge", "xhigh"), - ), - ), - ) - scoring_manifest = replace( - self.manifest, - evaluator=replace(self.manifest.evaluator, iop=evaluator.iop), - ) - environment = self._live_environment(manifest=scoring_manifest) - locator_ready = threading.Event() - locators = [] - worker_results = [] - workers = [] - first = True - - def invoker(invocation, on_started): - nonlocal first - if not first: - self.assertTrue(workers) - self.assertFalse(workers[0].is_alive()) - lifecycle = _invocation_result() - binding = ( - evaluator.iop.route_kind, - evaluator.iop.route_id, - evaluator.iop.request_model, - evaluator.iop.requested_effort, - ) - return CodexInvocationResult(lifecycle, binding) - first = False - spec = replace( - invocation.spec, - argv=( - sys.executable, - "-u", - "-c", - "import sys,time; sys.stdin.buffer.read(); " - "print('START', flush=True); time.sleep(30)", - ), - env=env_pairs( - {"PATH": os.environ.get("PATH", "/usr/bin:/bin")} - ), - task_payload=b"evaluate", - ) - - def run(): - worker_results.append( - run_invocation( - spec, - parse_event=lambda _stream, _line: None, - on_started=lambda locator: ( - on_started(locator), - locators.append(locator), - locator_ready.set(), - ), - ) - ) - - worker = threading.Thread(target=run) - workers.append(worker) - worker.start() - if not locator_ready.wait(5): - self.fail("live evaluator locator was not published") - deadline = time.monotonic() + 5 - while not recover_invocation(locators[0], stop=False).caller_launched: - if time.monotonic() >= deadline: - self.fail("live evaluator did not launch") - time.sleep(0.01) - raise KeyboardInterrupt("simulated live scoring controller loss") - - adapter = live_iop.build_live_scoring_adapter( - environment, - observer=lambda _runtime: live_iop._Observation( - (evaluator.iop.request_model,), - "sha256:" + "b" * 64, - True, - ), - invoker=invoker, - ) - self.assertEqual(adapter.preflight(evaluator).result.status, "ready") - blind_root = self.root / "recovery-run" / "blind" / "blind-recovery" - for name in ("input", "session", "output"): - (blind_root / name).mkdir(parents=True, exist_ok=True) - blind = BlindWorkspace( - "blind-recovery", - str(blind_root), - str(blind_root / "input"), - str(blind_root / "session"), - str(blind_root / "output"), - "sha256:" + "c" * 64, - "sha256:" + "d" * 64, - ) - - def cleanup(): - if workers and workers[0].is_alive() and locators: - try: - recover_invocation(locators[0], stop=True) - except Exception: - pass - workers[0].join(5) - try: - adapter.finalize_evidence(blind) - except Exception: - pass - - self.addCleanup(cleanup) - with self.assertRaises(KeyboardInterrupt): - adapter.invoke( - evaluator, - blind, - b"Evaluate anonymous output.", - self.manifest.timeout, - lambda locator, digest: self.assertRegex( - digest, r"^sha256:[0-9a-f]{64}$" - ), - ) - try: - stopped = recover_invocation(locators[0], stop=True) - except LifecycleRecoveryError: - stopped = None - if stopped is not None: - self.assertTrue(stopped.cleanup_complete) - self.assertFalse(stopped.process_group_alive) - workers[0].join(5) - self.assertFalse(workers[0].is_alive()) - self.assertEqual(len(worker_results), 1) - self.assertTrue(worker_results[0].harness.cleanup_complete) - self.assertFalse(worker_results[0].process_group_alive) - receipt = json.loads( - ( - blind_root / "output" / "codex-control" / "cleanup-receipt.json" - ).read_text() - ) - self.assertTrue(receipt["cleanup_complete"]) - self.assertFalse(receipt["process_group_alive"]) - self.assertTrue(adapter.finalize_evidence(blind).safe) - - retry = adapter.invoke( - evaluator, - blind, - b"Evaluate anonymous output.", - self.manifest.timeout, - lambda *_args: None, - ) - self.assertEqual( - (retry.product, retry.harness, retry.process), - ("succeeded", "passed", "exited"), - ) - self.assertTrue(adapter.finalize_evidence(blind).safe) - - def test_catalog_only_never_creates_ready_binding(self) -> None: - environment = self._live_environment() - del environment["IOP_BENCH_CONFIG_OBSERVATION_ENV"] - registry = live_iop.build_live_adapter_registry( - environment, - observer=lambda _runtime: live_iop._Observation( - tuple(sorted(cell.iop.request_model for cell in self.manifest.matrix)), - "sha256:" + "d" * 64, - True, - ), - ) - cell = next(cell for cell in self.manifest.matrix if cell.caller == "claude") - result = registry["claude"].preflight(cell).result - self.assertEqual(result.status, "registration_required") - self.assertEqual([item.code for item in result.issues], ["route_missing"]) - self.assertIsNone(result.binding.effective_model) - - def test_config_owner_binding_is_passed_without_manifest_synthesis(self) -> None: - environment = self._live_environment() - cell = next(cell for cell in self.manifest.matrix if cell.id == "claude-sonnet-direct") - registry = live_iop.build_live_adapter_registry( - environment, - observer=lambda _runtime: live_iop._Observation( - tuple(sorted(item.iop.request_model for item in self.manifest.matrix)), - "sha256:" + "e" * 64, - True, - ), - ) - result = registry["claude"].preflight(cell).result - self.assertEqual(result.status, "ready") - self.assertEqual(result.binding.effective_route_id, cell.iop.route_id) - self.assertEqual(result.binding.effective_model, cell.iop.request_model) - self.assertEqual( - result.binding.effective_bindings, - (EffectiveBinding("request", cell.iop.request_model, cell.iop.requested_effort),), - ) - - def test_live_scoring_preset_requires_observed_stage_bindings(self) -> None: - raw = json.loads(self.path.read_text()) - raw["evaluator"]["iop"] = { - "request_model": "judge", - "requested_effort": "xhigh", - "route_kind": "execution_preset", - "route_id": "judge-preset", - "expected_bindings": [ - {"stage": stage, "model": "judge"} - for stage in ("selector", "plan", "work", "review") - ], - } - raw["output_root"] = "agent-test/runs/preset-observation" - path = self.root / "preset-observation.json" - path.write_text(json.dumps(raw), encoding="utf-8") - manifest = load_manifest(path, repo_root=self.root) - evaluator = MatrixCell( - "evaluator", manifest.evaluator.caller, manifest.evaluator.iop - ) - environment = self._live_environment(manifest=manifest) - observed = lambda _runtime: live_iop._Observation( - ("judge",), "sha256:" + "8" * 64, True - ) - - ready = live_iop.build_live_scoring_adapter( - environment, observer=observed - ).preflight(evaluator) - self.assertEqual(ready.result.status, "ready") - self.assertEqual( - ready.result.binding.effective_bindings, - tuple( - EffectiveBinding(item.stage, item.model, item.effort) - for item in evaluator.iop.expected_bindings - ), - ) - - routes = json.loads(environment["BENCH_CONFIG"])["routes"] - index = next( - i for i, item in enumerate(routes) if item["route_id"] == "judge-preset" - ) - cases = { - "missing": routes[index]["bindings"][:-1], - "reordered": list(reversed(routes[index]["bindings"])), - "substituted": [ - ( - {**binding, "model": "other-model"} - if binding["stage"] == "work" - else dict(binding) - ) - for binding in routes[index]["bindings"] - ], - } - for name, bindings in cases.items(): - with self.subTest(case=name): - changed = [dict(item) for item in routes] - changed[index] = {**changed[index], "bindings": bindings} - candidate = { - **environment, - "BENCH_CONFIG": json.dumps( - {"schema_version": "1", "routes": changed}, - sort_keys=True, - ), - } - result = live_iop.build_live_scoring_adapter( - candidate, observer=observed - ).preflight(evaluator).result - self.assertEqual(result.status, "implementation_gap") - self.assertEqual( - [issue.code for issue in result.issues], - ["protocol_incompatible"], - ) - self.assertIsNone(result.binding.effective_model) - - def test_live_scoring_metrics_match_any_admitted_stage_model(self) -> None: - direct_evaluator = MatrixCell( - "evaluator", - self.manifest.evaluator.caller, - replace( - self.manifest.evaluator.iop, - requested_effort="xhigh", - expected_bindings=(ExpectedBinding("request", "judge", "xhigh"),), - ), - ) - direct_manifest = replace( - self.manifest, - evaluator=replace(self.manifest.evaluator, iop=direct_evaluator.iop), - ) - - raw = json.loads(self.path.read_text()) - raw["evaluator"]["iop"] = { - "request_model": "judge-selector", - "requested_effort": "xhigh", - "route_kind": "execution_preset", - "route_id": "judge-heterogeneous", - "expected_bindings": [ - {"stage": "selector", "model": "judge-selector"}, - {"stage": "plan", "model": "judge-plan"}, - {"stage": "work", "model": "judge-work"}, - {"stage": "review", "model": "judge-review"}, - ], - } - raw["output_root"] = "agent-test/runs/heterogeneous-metrics" - path = self.root / "heterogeneous-metrics.json" - path.write_text(json.dumps(raw), encoding="utf-8") - preset_manifest = load_manifest(path, repo_root=self.root) - - def exercise( - manifest, metric_model: str, suffix: str, metric_stage: str | None = None - ): - evaluator = MatrixCell( - "evaluator", manifest.evaluator.caller, manifest.evaluator.iop - ) - - def invoke(_invocation, _on_started): - metric = ParsedMetric( - "model_duration", - 1, - UNIT_NANOSECONDS, - CLOCK_HARNESS_MONOTONIC, - SOURCE_HARNESS, - ( - metric_stage - if metric_stage is not None - else ( - "work" - if evaluator.iop.route_kind == "execution_preset" - else "request" - ) - ), - metric_model, - "call-1", - ) - lifecycle = _invocation_result(metrics=(metric,)) - binding = ( - evaluator.iop.route_kind, - evaluator.iop.route_id, - evaluator.iop.request_model, - evaluator.iop.requested_effort, - ) - return CodexInvocationResult(lifecycle, binding) - - adapter = live_iop.build_live_scoring_adapter( - self._live_environment(manifest=manifest), - observer=lambda _runtime: live_iop._Observation( - (evaluator.iop.request_model,), "sha256:" + "b" * 64, True - ), - invoker=invoke, - ) - self.assertEqual(adapter.preflight(evaluator).result.status, "ready") - blind_root = self.root / "metric-run" / suffix - for name in ("input", "session", "output"): - (blind_root / name).mkdir(parents=True, exist_ok=True) - blind = BlindWorkspace( - "blind-" + suffix, - str(blind_root), - str(blind_root / "input"), - str(blind_root / "session"), - str(blind_root / "output"), - "sha256:" + "c" * 64, - "sha256:" + "d" * 64, - ) - try: - return adapter.invoke( - evaluator, - blind, - b"Evaluate anonymous output.", - manifest.timeout, - lambda *_args: None, - ) - finally: - adapter.finalize_evidence(blind) - - self.assertEqual( - exercise(direct_manifest, "judge", "direct").product, - "succeeded", - ) - self.assertEqual( - exercise(preset_manifest, "judge-work", "preset-work").product, - "succeeded", - ) - self.assertEqual( - exercise( - preset_manifest, "judge-plan", "preset-unqualified", metric_stage="" - ).product, - "succeeded", - ) - for stage, model in (("plan", "judge-work"), ("work", "judge-plan")): - with self.subTest(stage=stage, model=model): - with self.assertRaises(live_iop.LiveIopError) as raised: - exercise( - preset_manifest, - model, - f"preset-cross-{stage}", - metric_stage=stage, - ) - self.assertEqual( - raised.exception.issue_code, "stream_incompatible" - ) - with self.assertRaises(live_iop.LiveIopError) as raised: - exercise(preset_manifest, "unadmitted-model", "preset-unknown") - self.assertEqual(raised.exception.issue_code, "stream_incompatible") - - def test_catalog_accepts_edge_routing_ids_and_rejects_malformed_records(self) -> None: - environment = self._live_environment() - runtime = live_iop._runtime_from_environment("claude", environment).runtime - self.assertIsNotNone(runtime) - - class Response: - status = 200 - - def __init__(self, records) -> None: - self.body = json.dumps({"object": "list", "data": records}).encode() - - def __enter__(self): return self - def __exit__(self, *_args): return False - def read(self): return self.body - - model_ids = ( - "claude-sonnet-5", - "gemini-3.6-flash", - "gpt-5.6-luna", - "qwen3.6:35b", - "ornith:35b", - "laguna-s:2.1", - ) - with mock.patch.object( - live_iop, - "urlopen", - return_value=Response([{"id": model_id} for model_id in model_ids]), - ): - models, _identity = live_iop._catalog(runtime) # type: ignore[arg-type] - self.assertEqual(models, tuple(sorted(model_ids))) - - malformed = { - "numeric": [{"id": 1}], - "empty": [{"id": ""}], - "whitespace": [{"id": " \t"}], - "duplicate": [{"id": "claude-sonnet-5"}, {"id": "claude-sonnet-5"}], - } - for name, records in malformed.items(): - with self.subTest(name=name): - with mock.patch.object(live_iop, "urlopen", return_value=Response(records)): - with self.assertRaises(live_iop.LiveIopError) as raised: - live_iop._catalog(runtime) # type: ignore[arg-type] - self.assertEqual(raised.exception.issue_code, "protocol_incompatible") - - def test_live_failure_taxonomy_is_exact(self) -> None: - """Every live setup/catalog boundary returns one closed issue/resume pair.""" - environment = self._live_environment(token="secret-must-not-appear") - runtime_cases = ( - ("missing-base", {"IOP_BENCH_CLAUDE_BASE_URL": ""}, "endpoint_incompatible"), - ("invalid-base", {"IOP_BENCH_CLAUDE_BASE_URL": "not-a-url"}, "endpoint_incompatible"), - ("invalid-secret-ref", {"IOP_BENCH_CLAUDE_SECRET_ENV": "1BAD"}, "credential_missing"), - ("missing-secret", {"BENCH_TOKEN": ""}, "credential_missing"), - ) - for _name, updates, expected in runtime_cases: - candidate = {**environment, **updates} - resolution = live_iop._runtime_from_environment("claude", candidate) - self.assertIsNone(resolution.runtime) - self.assertEqual(resolution.issue_code, expected) - - runtime = live_iop._runtime_from_environment("claude", environment).runtime - self.assertIsNotNone(runtime) - - class Response: - def __init__(self, status, body): self.status, self.body = status, body - def __enter__(self): return self - def __exit__(self, *_args): return False - def read(self): return self.body - - catalog_cases = ( - (HTTPError("http://invalid", 401, "", None, None), "auth_incompatible"), - (HTTPError("http://invalid", 403, "", None, None), "auth_incompatible"), - (OSError("unreachable"), "endpoint_incompatible"), - (Response(502, b"{}"), "endpoint_incompatible"), - (Response(200, b"not-json"), "protocol_incompatible"), - (Response(200, b'{"data":[{"id":1}]}'), "protocol_incompatible"), - ) - for outcome, expected in catalog_cases: - patch_kwargs = {"side_effect": outcome} if isinstance(outcome, BaseException) else {"return_value": outcome} - with mock.patch.object(live_iop, "urlopen", **patch_kwargs): - with self.assertRaises(live_iop.LiveIopError) as raised: - live_iop._catalog(runtime) # type: ignore[arg-type] - self.assertEqual(raised.exception.issue_code, expected) - - claude = next(cell for cell in self.manifest.matrix if cell.caller == "claude") - routes = json.loads(environment["BENCH_CONFIG"])["routes"] - no_route = {**environment, "BENCH_CONFIG": json.dumps({"schema_version": "1", "routes": [item for item in routes if item["route_id"] != claude.iop.route_id]})} - no_model_routes = [ - {**item, "model": "other-model"} if item["route_id"] == claude.iop.route_id else dict(item) - for item in routes - ] - no_model = {**environment, "BENCH_CONFIG": json.dumps({"schema_version": "1", "routes": no_model_routes})} - unsupported = replace(next(cell for cell in self.manifest.matrix if cell.caller == "agy"), iop=replace(next(cell for cell in self.manifest.matrix if cell.caller == "agy").iop, requested_effort="max")) - observed = lambda _runtime: live_iop._Observation(tuple(sorted(cell.iop.request_model for cell in self.manifest.matrix)), "sha256:" + "1" * 64, True, "agy 1.1.12", "--print --output-format --dangerously-skip-permissions --model --effort stream-json") - checks = ( - (live_iop.build_live_adapter_registry(no_route, observer=observed)["claude"].preflight(claude).result, "route_missing"), - (live_iop.build_live_adapter_registry(no_model, observer=observed)["claude"].preflight(claude).result, "model_missing"), - (live_iop.build_live_adapter_registry(environment, observer=lambda _runtime: (_ for _ in ()).throw(live_iop.LiveIopError("stream_incompatible")))["claude"].preflight(claude).result, "stream_incompatible"), - ) - for result, issue_code in checks: - self.assertEqual( - (result.status, tuple((item.code, item.resume_code) for item in result.issues)), - ("registration_required" if issue_code in ISSUE_RESUME_CODES and issue_code in {"route_missing", "model_missing", "effort_unsupported"} else "implementation_gap", ((issue_code, ISSUE_RESUME_CODES[issue_code]),)), - ) - _, unsupported_issues = live_iop._binding_from_config( - unsupported, - CallerCapability("agy", ("direct", "execution_preset"), ("high", "low", "medium")), - live_iop._runtime_from_environment("agy", environment).runtime.config, # type: ignore[union-attr] - ) - self.assertEqual( - tuple((item.code, item.resume_code) for item in unsupported_issues), - (("effort_unsupported", ISSUE_RESUME_CODES["effort_unsupported"]),), - ) - - def test_live_invocation_uses_config_owned_codex_binding_and_rejects_mismatch(self) -> None: - environment = self._live_environment() - observed = lambda _runtime: live_iop._Observation( - tuple(sorted(cell.iop.request_model for cell in self.manifest.matrix)), - "sha256:" + "2" * 64, True, "agy 1.1.12", - "--print --output-format --dangerously-skip-permissions --model --effort stream-json", - ) - registry = live_iop.build_live_adapter_registry( - environment, observer=observed, binary_resolver=lambda _name: "/bin/true" - ) - agy = next(cell for cell in self.manifest.matrix if cell.caller == "agy") - codex = next(cell for cell in self.manifest.matrix if cell.caller == "codex") - self.assertEqual(registry["agy"].preflight(agy).result.status, "ready") - self.assertEqual(registry["codex"].preflight(codex).result.status, "ready") - - agy_adapter = registry["agy"] - codex_adapter = registry["codex"] - agy_adapter._invokers = live_iop._InvokerSeams( # type: ignore[attr-defined] - live_iop._DEFAULT_INVOKERS.claude, - lambda *_args: None, - live_iop._DEFAULT_INVOKERS.codex, - ) - with mock.patch.object(live_iop, "build_agy_invocation", return_value=object()): - with self.assertRaises(live_iop.LiveIopError) as raised: - agy_adapter.invoke( - agy, object(), object(), "/tmp/control", b"task", - self.manifest.timeout, lambda *_args: None, - ) - self.assertEqual(raised.exception.issue_code, "stream_incompatible") - - failed_lifecycle = _invocation_result( - product_status="unknown", - harness_reason="nonzero_exit", - exit_code=1, - spec_digest_value="sha256:" + "9" * 64, - ) - agy_adapter._invokers = live_iop._InvokerSeams( # type: ignore[attr-defined] - live_iop._DEFAULT_INVOKERS.claude, - lambda *_args: failed_lifecycle, - live_iop._DEFAULT_INVOKERS.codex, - ) - with ( - mock.patch.object(live_iop, "build_agy_invocation", return_value=object()), - mock.patch.object(live_iop, "_bind_live_spec", side_effect=lambda *_args: _args[-1]), - ): - failed = agy_adapter.invoke( - agy, object(), object(), "/tmp/control", b"task", - self.manifest.timeout, lambda *_args: None, - ) - self.assertIs(failed, failed_lifecycle) - - lifecycle = _invocation_result() - invocation = CodexInvocation( - InvocationSpec( - argv=("codex",), cwd="/tmp", env=(), - submission_mode=SUBMISSION_STDIN_ONCE, - completion_mode=COMPLETION_EXIT_AFTER_IDLE, - timeout=self.manifest.timeout, evidence_dir="/tmp", - ), - None, # type: ignore[arg-type] - lambda line: line, - ) - observed_binding = {"value": None} - - def invoke_codex(*_args): - return CodexInvocationResult(lifecycle, observed_binding["value"]) - - codex_adapter._invokers = live_iop._InvokerSeams( # type: ignore[attr-defined] - live_iop._DEFAULT_INVOKERS.claude, - live_iop._DEFAULT_INVOKERS.agy, - invoke_codex, - ) - with ( - mock.patch.object(live_iop, "build_codex_invocation", return_value=invocation), - mock.patch.object(live_iop, "_bind_live_spec", side_effect=lambda *_args: _args[-1]), - ): - result = codex_adapter.invoke( - codex, object(), object(), "/tmp/control", b"task", - self.manifest.timeout, lambda *_args: None, - ) - self.assertIs(result, lifecycle) - - observed_binding["value"] = ( - codex.iop.route_kind, - "contradictory-route", - codex.iop.request_model, - codex.iop.requested_effort, - ) - with self.assertRaises(live_iop.LiveIopError) as raised: - codex_adapter.invoke( - codex, object(), object(), "/tmp/control", b"task", - self.manifest.timeout, lambda *_args: None, - ) - self.assertEqual(raised.exception.issue_code, "stream_incompatible") - - def test_live_agy_multiple_cells_consume_their_own_preflight_state(self) -> None: - manifest, _, _ = _write_manifest( - self.root, - [ - _cell("agy-direct", "agy", "gemini-3.6-flash", "high"), - _preset("agy-preset", "agy", "gemini-hybrid", "high"), - ], - output_id="agy-cell-state", - ) - environment = self._live_environment(manifest=manifest) - observed_models = tuple( - sorted(cell.iop.request_model for cell in manifest.matrix) - ) - ready_observation = lambda _runtime: live_iop._Observation( - observed_models, - "sha256:" + "2" * 64, - True, - "agy 1.1.12", - "--print --output-format --dangerously-skip-permissions --model --effort stream-json", - ) - adapter = live_iop.build_live_adapter_registry( - environment, - observer=ready_observation, - binary_resolver=lambda _name: "/bin/true", - )["agy"] - cells = {cell.id: cell for cell in manifest.matrix} - for cell in manifest.matrix: - self.assertEqual(adapter.preflight(cell).result.status, "ready") - - preflights = dict(adapter._agy_preflights) # type: ignore[attr-defined] - self.assertEqual(set(preflights), set(cells)) - self.assertIsNot(preflights["agy-direct"], preflights["agy-preset"]) - self.assertEqual( - preflights["agy-direct"].runtime.observation.cell_id, - "agy-direct", - ) - self.assertEqual( - preflights["agy-preset"].runtime.observation.cell_id, - "agy-preset", - ) - - lifecycle = _invocation_result( - started_at="2026-08-12T00:00:00+00:00", - ended_at="2026-08-12T00:00:01+00:00", - ) - built: list[tuple[str, object]] = [] - invoked: list[tuple[str, object]] = [] - - def build(cell, _prepared, _task, _timeout, preflight): - built.append((cell.id, preflight)) - return object() - - def invoke(_spec, parser, preflight, _on_started): - invoked.append((parser._cell.id, preflight)) - return lifecycle - - def observed_result(parser, capability, _result): - cell = parser._cell - admitted = adapter._admitted_bindings[cell.id] # type: ignore[attr-defined] - caller_capability = CallerCapability( - "agy", capability.route_kinds, capability.efforts - ) - return make_result(cell, caller_capability, admitted) - - adapter._invokers = live_iop._InvokerSeams( # type: ignore[attr-defined] - live_iop._DEFAULT_INVOKERS.claude, - invoke, - live_iop._DEFAULT_INVOKERS.codex, - ) - with ( - mock.patch.object(live_iop, "build_agy_invocation", side_effect=build), - mock.patch.object(live_iop, "_bind_live_spec", side_effect=lambda *_args: _args[-1]), - mock.patch.object( - live_iop.AgyEventParser, - "observed_result", - new=observed_result, - ), - ): - for cell_id in ("agy-preset", "agy-direct"): - cell = cells[cell_id] - result = adapter.invoke( - cell, - object(), - object(), - "/tmp/control", - b"task", - manifest.timeout, - lambda *_args: None, - ) - self.assertIs(result, lifecycle) - - self.assertEqual( - built, - [ - ("agy-preset", preflights["agy-preset"]), - ("agy-direct", preflights["agy-direct"]), - ], - ) - self.assertEqual(invoked, built) - - adapter._observer = lambda _runtime: live_iop._Observation( # type: ignore[attr-defined] - observed_models, - "sha256:" + "2" * 64, - False, - ) - self.assertEqual( - adapter.preflight(cells["agy-direct"]).result.status, - "implementation_gap", - ) - with self.assertRaises(live_iop.LiveIopError) as raised: - adapter.invoke( - cells["agy-direct"], - object(), - object(), - "/tmp/control", - b"task", - manifest.timeout, - lambda *_args: None, - ) - self.assertEqual(raised.exception.issue_code, "stream_incompatible") - - def test_cli_missing_live_input_fails_closed_without_secret_or_attempt(self) -> None: - sentinel = "missing-input-token" - stdout = io.StringIO() - stderr = io.StringIO() - with ( - mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root), - contextlib.redirect_stdout(stdout), - contextlib.redirect_stderr(stderr), - ): - exit_code = benchmark_cli.main(["run", "--manifest", str(self.path)]) - self.assertEqual(exit_code, 69) - self.assertEqual(stdout.getvalue(), "") - self.assertIn("error: preflight blocked", stderr.getvalue()) - self.assertNotIn(sentinel, stderr.getvalue()) - run_roots = list((self.root / self.manifest.output_root).glob("run-*")) - self.assertEqual(len(run_roots), 1) - self.assertFalse((run_roots[0] / "cells").exists()) - - def test_concurrent_writer_fails_fast_without_partial_record(self) -> None: - registry = self._registry() - observations = collect_preflight_observations(self.manifest, registry) - run = self.store.create(self.manifest, self.raw) - result: list[BaseException] = [] - - def append() -> None: - try: - self.store.record_preflight(run, self.manifest, observations) - except BaseException as exc: - result.append(exc) - - with self.store.writer(run): - worker = threading.Thread(target=append) - worker.start() - worker.join(5) - self.assertFalse(worker.is_alive()) - self.assertEqual(len(result), 1) - self.assertIsInstance(result[0], RunBusyError) - self.assertEqual(self.store.preflights(run, self.manifest), ()) - - def test_cli_fake_registry_reports_only_closed_summary(self) -> None: - sentinel = "private_endpoint_and_token_must_not_appear" - registry = self._registry( - {"codex-gpt-direct": ("credential_missing",)}, sentinel=sentinel - ) - stdout = io.StringIO() - stderr = io.StringIO() - with ( - mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root), - mock.patch.object(benchmark_cli, "build_adapter_registry", return_value=registry), - contextlib.redirect_stdout(stdout), - contextlib.redirect_stderr(stderr), - ): - exit_code = benchmark_cli.main( - ["preflight", "--manifest", str(self.path)] - ) - self.assertEqual(exit_code, 69) - self.assertEqual(stdout.getvalue(), "") - self.assertIn("status=registration_required", stderr.getvalue()) - self.assertIn("registration_required=1", stderr.getvalue()) - self.assertNotIn(sentinel, stderr.getvalue()) - run_roots = list((self.root / self.manifest.output_root).glob("run-*")) - self.assertEqual(len(run_roots), 1) - durable = b"".join( - path.read_bytes() for path in run_roots[0].rglob("*") if path.is_file() - ) - self.assertNotIn(sentinel.encode("ascii"), durable) - self.assertFalse((run_roots[0] / "cells").exists()) - - def test_cli_score_missing_run_fails_closed_with_run_id(self) -> None: - run_id = "run-20260811T010203Z-000000000000" - stdout = io.StringIO() - stderr = io.StringIO() - with ( - mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root), - contextlib.redirect_stdout(stdout), - contextlib.redirect_stderr(stderr), - ): - exit_code = benchmark_cli.main( - ["score", "--manifest", str(self.path), "--run-id", run_id] - ) - self.assertEqual(exit_code, 69) - self.assertEqual(stdout.getvalue(), "") - self.assertEqual( - stderr.getvalue(), - f"error: benchmark scoring is unavailable run_id={run_id}\n", - ) - - def test_cli_score_prints_only_closed_counts_and_forwards_retry(self) -> None: - run = self.store.create(self.manifest, self.raw) - cases = ( - (ScoringSummary(run.run_id, 2, 1, 0, 0), 0, "ok: score "), - (ScoringSummary(run.run_id, 0, 1, 1, 2), 69, "error: benchmark scoring failed "), - ) - for summary, expected_exit, prefix in cases: - with self.subTest(summary=summary): - stdout = io.StringIO() - stderr = io.StringIO() - with ( - mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root), - mock.patch.object( - benchmark_cli, "build_live_scoring_adapter", - return_value=object(), - ), - mock.patch.object( - benchmark_cli, "score_run", return_value=summary - ) as score, - contextlib.redirect_stdout(stdout), - contextlib.redirect_stderr(stderr), - ): - exit_code = benchmark_cli.main( - [ - "score", "--manifest", str(self.path), - "--run-id", run.run_id, "--retry-scoring-failed", - ] - ) - self.assertEqual(exit_code, expected_exit) - rendered = stdout.getvalue() or stderr.getvalue() - self.assertEqual( - rendered, - prefix - + f"run_id={run.run_id} scored={summary.scored} " - + f"unscored={summary.unscored} " - + f"scoring_failed={summary.scoring_failed} " - + f"blocked={summary.blocked}\n", - ) - score.assert_called_once() - self.assertTrue(score.call_args.kwargs["retry_scoring_failed"]) - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/agent_benchmark/connectivity_test.py b/scripts/agent_benchmark/connectivity_test.py deleted file mode 100644 index c7458b15..00000000 --- a/scripts/agent_benchmark/connectivity_test.py +++ /dev/null @@ -1,600 +0,0 @@ -"""Deterministic tests for the closed benchmark connectivity contract.""" - -from __future__ import annotations - -import copy -import dataclasses -import hashlib -import json -import os -import tempfile -import unittest -from dataclasses import FrozenInstanceError -from pathlib import Path - -from scripts.agent_benchmark.connectivity import ( - ISSUE_RESUME_CODES, - MAX_EVIDENCE_BYTES, - CallerCapability, - ConnectivityEvidenceError, - ConnectivityIssue, - ConnectivityResult, - ConnectivityValidationError, - EffectiveBinding, - RequestedEffectiveBinding, - canonical_evidence_bytes, - classify_issues, - make_result, - read_evidence, - validate_binding, - validate_result, - write_evidence, -) -from scripts.agent_benchmark.manifest import ExpectedBinding, IopCell, MatrixCell - -EFFECTIVE_SCALARS = ( - "effective_route_kind", - "effective_route_id", - "effective_model", - "effective_effort", -) - - -def _identity(label: str) -> str: - return "sha256:" + hashlib.sha256(label.encode("ascii")).hexdigest() - - -def _cell(route_kind: str = "direct") -> MatrixCell: - if route_kind == "direct": - bindings = (ExpectedBinding("request", "gpt", "xhigh"),) - else: - bindings = ( - ExpectedBinding("selector", "gpt", "xhigh"), - ExpectedBinding("plan", "gpt", "high"), - ExpectedBinding("work", "gpt", None), - ExpectedBinding("review", "gpt", "high"), - ) - return MatrixCell("cell", "codex", IopCell("gpt", "xhigh", route_kind, "route", bindings)) - - -def _capability() -> CallerCapability: - return CallerCapability("codex", ("direct", "execution_preset"), ("high", "xhigh")) - - -def _requested_only(cell: MatrixCell) -> RequestedEffectiveBinding: - """Blocked observation: requested identity only, no effective group at all.""" - return RequestedEffectiveBinding( - cell.id, cell.caller, cell.iop.route_kind, cell.iop.route_id, - cell.iop.request_model, cell.iop.requested_effort, - ) - - -def _binding(cell: MatrixCell) -> RequestedEffectiveBinding: - return RequestedEffectiveBinding( - cell.id, cell.caller, cell.iop.route_kind, cell.iop.route_id, - cell.iop.request_model, cell.iop.requested_effort, - cell.iop.route_kind, cell.iop.route_id, - cell.iop.request_model, cell.iop.requested_effort, - tuple(EffectiveBinding(item.stage, item.model, item.effort) for item in cell.iop.expected_bindings), - ) - - -def _issue(code: str) -> ConnectivityIssue: - return ConnectivityIssue(code, ISSUE_RESUME_CODES[code]) - - -class ConnectivityContractTest(unittest.TestCase): - def test_direct_and_preset_exact_contracts_are_frozen(self): - for route_kind in ("direct", "execution_preset"): - cell = _cell(route_kind) - result = make_result(cell, _capability(), _binding(cell)) - self.assertEqual(result.status, "ready") - self.assertEqual(result.binding.effective_bindings, tuple( - EffectiveBinding(item.stage, item.model, item.effort) - for item in cell.iop.expected_bindings - )) - with self.assertRaises(FrozenInstanceError): - result.status = "implementation_gap" # type: ignore[misc] - - def test_every_requested_or_effective_substitution_fails_closed(self): - cell = _cell() - binding = _binding(cell) - substitutions = ( - {"cell_id": "other"}, {"caller": "agy"}, - {"requested_route_kind": "execution_preset"}, {"requested_route_id": "other"}, - {"requested_model": "alias"}, {"requested_effort": "high"}, - {"effective_route_kind": "execution_preset"}, {"effective_route_id": "other"}, - {"effective_model": "alias"}, {"effective_effort": "high"}, - ) - for replacement in substitutions: - with self.subTest(replacement=replacement): - mutated = RequestedEffectiveBinding(**{**binding.__dict__, **replacement}) - with self.assertRaises(ConnectivityValidationError): - validate_binding(cell, _capability(), mutated) - - def test_missing_extra_and_reordered_stage_bindings_fail_closed(self): - cell = _cell("execution_preset") - binding = _binding(cell) - cases = ( - binding.effective_bindings[:-1], - binding.effective_bindings + (EffectiveBinding("repair", "gpt", "high"),), - tuple(reversed(binding.effective_bindings)), - binding.effective_bindings[:-1] + (EffectiveBinding("review", "alias", "high"),), - ) - for stages in cases: - with self.subTest(stages=stages): - observed = RequestedEffectiveBinding(**{**binding.__dict__, "effective_bindings": stages}) - with self.assertRaises(ConnectivityValidationError): - validate_binding(cell, _capability(), observed) - - def test_capability_is_closed_and_requires_requested_effort(self): - cell = _cell() - with self.assertRaises(ConnectivityValidationError): - validate_binding(cell, CallerCapability("codex", ("direct",), ("high",)), _binding(cell)) - with self.assertRaises(ConnectivityValidationError): - validate_binding(cell, CallerCapability("codex", ("execution_preset", "direct"), ("high", "xhigh")), _binding(cell)) - - def test_malformed_capability_entries_raise_closed_error(self): - cell = _cell() - binding = _binding(cell) - capabilities = ( - CallerCapability("codex", ("unknown", "direct"), ("high", "xhigh")), - CallerCapability("codex", ("direct", ["execution_preset"]), ("high", "xhigh")), - CallerCapability("codex", ("direct", "direct"), ("high", "xhigh")), - CallerCapability("codex", (), ("high", "xhigh")), - CallerCapability("codex", ["direct"], ("high", "xhigh")), - CallerCapability("codex", ("direct",), ("high", 3)), - CallerCapability("codex", ("direct",), (["xhigh"],)), - CallerCapability("codex", ("direct",), ("xhigh", "high")), - CallerCapability("codex", ("direct",), ("Xhigh",)), - CallerCapability("unknown", ("direct",), ("xhigh",)), - "codex", - ) - for capability in capabilities: - with self.subTest(capability=repr(capability)): - with self.assertRaises(ConnectivityValidationError) as caught: - validate_binding(cell, capability, binding) - self.assertNotIsInstance(caught.exception, (ValueError, TypeError, KeyError)) - - def test_classifier_is_closed_and_implementation_gap_has_precedence(self): - registration = (_issue("credential_missing"),) - implementation = (_issue("stream_incompatible"),) - self.assertEqual(classify_issues(()), "ready") - self.assertEqual(classify_issues(registration), "registration_required") - self.assertEqual(classify_issues(implementation), "implementation_gap") - self.assertEqual(classify_issues(registration + implementation), "implementation_gap") - with self.assertRaises(ConnectivityValidationError): - classify_issues((ConnectivityIssue("unknown", "register_credential"),)) - with self.assertRaises(ConnectivityValidationError): - classify_issues([_issue("credential_missing")]) - - def test_issue_resume_pairs_are_closed_and_canonically_ordered(self): - for code, resume_code in ISSUE_RESUME_CODES.items(): - with self.subTest(code=code): - self.assertIn( - classify_issues((ConnectivityIssue(code, resume_code),)), - ("registration_required", "implementation_gap"), - ) - for other_code, other_resume in ISSUE_RESUME_CODES.items(): - if other_code == code: - continue - with self.assertRaises(ConnectivityValidationError): - classify_issues((ConnectivityIssue(code, other_resume),)) - for text in ("register credential", "sk-live-0000", "", "register_credential ", None): - with self.assertRaises(ConnectivityValidationError): - classify_issues((ConnectivityIssue(code, text),)) - canonical = (_issue("model_missing"), _issue("stream_incompatible")) - self.assertEqual(classify_issues(canonical), "implementation_gap") - with self.assertRaises(ConnectivityValidationError): - classify_issues(tuple(reversed(canonical))) - with self.assertRaises(ConnectivityValidationError): - classify_issues(canonical + canonical[:1]) - - def test_malformed_issue_entries_raise_closed_error(self): - cell = _cell() - capability = _capability() - binding = _binding(cell) - valid_resume = ISSUE_RESUME_CODES["credential_missing"] - other_resume = ISSUE_RESUME_CODES["model_missing"] - bad_code_entries = ( - ConnectivityIssue(["credential_missing"], valid_resume), - ConnectivityIssue({"code": "credential_missing"}, valid_resume), - ConnectivityIssue(42, valid_resume), - ConnectivityIssue(None, valid_resume), - ConnectivityIssue("unknown", valid_resume), - ) - bad_resume_entries = ( - ConnectivityIssue("credential_missing", ["register_credential"]), - ConnectivityIssue("credential_missing", {"resume_code": "register_credential"}), - ConnectivityIssue("credential_missing", 42), - ConnectivityIssue("credential_missing", None), - ConnectivityIssue("credential_missing", "register credential"), - ConnectivityIssue("credential_missing", "sk-live-0000"), - ConnectivityIssue("credential_missing", ""), - ConnectivityIssue("credential_missing", other_resume), - ) - non_issue_objects = ( - {"code": "credential_missing", "resume_code": valid_resume}, - "credential_missing", - 42, - None, - ("credential_missing", valid_resume), - ) - expected_message = { - "code": "invalid issue code", - "resume": "invalid issue resume_code", - "object": "invalid issue", - } - forbidden_raw_values = ( - "sk-live-0000", "register credential", "credential_missing", - "register_credential", "register_model", "42", "None", - ) - labelled = ( - *(("code", item) for item in bad_code_entries), - *(("resume", item) for item in bad_resume_entries), - *(("object", item) for item in non_issue_objects), - ) - for label, entry in labelled: - with self.subTest(label=label, entry=repr(entry)): - issues = (entry,) - with self.assertRaises(ConnectivityValidationError) as classifier_caught: - classify_issues(issues) - self.assertNotIsInstance( - classifier_caught.exception, - (TypeError, KeyError, ValueError, AttributeError), - ) - self.assertEqual(str(classifier_caught.exception), expected_message[label]) - for token in forbidden_raw_values: - self.assertNotIn(token, str(classifier_caught.exception)) - self.assertNotIn(repr(entry), str(classifier_caught.exception)) - with self.assertRaises(ConnectivityValidationError) as result_caught: - make_result(cell, capability, binding, issues) - self.assertNotIsInstance( - result_caught.exception, - (TypeError, KeyError, ValueError, AttributeError), - ) - self.assertEqual(str(result_caught.exception), expected_message[label]) - for token in forbidden_raw_values: - self.assertNotIn(token, str(result_caught.exception)) - self.assertNotIn(repr(entry), str(result_caught.exception)) - - def test_ready_requires_complete_exact_effective_observation(self): - cell = _cell("execution_preset") - binding = _binding(cell) - with self.assertRaises(ConnectivityValidationError): - make_result(cell, _capability(), _requested_only(cell)) - partials = [{name: None} for name in EFFECTIVE_SCALARS] - partials.append({"effective_bindings": ()}) - partials.append({name: None for name in EFFECTIVE_SCALARS}) - partials.append({"effective_model": "alias"}) - partials.append({"effective_bindings": binding.effective_bindings[:-1]}) - for replacement in partials: - with self.subTest(replacement=replacement): - mutated = RequestedEffectiveBinding(**{**binding.__dict__, **replacement}) - with self.assertRaises(ConnectivityValidationError): - make_result(cell, _capability(), mutated) - with self.assertRaises(ConnectivityValidationError): - make_result(cell, _capability(), mutated, (_issue("model_missing"),)) - - def test_blocked_results_stay_blocked_and_cannot_be_forged_ready(self): - cell = _cell() - issues = (_issue("credential_missing"),) - blocked = make_result(cell, _capability(), _requested_only(cell), issues) - self.assertEqual(blocked.status, "registration_required") - self.assertIsNone(blocked.binding.effective_model) - self.assertEqual(blocked.binding.effective_bindings, ()) - validate_result(cell, blocked) - forgeries = ( - ConnectivityResult(_capability(), _requested_only(cell), issues, "ready"), - ConnectivityResult(_capability(), _binding(cell), issues, "ready"), - ConnectivityResult(_capability(), _binding(cell), (), "blocked"), - ConnectivityResult(_capability(), _requested_only(cell), (), "ready"), - ) - for forged in forgeries: - with self.subTest(status=forged.status, issues=len(forged.issues)): - with self.assertRaises(ConnectivityValidationError): - validate_result(cell, forged) - - def test_no_contract_field_accepts_opaque_caller_text(self): - self.assertEqual( - {field.name for field in dataclasses.fields(ConnectivityIssue)}, - {"code", "resume_code"}, - ) - cell = _cell() - binding = _binding(cell) - opaque = "sk-live-0000000000000000" - secret = "Authorization: Bearer should-not-appear" - for text in (secret, opaque, "https://private.invalid"): - with self.subTest(text=text): - with self.assertRaises(ConnectivityValidationError) as caught: - make_result(cell, _capability(), binding, (ConnectivityIssue("credential_missing", text),)) - self.assertNotIn(text, str(caught.exception)) - for replacement in ( - {"requested_model": opaque}, {"effective_model": opaque}, - {"effective_route_id": "private-endpoint"}, - ): - with self.subTest(replacement=replacement): - mutated = RequestedEffectiveBinding(**{**binding.__dict__, **replacement}) - with self.assertRaises(ConnectivityValidationError) as caught: - make_result(cell, _capability(), mutated) - self.assertNotIn(opaque, str(caught.exception)) - self.assertNotIn("private-endpoint", str(caught.exception)) - - -class ConnectivityEvidenceTest(unittest.TestCase): - def setUp(self): - self.temp = tempfile.TemporaryDirectory(dir="/tmp", prefix="connectivity-") - self.base = Path(os.path.realpath(self.temp.name)) - self.root = self.base / "evidence" - self.root.mkdir() - self.cell = _cell() - self.result = make_result(self.cell, _capability(), _binding(self.cell)) - self.endpoint_identity = _identity("endpoint") - self.config_identity = _identity("config") - - def tearDown(self): - self.temp.cleanup() - - def _write(self, root, relative_path, cell=None, result=None): - write_evidence( - root, relative_path, cell or self.cell, result or self.result, - self.endpoint_identity, self.config_identity, - ) - - def test_canonical_evidence_is_deterministic_and_secret_safe(self): - one = canonical_evidence_bytes(self.cell, self.result, self.endpoint_identity, self.config_identity) - two = canonical_evidence_bytes(self.cell, self.result, self.endpoint_identity, self.config_identity) - self.assertEqual(one, two) - self.assertIn(self.endpoint_identity.encode(), one) - for forbidden in (b"Authorization", b"Bearer", b"https://", b"prompt", b"tool"): - self.assertNotIn(forbidden, one) - - def test_blocked_results_omit_effective_observations_and_round_trip(self): - cases = ( - ("direct", _issue("model_missing"), "registration_required"), - ("execution_preset", _issue("endpoint_incompatible"), "implementation_gap"), - ) - for index, (route_kind, issue, status) in enumerate(cases): - with self.subTest(route_kind=route_kind): - cell = _cell(route_kind) - result = make_result(cell, _capability(), _requested_only(cell), (issue,)) - self.assertEqual(result.status, status) - raw = canonical_evidence_bytes(cell, result, self.endpoint_identity, self.config_identity) - for absent in EFFECTIVE_SCALARS: - self.assertIn(f'"{absent}":null'.encode("ascii"), raw) - self.assertIn(b'"effective_bindings":[]', raw) - name = f"blocked-{index}.json" - self._write(self.root, name, cell, result) - parsed = read_evidence(self.root, name, cell) - self.assertEqual(parsed["status"], status) - self.assertIsNone(parsed["binding"]["effective_model"]) - self.assertEqual(parsed["binding"]["effective_bindings"], []) - self.assertEqual( - parsed["issues"], [{"code": issue.code, "resume_code": issue.resume_code}] - ) - forged = ( - (self.root / name).read_text(encoding="ascii") - .replace(f'"status":"{status}"', '"status":"ready"') - .replace(f'{{"code":"{issue.code}","resume_code":"{issue.resume_code}"}}', "") - ) - (self.root / f"forged-{index}.json").write_text(forged, encoding="ascii") - with self.assertRaises(ConnectivityEvidenceError): - read_evidence(self.root, f"forged-{index}.json", cell) - - def test_write_is_no_overwrite_and_read_rejects_corruption(self): - self._write(self.root, "nested/preflight.json") - raw = (self.root / "nested/preflight.json").read_bytes() - self.assertEqual(read_evidence(self.root, "nested/preflight.json", self.cell)["status"], "ready") - with self.assertRaises(ConnectivityEvidenceError): - self._write(self.root, "nested/preflight.json") - (self.root / "nested/preflight.json").write_bytes(raw + b" ") - with self.assertRaises(ConnectivityEvidenceError): - read_evidence(self.root, "nested/preflight.json", self.cell) - - def test_reader_rejects_canonical_schema_drift(self): - self._write(self.root, "preflight.json") - raw = (self.root / "preflight.json").read_text(encoding="ascii") - (self.root / "preflight.json").write_text(raw.replace('"status":"ready"', '"status":"registration_required"'), encoding="ascii") - with self.assertRaises(ConnectivityEvidenceError): - read_evidence(self.root, "preflight.json", self.cell) - - def test_reader_rejects_canonical_binding_semantic_substitution(self): - def _reseat(parsed: dict) -> bytes: - return ( - json.dumps( - parsed, sort_keys=True, separators=(",", ":"), ensure_ascii=True - ).encode("ascii") - + b"\n" - ) - - def _write_mutant(cell, result, name, mutate): - self._write(self.root, f"{name}.json", cell, result) - baseline = read_evidence(self.root, f"{name}.json", cell) - self.assertEqual(baseline["status"], result.status) - original = (self.root / f"{name}.json").read_text(encoding="ascii") - parsed = json.loads(original) - mutant = copy.deepcopy(parsed) - mutate(mutant) - mutant_name = f"mutant-{name}.json" - (self.root / mutant_name).write_bytes(_reseat(mutant)) - return mutant_name - - cases: list[tuple] = [] - - direct_cell = _cell("direct") - direct_result = make_result(direct_cell, _capability(), _binding(direct_cell)) - direct_scalars = ( - ("requested_route_kind", "execution_preset"), - ("requested_route_id", "route2"), - ("requested_model", "gpth"), - ("requested_effort", "high"), - ("effective_route_kind", "execution_preset"), - ("effective_route_id", "route2"), - ("effective_model", "gpth"), - ("effective_effort", "high"), - ) - for field, value in direct_scalars: - cases.append(( - "direct-ready-scalar", direct_cell, direct_result, f"direct-{field}", - lambda p, f=field, v=value: p["binding"].__setitem__(f, v), - )) - cases.append(( - "direct-ready-stage", direct_cell, direct_result, "direct-stage", - lambda p: p["binding"]["effective_bindings"].__setitem__( - 0, {"stage": "request", "model": "gpth", "effort": "xhigh"} - ), - )) - - preset_cell = _cell("execution_preset") - preset_result = make_result(preset_cell, _capability(), _binding(preset_cell)) - preset_scalars = ( - ("requested_route_kind", "direct"), - ("requested_route_id", "route2"), - ("requested_model", "gpth"), - ("requested_effort", "high"), - ("effective_route_kind", "direct"), - ("effective_route_id", "route2"), - ("effective_model", "gpth"), - ("effective_effort", "high"), - ) - for field, value in preset_scalars: - cases.append(( - "preset-ready-scalar", preset_cell, preset_result, f"preset-{field}", - lambda p, f=field, v=value: p["binding"].__setitem__(f, v), - )) - cases.append(( - "preset-ready-stage-model", preset_cell, preset_result, "preset-stage-model", - lambda p: p["binding"]["effective_bindings"].__setitem__( - 0, {"stage": "selector", "model": "gpth", "effort": "xhigh"} - ), - )) - cases.append(( - "preset-ready-stage-effort", preset_cell, preset_result, "preset-stage-effort", - lambda p: p["binding"]["effective_bindings"].__setitem__( - 1, {"stage": "plan", "model": "gpt", "effort": "xhigh"} - ), - )) - cases.append(( - "preset-ready-stage-set", preset_cell, preset_result, "preset-stage-set", - lambda p: p["binding"]["effective_bindings"].__setitem__( - 0, {"stage": "request", "model": "gpt", "effort": "xhigh"} - ), - )) - cases.append(( - "preset-ready-stage-order", preset_cell, preset_result, "preset-stage-order", - lambda p: p["binding"]["effective_bindings"].reverse(), - )) - cases.append(( - "preset-ready-stage-extra", preset_cell, preset_result, "preset-stage-extra", - lambda p: p["binding"]["effective_bindings"].append( - {"stage": "repair", "model": "gpt", "effort": "high"} - ), - )) - cases.append(( - "preset-ready-stage-missing", preset_cell, preset_result, "preset-stage-missing", - lambda p: p["binding"]["effective_bindings"].pop(), - )) - - blocked_cell = _cell("execution_preset") - blocked_result = make_result( - blocked_cell, _capability(), _binding(blocked_cell), (_issue("credential_missing"),) - ) - self.assertEqual(blocked_result.status, "registration_required") - cases.append(( - "blocked-observed-stage-model", blocked_cell, blocked_result, "blocked-stage-model", - lambda p: p["binding"]["effective_bindings"].__setitem__( - 0, {"stage": "selector", "model": "gpth", "effort": "xhigh"} - ), - )) - cases.append(( - "blocked-observed-stage-order", blocked_cell, blocked_result, "blocked-stage-order", - lambda p: p["binding"]["effective_bindings"].reverse(), - )) - - for label, cell, result, name, mutate in cases: - with self.subTest(label=label, name=name): - mutant_path = _write_mutant(cell, result, name, mutate) - with self.assertRaises(ConnectivityEvidenceError): - read_evidence(self.root, mutant_path, cell) - - def test_reader_rejects_noncanonical_issue_order(self): - cell = _cell() - issues = (_issue("model_missing"), _issue("stream_incompatible")) - result = make_result(cell, _capability(), _requested_only(cell), issues) - self._write(self.root, "ordered.json", cell, result) - raw = (self.root / "ordered.json").read_text(encoding="ascii") - first = '{"code":"model_missing","resume_code":"register_model"}' - second = '{"code":"stream_incompatible","resume_code":"implement_stream_adapter"}' - self.assertIn(f"{first},{second}", raw) - (self.root / "swapped.json").write_text(raw.replace(f"{first},{second}", f"{second},{first}"), encoding="ascii") - with self.assertRaises(ConnectivityEvidenceError): - read_evidence(self.root, "swapped.json", cell) - with self.assertRaises(ConnectivityValidationError): - make_result(cell, _capability(), _requested_only(cell), tuple(reversed(issues))) - - def test_symlinked_roots_parents_and_targets_are_rejected(self): - outside = self.base / "outside" - outside.mkdir() - real_root = self.base / "real" / "evidence" - real_root.mkdir(parents=True) - (self.base / "link").symlink_to(self.base / "real", target_is_directory=True) - with self.assertRaises(ConnectivityEvidenceError): - self._write(self.base / "link" / "evidence", "preflight.json") - with self.assertRaises(ConnectivityEvidenceError): - read_evidence(self.base / "link" / "evidence", "preflight.json", self.cell) - self.assertFalse((real_root / "preflight.json").exists()) - self._write(real_root, "preflight.json") - self.assertEqual(read_evidence(real_root, "preflight.json", self.cell)["status"], "ready") - - (self.base / "root-link").symlink_to(self.root, target_is_directory=True) - with self.assertRaises(ConnectivityEvidenceError): - self._write(self.base / "root-link", "preflight.json") - - (self.root / "nested").symlink_to(outside, target_is_directory=True) - with self.assertRaises(ConnectivityEvidenceError): - self._write(self.root, "nested/preflight.json") - self.assertFalse((outside / "preflight.json").exists()) - - (self.root / "link.json").symlink_to(outside / "escape.json") - with self.assertRaises(ConnectivityEvidenceError): - self._write(self.root, "link.json") - with self.assertRaises(ConnectivityEvidenceError): - read_evidence(self.root, "link.json", self.cell) - self.assertFalse((outside / "escape.json").exists()) - - def test_oversized_directory_and_non_regular_targets_are_rejected(self): - self._write(self.root, "small.json") - self.assertEqual(read_evidence(self.root, "small.json", self.cell)["status"], "ready") - (self.root / "big.json").write_bytes(b"{" + b" " * (MAX_EVIDENCE_BYTES + 16) + b"}") - with self.assertRaises(ConnectivityEvidenceError): - read_evidence(self.root, "big.json", self.cell) - (self.root / "dir.json").mkdir() - with self.assertRaises(ConnectivityEvidenceError): - read_evidence(self.root, "dir.json", self.cell) - with self.assertRaises(ConnectivityEvidenceError): - self._write(self.root, "dir.json") - if hasattr(os, "mkfifo"): - os.mkfifo(self.root / "fifo.json") - with self.assertRaises(ConnectivityEvidenceError): - read_evidence(self.root, "fifo.json", self.cell) - with self.assertRaises(ConnectivityEvidenceError): - self._write(self.root, "fifo.json") - - def test_private_identity_and_escaping_paths_are_rejected(self): - for identity in ("https://private.invalid", "sha256:" + "z" * 64, "", "sha256:abcd", None): - with self.subTest(identity=identity): - with self.assertRaises(ConnectivityEvidenceError): - canonical_evidence_bytes(self.cell, self.result, identity, self.config_identity) - with self.assertRaises(ConnectivityEvidenceError): - canonical_evidence_bytes(self.cell, self.result, self.endpoint_identity, identity) - for relative in ("../escape.json", "/abs.json", "nested/../escape.json", "preflight.txt", "preflight", "", ".json"): - with self.subTest(relative=relative): - with self.assertRaises(ConnectivityEvidenceError): - self._write(self.root, relative) - with self.assertRaises(ConnectivityEvidenceError): - read_evidence(self.root, relative, self.cell) - self.assertFalse((self.base / "escape.json").exists()) - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/agent_benchmark/lifecycle.py b/scripts/agent_benchmark/lifecycle.py deleted file mode 100644 index ef6f44a8..00000000 --- a/scripts/agent_benchmark/lifecycle.py +++ /dev/null @@ -1,2431 +0,0 @@ -#!/usr/bin/env python3 -""" -lifecycle.py - Generic bounded caller invocation lifecycle. - -Owns exactly one caller invocation with exactly one harness-owned task -submission, normalized finish/idle terminal evidence, bounded and redacted -output capture, closed completion policies, single-owner terminal -arbitration, and verified owned-process-group cleanup on every return path. - -The controller never launches the caller directly. It launches an internal -supervisor (this module in supervisor mode) in a new POSIX session, receives a -durable authenticated locator, commits it through ``on_started`` and only then -authorizes the caller launch. This module encodes no caller-specific command -line or protocol; adapters inject an event parser and a redactor. -""" - -from __future__ import annotations - -import ctypes -import datetime -import errno -import hashlib -import hmac -import json -import os -import queue -import re -import secrets -import shutil -import signal -import socket -import stat -import subprocess -import sys -import tempfile -import threading -import time -from dataclasses import dataclass -from decimal import Decimal, InvalidOperation -from pathlib import Path -from typing import Any, Callable, Mapping, Optional - -from scripts.agent_benchmark.manifest import Timeout - -# --------------------------------------------------------------------------- -# Closed vocabularies -# --------------------------------------------------------------------------- - -SUBMISSION_ARGV_TASK = "argv_task" -SUBMISSION_STDIN_ONCE = "stdin_once" -SUBMISSION_MODES = (SUBMISSION_ARGV_TASK, SUBMISSION_STDIN_ONCE) - -COMPLETION_EXIT_AFTER_IDLE = "exit_after_idle" -COMPLETION_STOP_AFTER_IDLE = "stop_after_idle" -COMPLETION_MODES = (COMPLETION_EXIT_AFTER_IDLE, COMPLETION_STOP_AFTER_IDLE) - -EVENT_SUBMITTED = "submitted" -EVENT_FIRST_OUTPUT = "first_output" -EVENT_FINISH = "finish" -EVENT_IDLE = "idle" -EVENT_QUIET = "quiet" -EVENT_EXITED = "exited" -EVENT_TERMINAL = "terminal" -EVENT_CALLER_TERMINAL = "caller_terminal" -PARSER_TERMINAL_KINDS = (EVENT_FINISH, EVENT_IDLE) -METRIC_PREFIX = "metric:" - -SOURCE_HARNESS = "harness" -SOURCE_CALLER_OUTPUT = "caller_output" -SOURCE_WORKSPACE_POLL = "workspace_poll" -METRIC_SOURCES = (SOURCE_HARNESS, SOURCE_CALLER_OUTPUT, SOURCE_WORKSPACE_POLL) - -# A metric value is meaningless without the clock that produced it. Counts are -# not temporal at all, so they carry the explicit ``none`` clock rather than an -# implied one, and no value from one clock is ever compared with another. -CLOCK_NONE = "none" -CLOCK_HARNESS_MONOTONIC = "harness_monotonic" -CLOCK_CALLER_REPORTED = "caller_reported" -CLOCK_FILESYSTEM_MTIME = "filesystem_mtime" -METRIC_CLOCKS = ( - CLOCK_NONE, CLOCK_HARNESS_MONOTONIC, CLOCK_CALLER_REPORTED, CLOCK_FILESYSTEM_MTIME, -) -TEMPORAL_CLOCKS = ( - CLOCK_HARNESS_MONOTONIC, CLOCK_CALLER_REPORTED, CLOCK_FILESYSTEM_MTIME, -) - -UNIT_NANOSECONDS = "ns" -UNIT_CALLS = "calls" -UNIT_TOKENS = "tokens" - -# The closed metric vocabulary. A name that is absent here can never become -# durable evidence, and each name owns exactly one unit. -METRIC_UNITS = { - "queue_duration": UNIT_NANOSECONDS, - "model_duration": UNIT_NANOSECONDS, - "tool_duration": UNIT_NANOSECONDS, - "total_duration": UNIT_NANOSECONDS, - "model_calls": UNIT_CALLS, - "tool_calls": UNIT_CALLS, - "input_tokens": UNIT_TOKENS, - "cached_input_tokens": UNIT_TOKENS, - "cache_write_tokens": UNIT_TOKENS, - "output_tokens": UNIT_TOKENS, - "reasoning_tokens": UNIT_TOKENS, - "total_tokens": UNIT_TOKENS, -} -METRIC_NAMES = tuple(sorted(METRIC_UNITS)) -DURATION_SCALES_NS = {"s": 10 ** 9, "ms": 10 ** 6, "us": 10 ** 3, "ns": 1} - -REASON_SUCCESS = "success" -REASON_START_CALLBACK_FAILED = "start_callback_failed" -REASON_LAUNCH_FAILED = "launch_failed" -REASON_NONZERO_EXIT = "nonzero_exit" -REASON_MISSING_IDLE = "missing_idle" -REASON_DUPLICATE_EVENT = "duplicate_event" -REASON_OUT_OF_ORDER_EVENT = "out_of_order_event" -REASON_MALFORMED_EVENT = "malformed_event" -REASON_PARSER_ERROR = "parser_error" -REASON_READER_ERROR = "reader_error" -REASON_TIMED_OUT = "timed_out" -REASON_CANCELLED = "cancelled" -REASON_CONTROLLER_LOST = "controller_lost" -REASON_RECOVERED_STOP = "recovered_stop" -REASON_CLEANUP_FAILED = "cleanup_failed" -REASON_SUPERVISOR_ERROR = "supervisor_error" -REASON_INTERRUPTED = "interrupted" -TERMINAL_REASONS = ( - REASON_SUCCESS, - REASON_START_CALLBACK_FAILED, - REASON_LAUNCH_FAILED, - REASON_NONZERO_EXIT, - REASON_MISSING_IDLE, - REASON_DUPLICATE_EVENT, - REASON_OUT_OF_ORDER_EVENT, - REASON_MALFORMED_EVENT, - REASON_PARSER_ERROR, - REASON_READER_ERROR, - REASON_TIMED_OUT, - REASON_CANCELLED, - REASON_CONTROLLER_LOST, - REASON_RECOVERED_STOP, - REASON_CLEANUP_FAILED, - REASON_SUPERVISOR_ERROR, - REASON_INTERRUPTED, -) - -CALLER_STATUS_SUCCEEDED = "succeeded" -CALLER_STATUS_FAILED = "failed" -CALLER_STATUSES = (CALLER_STATUS_SUCCEEDED, CALLER_STATUS_FAILED) -CALLER_REASON_SUCCESS = "caller_success" -CALLER_REASON_ERROR = "caller_error" -CALLER_REASONS = (CALLER_REASON_SUCCESS, CALLER_REASON_ERROR) - -PRODUCT_STATUS_SUCCEEDED = "succeeded" -PRODUCT_STATUS_FAILED = "failed" -PRODUCT_STATUS_UNKNOWN = "unknown" -PRODUCT_STATUSES = ( - PRODUCT_STATUS_SUCCEEDED, PRODUCT_STATUS_FAILED, PRODUCT_STATUS_UNKNOWN, -) -PRODUCT_REASON_UNAVAILABLE = "unavailable" -PRODUCT_REASONS = ( - CALLER_REASON_SUCCESS, CALLER_REASON_ERROR, PRODUCT_REASON_UNAVAILABLE, -) - -HARNESS_STATUS_PASSED = "passed" -HARNESS_STATUS_FAILED = "failed" -HARNESS_STATUSES = (HARNESS_STATUS_PASSED, HARNESS_STATUS_FAILED) -HARNESS_REASONS = TERMINAL_REASONS - -PROCESS_STATUS_EXITED = "exited" -PROCESS_STATUS_SIGNALLED = "signalled" -PROCESS_STATUS_TIMED_OUT = "timed_out" -PROCESS_STATUS_CANCELLED = "cancelled" -PROCESS_STATUS_NOT_STARTED = "not_started" -PROCESS_STATUSES = ( - PROCESS_STATUS_EXITED, - PROCESS_STATUS_SIGNALLED, - PROCESS_STATUS_TIMED_OUT, - PROCESS_STATUS_CANCELLED, - PROCESS_STATUS_NOT_STARTED, -) - -FAULT_NONE = "" -FAULT_READER_ERROR = "reader_error" -FAULT_MODES = (FAULT_NONE, FAULT_READER_ERROR) - -DEFAULT_ENV_ALLOWLIST = ( - "PATH", "HOME", "LANG", "LC_ALL", "LC_CTYPE", "TZ", "TERM", "TMPDIR", - "USER", "LOGNAME", "SHELL", "PWD", "PYTHONPATH", "PYTHONHASHSEED", - "NO_COLOR", "CI", -) -TLS_CA_ENV_KEYS = ("SSL_CERT_FILE", "NODE_EXTRA_CA_CERTS") - -ENV_KEY_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]{0,63}$") -METRIC_KIND_RE = re.compile(r"^metric:[a-z0-9][a-z0-9_.+-]{0,63}$") -SAFE_LABEL_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.:+-]{0,63}$") - -JOURNAL_FILENAME = "lifecycle-journal.jsonl" -RESULT_FILENAME = "lifecycle-result.json" -LOCATOR_FILENAME = "locator.json" -RECEIPT_FILENAME = "cleanup-receipt.json" -SOCKET_FILENAME = "control.sock" -SUPERVISOR_ERR_FILENAME = "supervisor.err" - -REDACTED = "[redacted]" -RECEIPT_VERSION = 1 -JOURNAL_VERSION = 2 - -MAX_TASK_PAYLOAD_BYTES = 1 << 20 -MAX_CAPTURE_BYTES_LIMIT = 1 << 24 -MAX_CAPTURE_LINES_LIMIT = 1 << 20 -MAX_EVENT_DETAIL_CHARS = 512 -MAX_METRIC_EVENTS = 1000 -MAX_METRIC_KIND_CHARS = len(METRIC_PREFIX) + 64 -MAX_PARSED_ITEMS = 16 -_MAX_CHUNK_BYTES = 1 << 16 -_PROXY_CAP_FACTOR = 4 -_POLL_INTERVAL_SECONDS = 0.02 -_REGISTER_TIMEOUT_SECONDS = 30.0 -_CONTROL_SOCKET_TIMEOUT_SECONDS = 20.0 -_READER_JOIN_SECONDS = 10.0 -_KILL_WAIT_SECONDS = 10.0 -_TERMINAL_SLACK_SECONDS = 20.0 -_SUPERVISOR_EXIT_SECONDS = 10.0 - -_REPO_ROOT = Path(__file__).resolve().parents[2] - -_FALLBACK_SECRET_PATTERNS = ( - re.compile(r"(?i)\bauthorization\s*:?[ \t]*bearer\s+\S+"), - re.compile(r"(?i)\bbearer\s+\S+"), - re.compile(r"(?i)\b(?:api[_-]?key|access[_-]?token|secret|password|passwd)\b" - r"\s*[:=]\s*\S+"), - re.compile(r"\b(?:sk|pk|rk)-[A-Za-z0-9_\-]{8,}"), - re.compile(r"\b(?:ghp|gho|ghs|ghu)_[A-Za-z0-9]{8,}"), - re.compile(r"\bxox[baprs]-[A-Za-z0-9\-]{8,}"), - re.compile(r"\biop_[A-Za-z0-9_\-]{12,}"), - re.compile(r"\bAKIA[0-9A-Z]{12,}"), -) - - -# --------------------------------------------------------------------------- -# Errors -# --------------------------------------------------------------------------- - -class LifecycleError(Exception): - """Base error for invocation lifecycle failures.""" - - -class LifecycleValidationError(LifecycleError): - """Raised when the invocation specification or environment fails preflight.""" - - -class LifecycleProtocolError(LifecycleError): - """Raised when the internal supervisor protocol is violated.""" - - -class LifecycleRecoveryError(LifecycleError): - """Raised when an authenticated recovery request cannot be trusted.""" - - -class LifecycleMetricError(LifecycleError): - """Raised when an observation cannot be represented without invention.""" - - -# --------------------------------------------------------------------------- -# Frozen contracts -# --------------------------------------------------------------------------- - -@dataclass(frozen=True) -class InvocationSpec: - """Immutable specification of exactly one bounded caller invocation.""" - - argv: tuple[str, ...] - cwd: str - env: tuple[tuple[str, str], ...] - submission_mode: str - completion_mode: str - timeout: Timeout - evidence_dir: str - task_payload: bytes = b"" - env_allowlist: tuple[str, ...] = () - max_capture_bytes: int = 1 << 20 - max_capture_lines: int = 10000 - control_dir: Optional[str] = None - caller_detaches: bool = False - fault_injection: str = FAULT_NONE - - -@dataclass(frozen=True) -class LifecycleEvent: - kind: str - source: str - stream: str - monotonic_ns: int - source_monotonic_ns: int - observed_at: str - detail: str - - -@dataclass(frozen=True) -class ParsedMetric: - """One validated numeric observation with its clock, source and binding. - - ``stage``, ``model`` and ``call_id`` are optional closed labels; an empty - label means the observation is an unqualified caller total. ``overlap`` - marks an interval that may be contained in another reported interval, so a - consumer can never treat the set as a partition to subtract. - """ - - name: str - value: int - unit: str - clock: str - source: str - stage: str = "" - model: str = "" - call_id: str = "" - overlap: bool = False - - -@dataclass(frozen=True) -class CallerEvent: - """One closed caller lifecycle observation.""" - - kind: str - - def __post_init__(self) -> None: - if self.kind not in PARSER_TERMINAL_KINDS: - raise LifecycleValidationError("caller event kind is invalid") - - -@dataclass(frozen=True) -class CallerTerminal: - """The caller-declared product outcome, independent of its process exit.""" - - status: str - reason: str - - def __post_init__(self) -> None: - expected = { - CALLER_STATUS_SUCCEEDED: CALLER_REASON_SUCCESS, - CALLER_STATUS_FAILED: CALLER_REASON_ERROR, - } - if self.status not in CALLER_STATUSES or self.reason != expected[self.status]: - raise LifecycleValidationError("caller terminal is invalid") - - -CallerObservation = CallerEvent | CallerTerminal | ParsedMetric - - -@dataclass(frozen=True) -class CaptureStream: - stream: str - text: str - line_count: int - byte_count: int - truncated: bool - - -@dataclass(frozen=True) -class SupervisorLocator: - supervisor_pid: int - start_identity: str - socket_path: str - challenge: str - control_dir: str - created_at: str - - -@dataclass(frozen=True) -class TerminalOutcome: - reason: str - exit_code: Optional[int] - signal: Optional[int] - caller_launched: bool - cleanup_complete: bool - process_group_alive: bool - receipt_path: str - - -@dataclass(frozen=True) -class ProductOutcome: - status: str - reason: str - - def __post_init__(self) -> None: - expected = { - PRODUCT_STATUS_SUCCEEDED: CALLER_REASON_SUCCESS, - PRODUCT_STATUS_FAILED: CALLER_REASON_ERROR, - PRODUCT_STATUS_UNKNOWN: PRODUCT_REASON_UNAVAILABLE, - } - if self.status not in PRODUCT_STATUSES or self.reason != expected[self.status]: - raise LifecycleValidationError("product outcome is invalid") - - -@dataclass(frozen=True) -class HarnessOutcome: - status: str - reason: str - ordered_terminal: bool - cleanup_complete: bool - - def __post_init__(self) -> None: - if ( - self.status not in HARNESS_STATUSES - or self.reason not in HARNESS_REASONS - or not isinstance(self.ordered_terminal, bool) - or not isinstance(self.cleanup_complete, bool) - or (self.status == HARNESS_STATUS_PASSED) != (self.reason == REASON_SUCCESS) - or (not self.cleanup_complete) != (self.reason == REASON_CLEANUP_FAILED) - or (self.status == HARNESS_STATUS_PASSED and not self.ordered_terminal) - ): - raise LifecycleValidationError("harness outcome is invalid") - - -@dataclass(frozen=True) -class ProcessOutcome: - status: str - exit_code: Optional[int] - signal: Optional[int] - - def __post_init__(self) -> None: - if self.status not in PROCESS_STATUSES: - raise LifecycleValidationError("process outcome is invalid") - if self.exit_code is not None and ( - not isinstance(self.exit_code, int) or isinstance(self.exit_code, bool) - ): - raise LifecycleValidationError("process exit code is invalid") - if self.signal is not None and ( - not isinstance(self.signal, int) or isinstance(self.signal, bool) - ): - raise LifecycleValidationError("process signal is invalid") - if self.status == PROCESS_STATUS_SIGNALLED and self.signal is None: - raise LifecycleValidationError("signalled process requires a signal") - if self.status in (PROCESS_STATUS_EXITED, PROCESS_STATUS_NOT_STARTED) and self.signal is not None: - raise LifecycleValidationError("process signal contradicts its status") - if self.status == PROCESS_STATUS_NOT_STARTED and self.exit_code is not None: - raise LifecycleValidationError("not-started process has an exit code") - - -@dataclass(frozen=True) -class InvocationResult: - """Independent product, harness, and process outcomes for one invocation.""" - - product: ProductOutcome - harness: HarnessOutcome - process: ProcessOutcome - submitted: bool - process_group_alive: bool - events: tuple[LifecycleEvent, ...] - stdout: CaptureStream - stderr: CaptureStream - journal_path: str - result_path: str - locator: Optional[SupervisorLocator] - spec_digest: str - started_at: str - ended_at: str - duration_ns: int - metrics: tuple[ParsedMetric, ...] = () - - -class CancellationToken: - """Thread-safe cancellation flag accepted by :func:`run_invocation`.""" - - def __init__(self) -> None: - self._event = threading.Event() - - def cancel(self) -> None: - self._event.set() - - def is_cancelled(self) -> bool: - return self._event.is_set() - - -# --------------------------------------------------------------------------- -# Small helpers -# --------------------------------------------------------------------------- - -def _utc_now() -> str: - return datetime.datetime.now(datetime.timezone.utc).isoformat() - - -def _safe_detail(exc: BaseException) -> str: - """Return a bounded, class-anchored error detail without raw inputs.""" - return f"{type(exc).__name__}"[:MAX_EVENT_DETAIL_CHARS] - - -@dataclass(frozen=True) -class _FileIdentity: - device: int - inode: int - - -def _fsync_directory(directory: Path) -> None: - dir_fd = os.open(str(directory), os.O_RDONLY) - try: - os.fsync(dir_fd) - finally: - os.close(dir_fd) - - -def _stage_bytes(directory: Path, data: bytes, mode: int) -> Path: - """Write and fsync private staging bytes in the target directory.""" - fd, tmp_name = tempfile.mkstemp(prefix=".tmp-", dir=str(directory)) - try: - with os.fdopen(fd, "wb") as handle: - handle.write(data) - handle.flush() - os.fsync(handle.fileno()) - os.chmod(tmp_name, mode) - except BaseException: - try: - os.unlink(tmp_name) - except OSError: - pass - raise - return Path(tmp_name) - - -def _identity(path: Path) -> _FileIdentity: - stat_result = path.stat(follow_symlinks=False) - return _FileIdentity(stat_result.st_dev, stat_result.st_ino) - - -def _rollback_owned(path: Path, identity: _FileIdentity) -> None: - """Remove path only while it still names the inode published by this call.""" - try: - if _identity(path) != identity: - return - path.unlink() - _fsync_directory(path.parent) - except FileNotFoundError: - return - - -def _publish_staged_no_replace(staged: Path, path: Path) -> _FileIdentity: - """Atomically link staged bytes into an absent target without replacement.""" - staged_identity = _identity(staged) - linked = False - try: - os.link(staged, path, follow_symlinks=False) - linked = True - if _identity(path) != staged_identity: - raise LifecycleError("published target identity changed concurrently") - _fsync_directory(path.parent) - return staged_identity - except BaseException: - if linked: - _rollback_owned(path, staged_identity) - raise - finally: - try: - staged.unlink() - except FileNotFoundError: - pass - - -def _write_bytes_no_replace(path: Path, data: bytes, mode: int = 0o600) -> _FileIdentity: - """Stage, fsync and atomically publish bytes only when target is absent.""" - staged = _stage_bytes(path.parent, data, mode) - return _publish_staged_no_replace(staged, path) - - -def _enable_child_subreaper() -> bool: - """Adopt owned orphan descendants on Linux so they can be reaped.""" - if not sys.platform.startswith("linux"): - return False - try: - libc = ctypes.CDLL(None, use_errno=True) - if libc.prctl(36, 1, 0, 0, 0) != 0: # PR_SET_CHILD_SUBREAPER - raise OSError(ctypes.get_errno(), "prctl(PR_SET_CHILD_SUBREAPER)") - except (AttributeError, OSError): - return False - return True - - -def _process_start_identity(pid: int) -> str: - """Return an OS start identity for pid, or '' when unavailable.""" - try: - raw = Path(f"/proc/{pid}/stat").read_text(encoding="utf-8", errors="replace") - except OSError: - return "" - try: - return raw.rsplit(") ", 1)[1].split()[19] - except (IndexError, ValueError): - return "" - - -def _proc_group_has_live_member(pgid: int) -> bool: - """Return True when /proc shows a non-zombie member of pgid.""" - proc = Path("/proc") - for entry in proc.iterdir(): - if not entry.name.isdigit(): - continue - try: - raw = (entry / "stat").read_text(encoding="utf-8", errors="replace") - fields = raw.rsplit(") ", 1)[1].split() - state, group = fields[0], int(fields[2]) - except (OSError, IndexError, ValueError): - continue - if group == pgid and state != "Z": - return True - return False - - -def _proc_group_has_member(pgid: int) -> bool: - """Return True when /proc still contains any member, including a zombie.""" - proc = Path("/proc") - if not proc.is_dir(): - return False - for entry in proc.iterdir(): - if not entry.name.isdigit(): - continue - try: - raw = (entry / "stat").read_text(encoding="utf-8", errors="replace") - group = int(raw.rsplit(") ", 1)[1].split()[2]) - except (OSError, IndexError, ValueError): - continue - if group == pgid: - return True - return False - - -def _group_alive(pgid: Optional[int]) -> bool: - """Return True when the owned process group still has a live member.""" - if not pgid: - return False - try: - os.killpg(pgid, 0) - except ProcessLookupError: - return False - except PermissionError: - return True - except OSError: - return True - if Path("/proc/self/stat").exists(): - return _proc_group_has_live_member(pgid) - return True - - -def _killpg_quiet(pgid: int, sig: int) -> None: - try: - os.killpg(pgid, sig) - except (ProcessLookupError, PermissionError, OSError): - pass - - -def exact_value_redactor(values: tuple[str, ...]) -> Callable[[str], str]: - """Build a redactor replacing every non-empty exact secret value.""" - ordered = tuple(sorted({v for v in values if v}, key=len, reverse=True)) - - def _redact(text: str) -> str: - for value in ordered: - text = text.replace(value, REDACTED) - return text - - return _redact - - -def fallback_redact(text: str) -> str: - """Redact secret-shaped substrings that no adapter redactor removed.""" - for pattern in _FALLBACK_SECRET_PATTERNS: - text = pattern.sub(REDACTED, text) - return text - - -def normalize_duration_ns(value: Any, reported_unit: str = "ms") -> int: - """Convert one reported non-negative duration into exact integer nanoseconds. - - Callers report durations as integers or decimals. The decimal text is the - authority, so the value is rebuilt with ``Decimal(str(value))`` and refused - whenever it cannot be represented in whole nanoseconds. - """ - scale = DURATION_SCALES_NS.get(reported_unit) - if scale is None: - raise LifecycleMetricError("duration unit is not a supported scale") - if isinstance(value, bool) or not isinstance(value, (int, float, str, Decimal)): - raise LifecycleMetricError("duration value is not a reported number") - try: - reported = Decimal(value) if isinstance(value, int) else Decimal(str(value)) - except (InvalidOperation, ValueError) as exc: - raise LifecycleMetricError("duration value is not a reported number") from exc - if not reported.is_finite() or reported < 0: - raise LifecycleMetricError("duration value is not finite and non-negative") - exact = reported * scale - if exact != exact.to_integral_value(): - raise LifecycleMetricError("duration precision is finer than one nanosecond") - return int(exact) - - -def is_reported_number(value: Any) -> bool: - """True only for a plain JSON number, so no wire string is coerced.""" - return not isinstance(value, bool) and isinstance(value, (int, float)) - - -def normalize_count(value: Any) -> int: - """Admit only a non-negative integer call or token count.""" - if isinstance(value, bool) or not isinstance(value, int) or value < 0: - raise LifecycleMetricError("count value must be a non-negative integer") - return value - - -def _safe_label(label: Any) -> bool: - if not isinstance(label, str): - return False - if not label: - return True - return SAFE_LABEL_RE.fullmatch(label) is not None and fallback_redact(label) == label - - -def validate_metric(metric: Any) -> ParsedMetric: - """Validate one observation against the closed metric contract.""" - if not isinstance(metric, ParsedMetric): - raise LifecycleMetricError("metric must be a ParsedMetric instance") - unit = METRIC_UNITS.get(metric.name) - if unit is None or metric.unit != unit: - raise LifecycleMetricError("metric name and unit are not a closed pair") - if metric.clock not in METRIC_CLOCKS or metric.source not in METRIC_SOURCES: - raise LifecycleMetricError("metric clock and source must be closed values") - if not isinstance(metric.overlap, bool): - raise LifecycleMetricError("metric overlap must be a boolean") - if unit == UNIT_NANOSECONDS: - if metric.clock not in TEMPORAL_CLOCKS: - raise LifecycleMetricError("a duration requires a temporal clock") - elif metric.clock != CLOCK_NONE or metric.overlap: - raise LifecycleMetricError("a count has no clock and cannot overlap") - if isinstance(metric.value, bool) or not isinstance(metric.value, int) or metric.value < 0: - raise LifecycleMetricError("metric value must be a non-negative integer") - if not all(_safe_label(label) for label in (metric.stage, metric.model, metric.call_id)): - raise LifecycleMetricError("metric labels must be safe closed identifiers") - return metric - - -def duration_metric( - name: str, - value: Any, - *, - reported_unit: str = "ms", - clock: str = CLOCK_CALLER_REPORTED, - source: str = SOURCE_CALLER_OUTPUT, - stage: str = "", - model: str = "", - call_id: str = "", - overlap: bool = False, -) -> ParsedMetric: - """Build one validated duration observation in integer nanoseconds.""" - return validate_metric(ParsedMetric( - name, normalize_duration_ns(value, reported_unit), UNIT_NANOSECONDS, - clock, source, stage, model, call_id, overlap, - )) - - -def count_metric( - name: str, - value: Any, - *, - source: str = SOURCE_CALLER_OUTPUT, - stage: str = "", - model: str = "", - call_id: str = "", -) -> ParsedMetric: - """Build one validated call or token count observation.""" - unit = METRIC_UNITS.get(name) - if unit not in (UNIT_CALLS, UNIT_TOKENS): - raise LifecycleMetricError("metric name is not a closed count") - return validate_metric(ParsedMetric( - name, normalize_count(value), unit, CLOCK_NONE, source, stage, model, call_id, False, - )) - - -def metric_record(metric: ParsedMetric) -> dict[str, Any]: - """Return the canonical durable projection of one validated observation.""" - validated = validate_metric(metric) - return { - "name": validated.name, - "value": validated.value, - "unit": validated.unit, - "clock": validated.clock, - "source": validated.source, - "stage": validated.stage, - "model": validated.model, - "call_id": validated.call_id, - "overlap": validated.overlap, - } - - -def publish_bytes_no_replace(path: Path, data: bytes, mode: int = 0o600) -> None: - """Publish bytes atomically and only into an absent target.""" - _write_bytes_no_replace(path, data, mode) - - -def spec_digest(spec: InvocationSpec) -> str: - """Compute a stable digest binding argv/env/payload without revealing them.""" - hasher = hashlib.sha256() - hasher.update(b"IOP-BENCH-INVOCATION\x00") - for item in spec.argv: - hasher.update(item.encode("utf-8") + b"\x00") - for key, value in spec.env: - hasher.update(key.encode("utf-8") + b"=" + value.encode("utf-8") + b"\x00") - hasher.update(spec.cwd.encode("utf-8") + b"\x00") - hasher.update(spec.submission_mode.encode("utf-8") + b"\x00") - hasher.update(spec.completion_mode.encode("utf-8") + b"\x00") - hasher.update(hashlib.sha256(spec.task_payload).digest()) - return "sha256:" + hasher.hexdigest() - - -def env_pairs(mapping: dict[str, str]) -> tuple[tuple[str, str], ...]: - """Freeze an environment mapping into canonical immutable pairs.""" - return tuple(sorted((str(k), str(v)) for k, v in mapping.items())) - - -def inherited_tls_ca_environment( - environment: Mapping[str, str] | None = None, -) -> tuple[tuple[str, str], ...]: - """Freeze only the two standard public CA bundle settings for a child.""" - source = os.environ if environment is None else environment - return tuple( - (key, value) - for key in TLS_CA_ENV_KEYS - if isinstance((value := source.get(key)), str) and value - ) - - -# --------------------------------------------------------------------------- -# Frame transport -# --------------------------------------------------------------------------- - -class _FrameWriter: - """Serialized newline-delimited JSON writer shared by supervisor threads.""" - - def __init__(self, handle: Any) -> None: - self._handle = handle - self._lock = threading.Lock() - self.alive = True - - def send(self, frame: dict[str, Any]) -> None: - payload = (json.dumps(frame, ensure_ascii=False) + "\n").encode("utf-8") - with self._lock: - if not self.alive: - return - try: - self._handle.write(payload) - self._handle.flush() - except (BrokenPipeError, ValueError, OSError): - self.alive = False - - -def _read_frame(handle: Any) -> Optional[dict[str, Any]]: - """Read one JSON frame; return None on EOF or a closed handle.""" - try: - line = handle.readline() - except (ValueError, OSError): - return None - if not line: - return None - try: - frame = json.loads(line) - except (json.JSONDecodeError, UnicodeDecodeError): - return {"op": "error", "detail": "malformed_frame"} - return frame if isinstance(frame, dict) else {"op": "error", "detail": "malformed_frame"} - - -def _send_json(handle: Any, payload: dict[str, Any]) -> None: - handle.write((json.dumps(payload, ensure_ascii=False) + "\n").encode("utf-8")) - handle.flush() - - -# --------------------------------------------------------------------------- -# Supervisor -# --------------------------------------------------------------------------- - -class _Supervisor: - """Registered owner of one caller process group and its terminal arbitration.""" - - def __init__(self, reader: Any, writer: Any, control_dir: Path) -> None: - self.reader = reader - self.writer = _FrameWriter(writer) - self.control_dir = control_dir - self.spec: dict[str, Any] = {} - self.challenge = secrets.token_hex(32) - self.start_identity = _process_start_identity(os.getpid()) - self.child: Optional[subprocess.Popen] = None - self.pgid: Optional[int] = None - self.readers: list[threading.Thread] = [] - self.submission_writer: Optional[threading.Thread] = None - self.exit_watcher: Optional[threading.Thread] = None - self.subreaper_enabled = False - self.forwarded: dict[str, int] = {"stdout": 0, "stderr": 0} - self.truncated: dict[str, bool] = {"stdout": False, "stderr": False} - self.sock: Optional[socket.socket] = None - self.terminal: Optional[dict[str, Any]] = None - self._arbiter_lock = threading.Lock() - self._terminal_sent = False - self._send_lock = threading.Lock() - - # -- registration ------------------------------------------------------ - - def _register(self) -> None: - socket_path = self.control_dir / SOCKET_FILENAME - self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - previous_umask = os.umask(0o177) - original_dir_fd = -1 - control_dir_fd = -1 - try: - # Some shared/container filesystems reject bind(2) when the socket - # pathname traverses the short attempt symlink, even though the - # resolved directory supports Unix sockets. Resolve the directory - # once with fchdir and bind the basename; the published locator - # remains the authenticated short absolute alias. - directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) - original_dir_fd = os.open(".", directory_flags) - control_dir_fd = os.open(self.control_dir, directory_flags) - os.fchdir(control_dir_fd) - self.sock.bind(SOCKET_FILENAME) - try: - os.chmod(SOCKET_FILENAME, 0o600) - except OSError as exc: - # A small class of shared filesystems permits Unix sockets but - # rejects chmod on the socket inode. The containing directory - # is still an exclusive 0700 security boundary. - directory = os.stat(".") - unsupported = {errno.EINVAL} - if hasattr(errno, "ENOTSUP"): - unsupported.add(errno.ENOTSUP) - if ( - exc.errno not in unsupported - or stat.S_IMODE(directory.st_mode) != 0o700 - or directory.st_uid != os.geteuid() - ): - raise - else: - socket_mode = stat.S_IMODE(os.lstat(SOCKET_FILENAME).st_mode) - if socket_mode & 0o077: - raise PermissionError("control socket permissions are too broad") - finally: - if original_dir_fd >= 0: - os.fchdir(original_dir_fd) - if control_dir_fd >= 0: - os.close(control_dir_fd) - if original_dir_fd >= 0: - os.close(original_dir_fd) - os.umask(previous_umask) - self.sock.listen(4) - locator = { - "supervisor_pid": os.getpid(), - "start_identity": self.start_identity, - "socket_path": str(socket_path), - "challenge": self.challenge, - "control_dir": str(self.control_dir), - "created_at": _utc_now(), - } - _write_bytes_no_replace( - self.control_dir / LOCATOR_FILENAME, - json.dumps(locator, ensure_ascii=False).encode("utf-8"), - ) - threading.Thread(target=self._serve_control, daemon=True).start() - self.writer.send({"op": "registered", "locator": locator}) - - # -- caller launch ----------------------------------------------------- - - def _launch(self) -> None: - spec = self.spec - mode = spec["submission_mode"] - stdin = subprocess.PIPE if mode == SUBMISSION_STDIN_ONCE else subprocess.DEVNULL - self.subreaper_enabled = _enable_child_subreaper() - self.child = subprocess.Popen( - list(spec["argv"]), - cwd=spec["cwd"], - env={key: value for key, value in spec["env"]}, - stdin=stdin, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - process_group=0, - close_fds=True, - ) - self.pgid = self.child.pid - for name in ("stdout", "stderr"): - thread = threading.Thread( - target=self._pump, args=(name, getattr(self.child, name)), daemon=True - ) - thread.start() - self.readers.append(thread) - self.exit_watcher = threading.Thread(target=self._await_exit, daemon=True) - self.exit_watcher.start() - if mode == SUBMISSION_STDIN_ONCE: - payload = bytes.fromhex(spec["task_payload_hex"]) - self.submission_writer = threading.Thread( - target=self._submit_stdin_once, args=(payload,), daemon=True - ) - self.submission_writer.start() - else: - self._send_started() - - def _send_started(self) -> None: - child = self.child - if child is None: - return - self.writer.send({ - "op": "started", - "pid": child.pid, - "pgid": self.pgid, - "ns": time.monotonic_ns(), - "submission_mode": self.spec["submission_mode"], - }) - - def _submit_stdin_once(self, payload: bytes) -> None: - child = self.child - if child is None or child.stdin is None: - return - complete = False - try: - written = child.stdin.write(payload) - child.stdin.flush() - complete = written == len(payload) - except (BrokenPipeError, ValueError, OSError): - complete = False - finally: - try: - child.stdin.close() - except (BrokenPipeError, ValueError, OSError): - complete = False - if complete: - self._send_started() - else: - self.writer.send({"op": "submission_error", "detail": "stdin_write_failed"}) - - def _pump(self, stream: str, pipe: Any) -> None: - cap = int(self.spec["max_capture_bytes"]) * _PROXY_CAP_FACTOR + _MAX_CHUNK_BYTES - injected = self.spec.get("fault_injection", FAULT_NONE) == FAULT_READER_ERROR - try: - while True: - chunk = pipe.readline(_MAX_CHUNK_BYTES) - if not chunk: - break - if injected and stream == "stdout": - raise OSError("injected reader failure") - now = time.monotonic_ns() - if self.forwarded[stream] < cap: - self.forwarded[stream] += len(chunk) - self.writer.send({ - "op": "output", - "stream": stream, - "ns": now, - "data": chunk.decode("utf-8", "replace"), - }) - elif not self.truncated[stream]: - self.truncated[stream] = True - self.writer.send({"op": "proxy_truncated", "stream": stream, "ns": now}) - except Exception as exc: # reader failure is terminal evidence, never silent - self.writer.send( - {"op": "reader_error", "stream": stream, "detail": _safe_detail(exc)} - ) - finally: - try: - pipe.close() - except OSError: - pass - self.writer.send({ - "op": "stream_eof", - "stream": stream, - "ns": time.monotonic_ns(), - }) - - def _await_exit(self) -> None: - child = self.child - if child is None: - return - try: - code = child.wait() - except OSError as exc: - self.writer.send({"op": "error", "detail": _safe_detail(exc)}) - return - self.writer.send({ - "op": "exited", - "exit_code": code if code >= 0 else None, - "signal": -code if code < 0 else None, - "ns": time.monotonic_ns(), - }) - - # -- terminal arbitration --------------------------------------------- - - def finish(self, reason: str) -> dict[str, Any]: - """Single-owner terminal arbiter: first reason wins, cleanup always runs.""" - with self._arbiter_lock: - if self.terminal is not None: - return self.terminal - exit_code: Optional[int] = None - signal_num: Optional[int] = None - group_alive = False - if self.child is not None: - grace = int(self.spec.get("cleanup_grace_seconds", 5)) - self._terminate_group(grace) - io_complete = self._join_io_threads() - descendants_reaped = self._reap_owned_descendants() - if self.child is not None: - # The exit watcher publishes the controller's exited frame. - # Join it before freezing the receipt so both durable views - # observe the same authoritative child return code. - self.child.poll() - code = self.child.returncode - if code is not None: - exit_code = code if code >= 0 else None - signal_num = -code if code < 0 else None - group_alive = _group_alive(self.pgid) - outcome = { - "reason": ( - reason - if not group_alive and io_complete and descendants_reaped - else REASON_CLEANUP_FAILED - ), - "exit_code": exit_code, - "signal": signal_num, - "caller_launched": self.child is not None, - "cleanup_complete": not group_alive and io_complete and descendants_reaped, - "process_group_alive": group_alive, - "receipt_path": str(self.control_dir / RECEIPT_FILENAME), - } - if not self._write_receipt(outcome): - outcome["reason"] = REASON_CLEANUP_FAILED - outcome["cleanup_complete"] = False - self.terminal = outcome - return outcome - - def _terminate_group(self, grace_seconds: int) -> None: - child = self.child - if child is None or self.pgid is None: - return - if not _group_alive(self.pgid): - child.poll() - return - _killpg_quiet(self.pgid, signal.SIGTERM) - if self._wait_group_gone(grace_seconds): - return - _killpg_quiet(self.pgid, signal.SIGKILL) - self._wait_group_gone(_KILL_WAIT_SECONDS) - - def _wait_group_gone(self, seconds: float) -> bool: - deadline = time.monotonic() + max(0.0, float(seconds)) - while True: - if self.child is not None: - self.child.poll() - if not _group_alive(self.pgid): - return True - if time.monotonic() >= deadline: - return False - time.sleep(_POLL_INTERVAL_SECONDS) - - def _join_io_threads(self) -> bool: - deadline = time.monotonic() + _READER_JOIN_SECONDS - threads = [self.submission_writer, *self.readers, self.exit_watcher] - for thread in threads: - if thread is None: - continue - thread.join(max(0.0, deadline - time.monotonic())) - return all(thread is None or not thread.is_alive() for thread in threads) - - def _reap_owned_descendants(self) -> bool: - if self.pgid is None or not self.subreaper_enabled: - return True - deadline = time.monotonic() + _KILL_WAIT_SECONDS - while True: - reaped = False - try: - while True: - pid, _ = os.waitpid(-self.pgid, os.WNOHANG) - if pid <= 0: - break - reaped = True - except ChildProcessError: - pass - if not _proc_group_has_member(self.pgid): - return True - if time.monotonic() >= deadline: - return False - if not reaped: - time.sleep(_POLL_INTERVAL_SECONDS) - - def _write_receipt(self, outcome: dict[str, Any]) -> bool: - receipt = { - "receipt_version": RECEIPT_VERSION, - "supervisor_pid": os.getpid(), - "challenge_digest": hashlib.sha256(self.challenge.encode("utf-8")).hexdigest(), - "reason": outcome["reason"], - "exit_code": outcome["exit_code"], - "signal": outcome["signal"], - "caller_launched": outcome["caller_launched"], - "cleanup_complete": outcome["cleanup_complete"], - "process_group_alive": outcome["process_group_alive"], - "completed_at": _utc_now(), - } - try: - _write_bytes_no_replace( - self.control_dir / RECEIPT_FILENAME, - json.dumps(receipt, ensure_ascii=False).encode("utf-8"), - ) - except (OSError, LifecycleError): - return False - return True - - def _send_terminal(self, outcome: dict[str, Any]) -> None: - with self._send_lock: - if self._terminal_sent: - return - self._terminal_sent = True - self.writer.send({"op": "terminal", "outcome": outcome}) - - # -- authenticated control endpoint ----------------------------------- - - def _serve_control(self) -> None: - while True: - try: - conn, _ = self.sock.accept() # type: ignore[union-attr] - except OSError: - return - threading.Thread( - target=self._handle_control, args=(conn,), daemon=True - ).start() - - def _handle_control(self, conn: socket.socket) -> None: - with conn: - conn.settimeout(_CONTROL_SOCKET_TIMEOUT_SECONDS) - stream = conn.makefile("rwb") - try: - self._control_exchange(stream) - except (OSError, ValueError, json.JSONDecodeError): - return - finally: - try: - stream.close() - except OSError: - pass - - def _control_exchange(self, stream: Any) -> None: - auth = _read_frame(stream) or {} - presented = str(auth.get("challenge", "")) - if auth.get("op") != "auth" or not hmac.compare_digest(presented, self.challenge): - _send_json(stream, {"ok": False, "error": "authentication_failed"}) - return - _send_json(stream, { - "ok": True, - "supervisor_pid": os.getpid(), - "start_identity": self.start_identity, - }) - request = _read_frame(stream) or {} - operation = request.get("op") - if operation == "status": - _send_json(stream, {"ok": True, "status": self._status()}) - return - if operation != "stop": - _send_json(stream, {"ok": False, "error": "unsupported_op"}) - return - outcome = self.finish(REASON_RECOVERED_STOP) - _send_json(stream, {"ok": True, "outcome": outcome}) - self._send_terminal(outcome) - self._exit_after_recovered_stop(stream) - - def _status(self) -> dict[str, Any]: - return { - "caller_launched": self.child is not None, - "process_group_alive": _group_alive(self.pgid), - "terminal_reason": None if self.terminal is None else self.terminal["reason"], - } - - def _exit_after_recovered_stop(self, stream: Any) -> None: - """Leave immediately after a recovered stop; cleanup and receipt are done.""" - for handle in (stream, self.sock): - try: - if handle is not None: - handle.close() - except OSError: - pass - os._exit(0) - - # -- main loop --------------------------------------------------------- - - def run(self) -> int: - try: - spec = _read_frame(self.reader) - if spec is None or spec.get("op") != "spec": - self._send_terminal(self.finish(REASON_CONTROLLER_LOST)) - return 0 - self.spec = spec - self._register() - except Exception as exc: - self.writer.send({"op": "error", "detail": _safe_detail(exc)}) - self._send_terminal(self.finish(REASON_SUPERVISOR_ERROR)) - return 1 - - gate = _read_frame(self.reader) - if gate is None: - self._send_terminal(self.finish(REASON_CONTROLLER_LOST)) - return 0 - if gate.get("op") != "start": - self._send_terminal(self.finish(REASON_START_CALLBACK_FAILED)) - return 0 - - try: - self._launch() - except Exception as exc: - self.writer.send({"op": "error", "detail": _safe_detail(exc)}) - self._send_terminal(self.finish(REASON_LAUNCH_FAILED)) - return 0 - - outcome = self._control_loop() - self._send_terminal(outcome) - self._close_socket() - return 0 - - def _control_loop(self) -> dict[str, Any]: - while True: - frame = _read_frame(self.reader) - if frame is None: - return self.finish(REASON_CONTROLLER_LOST) - if frame.get("op") == "stop": - reason = str(frame.get("reason") or REASON_SUPERVISOR_ERROR) - if reason not in TERMINAL_REASONS: - reason = REASON_SUPERVISOR_ERROR - return self.finish(reason) - - def _close_socket(self) -> None: - if self.sock is None: - return - try: - self.sock.close() - except OSError: - pass - try: - os.unlink(self.control_dir / SOCKET_FILENAME) - except OSError: - pass - - -def _supervisor_main(argv: list[str]) -> int: - options: dict[str, str] = {} - for item in argv: - key, _, value = item.partition("=") - options[key] = value - read_fd = int(options["--read-fd"]) - write_fd = int(options["--write-fd"]) - control_dir = Path(options["--control-dir"]) - reader = os.fdopen(read_fd, "rb") - writer = os.fdopen(write_fd, "wb") - return _Supervisor(reader, writer, control_dir).run() - - -# --------------------------------------------------------------------------- -# Controller-side capture -# --------------------------------------------------------------------------- - -class _StreamCapture: - """Line assembler applying redaction and hard byte/line capture bounds.""" - - def __init__(self, stream: str, max_bytes: int, max_lines: int) -> None: - self.stream = stream - self.max_bytes = max_bytes - self.max_lines = max_lines - self.parts: list[str] = [] - self.byte_count = 0 - self.line_count = 0 - self.truncated = False - self._pending = "" - - def add_chunk(self, text: str) -> list[str]: - """Buffer a proxied chunk and return every newly completed raw line.""" - self._pending += text - lines: list[str] = [] - while "\n" in self._pending: - line, _, self._pending = self._pending.partition("\n") - lines.append(line) - if len(self._pending) > _MAX_CHUNK_BYTES: - lines.append(self._pending) - self._pending = "" - return lines - - def flush(self) -> list[str]: - if not self._pending: - return [] - line, self._pending = self._pending, "" - return [line] - - def record(self, redacted_line: str) -> None: - """Append one redacted line while enforcing the capture bounds.""" - if self.line_count >= self.max_lines or self.byte_count >= self.max_bytes: - self.truncated = True - return - encoded = len(redacted_line.encode("utf-8")) + 1 - remaining = self.max_bytes - self.byte_count - if encoded > remaining: - self.parts.append(redacted_line.encode("utf-8")[:remaining].decode("utf-8", "ignore")) - self.byte_count = self.max_bytes - self.line_count += 1 - self.truncated = True - return - self.parts.append(redacted_line) - self.byte_count += encoded - self.line_count += 1 - - def freeze(self) -> CaptureStream: - text = "\n".join(self.parts) - if self.truncated: - text = text + "\n[truncated]" - return CaptureStream( - stream=self.stream, - text=text, - line_count=self.line_count, - byte_count=self.byte_count, - truncated=self.truncated, - ) - - -# --------------------------------------------------------------------------- -# Controller -# --------------------------------------------------------------------------- - -class _Invocation: - """Controller state machine for one registered, bounded caller invocation.""" - - def __init__( - self, - spec: InvocationSpec, - parse_event: Callable[[str, str], Any], - redact: Optional[Callable[[str], str]], - cancellation: Any, - on_started: Callable[[SupervisorLocator], None], - ) -> None: - self.spec = spec - self.parse_event = parse_event - self.redact = redact - self.cancellation = cancellation - self.on_started = on_started - self.queue: "queue.Queue[Optional[dict[str, Any]]]" = queue.Queue() - self.captures = { - name: _StreamCapture(name, spec.max_capture_bytes, spec.max_capture_lines) - for name in ("stdout", "stderr") - } - self.events: list[LifecycleEvent] = [] - self.metrics: list[ParsedMetric] = [] - self.metric_events = 0 - self.first_output_at: Optional[float] = None - self.stream_eof: set[str] = set() - self.locator: Optional[SupervisorLocator] = None - self.reason: Optional[str] = None - self.external_outcome: Optional[dict[str, Any]] = None - self.submitted = False - self.finish_at: Optional[float] = None - self.idle_at: Optional[float] = None - self.caller_terminal: Optional[CallerTerminal] = None - self.last_output_at: Optional[float] = None - self.quiet = False - self.exited = False - self.exit_code: Optional[int] = None - self.signal: Optional[int] = None - self.process_status_hint: Optional[str] = None - self.run_deadline = 0.0 - self.control_dir: Optional[Path] = None - self.owns_control_dir = False - self.supervisor: Optional[subprocess.Popen] = None - self.err_handle: Optional[Any] = None - self.to_supervisor: Optional[Any] = None - self.from_supervisor: Optional[Any] = None - self.started_at = "" - self.start_ns = 0 - - # -- public entry ------------------------------------------------------ - - def run(self) -> InvocationResult: - _preflight(self.spec) - self.started_at = _utc_now() - self.start_ns = time.monotonic_ns() - self._spawn_supervisor() - try: - if self._register_and_start(): - self._pump_until_terminal() - outcome = self._request_terminal() - finally: - self._shutdown_supervisor() - return self._publish(outcome) - - # -- supervisor process ------------------------------------------------ - - def _spawn_supervisor(self) -> None: - if self.spec.control_dir: - # Keep a caller-supplied short pathname for the AF_UNIX endpoint. - # Resolving a containment-preserving alias here can exceed the - # platform socket-path limit before the supervisor is registered. - self.control_dir = Path(self.spec.control_dir) - try: - self.control_dir.mkdir(mode=0o700) - except FileExistsError as exc: - raise LifecycleValidationError( - "control_dir must be absent so the invocation can own it exclusively" - ) from exc - else: - self.control_dir = Path(tempfile.mkdtemp(prefix="iop-bench-lifecycle-")) - self.owns_control_dir = True - os.chmod(self.control_dir, 0o700) - controller_read, supervisor_write = os.pipe() - supervisor_read, controller_write = os.pipe() - self.err_handle = open(self.control_dir / SUPERVISOR_ERR_FILENAME, "xb") - argv = [ - sys.executable, "-m", "scripts.agent_benchmark.lifecycle", - f"--read-fd={supervisor_read}", - f"--write-fd={supervisor_write}", - f"--control-dir={self.control_dir}", - ] - env = { - "PATH": os.environ.get("PATH", ""), - "PYTHONPATH": str(_REPO_ROOT), - "LANG": os.environ.get("LANG", "C"), - "TMPDIR": os.environ.get("TMPDIR", tempfile.gettempdir()), - } - try: - self.supervisor = subprocess.Popen( - argv, - cwd=str(_REPO_ROOT), - env=env, - stdin=subprocess.DEVNULL, - stdout=subprocess.DEVNULL, - stderr=self.err_handle, - pass_fds=(supervisor_read, supervisor_write), - start_new_session=True, - close_fds=True, - ) - finally: - os.close(supervisor_read) - os.close(supervisor_write) - self.to_supervisor = os.fdopen(controller_write, "wb") - self.from_supervisor = os.fdopen(controller_read, "rb") - threading.Thread(target=self._frame_reader, daemon=True).start() - - def _frame_reader(self) -> None: - handle = self.from_supervisor - try: - while True: - frame = _read_frame(handle) - if frame is None: - break - self.queue.put(frame) - finally: - self.queue.put(None) - - def _send(self, frame: dict[str, Any]) -> None: - handle = self.to_supervisor - if handle is None: - return - try: - handle.write((json.dumps(frame, ensure_ascii=False) + "\n").encode("utf-8")) - handle.flush() - except (BrokenPipeError, ValueError, OSError): - self.reason = self.reason or REASON_SUPERVISOR_ERROR - - # -- registration gate ------------------------------------------------- - - def _register_and_start(self) -> bool: - self._send({ - "op": "spec", - "argv": list(self.spec.argv), - "cwd": self.spec.cwd, - "env": [list(pair) for pair in self.spec.env], - "submission_mode": self.spec.submission_mode, - "task_payload_hex": self.spec.task_payload.hex(), - "max_capture_bytes": self.spec.max_capture_bytes, - "cleanup_grace_seconds": self.spec.timeout.cleanup_grace_seconds, - "fault_injection": self.spec.fault_injection, - }) - frame = self._await_frame("registered", _REGISTER_TIMEOUT_SECONDS) - if frame is None: - self.reason = REASON_SUPERVISOR_ERROR - return False - raw = frame.get("locator") or {} - self.locator = SupervisorLocator( - supervisor_pid=int(raw.get("supervisor_pid", 0)), - start_identity=str(raw.get("start_identity", "")), - socket_path=str(raw.get("socket_path", "")), - challenge=str(raw.get("challenge", "")), - control_dir=str(raw.get("control_dir", "")), - created_at=str(raw.get("created_at", "")), - ) - try: - self.on_started(self.locator) - except Exception: - self._send({"op": "abort"}) - self.reason = REASON_START_CALLBACK_FAILED - return False - self.run_deadline = time.monotonic() + self.spec.timeout.run_seconds - self._send({"op": "start"}) - return True - - def _await_frame(self, expected: str, timeout: float) -> Optional[dict[str, Any]]: - deadline = time.monotonic() + timeout - while time.monotonic() < deadline: - try: - frame = self.queue.get(timeout=_POLL_INTERVAL_SECONDS) - except queue.Empty: - continue - if frame is None: - return None - if frame.get("op") == expected: - return frame - if frame.get("op") in ("error", "terminal"): - return None - return None - - # -- main pump --------------------------------------------------------- - - def _pump_until_terminal(self) -> None: - while self.reason is None and self.external_outcome is None: - self._check_deadlines() - if self.reason is not None: - break - try: - frame = self.queue.get(timeout=_POLL_INTERVAL_SECONDS) - except queue.Empty: - continue - if frame is None: - self.reason = self.reason or REASON_SUPERVISOR_ERROR - break - self._handle_frame(frame) - - def _handle_frame(self, frame: dict[str, Any]) -> None: - operation = frame.get("op") - if operation == "started": - self._record_submitted(frame) - elif operation == "output": - self._handle_output(frame) - elif operation == "stream_eof": - self._handle_stream_eof(frame) - elif operation == "submission_error": - self.reason = self.reason or REASON_LAUNCH_FAILED - elif operation == "proxy_truncated": - self.captures[str(frame.get("stream", "stdout"))].truncated = True - elif operation == "exited": - self._handle_exit(frame) - elif operation == "reader_error": - self.reason = self.reason or REASON_READER_ERROR - elif operation == "terminal": - self.external_outcome = frame.get("outcome") or {} - elif operation == "error": - self.reason = self.reason or REASON_SUPERVISOR_ERROR - - def _record_submitted(self, frame: dict[str, Any]) -> None: - if self.submitted: - self.reason = self.reason or REASON_DUPLICATE_EVENT - return - self.submitted = True - self._add_event(EVENT_SUBMITTED, SOURCE_HARNESS, "", frame, self.spec.submission_mode) - - def _handle_output(self, frame: dict[str, Any]) -> None: - stream = str(frame.get("stream", "stdout")) - capture = self.captures.get(stream) - if capture is None: - return - self.last_output_at = time.monotonic() - data = str(frame.get("data", "")) - self._record_first_output(stream, frame, data) - for line in capture.add_chunk(data): - self._consume_line(stream, line, frame) - - def _record_first_output( - self, stream: str, frame: dict[str, Any], data: str - ) -> None: - """Record the first non-empty caller frame exactly once, before parsing.""" - if self.first_output_at is not None or not data.strip(): - return - self.first_output_at = time.monotonic() - self._add_event(EVENT_FIRST_OUTPUT, SOURCE_HARNESS, stream, frame, "", safe=True) - - def _handle_stream_eof(self, frame: dict[str, Any]) -> None: - stream = str(frame.get("stream", "")) - capture = self.captures.get(stream) - if capture is None or stream in self.stream_eof: - self.reason = self.reason or REASON_READER_ERROR - return - for line in capture.flush(): - self._consume_line(stream, line, frame) - self.stream_eof.add(stream) - - def _consume_line(self, stream: str, line: str, frame: dict[str, Any]) -> None: - capture = self.captures[stream] - redacted = self._redact(line) - capture.record(redacted) - try: - parsed = self.parse_event(stream, line) - except Exception: - self.reason = self.reason or REASON_PARSER_ERROR - return - self._apply_parsed(parsed, stream, frame, redacted) - - def _apply_parsed( - self, parsed: Any, stream: str, frame: dict[str, Any], redacted: str - ) -> None: - """Apply one closed caller observation or a bounded tuple of them.""" - if parsed is None: - return - items = parsed if isinstance(parsed, tuple) else (parsed,) - if not items or len(items) > MAX_PARSED_ITEMS: - self.reason = self.reason or REASON_MALFORMED_EVENT - return - # A failure inside this line stops the rest of the line; a reason - # latched by an earlier line keeps the pre-existing consume behaviour. - entry_reason = self.reason - for item in items: - if self.reason is not entry_reason: - return - self._apply_item(item, stream, frame, redacted) - - def _apply_item( - self, item: Any, stream: str, frame: dict[str, Any], redacted: str - ) -> None: - if isinstance(item, ParsedMetric): - self._record_metric(item, stream, frame) - return - if isinstance(item, CallerEvent): - self._apply_terminal_evidence(item.kind, stream, frame, redacted) - return - if isinstance(item, CallerTerminal): - self._apply_caller_terminal(item, stream, frame) - return - self.reason = self.reason or REASON_MALFORMED_EVENT - - def _apply_caller_terminal( - self, terminal: CallerTerminal, stream: str, frame: dict[str, Any] - ) -> None: - if self.caller_terminal is not None: - self.reason = self.reason or REASON_DUPLICATE_EVENT - return - self.caller_terminal = terminal - self._add_event( - EVENT_CALLER_TERMINAL, - SOURCE_CALLER_OUTPUT, - stream, - frame, - f"status={terminal.status} reason={terminal.reason}", - safe=True, - ) - - def _record_metric( - self, metric: ParsedMetric, stream: str, frame: dict[str, Any] - ) -> None: - """Record one typed observation whose fields are closed and safe.""" - try: - validated = validate_metric(metric) - detail = json.dumps( - metric_record(validated), sort_keys=True, separators=(",", ":") - ) - except LifecycleMetricError: - self.reason = self.reason or REASON_MALFORMED_EVENT - return - if self.metric_events >= MAX_METRIC_EVENTS: - # Truncating a typed observation stream could leave a successful - # terminal that claims a complete measurement. Retain the first - # bounded diagnostics, but fail this invocation closed. - self.reason = self.reason or REASON_MALFORMED_EVENT - return - self.metric_events += 1 - self.metrics.append(validated) - self._add_event( - METRIC_PREFIX + validated.name, validated.source, stream, frame, - detail, safe=True, - ) - - def _validate_metric_kind(self, parsed: str) -> Optional[str]: - if len(parsed) > MAX_METRIC_KIND_CHARS or METRIC_KIND_RE.fullmatch(parsed) is None: - return None - return parsed if self._redact(parsed) == parsed else None - - def _apply_terminal_evidence( - self, kind: str, stream: str, frame: dict[str, Any], redacted: str - ) -> None: - now = time.monotonic() - if kind == EVENT_FINISH: - if self.finish_at is not None: - self.reason = self.reason or REASON_DUPLICATE_EVENT - return - if self.idle_at is not None: - self.reason = self.reason or REASON_OUT_OF_ORDER_EVENT - return - self.finish_at = now - else: - if self.idle_at is not None: - self.reason = self.reason or REASON_DUPLICATE_EVENT - return - if self.finish_at is None: - self.reason = self.reason or REASON_OUT_OF_ORDER_EVENT - return - self.idle_at = now - self._add_event(kind, SOURCE_CALLER_OUTPUT, stream, frame, redacted) - - def _handle_exit(self, frame: dict[str, Any]) -> None: - self.exited = True - raw_code = frame.get("exit_code") - raw_signal = frame.get("signal") - self.exit_code = None if raw_code is None else int(raw_code) - self.signal = None if raw_signal is None else int(raw_signal) - self._add_event( - EVENT_EXITED, SOURCE_HARNESS, "", frame, - f"exit_code={self.exit_code} signal={self.signal}", - ) - - # -- completion policy ------------------------------------------------- - - def _check_deadlines(self) -> None: - now = time.monotonic() - if _is_cancelled(self.cancellation): - self.process_status_hint = PROCESS_STATUS_CANCELLED - self.reason = REASON_CANCELLED - return - if now >= self.run_deadline: - self.process_status_hint = PROCESS_STATUS_TIMED_OUT - self.reason = REASON_TIMED_OUT - return - if ( - self.finish_at is not None - and self.idle_at is None - and now - self.finish_at >= self.spec.timeout.idle_seconds - ): - self.reason = REASON_MISSING_IDLE - return - self._check_quiescence(now) - if ( - self.reason is None - and self.exited - and len(self.stream_eof) == len(self.captures) - and not self.quiet - ): - if self.caller_terminal is not None and self.idle_at is not None: - # A caller-declared product failure commonly exits non-zero. - # Preserve that code on the process axis while still allowing - # the ordered terminal stream to reach quiet. - return - if self.exit_code != 0: - self.reason = REASON_NONZERO_EXIT - elif self.idle_at is None: - self.reason = REASON_MISSING_IDLE - - def _check_quiescence(self, now: float) -> None: - if self.idle_at is None or self.quiet: - return - last_output = self.last_output_at if self.last_output_at is not None else self.idle_at - if now - last_output < self.spec.timeout.quiet_seconds: - return - self.quiet = True - self._add_event(EVENT_QUIET, SOURCE_HARNESS, "", {"ns": time.monotonic_ns()}, "") - if self.spec.completion_mode == COMPLETION_STOP_AFTER_IDLE: - self.reason = ( - REASON_SUCCESS - if self.caller_terminal is not None - else REASON_MISSING_IDLE - ) - return - if self.exited: - self.reason = ( - REASON_SUCCESS - if self.caller_terminal is not None - else (REASON_NONZERO_EXIT if self.exit_code != 0 else REASON_MISSING_IDLE) - ) - - # -- terminal handshake ------------------------------------------------ - - def _request_terminal(self) -> dict[str, Any]: - if self.external_outcome is not None: - self.reason = str(self.external_outcome.get("reason") or REASON_SUPERVISOR_ERROR) - return self.external_outcome - reason = self.reason or REASON_SUPERVISOR_ERROR - self.reason = reason - self._send({"op": "stop", "reason": reason}) - wait_seconds = ( - self.spec.timeout.cleanup_grace_seconds - + _KILL_WAIT_SECONDS - + _TERMINAL_SLACK_SECONDS - ) - frame = self._await_terminal(wait_seconds, reason) - if frame is None: - self.reason = REASON_CLEANUP_FAILED - return { - "reason": REASON_CLEANUP_FAILED, - "exit_code": self.exit_code, - "signal": self.signal, - "caller_launched": self.submitted, - "cleanup_complete": False, - "process_group_alive": True, - "receipt_path": "", - } - outcome = frame.get("outcome") or {} - self.reason = str(outcome.get("reason") or reason) - return outcome - - def _await_terminal( - self, timeout: float, frozen_reason: str - ) -> Optional[dict[str, Any]]: - """Drain all frames preceding terminal while preserving the first reason.""" - deadline = time.monotonic() + timeout - while time.monotonic() < deadline: - try: - frame = self.queue.get(timeout=_POLL_INTERVAL_SECONDS) - except queue.Empty: - continue - if frame is None: - return None - if frame.get("op") == "terminal": - self.external_outcome = frame.get("outcome") or {} - return frame - self._handle_frame(frame) - self.reason = frozen_reason - return None - - def _shutdown_supervisor(self) -> None: - for handle in (self.to_supervisor, self.from_supervisor): - try: - if handle is not None: - handle.close() - except OSError: - pass - supervisor = self.supervisor - if supervisor is not None: - try: - supervisor.wait(timeout=_SUPERVISOR_EXIT_SECONDS) - except subprocess.TimeoutExpired: - supervisor.kill() - try: - supervisor.wait(timeout=_SUPERVISOR_EXIT_SECONDS) - except subprocess.TimeoutExpired: - pass - self.reason = REASON_CLEANUP_FAILED - if self.err_handle is not None: - try: - self.err_handle.close() - except OSError: - pass - - # -- evidence ---------------------------------------------------------- - - def _redact(self, text: str) -> str: - if self.redact is not None: - try: - text = self.redact(text) - except Exception: - text = REDACTED - return fallback_redact(text) - - def _add_event( - self, kind: str, source: str, stream: str, frame: dict[str, Any], detail: str, - *, safe: bool = False, - ) -> None: - # ``safe`` details are built from already validated closed fields, so the - # adapter redactor - which only understands raw caller lines - must not - # rewrite them. The fallback secret sweep still applies. - text = fallback_redact(detail) if safe else self._redact(detail) - self.events.append(LifecycleEvent( - kind=kind, - source=source, - stream=stream, - monotonic_ns=time.monotonic_ns(), - source_monotonic_ns=int(frame.get("ns") or 0), - observed_at=_utc_now(), - detail=text[:MAX_EVENT_DETAIL_CHARS], - )) - - def _publish(self, outcome: dict[str, Any]) -> InvocationResult: - reason = str(outcome.get("reason") or self.reason or REASON_SUPERVISOR_ERROR) - self.reason = reason - cleanup_complete = bool(outcome.get("cleanup_complete")) - group_alive = bool(outcome.get("process_group_alive")) - ordered = bool(self.finish_at is not None and self.idle_at is not None and self.quiet) - harness_passed = ( - reason == REASON_SUCCESS - and cleanup_complete - and not group_alive - and ordered - and self.caller_terminal is not None - ) - product_evidence_valid = ( - self.caller_terminal is not None - and self.finish_at is not None - and self.idle_at is not None - and reason in {REASON_SUCCESS, REASON_CLEANUP_FAILED} - ) - if not product_evidence_valid: - product = ProductOutcome(PRODUCT_STATUS_UNKNOWN, PRODUCT_REASON_UNAVAILABLE) - elif self.caller_terminal.status == CALLER_STATUS_SUCCEEDED: - product = ProductOutcome(PRODUCT_STATUS_SUCCEEDED, CALLER_REASON_SUCCESS) - else: - product = ProductOutcome(PRODUCT_STATUS_FAILED, CALLER_REASON_ERROR) - exit_code = self.exit_code if self.exit_code is not None else outcome.get("exit_code") - process_signal = self.signal if self.signal is not None else outcome.get("signal") - if self.process_status_hint is not None: - process_status = self.process_status_hint - elif not self.submitted: - process_status = PROCESS_STATUS_NOT_STARTED - exit_code = None - process_signal = None - elif process_signal is not None: - process_status = PROCESS_STATUS_SIGNALLED - else: - process_status = PROCESS_STATUS_EXITED - evidence_dir = Path(self.spec.evidence_dir) - result = InvocationResult( - product=product, - harness=HarnessOutcome( - HARNESS_STATUS_PASSED if harness_passed else HARNESS_STATUS_FAILED, - REASON_SUCCESS if harness_passed else reason, - ordered, - cleanup_complete, - ), - process=ProcessOutcome(process_status, exit_code, process_signal), - submitted=self.submitted, - process_group_alive=group_alive, - events=tuple(self.events), - stdout=self.captures["stdout"].freeze(), - stderr=self.captures["stderr"].freeze(), - journal_path=str(evidence_dir / JOURNAL_FILENAME), - result_path=str(evidence_dir / RESULT_FILENAME), - locator=self.locator, - spec_digest=spec_digest(self.spec), - started_at=self.started_at, - ended_at=_utc_now(), - duration_ns=time.monotonic_ns() - self.start_ns, - metrics=tuple(self.metrics), - ) - try: - _publish_evidence(result, self.spec) - finally: - self._discard_owned_control_dir() - return result - - def _discard_owned_control_dir(self) -> None: - if self.owns_control_dir and self.control_dir is not None: - allowed = { - LOCATOR_FILENAME, - RECEIPT_FILENAME, - SUPERVISOR_ERR_FILENAME, - SOCKET_FILENAME, - } - try: - entries = tuple(self.control_dir.iterdir()) - except OSError: - return - if any(entry.name not in allowed for entry in entries): - return - shutil.rmtree(self.control_dir, ignore_errors=True) - - -def _is_cancelled(token: Any) -> bool: - if token is None: - return False - for attribute in ("is_cancelled", "is_set"): - probe = getattr(token, attribute, None) - if callable(probe): - return bool(probe()) - return bool(token() if callable(token) else token) - - -def _locator_public(locator: Optional[SupervisorLocator]) -> Optional[dict[str, Any]]: - """Return locator evidence with the challenge marker reduced to a digest.""" - if locator is None: - return None - return { - "supervisor_pid": locator.supervisor_pid, - "start_identity": locator.start_identity, - "socket_path": locator.socket_path, - "control_dir": locator.control_dir, - "challenge_digest": hashlib.sha256(locator.challenge.encode("utf-8")).hexdigest(), - "created_at": locator.created_at, - } - - -def _event_record(event: LifecycleEvent) -> dict[str, Any]: - return { - "record": "event", - "kind": event.kind, - "source": event.source, - "stream": event.stream, - "monotonic_ns": event.monotonic_ns, - "source_monotonic_ns": event.source_monotonic_ns, - "observed_at": event.observed_at, - "detail": event.detail, - } - - -def _result_record(result: InvocationResult, spec: InvocationSpec) -> dict[str, Any]: - return { - "record": "result", - "product": _product_record(result.product), - "harness": _harness_record(result.harness), - "process": _process_record(result.process), - "submitted": result.submitted, - "process_group_alive": result.process_group_alive, - "submission_mode": spec.submission_mode, - "completion_mode": spec.completion_mode, - "spec_digest": result.spec_digest, - "locator": _locator_public(result.locator), - "started_at": result.started_at, - "ended_at": result.ended_at, - "duration_ns": result.duration_ns, - "stdout": _capture_record(result.stdout), - "stderr": _capture_record(result.stderr), - "events": [_event_record(event) for event in result.events], - } - - -def _product_record(outcome: ProductOutcome) -> dict[str, Any]: - return {"status": outcome.status, "reason": outcome.reason} - - -def _harness_record(outcome: HarnessOutcome) -> dict[str, Any]: - return { - "status": outcome.status, - "reason": outcome.reason, - "ordered_terminal": outcome.ordered_terminal, - "cleanup_complete": outcome.cleanup_complete, - } - - -def _process_record(outcome: ProcessOutcome) -> dict[str, Any]: - return { - "status": outcome.status, - "exit_code": outcome.exit_code, - "signal": outcome.signal, - } - - -def _capture_record(capture: CaptureStream) -> dict[str, Any]: - return { - "stream": capture.stream, - "text": capture.text, - "line_count": capture.line_count, - "byte_count": capture.byte_count, - "truncated": capture.truncated, - } - - -def _publish_evidence(result: InvocationResult, spec: InvocationSpec) -> None: - """Publish a no-clobber journal/result pair after terminal cleanup.""" - header = { - "record": "header", - "journal_version": JOURNAL_VERSION, - "spec_digest": result.spec_digest, - "submission_mode": spec.submission_mode, - "completion_mode": spec.completion_mode, - "started_at": result.started_at, - } - terminal = { - "record": "terminal", - "product": _product_record(result.product), - "harness": _harness_record(result.harness), - "process": _process_record(result.process), - "process_group_alive": result.process_group_alive, - "ended_at": result.ended_at, - } - lines = [header] + [_event_record(event) for event in result.events] + [terminal] - journal = "".join(json.dumps(line, ensure_ascii=False) + "\n" for line in lines) - journal_path = Path(result.journal_path) - result_path = Path(result.result_path) - staged: dict[Path, Path] = {} - published: list[tuple[Path, _FileIdentity]] = [] - try: - staged[journal_path] = _stage_bytes( - journal_path.parent, journal.encode("utf-8"), 0o600 - ) - staged[result_path] = _stage_bytes( - result_path.parent, - json.dumps(_result_record(result, spec), ensure_ascii=False, indent=2).encode( - "utf-8" - ), - 0o600, - ) - for target in (journal_path, result_path): - published.append((target, _publish_staged_no_replace(staged[target], target))) - except OSError as exc: - for target, identity in reversed(published): - _rollback_owned(target, identity) - raise LifecycleError("evidence publication refused an existing target") from exc - except BaseException: - for target, identity in reversed(published): - _rollback_owned(target, identity) - raise - finally: - for stage in staged.values(): - try: - stage.unlink() - except FileNotFoundError: - pass - - -# --------------------------------------------------------------------------- -# Preflight -# --------------------------------------------------------------------------- - -def _preflight(spec: InvocationSpec) -> None: - """Validate platform and specification before any process is created.""" - if os.name != "posix" or not hasattr(os, "killpg") or not hasattr(socket, "AF_UNIX"): - raise LifecycleValidationError("bounded lifecycle requires a POSIX platform") - if sys.version_info < (3, 11): - raise LifecycleValidationError("bounded lifecycle requires Python 3.11 or newer") - if not isinstance(spec, InvocationSpec): - raise LifecycleValidationError("spec must be an InvocationSpec instance") - if spec.caller_detaches: - raise LifecycleValidationError( - "callers that detach from the owned process group are unsupported" - ) - if spec.fault_injection not in FAULT_MODES: - raise LifecycleValidationError("fault_injection must be a closed fault mode") - _preflight_invocation(spec) - _preflight_bounds(spec) - _preflight_evidence(spec) - - -def _preflight_invocation(spec: InvocationSpec) -> None: - if not isinstance(spec.argv, tuple) or not spec.argv: - raise LifecycleValidationError("argv must be a non-empty tuple") - if not all(isinstance(item, str) and item for item in spec.argv): - raise LifecycleValidationError("argv entries must be non-empty strings") - if spec.submission_mode not in SUBMISSION_MODES: - raise LifecycleValidationError(f"submission_mode must be one of {SUBMISSION_MODES}") - if spec.completion_mode not in COMPLETION_MODES: - raise LifecycleValidationError(f"completion_mode must be one of {COMPLETION_MODES}") - if spec.submission_mode == SUBMISSION_ARGV_TASK and spec.task_payload: - raise LifecycleValidationError("argv_task carries the task in argv and takes no payload") - if spec.submission_mode == SUBMISSION_STDIN_ONCE and not spec.task_payload: - raise LifecycleValidationError("stdin_once requires exactly one non-empty task payload") - if len(spec.task_payload) > MAX_TASK_PAYLOAD_BYTES: - raise LifecycleValidationError("task payload exceeds the bounded submission size") - cwd = Path(spec.cwd) - if not spec.cwd or not cwd.is_dir(): - raise LifecycleValidationError("cwd must be an existing directory") - if not isinstance(spec.env, tuple): - raise LifecycleValidationError("env must be a tuple of key/value pairs") - allowed = set(DEFAULT_ENV_ALLOWLIST) | set(spec.env_allowlist) - seen_keys: set[str] = set() - for pair in spec.env: - if not isinstance(pair, tuple) or len(pair) != 2: - raise LifecycleValidationError("environment entries must be key/value pairs") - key, value = pair - if not isinstance(key, str) or not isinstance(value, str): - raise LifecycleValidationError("environment keys and values must be strings") - if not ENV_KEY_RE.match(key): - raise LifecycleValidationError("environment keys must be POSIX identifiers") - if key not in allowed: - raise LifecycleValidationError(f"environment key '{key}' is not allowlisted") - if key in seen_keys: - raise LifecycleValidationError("environment keys must be unique") - seen_keys.add(key) - - -def _preflight_bounds(spec: InvocationSpec) -> None: - timeout = spec.timeout - if not isinstance(timeout, Timeout): - raise LifecycleValidationError("timeout must be a manifest Timeout instance") - values = ( - timeout.run_seconds, timeout.idle_seconds, - timeout.quiet_seconds, timeout.cleanup_grace_seconds, - ) - if any(not isinstance(v, int) or isinstance(v, bool) or v <= 0 for v in values): - raise LifecycleValidationError("every timeout bound must be a positive integer") - if not 0 < spec.max_capture_bytes <= MAX_CAPTURE_BYTES_LIMIT: - raise LifecycleValidationError("max_capture_bytes is out of bounds") - if not 0 < spec.max_capture_lines <= MAX_CAPTURE_LINES_LIMIT: - raise LifecycleValidationError("max_capture_lines is out of bounds") - - -def _preflight_evidence(spec: InvocationSpec) -> None: - evidence_dir = Path(spec.evidence_dir) - if not spec.evidence_dir or not evidence_dir.is_dir(): - raise LifecycleValidationError("evidence_dir must be an existing directory") - for name in (JOURNAL_FILENAME, RESULT_FILENAME): - target = evidence_dir / name - if target.exists() or target.is_symlink(): - raise LifecycleValidationError( - f"evidence '{name}' already exists and must never be overwritten" - ) - if spec.control_dir: - control_dir = Path(spec.control_dir) - if control_dir.exists() or control_dir.is_symlink(): - raise LifecycleValidationError( - "control_dir must be absent so the invocation can own it exclusively" - ) - if not control_dir.parent.is_dir(): - raise LifecycleValidationError("control_dir parent must be an existing directory") - - -# --------------------------------------------------------------------------- -# Public API -# --------------------------------------------------------------------------- - -def run_invocation( - spec: InvocationSpec, - *, - parse_event: Callable[[str, str], Any], - on_started: Callable[[SupervisorLocator], None], - redact: Optional[Callable[[str], str]] = None, - cancellation: Any = None, -) -> InvocationResult: - """Execute exactly one bounded caller invocation and publish its evidence. - - The caller is launched only after ``on_started`` durably commits the - supervisor locator. Terminal outcome, owned-process-group cleanup and - atomic evidence publication happen on every return path. - - Args: - spec: Frozen invocation specification. - parse_event: Adapter parser mapping ``(stream, line)`` to ``None``, - a typed ``CallerEvent``, ``CallerTerminal``, ``ParsedMetric``, or a - bounded tuple of those items when one caller line carries multiple - observations. - on_started: Required durable locator commit callback. - redact: Optional adapter redactor for exact secret values. - cancellation: Optional cancellation token, event or predicate. - - Returns: - Frozen InvocationResult. - - Raises: - LifecycleValidationError: If platform or specification preflight fails. - """ - if not callable(parse_event): - raise LifecycleValidationError("parse_event must be callable") - if not callable(on_started): - raise LifecycleValidationError("on_started must be callable") - return _Invocation(spec, parse_event, redact, cancellation, on_started).run() - - -def recover_invocation( - locator: SupervisorLocator, stop: bool = True -) -> TerminalOutcome: - """Authenticate a recorded supervisor and request status or bounded cleanup. - - Authentication is the marker challenge over the recorded control endpoint. - Process identity is corroboration only and never authorizes a signal. - - Raises: - LifecycleRecoveryError: If the locator is stale, forged or mismatched. - """ - _validate_locator_endpoint(locator) - connection = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - connection.settimeout(_CONTROL_SOCKET_TIMEOUT_SECONDS) - try: - try: - connection.connect(locator.socket_path) - except OSError as exc: - raise LifecycleRecoveryError("supervisor control endpoint is stale") from exc - stream = connection.makefile("rwb") - _authenticate(stream, locator) - if not stop: - _send_json(stream, {"op": "status"}) - reply = _read_frame(stream) or {} - if not reply.get("ok"): - raise LifecycleRecoveryError("supervisor refused the status request") - status = reply.get("status") or {} - return TerminalOutcome( - reason=str(status.get("terminal_reason") or ""), - exit_code=None, - signal=None, - caller_launched=bool(status.get("caller_launched")), - cleanup_complete=not bool(status.get("process_group_alive")), - process_group_alive=bool(status.get("process_group_alive")), - receipt_path="", - ) - _send_json(stream, {"op": "stop", "reason": REASON_RECOVERED_STOP}) - reply = _read_frame(stream) or {} - if not reply.get("ok"): - raise LifecycleRecoveryError("supervisor refused the cleanup request") - outcome = reply.get("outcome") or {} - finally: - connection.close() - _verify_receipt(locator, outcome) - return TerminalOutcome( - reason=str(outcome.get("reason") or REASON_RECOVERED_STOP), - exit_code=outcome.get("exit_code"), - signal=outcome.get("signal"), - caller_launched=bool(outcome.get("caller_launched")), - cleanup_complete=bool(outcome.get("cleanup_complete")), - process_group_alive=bool(outcome.get("process_group_alive")), - receipt_path=str(outcome.get("receipt_path") or ""), - ) - - -def _validate_locator_endpoint(locator: SupervisorLocator) -> None: - if not isinstance(locator, SupervisorLocator): - raise LifecycleRecoveryError("locator must be a SupervisorLocator instance") - if not locator.challenge or not locator.socket_path: - raise LifecycleRecoveryError("locator is missing its authenticated endpoint") - path = Path(locator.socket_path) - if not path.is_socket(): - raise LifecycleRecoveryError("locator socket is missing or not a socket") - live_identity = _process_start_identity(locator.supervisor_pid) - if live_identity and locator.start_identity and live_identity != locator.start_identity: - raise LifecycleRecoveryError("supervisor start identity does not match the locator") - - -def _authenticate(stream: Any, locator: SupervisorLocator) -> None: - _send_json(stream, {"op": "auth", "challenge": locator.challenge}) - reply = _read_frame(stream) or {} - if not reply.get("ok"): - raise LifecycleRecoveryError("supervisor challenge authentication failed") - if int(reply.get("supervisor_pid", -1)) != locator.supervisor_pid: - raise LifecycleRecoveryError("supervisor pid does not match the locator") - if str(reply.get("start_identity", "")) != locator.start_identity: - raise LifecycleRecoveryError("supervisor start identity does not match the locator") - - -def _verify_receipt(locator: SupervisorLocator, outcome: dict[str, Any]) -> None: - receipt_path = Path(str(outcome.get("receipt_path") or "")) - if not receipt_path.is_file(): - raise LifecycleRecoveryError("cleanup receipt is missing") - try: - receipt = json.loads(receipt_path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError) as exc: - raise LifecycleRecoveryError("cleanup receipt is unreadable") from exc - expected = hashlib.sha256(locator.challenge.encode("utf-8")).hexdigest() - if receipt.get("challenge_digest") != expected: - raise LifecycleRecoveryError("cleanup receipt does not match the locator") - if not receipt.get("cleanup_complete") or receipt.get("process_group_alive"): - raise LifecycleRecoveryError("cleanup receipt does not prove owned-group cleanup") - - -def read_locator(control_dir: str | Path) -> SupervisorLocator: - """Load a durably registered locator from a supervisor control directory.""" - raw = json.loads((Path(control_dir) / LOCATOR_FILENAME).read_text(encoding="utf-8")) - return SupervisorLocator( - supervisor_pid=int(raw["supervisor_pid"]), - start_identity=str(raw["start_identity"]), - socket_path=str(raw["socket_path"]), - challenge=str(raw["challenge"]), - control_dir=str(raw["control_dir"]), - created_at=str(raw["created_at"]), - ) - - -if __name__ == "__main__": - sys.exit(_supervisor_main(sys.argv[1:])) diff --git a/scripts/agent_benchmark/lifecycle_test.py b/scripts/agent_benchmark/lifecycle_test.py deleted file mode 100644 index 63b8d6b1..00000000 --- a/scripts/agent_benchmark/lifecycle_test.py +++ /dev/null @@ -1,1028 +0,0 @@ -"""Deterministic subprocess coverage for the bounded benchmark lifecycle.""" - -from __future__ import annotations - -import json -import os -import select -import signal -import subprocess -import sys -import tempfile -import threading -import time -import unittest -from dataclasses import replace -from pathlib import Path - -from scripts.agent_benchmark.lifecycle import ( - CALLER_REASON_ERROR, - CALLER_REASON_SUCCESS, - CALLER_STATUS_FAILED, - CALLER_STATUS_SUCCEEDED, - COMPLETION_EXIT_AFTER_IDLE, - COMPLETION_STOP_AFTER_IDLE, - REASON_CANCELLED, - REASON_CLEANUP_FAILED, - REASON_CONTROLLER_LOST, - REASON_DUPLICATE_EVENT, - REASON_MALFORMED_EVENT, - REASON_MISSING_IDLE, - REASON_NONZERO_EXIT, - REASON_OUT_OF_ORDER_EVENT, - REASON_PARSER_ERROR, - REASON_READER_ERROR, - REASON_RECOVERED_STOP, - REASON_START_CALLBACK_FAILED, - REASON_TIMED_OUT, - MAX_METRIC_EVENTS, - SUBMISSION_ARGV_TASK, - SUBMISSION_STDIN_ONCE, - CancellationToken, - CallerEvent, - CallerTerminal, - HarnessOutcome, - InvocationSpec, - LifecycleError, - LifecycleRecoveryError, - LifecycleValidationError, - ParsedMetric, - SupervisorLocator, - count_metric, - duration_metric, - env_pairs, - exact_value_redactor, - read_locator, - recover_invocation, - run_invocation, -) -from scripts.agent_benchmark.manifest import Timeout - - -def _events(_: str, line: str): - return { - "FINISH": ( - CallerTerminal(CALLER_STATUS_SUCCEEDED, CALLER_REASON_SUCCESS), - CallerEvent("finish"), - ), - "IDLE": CallerEvent("idle"), - }.get(line) - - -class LifecycleTest(unittest.TestCase): - """Each test owns a temporary evidence directory and real process group.""" - - def setUp(self) -> None: - self.tmp = tempfile.TemporaryDirectory() - self.root = Path(self.tmp.name) - - def tearDown(self) -> None: - self.tmp.cleanup() - - def _spec( - self, - source: str, - *, - submission_mode: str = SUBMISSION_ARGV_TASK, - completion_mode: str = COMPLETION_EXIT_AFTER_IDLE, - payload: bytes = b"", - run_seconds: int = 5, - max_capture_bytes: int = 4096, - max_capture_lines: int = 100, - fault_injection: str = "", - ) -> InvocationSpec: - return InvocationSpec( - argv=(sys.executable, "-u", "-c", source), - cwd=str(self.root), - env=env_pairs({"PATH": os.environ.get("PATH", "/usr/bin:/bin")}), - submission_mode=submission_mode, - completion_mode=completion_mode, - timeout=Timeout(run_seconds, 2, 1, 1), - evidence_dir=str(self.root), - task_payload=payload, - max_capture_bytes=max_capture_bytes, - max_capture_lines=max_capture_lines, - fault_injection=fault_injection, - ) - - def _run(self, spec: InvocationSpec, **kwargs: object): - return run_invocation(spec, parse_event=_events, on_started=lambda _: None, **kwargs) - - def _start_supervisor(self, control_dir: Path): - control_dir.mkdir() - controller_read, supervisor_write = os.pipe() - supervisor_read, controller_write = os.pipe() - process = subprocess.Popen( - ( - sys.executable, - "-m", - "scripts.agent_benchmark.lifecycle", - f"--read-fd={supervisor_read}", - f"--write-fd={supervisor_write}", - f"--control-dir={control_dir}", - ), - cwd=str(Path(__file__).resolve().parents[2]), - env={ - "PATH": os.environ.get("PATH", "/usr/bin:/bin"), - "PYTHONPATH": str(Path(__file__).resolve().parents[2]), - }, - pass_fds=(supervisor_read, supervisor_write), - start_new_session=True, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ) - os.close(supervisor_read) - os.close(supervisor_write) - return ( - process, - os.fdopen(controller_write, "wb", buffering=0), - os.fdopen(controller_read, "rb", buffering=0), - ) - - def _send_supervisor_spec(self, writer: object, source: str) -> None: - self._write_frame(writer, { - "op": "spec", - "argv": [sys.executable, "-u", "-c", source], - "cwd": str(self.root), - "env": [["PATH", os.environ.get("PATH", "/usr/bin:/bin")]], - "submission_mode": SUBMISSION_ARGV_TASK, - "task_payload_hex": "", - "max_capture_bytes": 1024, - "cleanup_grace_seconds": 1, - "fault_injection": "", - }) - - def _close_supervisor( - self, process: subprocess.Popen, writer: object, reader: object - ) -> None: - if process.poll() is None: - for candidate in self.root.glob("*-control/locator.json"): - if candidate.is_file(): - try: - recover_invocation(read_locator(candidate.parent)) - except LifecycleRecoveryError: - pass - if process.poll() is not None: - break - for handle in (writer, reader): - try: - handle.close() # type: ignore[attr-defined] - except OSError: - pass - if process.poll() is None: - process.kill() - process.wait(timeout=5) - - def test_exit_after_idle_publishes_ordered_atomic_evidence(self) -> None: - result = self._run(self._spec("print('FINISH'); print('IDLE')")) - - self.assertTrue(result.product.status == "succeeded") - self.assertTrue(result.harness.cleanup_complete) - self.assertFalse(result.process_group_alive) - self.assertTrue(result.harness.ordered_terminal) - self.assertEqual([event.kind for event in result.events], [ - "submitted", "first_output", "caller_terminal", "finish", "idle", - "exited", "quiet", - ]) - self.assertTrue(Path(result.journal_path).is_file()) - published = json.loads(Path(result.result_path).read_text(encoding="utf-8")) - self.assertNotIn("argv", published) - self.assertNotIn("env", published) - - def test_stdin_once_submits_exactly_once_and_closes_input(self) -> None: - source = "import sys; print(sys.stdin.read()); print('FINISH'); print('IDLE')" - result = self._run(self._spec( - source, - submission_mode=SUBMISSION_STDIN_ONCE, - payload=b"single task payload", - )) - - self.assertTrue(result.product.status == "succeeded") - self.assertIn("single task payload", result.stdout.text) - self.assertEqual(sum(event.kind == "submitted" for event in result.events), 1) - - def test_stdin_once_non_reader_times_out_and_cleans_group(self) -> None: - started = time.monotonic() - result = self._run(self._spec( - "import time; time.sleep(30)", - submission_mode=SUBMISSION_STDIN_ONCE, - payload=b"x" * (1 << 20), - run_seconds=1, - )) - - self.assertLess(time.monotonic() - started, 6) - self.assertEqual(result.harness.reason, REASON_TIMED_OUT) - self.assertFalse(result.submitted) - self.assertEqual(sum(event.kind == "submitted" for event in result.events), 0) - self.assertTrue(result.harness.cleanup_complete) - self.assertFalse(result.process_group_alive) - - def test_unterminated_final_idle_is_consumed_before_terminal(self) -> None: - source = "import sys; sys.stdout.write('FINISH\\nIDLE'); sys.stdout.flush()" - for index, mode in enumerate( - (COMPLETION_EXIT_AFTER_IDLE, COMPLETION_STOP_AFTER_IDLE) - ): - with self.subTest(completion_mode=mode): - evidence = self.root / f"unterminated-{index}" - evidence.mkdir() - result = self._run(replace( - self._spec(source, completion_mode=mode), - evidence_dir=str(evidence), - )) - - self.assertTrue(result.product.status == "succeeded") - kinds = [event.kind for event in result.events] - self.assertLess(kinds.index("finish"), kinds.index("idle")) - self.assertLess(kinds.index("idle"), kinds.index("quiet")) - records = [ - json.loads(line) - for line in Path(result.journal_path).read_text(encoding="utf-8").splitlines() - ] - self.assertEqual(sum(record.get("record") == "terminal" for record in records), 1) - self.assertEqual(records[-1]["record"], "terminal") - - def test_stop_after_idle_gracefully_stops_live_caller(self) -> None: - started = time.monotonic() - result = self._run(self._spec( - "import time; print('FINISH'); print('IDLE'); time.sleep(30)", - completion_mode=COMPLETION_STOP_AFTER_IDLE, - )) - - self.assertTrue(result.product.status == "succeeded") - self.assertLess(time.monotonic() - started, 8) - self.assertTrue(result.harness.cleanup_complete) - self.assertFalse(result.process_group_alive) - - def test_caller_output_cannot_synthesize_submission(self) -> None: - result = self._run(self._spec("print('submitted'); print('FINISH'); print('IDLE')")) - - self.assertTrue(result.product.status == "succeeded") - self.assertEqual(sum(event.kind == "submitted" for event in result.events), 1) - self.assertEqual(result.events[0].source, "harness") - - def test_invalid_terminal_sequences_fail_closed(self) -> None: - cases = ( - ("print('IDLE')", REASON_OUT_OF_ORDER_EVENT), - ("print('FINISH'); print('FINISH')", REASON_DUPLICATE_EVENT), - ("print('FINISH')", REASON_MISSING_IDLE), - ) - for source, reason in cases: - with self.subTest(reason=reason): - evidence = self.root / reason - evidence.mkdir() - spec = replace(self._spec(source), evidence_dir=str(evidence)) - result = self._run(spec) - self.assertFalse(result.product.status == "succeeded") - self.assertEqual(result.harness.reason, reason) - self.assertTrue(result.harness.cleanup_complete) - - def test_malformed_parser_and_nonzero_exit_fail_closed(self) -> None: - malformed = run_invocation( - self._spec("print('UNKNOWN')"), - parse_event=lambda _stream, _line: "submitted", - on_started=lambda _: None, - ) - self.assertEqual(malformed.harness.reason, REASON_MALFORMED_EVENT) - self.assertTrue(malformed.harness.cleanup_complete) - - evidence = self.root / "nonzero" - evidence.mkdir() - failed = self._run(replace( - self._spec("print('FINISH'); print('IDLE'); raise SystemExit(7)"), - evidence_dir=str(evidence), - )) - self.assertEqual(failed.product.status, "succeeded") - self.assertEqual(failed.harness.status, "passed") - self.assertEqual(failed.process.exit_code, 7) - self.assertTrue(failed.harness.cleanup_complete) - - def test_product_error_can_have_clean_harness_and_process(self) -> None: - def parse_error(_stream: str, line: str): - return { - "ERROR": ( - CallerTerminal(CALLER_STATUS_FAILED, CALLER_REASON_ERROR), - CallerEvent("finish"), - ), - "IDLE": CallerEvent("idle"), - }.get(line) - - result = run_invocation( - self._spec("print('ERROR'); print('IDLE')"), - parse_event=parse_error, - on_started=lambda _: None, - ) - - self.assertEqual((result.product.status, result.product.reason), ( - "failed", CALLER_REASON_ERROR, - )) - self.assertEqual((result.harness.status, result.harness.reason), ( - "passed", "success", - )) - self.assertEqual((result.process.status, result.process.exit_code), ( - "exited", 0, - )) - - with self.assertRaises(LifecycleValidationError): - HarnessOutcome("failed", "success", True, True) - with self.assertRaises(LifecycleValidationError): - HarnessOutcome("passed", "success", False, True) - - def test_parser_failure_leaves_product_unknown(self) -> None: - def broken_parser(_stream: str, _line: str): - raise ValueError("synthetic parser failure") - - result = run_invocation( - self._spec("print('BROKEN')"), - parse_event=broken_parser, - on_started=lambda _: None, - ) - - self.assertEqual((result.product.status, result.product.reason), ( - "unknown", "unavailable", - )) - self.assertEqual((result.harness.status, result.harness.reason), ( - "failed", REASON_PARSER_ERROR, - )) - - def test_timeout_cancel_and_cleanup_do_not_fabricate_product(self) -> None: - timeout = self._run(self._spec("import time; time.sleep(30)", run_seconds=1)) - - token = CancellationToken() - timer = threading.Timer(0.2, token.cancel) - timer.start() - try: - cancel_root = self.root / "independent-cancel" - cancel_root.mkdir() - cancelled = self._run( - replace( - self._spec("import time; time.sleep(30)"), - evidence_dir=str(cancel_root), - ), - cancellation=token, - ) - finally: - timer.cancel() - - cleanup_root = self.root / "independent-cleanup" - cleanup_root.mkdir() - cleanup_control = self.root / "independent-cleanup-control" - - def collide_receipt(locator: SupervisorLocator) -> None: - (Path(locator.control_dir) / "cleanup-receipt.json").write_bytes( - b"collision" - ) - - cleanup = run_invocation( - replace( - self._spec("print('FINISH'); print('IDLE')"), - evidence_dir=str(cleanup_root), - control_dir=str(cleanup_control), - ), - parse_event=_events, - on_started=collide_receipt, - ) - - for result, product_status, process_status, reason in ( - (timeout, "unknown", "timed_out", REASON_TIMED_OUT), - (cancelled, "unknown", "cancelled", REASON_CANCELLED), - (cleanup, "succeeded", "exited", REASON_CLEANUP_FAILED), - ): - with self.subTest(reason=reason): - self.assertEqual(result.product.status, product_status) - self.assertEqual(result.harness.reason, reason) - self.assertEqual(result.process.status, process_status) - - def test_timeout_cancel_and_reader_error_all_cleanup(self) -> None: - timeout = self._run(self._spec("import time; time.sleep(30)", run_seconds=1)) - self.assertEqual(timeout.harness.reason, REASON_TIMED_OUT) - self.assertTrue(timeout.harness.cleanup_complete) - - token = CancellationToken() - timer = threading.Timer(0.2, token.cancel) - timer.start() - try: - evidence = self.root / "cancel" - evidence.mkdir() - cancelled = self._run(replace( - self._spec("import time; time.sleep(30)"), evidence_dir=str(evidence)), - cancellation=token, - ) - finally: - timer.cancel() - self.assertEqual(cancelled.harness.reason, REASON_CANCELLED) - self.assertTrue(cancelled.harness.cleanup_complete) - - evidence = self.root / "reader" - evidence.mkdir() - reader_error = self._run(replace( - self._spec("print('FINISH'); print('IDLE')", fault_injection="reader_error"), - evidence_dir=str(evidence), - )) - self.assertEqual(reader_error.harness.reason, REASON_READER_ERROR) - self.assertTrue(reader_error.harness.cleanup_complete) - - def test_redaction_and_capture_bounds_apply_before_publication(self) -> None: - secret = "EXACT_SECRET_123456789" - source = ( - f"print('{secret}'); print('Authorization Bearer fallback-token-123456789'); " - "print('FINISH'); print('IDLE')" - ) - result = self._run( - self._spec(source, max_capture_bytes=4096, max_capture_lines=3), - redact=exact_value_redactor((secret,)), - ) - - evidence = Path(result.result_path).read_text(encoding="utf-8") - self.assertNotIn(secret, evidence) - self.assertNotIn("fallback-token-123456789", evidence) - self.assertIn("[redacted]", evidence) - self.assertTrue(result.stdout.truncated) - - def test_concurrent_evidence_collision_preserves_existing_files(self) -> None: - occupied_control = self.root / "occupied-control" - occupied_control.mkdir() - locator_sentinel = occupied_control / "locator.json" - locator_sentinel.write_bytes(b"locator-sentinel") - with self.assertRaises(LifecycleValidationError): - self._run(replace( - self._spec("print('FINISH'); print('IDLE')"), - control_dir=str(occupied_control), - )) - self.assertEqual(locator_sentinel.read_bytes(), b"locator-sentinel") - - racing_control = self.root / "racing-locator-control" - process, writer, reader = self._start_supervisor(racing_control) - racing_locator = racing_control / "locator.json" - racing_locator.write_bytes(b"concurrent-locator") - try: - self._send_supervisor_spec(writer, "print('FINISH'); print('IDLE')") - frames = [] - while True: - frame = self._read_frame(reader, 5) - frames.append(frame) - if frame.get("op") == "terminal": - break - self.assertEqual(process.wait(timeout=8), 1) - self.assertTrue(any(frame.get("op") == "error" for frame in frames)) - finally: - self._close_supervisor(process, writer, reader) - self.assertEqual(racing_locator.read_bytes(), b"concurrent-locator") - - for index, collision_name in enumerate( - ("lifecycle-journal.jsonl", "lifecycle-result.json") - ): - with self.subTest(collision=collision_name): - evidence = self.root / f"collision-{index}" - evidence.mkdir() - control = self.root / f"collision-control-{index}" - sentinel = evidence / collision_name - unrelated = evidence / "unrelated.txt" - sentinel_bytes = f"sentinel-{index}".encode() - unrelated.write_bytes(b"unrelated") - - def collide(_: SupervisorLocator) -> None: - sentinel.write_bytes(sentinel_bytes) - - with self.assertRaises(LifecycleError): - run_invocation( - replace( - self._spec("print('FINISH'); print('IDLE')"), - evidence_dir=str(evidence), - control_dir=str(control), - ), - parse_event=_events, - on_started=collide, - ) - - self.assertEqual(sentinel.read_bytes(), sentinel_bytes) - self.assertEqual(unrelated.read_bytes(), b"unrelated") - other_name = ( - "lifecycle-result.json" - if collision_name.endswith("jsonl") - else "lifecycle-journal.jsonl" - ) - self.assertFalse((evidence / other_name).exists()) - - receipt_evidence = self.root / "receipt-collision-evidence" - receipt_evidence.mkdir() - receipt_control = self.root / "receipt-collision-control" - receipt_sentinel = b"receipt-sentinel" - - def collide_receipt(locator: SupervisorLocator) -> None: - (Path(locator.control_dir) / "cleanup-receipt.json").write_bytes( - receipt_sentinel - ) - - receipt_result = run_invocation( - replace( - self._spec("print('FINISH'); print('IDLE')"), - evidence_dir=str(receipt_evidence), - control_dir=str(receipt_control), - ), - parse_event=_events, - on_started=collide_receipt, - ) - self.assertEqual(receipt_result.harness.reason, REASON_CLEANUP_FAILED) - self.assertEqual(receipt_result.product.status, "succeeded") - self.assertFalse(receipt_result.harness.cleanup_complete) - self.assertFalse(receipt_result.process_group_alive) - self.assertEqual( - (receipt_control / "cleanup-receipt.json").read_bytes(), receipt_sentinel - ) - published = json.loads(Path(receipt_result.result_path).read_text(encoding="utf-8")) - self.assertEqual(published["harness"]["reason"], REASON_CLEANUP_FAILED) - journal = [ - json.loads(line) - for line in Path(receipt_result.journal_path).read_text(encoding="utf-8").splitlines() - ] - terminals = [record for record in journal if record.get("record") == "terminal"] - self.assertEqual(len(terminals), 1) - self.assertEqual(terminals[0]["harness"]["reason"], REASON_CLEANUP_FAILED) - - def test_metric_kind_cannot_leak_secret(self) -> None: - cases = ( - ("exact_secret_123456789", exact_value_redactor(("exact_secret_123456789",))), - ("iop_abcdefghijklmnop", None), - ("a" * 65, None), - ) - for index, (metric_name, redactor) in enumerate(cases): - with self.subTest(metric=metric_name[:24]): - evidence = self.root / f"metric-invalid-{index}" - evidence.mkdir() - - def parse_metric(_stream: str, line: str) -> str | None: - terminal = _events(_stream, line) - return terminal if terminal is not None else f"metric:{line}" - - result = run_invocation( - replace( - self._spec( - f"print({metric_name!r}); print('FINISH'); print('IDLE')" - ), - evidence_dir=str(evidence), - ), - parse_event=parse_metric, - on_started=lambda _: None, - redact=redactor, - ) - self.assertEqual(result.harness.reason, REASON_MALFORMED_EVENT) - self.assertFalse(any(event.kind == f"metric:{metric_name}" for event in result.events)) - persisted = "\n".join( - path.read_text(encoding="utf-8") - for path in (Path(result.journal_path), Path(result.result_path)) - ) - if index < 2: - self.assertNotIn(metric_name, persisted) - - evidence = self.root / "metric-valid" - evidence.mkdir() - - def parse_valid(_stream: str, line: str) -> str | None: - terminal = _events(_stream, line) - return terminal if terminal is not None else f"metric:{line}" - - valid = run_invocation( - replace( - self._spec("print('duration_ms'); print('FINISH'); print('IDLE')"), - evidence_dir=str(evidence), - ), - parse_event=parse_valid, - on_started=lambda _: None, - ) - self.assertEqual(valid.product.status, "unknown") - self.assertEqual(valid.harness.reason, REASON_MALFORMED_EVENT) - self.assertNotIn("metric:duration_ms", [event.kind for event in valid.events]) - - def test_first_output_is_recorded_once_before_terminal_evidence(self) -> None: - source = ( - "import sys; sys.stdout.write(' \\n'); sys.stdout.flush(); " - "print('chatter'); print('more chatter'); print('FINISH'); print('IDLE')" - ) - result = self._run(self._spec(source)) - - self.assertTrue(result.product.status == "succeeded") - kinds = [event.kind for event in result.events] - self.assertEqual(kinds.count("first_output"), 1) - self.assertLess(kinds.index("submitted"), kinds.index("first_output")) - self.assertLess(kinds.index("first_output"), kinds.index("finish")) - first_output = result.events[kinds.index("first_output")] - # The instant is the harness observation of a caller frame, so its - # source is the harness and its stream is the observed caller stream. - self.assertEqual((first_output.source, first_output.stream), ("harness", "stdout")) - self.assertGreater(first_output.monotonic_ns, 0) - - def test_silent_caller_records_no_first_output(self) -> None: - result = self._run(self._spec("import time; time.sleep(30)", run_seconds=1)) - self.assertEqual(result.harness.reason, REASON_TIMED_OUT) - self.assertNotIn("first_output", [event.kind for event in result.events]) - - def test_typed_observations_are_published_with_terminal_evidence(self) -> None: - def parse_metric(_stream: str, line: str) -> object: - if line.strip() != "REPORT": - return _events(_stream, line) - return ( - duration_metric("total_duration", "12.5", model="claude-sonnet"), - count_metric("input_tokens", 11, model="claude-sonnet"), - ) - - result = run_invocation( - self._spec("print('REPORT'); print('FINISH'); print('IDLE')"), - parse_event=parse_metric, - on_started=lambda _: None, - ) - self.assertTrue(result.product.status == "succeeded") - self.assertEqual( - [(metric.name, metric.value) for metric in result.metrics], - [("total_duration", 12_500_000), ("input_tokens", 11)], - ) - kinds = [event.kind for event in result.events] - self.assertEqual(kinds.count("metric:total_duration"), 1) - journal = Path(result.journal_path).read_text(encoding="utf-8") - self.assertIn('\\"clock\\":\\"caller_reported\\"', journal) - self.assertIn('\\"model\\":\\"claude-sonnet\\"', journal) - self.assertIn('\\"unit\\":\\"tokens\\"', journal) - - def test_invalid_or_oversized_observation_sets_fail_closed(self) -> None: - forged = ParsedMetric( - "total_duration", 5, "tokens", "caller_reported", "caller_output" - ) - cases = { - "wrong-unit": forged, - "empty-set": (), - "oversized-set": tuple( - count_metric("input_tokens", index) for index in range(17) - ), - "nested-set": ((count_metric("input_tokens", 1),),), - } - for name, parsed in cases.items(): - with self.subTest(name=name): - evidence = self.root / f"observation-{name}" - evidence.mkdir() - result = run_invocation( - replace( - self._spec("print('REPORT'); print('FINISH'); print('IDLE')"), - evidence_dir=str(evidence), - ), - parse_event=lambda _stream, line, parsed=parsed: ( - parsed if line.strip() == "REPORT" else _events(_stream, line) - ), - on_started=lambda _: None, - ) - self.assertEqual(result.harness.reason, REASON_MALFORMED_EVENT) - self.assertEqual(result.metrics, ()) - - def test_metric_event_overflow_is_malformed_after_retaining_the_bound(self) -> None: - def parse_metric(_stream: str, line: str) -> object: - if line == "REPORT": - return count_metric("input_tokens", 1) - return _events(_stream, line) - - reports = "\n".join(["print('REPORT')"] * (MAX_METRIC_EVENTS + 1)) - result = run_invocation( - self._spec(reports + "; print('FINISH'); print('IDLE')"), - parse_event=parse_metric, - on_started=lambda _: None, - ) - self.assertEqual(result.harness.reason, REASON_MALFORMED_EVENT) - self.assertEqual(len(result.metrics), MAX_METRIC_EVENTS) - - def test_adapter_redactor_cannot_corrupt_a_validated_observation(self) -> None: - secret = "EXACT_SECRET_123456789" - - def parse_metric(_stream: str, line: str) -> object: - if line.strip().startswith("REPORT"): - return (count_metric("output_tokens", 22, model="claude-sonnet"), ) - return _events(_stream, line) - - result = run_invocation( - self._spec(f"print('REPORT {secret}'); print('FINISH'); print('IDLE')"), - parse_event=parse_metric, - on_started=lambda _: None, - # A structural redactor that rewrites every raw caller line must not - # rewrite a detail built from already validated closed fields. - redact=lambda _line: "[structural]", - ) - self.assertTrue(result.product.status == "succeeded") - detail = next( - event.detail for event in result.events if event.kind == "metric:output_tokens" - ) - self.assertEqual(json.loads(detail)["value"], 22) - self.assertNotIn(secret, Path(result.result_path).read_text(encoding="utf-8")) - - def test_callback_failure_launches_no_caller_and_persists_failure(self) -> None: - marker = self.root / "caller-ran" - spec = self._spec(f"from pathlib import Path; Path({str(marker)!r}).write_text('ran')") - - result = run_invocation( - spec, - parse_event=_events, - on_started=lambda _locator: (_ for _ in ()).throw(RuntimeError("durable write failed")), - ) - - self.assertEqual(result.harness.reason, REASON_START_CALLBACK_FAILED) - self.assertFalse(result.submitted) - self.assertFalse(marker.exists()) - self.assertTrue(result.harness.cleanup_complete) - - def test_forged_live_locator_refuses_recovery(self) -> None: - checked: list[SupervisorLocator] = [] - - def verify(locator: SupervisorLocator) -> None: - self.assertEqual(read_locator(locator.control_dir), locator) - checked.append(locator) - with self.assertRaises(LifecycleRecoveryError): - recover_invocation(replace(locator, challenge="forged-marker")) - - result = run_invocation( - self._spec("print('FINISH'); print('IDLE')"), - parse_event=_events, - on_started=verify, - ) - self.assertTrue(checked) - self.assertTrue(result.product.status == "succeeded") - - def test_control_socket_bind_supports_short_symlink_alias(self) -> None: - with tempfile.TemporaryDirectory( - dir=Path.cwd(), prefix=".lifecycle-symlink-target-" - ) as target_name, tempfile.TemporaryDirectory( - dir=tempfile.gettempdir(), prefix="iop-life-alias-parent-" - ) as alias_parent_name: - target = Path(target_name) - alias = Path(alias_parent_name) / "attempt" - alias.symlink_to(target.resolve(), target_is_directory=True) - evidence = target / "evidence" - evidence.mkdir() - result = self._run(replace( - self._spec("print('FINISH'); print('IDLE')"), - evidence_dir=str(evidence), - control_dir=str(alias / "control"), - )) - self.assertTrue(result.product.status == "succeeded") - self.assertTrue((target / "control" / "locator.json").is_file()) - self.assertFalse((target / "control" / "control.sock").exists()) - - def test_owned_descendant_ignoring_term_is_killed_and_reaped(self) -> None: - descendant_pid_path = self.root / "descendant.pid" - descendant_source = ( - "import signal,time; from pathlib import Path; " - "signal.signal(signal.SIGTERM, signal.SIG_IGN); " - f"Path({str(descendant_pid_path)!r}).write_text(str(__import__('os').getpid())); " - "time.sleep(30)" - ) - caller_source = ( - "import subprocess,sys,time; from pathlib import Path; " - f"p=subprocess.Popen([sys.executable,'-c',{descendant_source!r}]); " - f"deadline=time.monotonic()+5; marker=Path({str(descendant_pid_path)!r}); " - "\nwhile not marker.exists() and time.monotonic() None: - control = self.root / "recovery-control" - process, writer, reader = self._start_supervisor(control) - try: - self._send_supervisor_spec(writer, "import time; time.sleep(30)") - self.assertEqual(self._read_frame(reader, 5).get("op"), "registered") - locator = read_locator(control) - self._write_frame(writer, {"op": "start"}) - self.assertEqual(self._read_frame(reader, 5).get("op"), "started") - - status = recover_invocation(locator, stop=False) - self.assertTrue(status.caller_launched) - self.assertTrue(status.process_group_alive) - stopped = recover_invocation(locator, stop=True) - self.assertEqual(stopped.reason, REASON_RECOVERED_STOP) - self.assertTrue(stopped.cleanup_complete) - self.assertFalse(stopped.process_group_alive) - self.assertEqual(process.wait(timeout=8), 0) - finally: - self._close_supervisor(process, writer, reader) - - receipt = json.loads((control / "cleanup-receipt.json").read_text(encoding="utf-8")) - self.assertEqual(receipt["reason"], REASON_RECOVERED_STOP) - self.assertTrue(receipt["cleanup_complete"]) - - def test_locator_identity_mismatches_refuse_recovery(self) -> None: - control = self.root / "mismatch-control" - process, writer, reader = self._start_supervisor(control) - independent = subprocess.Popen( - (sys.executable, "-c", "import time; time.sleep(30)"), - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ) - try: - self._send_supervisor_spec(writer, "import time; time.sleep(30)") - self.assertEqual(self._read_frame(reader, 5).get("op"), "registered") - locator = read_locator(control) - self._write_frame(writer, {"op": "start"}) - self.assertEqual(self._read_frame(reader, 5).get("op"), "started") - - with self.assertRaises(LifecycleRecoveryError): - recover_invocation(replace(locator, start_identity="mismatched-start")) - with self.assertRaises(LifecycleRecoveryError): - recover_invocation(replace(locator, supervisor_pid=independent.pid)) - status = recover_invocation(locator, stop=False) - self.assertTrue(status.process_group_alive) - stopped = recover_invocation(locator) - self.assertTrue(stopped.cleanup_complete) - self.assertFalse(stopped.process_group_alive) - self.assertEqual(process.wait(timeout=8), 0) - finally: - independent.terminate() - independent.wait(timeout=5) - self._close_supervisor(process, writer, reader) - - def test_controller_eof_before_start_launches_no_caller(self) -> None: - control = self.root / "pre-start-control" - marker = self.root / "pre-start-caller-ran" - process, writer, reader = self._start_supervisor(control) - try: - self._send_supervisor_spec( - writer, - f"from pathlib import Path; Path({str(marker)!r}).write_text('ran')", - ) - self.assertEqual(self._read_frame(reader, 5).get("op"), "registered") - writer.close() - self.assertEqual(process.wait(timeout=8), 0) - finally: - self._close_supervisor(process, writer, reader) - - receipt = json.loads((control / "cleanup-receipt.json").read_text(encoding="utf-8")) - self.assertEqual(receipt["reason"], REASON_CONTROLLER_LOST) - self.assertFalse(receipt["caller_launched"]) - self.assertTrue(receipt["cleanup_complete"]) - self.assertFalse(marker.exists()) - - def test_near_deadline_terminal_reason_and_receipt_are_consistent(self) -> None: - cases = ( - ( - "timeout", - "import time; time.sleep(.85); print('FINISH'); print('IDLE')", - REASON_TIMED_OUT, - None, - ), - ( - "cancel", - "import time; time.sleep(.15); print('FINISH'); print('IDLE'); time.sleep(30)", - REASON_CANCELLED, - .9, - ), - ) - for index, (name, source, expected, cancel_after) in enumerate(cases): - with self.subTest(race=name): - evidence = self.root / f"race-evidence-{index}" - evidence.mkdir() - control = self.root / f"race-control-{index}" - token = CancellationToken() if cancel_after is not None else None - timer = ( - threading.Timer(cancel_after, token.cancel) - if cancel_after is not None and token is not None - else None - ) - if timer is not None: - timer.start() - try: - result = self._run( - replace( - self._spec( - source, - completion_mode=COMPLETION_STOP_AFTER_IDLE, - run_seconds=1 if name == "timeout" else 5, - ), - evidence_dir=str(evidence), - control_dir=str(control), - ), - cancellation=token, - ) - finally: - if timer is not None: - timer.cancel() - - receipt = json.loads( - (control / "cleanup-receipt.json").read_text(encoding="utf-8") - ) - published = json.loads(Path(result.result_path).read_text(encoding="utf-8")) - journal = [ - json.loads(line) - for line in Path(result.journal_path).read_text(encoding="utf-8").splitlines() - ] - terminals = [record for record in journal if record.get("record") == "terminal"] - self.assertEqual(result.harness.reason, expected) - self.assertEqual(receipt["reason"], expected) - self.assertEqual(published["harness"]["reason"], expected) - self.assertEqual(receipt["exit_code"], published["process"]["exit_code"]) - self.assertEqual(receipt["signal"], published["process"]["signal"]) - self.assertEqual(len(terminals), 1) - self.assertEqual(terminals[0]["harness"]["reason"], expected) - self.assertEqual(journal[-1]["record"], "terminal") - self.assertTrue(result.harness.cleanup_complete) - self.assertFalse(result.process_group_alive) - - def test_controller_eof_routes_supervisor_through_cleanup(self) -> None: - """Exercise the crash window directly: EOF after START must clean the group.""" - control_dir = self.root / "control" - control_dir.mkdir() - controller_read, supervisor_write = os.pipe() - supervisor_read, controller_write = os.pipe() - process = subprocess.Popen( - ( - sys.executable, - "-m", - "scripts.agent_benchmark.lifecycle", - f"--read-fd={supervisor_read}", - f"--write-fd={supervisor_write}", - f"--control-dir={control_dir}", - ), - cwd=str(Path(__file__).resolve().parents[2]), - env={"PATH": os.environ.get("PATH", "/usr/bin:/bin"), "PYTHONPATH": str(Path(__file__).resolve().parents[2])}, - pass_fds=(supervisor_read, supervisor_write), - start_new_session=True, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ) - os.close(supervisor_read) - os.close(supervisor_write) - writer = os.fdopen(controller_write, "wb", buffering=0) - reader = os.fdopen(controller_read, "rb", buffering=0) - try: - self._write_frame(writer, { - "op": "spec", - "argv": [sys.executable, "-u", "-c", "import time; time.sleep(30)"], - "cwd": str(self.root), - "env": [["PATH", os.environ.get("PATH", "/usr/bin:/bin")]], - "submission_mode": SUBMISSION_ARGV_TASK, - "task_payload_hex": "", - "max_capture_bytes": 1024, - "cleanup_grace_seconds": 1, - "fault_injection": "", - }) - self.assertEqual(self._read_frame(reader, 5).get("op"), "registered") - self._write_frame(writer, {"op": "start"}) - self.assertEqual(self._read_frame(reader, 5).get("op"), "started") - writer.close() # Simulated controller loss. - self.assertEqual(process.wait(timeout=8), 0) - finally: - reader.close() - if process.poll() is None: - process.kill() - process.wait(timeout=5) - receipt = json.loads((control_dir / "cleanup-receipt.json").read_text(encoding="utf-8")) - self.assertEqual(receipt["reason"], REASON_CONTROLLER_LOST) - self.assertTrue(receipt["caller_launched"]) - self.assertTrue(receipt["cleanup_complete"]) - self.assertFalse(receipt["process_group_alive"]) - - @staticmethod - def _write_frame(handle: object, value: dict[str, object]) -> None: - handle.write((json.dumps(value) + "\n").encode("utf-8")) # type: ignore[attr-defined] - handle.flush() # type: ignore[attr-defined] - - @staticmethod - def _read_frame(handle: object, timeout: float) -> dict[str, object]: - ready, _, _ = select.select([handle], [], [], timeout) - if not ready: - raise AssertionError("timed out waiting for supervisor frame") - raw = handle.readline() # type: ignore[attr-defined] - if not raw: - raise AssertionError("supervisor closed its frame stream") - return json.loads(raw) - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/agent_benchmark/live_iop.py b/scripts/agent_benchmark/live_iop.py deleted file mode 100644 index a15aa89a..00000000 --- a/scripts/agent_benchmark/live_iop.py +++ /dev/null @@ -1,1202 +0,0 @@ -"""Explicit, secret-safe live IOP boundary for benchmark callers. - -Only the six ``IOP_BENCH__{BASE_URL,SECRET_ENV}`` values select this -boundary. The referenced secret is retained in a private runtime object and -is never included in connectivity evidence, command output, or run metadata. -""" - -from __future__ import annotations - -import hashlib -import json -import os -import re -import shutil -import stat -import subprocess -import tempfile -from collections.abc import Callable, Mapping -from contextlib import ExitStack, contextmanager -from dataclasses import dataclass, replace -from pathlib import Path -from typing import Any, Iterator -from urllib.parse import urlsplit, urlunsplit -from urllib.error import HTTPError -from urllib.request import Request, urlopen - -from scripts.agent_benchmark.agy_iop import ( - AGY_CALLER, - AgyEventParser, - AgyPreflightResult, - AgyRuntimeInputs, - AgyRuntimeObservation, - build_agy_invocation, - inspect_agy_iop_capability, - preflight_agy_iop, - run_agy_invocation, - _runtime_identity as _agy_runtime_identity, -) -from scripts.agent_benchmark.attempts import Attempt, ExecutionAdapter, PreflightObservation -from scripts.agent_benchmark.claude_iop import ClaudeIopAdapter, ClaudeIopRuntime, claude_capability -from scripts.agent_benchmark.codex_iop import ( - BASE_URL_ENV_KEY, - SECRET_ENV_KEY, - build_codex_invocation, - codex_capability, - run_codex_invocation, - runtime_from_environment, -) -from scripts.agent_benchmark.connectivity import ( - ISSUE_CODE_ORDER, - ISSUE_RESUME_CODES, - CallerCapability, - ConnectivityIssue, - EffectiveBinding, - RequestedEffectiveBinding, - make_result, -) -from scripts.agent_benchmark.lifecycle import InvocationResult, run_invocation, spec_digest -from scripts.agent_benchmark.manifest import ( - CALLER_ENUM, - STAGE_ENUM, - MatrixCell, - TOKEN_RE, - Timeout, -) -from scripts.agent_benchmark.scoring import ( - BlindWorkspace, - ScoringAdapter, - ScoringEvidenceFinalization, - ScoringInvocationResult, -) -from scripts.agent_benchmark.workspace import ( - AttemptIdentity, - PreparedWorkspace, - TestbedProvenance, -) - - -_ENV_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]{0,63}$") -_CALLERS = ("claude", "agy", "codex") -_TIMEOUT_SECONDS = 10 -_SECRET_SCAN_MAX_ENTRIES = 100_000 -_SECRET_SCAN_MAX_DEPTH = 64 -_SECRET_SCAN_MAX_FILE_BYTES = 32 * 1024 * 1024 - - -class LiveIopError(Exception): - """Private boundary failure converted into one closed public issue.""" - - def __init__(self, issue_code: str) -> None: - if issue_code not in ISSUE_RESUME_CODES: - raise ValueError("invalid live IOP issue") - self.issue_code = issue_code - super().__init__(issue_code) - - -@dataclass(frozen=True) -class _Runtime: - caller: str - base_url: str - secret: str - endpoint_identity: str - config: "_ConfigObservation" - - -@dataclass(frozen=True) -class _RouteObservation: - route_kind: str - route_id: str - model: str - bindings: tuple[EffectiveBinding, ...] - - -@dataclass(frozen=True) -class _ConfigObservation: - routes: tuple[_RouteObservation, ...] - identity: str - - -@dataclass(frozen=True) -class _Observation: - catalog_models: tuple[str, ...] - config_identity: str - caller_ready: bool - agy_version: str = "" - agy_help: str = "" - - -@dataclass(frozen=True) -class _RuntimeResolution: - runtime: _Runtime | None - issue_code: str | None - - -@dataclass(frozen=True) -class _InvokerSeams: - """Test seams below registry construction; production uses real callers.""" - - claude: Callable[..., InvocationResult] - agy: Callable[..., InvocationResult] - codex: Callable[..., Any] - - -def _identity(label: str, value: str) -> str: - return "sha256:" + hashlib.sha256( - b"iop-benchmark-live-v1\0" + label.encode("ascii") + b"\0" + value.encode("utf-8") - ).hexdigest() - - -def _base_url(value: str) -> str: - if not isinstance(value, str) or not value: - raise LiveIopError("endpoint_incompatible") - parsed = urlsplit(value) - if parsed.scheme not in ("http", "https") or not parsed.netloc or parsed.query or parsed.fragment: - raise LiveIopError("endpoint_incompatible") - return value.rstrip("/") - - -def _models_url(base_url: str) -> str: - parsed = urlsplit(base_url) - path = parsed.path.rstrip("/") - marker = "/gemini/" - if marker in path: - path = path.split(marker, 1)[0] - if path.endswith("/v1"): - path += "/models" - else: - path += "/v1/models" - return urlunsplit((parsed.scheme, parsed.netloc, path, "", "")) - - -def _agy_route_base(base_url: str, route_id: str) -> str: - parsed = urlsplit(base_url) - path = parsed.path.rstrip("/") - marker = "/gemini/" - if marker in path: - path = path.split(marker, 1)[0] - if path.endswith("/v1"): - path = path[:-3] - path = path.rstrip("/") + f"/gemini/{route_id}" - return urlunsplit((parsed.scheme, parsed.netloc, path, "", "")) - - -def _command(argv: tuple[str, ...]) -> str: - try: - completed = subprocess.run( - argv, check=False, capture_output=True, text=True, timeout=_TIMEOUT_SECONDS, - env={"PATH": os.environ.get("PATH", "/usr/bin:/bin"), "LANG": "C.UTF-8", "LC_ALL": "C.UTF-8"}, - ) - except (OSError, subprocess.SubprocessError) as exc: - raise LiveIopError("stream_incompatible") from exc - if completed.returncode != 0: - raise LiveIopError("stream_incompatible") - return completed.stdout + completed.stderr - - -def _caller_binary(name: str) -> str: - path = shutil.which(name) - if not path or not os.path.isfile(path) or not os.access(path, os.X_OK): - raise LiveIopError("stream_incompatible") - return os.path.realpath(path) - - -def _catalog(runtime: _Runtime) -> tuple[tuple[str, ...], str]: - request = Request(_models_url(runtime.base_url), headers={"Authorization": f"Bearer {runtime.secret}"}) - try: - with urlopen(request, timeout=_TIMEOUT_SECONDS) as response: - if response.status in (401, 403): - raise LiveIopError("auth_incompatible") - if response.status != 200: - raise LiveIopError("endpoint_incompatible") - payload = json.loads(response.read().decode("utf-8")) - except HTTPError as exc: - if exc.code in (401, 403): - raise LiveIopError("auth_incompatible") from exc - raise LiveIopError("endpoint_incompatible") from exc - except LiveIopError: - raise - except OSError as exc: - # Connection refusal, DNS failure, and timeout all mean that this - # boundary cannot reach a compatible endpoint. They are distinct - # from a reachable endpoint with an invalid response schema. - raise LiveIopError("endpoint_incompatible") from exc - except ValueError as exc: - raise LiveIopError("protocol_incompatible") from exc - records = payload.get("data") if isinstance(payload, dict) else None - if not isinstance(records, list): - raise LiveIopError("protocol_incompatible") - model_ids: list[str] = [] - for item in records: - model_id = item.get("id") if isinstance(item, dict) else None - if not isinstance(model_id, str) or not model_id.strip(): - raise LiveIopError("protocol_incompatible") - model_ids.append(model_id) - models = tuple(sorted(model_ids)) - if not models or len(set(models)) != len(models): - raise LiveIopError("protocol_incompatible") - return models, _identity("catalog", "\n".join(models)) - - -def _observe(runtime: _Runtime) -> _Observation: - models, config_identity = _catalog(runtime) - if runtime.caller == "claude": - _command(("claude", "--version")) - _command(("claude", "--help")) - return _Observation(models, config_identity, True) - if runtime.caller == AGY_CALLER: - version = _command(("agy", "--version")) - help_output = _command(("agy", "--help")) - return _Observation(models, config_identity, True, version, help_output) - if runtime.caller == "codex": - _command(("codex", "--version")) - _command(("codex", "exec", "--help")) - return _Observation(models, config_identity, True) - raise LiveIopError("protocol_incompatible") - - -def _config_from_environment(environment: Mapping[str, str]) -> _ConfigObservation: - reference = environment.get("IOP_BENCH_CONFIG_OBSERVATION_ENV") - if not isinstance(reference, str) or not _ENV_NAME.fullmatch(reference): - raise LiveIopError("route_missing") - raw = environment.get(reference) - if not isinstance(raw, str) or not raw: - raise LiveIopError("route_missing") - try: - value = json.loads(raw) - except (TypeError, json.JSONDecodeError) as exc: - raise LiveIopError("protocol_incompatible") from exc - if not isinstance(value, dict) or set(value) != {"schema_version", "routes"} or value.get("schema_version") != "1": - raise LiveIopError("protocol_incompatible") - raw_routes = value.get("routes") - if not isinstance(raw_routes, list) or not raw_routes: - raise LiveIopError("route_missing") - routes: list[_RouteObservation] = [] - for item in raw_routes: - if not isinstance(item, dict) or set(item) != { - "route_kind", "route_id", "model", "bindings" - }: - raise LiveIopError("protocol_incompatible") - route_kind, route_id, model = (item.get(name) for name in ("route_kind", "route_id", "model")) - if route_kind not in ("direct", "execution_preset") or not all(isinstance(field, str) and TOKEN_RE.fullmatch(field) for field in (route_id, model)): - raise LiveIopError("protocol_incompatible") - raw_bindings = item.get("bindings") - if not isinstance(raw_bindings, list) or not raw_bindings: - raise LiveIopError("protocol_incompatible") - bindings: list[EffectiveBinding] = [] - stages: set[str] = set() - for raw_binding in raw_bindings: - if not isinstance(raw_binding, dict) or set(raw_binding) != { - "stage", "model", "effort" - }: - raise LiveIopError("protocol_incompatible") - stage = raw_binding.get("stage") - bound_model = raw_binding.get("model") - effort = raw_binding.get("effort") - if ( - stage not in STAGE_ENUM - or stage in stages - or not isinstance(bound_model, str) - or not TOKEN_RE.fullmatch(bound_model) - or ( - effort is not None - and ( - not isinstance(effort, str) - or not TOKEN_RE.fullmatch(effort) - ) - ) - ): - raise LiveIopError("protocol_incompatible") - stages.add(stage) - bindings.append(EffectiveBinding(stage, bound_model, effort)) - routes.append( - _RouteObservation(route_kind, route_id, model, tuple(bindings)) - ) - if len({(item.route_kind, item.route_id) for item in routes}) != len(routes): - raise LiveIopError("protocol_incompatible") - routes.sort(key=lambda item: (item.route_kind, item.route_id, item.model)) - canonical_routes = [ - { - "route_kind": item.route_kind, - "route_id": item.route_id, - "model": item.model, - "bindings": [ - { - "stage": binding.stage, - "model": binding.model, - "effort": binding.effort, - } - for binding in item.bindings - ], - } - for item in routes - ] - canonical = json.dumps( - {"schema_version": "1", "routes": canonical_routes}, - sort_keys=True, - separators=(",", ":"), - ensure_ascii=True, - ) - return _ConfigObservation(tuple(routes), _identity("config", canonical)) - - -def _runtime_from_environment(caller: str, environment: Mapping[str, str]) -> _RuntimeResolution: - prefix = f"IOP_BENCH_{caller.upper()}_" - base_key = prefix + "BASE_URL" - secret_ref_key = prefix + "SECRET_ENV" - base_url = environment.get(base_key) - secret_ref = environment.get(secret_ref_key) - try: - normalized = _base_url(base_url) - except LiveIopError as exc: - return _RuntimeResolution(None, exc.issue_code) - if not isinstance(secret_ref, str) or not _ENV_NAME.fullmatch(secret_ref): - return _RuntimeResolution(None, "credential_missing") - secret = environment.get(secret_ref) - if not isinstance(secret, str) or not secret: - return _RuntimeResolution(None, "credential_missing") - try: - config = _config_from_environment(environment) - except LiveIopError as exc: - return _RuntimeResolution(None, exc.issue_code) - return _RuntimeResolution(_Runtime(caller, normalized, secret, _identity("endpoint", normalized), config), None) - - -def _requested(cell: MatrixCell) -> RequestedEffectiveBinding: - return RequestedEffectiveBinding( - cell.id, cell.caller, cell.iop.route_kind, cell.iop.route_id, - cell.iop.request_model, cell.iop.requested_effort, - ) - - -def _issues(*codes: str) -> tuple[ConnectivityIssue, ...]: - selected = set(codes) - return tuple(ConnectivityIssue(code, ISSUE_RESUME_CODES[code]) for code in ISSUE_CODE_ORDER if code in selected) - - -def _binding_from_config(cell: MatrixCell, capability: CallerCapability, config: _ConfigObservation) -> tuple[RequestedEffectiveBinding, tuple[ConnectivityIssue, ...]]: - """Admit a manifest route from config ownership and caller capability only.""" - requested = _requested(cell) - if cell.iop.route_kind not in capability.route_kinds: - return requested, _issues("protocol_incompatible") - if cell.iop.requested_effort not in capability.efforts: - return requested, _issues("effort_unsupported") - route = next((item for item in config.routes if (item.route_kind, item.route_id) == (cell.iop.route_kind, cell.iop.route_id)), None) - if route is None: - return requested, _issues("route_missing") - if route.model != cell.iop.request_model: - return requested, _issues("model_missing") - expected = tuple( - EffectiveBinding(binding.stage, binding.model, binding.effort) - for binding in cell.iop.expected_bindings - ) - if route.bindings != expected: - return requested, _issues("protocol_incompatible") - return RequestedEffectiveBinding( - cell.id, cell.caller, cell.iop.route_kind, cell.iop.route_id, - cell.iop.request_model, cell.iop.requested_effort, - route.route_kind, route.route_id, route.model, cell.iop.requested_effort, - route.bindings, - ), () - - -def _bound_observations( - result: InvocationResult, admitted: RequestedEffectiveBinding -) -> InvocationResult: - """Return the lifecycle result only when every observation stays bound. - - Typed observations ride on the lifecycle result, so this boundary keeps the - result intact and refuses any observation whose model label names something - other than the admitted binding. - """ - if not isinstance(result, InvocationResult): - raise LiveIopError("stream_incompatible") - admitted_models = { - value for value in (admitted.effective_model, admitted.requested_model) if value - } - admitted_models.update( - binding.model for binding in admitted.effective_bindings if binding.model - ) - admitted_by_stage = { - binding.stage: binding.model - for binding in admitted.effective_bindings - if binding.stage and binding.model - } - for metric in result.metrics: - if ( - metric.stage - and metric.model - and admitted_by_stage.get(metric.stage) != metric.model - ): - raise LiveIopError("stream_incompatible") - if metric.model and metric.model not in admitted_models: - raise LiveIopError("stream_incompatible") - return result - - -def _default_claude_invoker(adapter: ClaudeIopAdapter, spec: Any, **kwargs: Any) -> InvocationResult: - return run_invocation(spec, **kwargs) - - -def _default_agy_invoker(spec: Any, parser: AgyEventParser, preflight: Any, on_started: Callable[..., None]) -> InvocationResult: - return run_agy_invocation(spec, parser, preflight, on_started) - - -def _default_codex_invoker(invocation: Any, on_started: Callable[..., None]) -> Any: - return run_codex_invocation(invocation, on_started) - - -_DEFAULT_INVOKERS = _InvokerSeams(_default_claude_invoker, _default_agy_invoker, _default_codex_invoker) - - -@contextmanager -def _temporary_owned_permissions( - path: Path, - required: int, - *, - expected: os.stat_result | None = None, -) -> Iterator[os.stat_result]: - """Grant minimum owner access and restore a retained inode exactly.""" - try: - info = os.lstat(path) - except OSError as exc: - raise LiveIopError("stream_incompatible") from exc - current_uid = getattr(os, "geteuid", lambda: info.st_uid)() - if ( - stat.S_ISLNK(info.st_mode) - or info.st_uid != current_uid - or ( - expected is not None - and (info.st_dev, info.st_ino) != (expected.st_dev, expected.st_ino) - ) - ): - raise LiveIopError("stream_incompatible") - original_mode = stat.S_IMODE(info.st_mode) - temporary_mode = original_mode | required - changed = temporary_mode != original_mode - try: - if changed: - os.chmod(path, temporary_mode, follow_symlinks=False) - current = os.lstat(path) - if ( - stat.S_ISLNK(current.st_mode) - or current.st_uid != current_uid - or (current.st_dev, current.st_ino) != (info.st_dev, info.st_ino) - or stat.S_IMODE(current.st_mode) & required != required - ): - raise LiveIopError("stream_incompatible") - yield current - except OSError as exc: - raise LiveIopError("stream_incompatible") from exc - finally: - if changed: - try: - current = os.lstat(path) - if ( - stat.S_ISLNK(current.st_mode) - or current.st_uid != current_uid - or (current.st_dev, current.st_ino) != (info.st_dev, info.st_ino) - ): - raise LiveIopError("stream_incompatible") - os.chmod(path, original_mode, follow_symlinks=False) - restored = os.lstat(path) - if ( - (restored.st_dev, restored.st_ino) != (info.st_dev, info.st_ino) - or stat.S_IMODE(restored.st_mode) != original_mode - ): - raise LiveIopError("stream_incompatible") - except OSError as exc: - raise LiveIopError("stream_incompatible") from exc - - -@contextmanager -def _temporary_directory_chain( - root: Path, parent: Path, *, writable_parent: bool = False -) -> Iterator[None]: - try: - relative = parent.relative_to(root) - except ValueError as exc: - raise LiveIopError("stream_incompatible") from exc - chain = [root] - current = root - for part in relative.parts: - current = current / part - chain.append(current) - with ExitStack() as stack: - for index, directory in enumerate(chain): - required = stat.S_IXUSR - if writable_parent and index == len(chain) - 1: - required |= stat.S_IWUSR - stack.enter_context(_temporary_owned_permissions(directory, required)) - yield - - -def _owned_plain_directory(path: Path) -> Path: - try: - info = os.lstat(path) - except OSError as exc: - raise LiveIopError("stream_incompatible") from exc - current_uid = getattr(os, "geteuid", lambda: info.st_uid)() - if ( - not stat.S_ISDIR(info.st_mode) - or stat.S_ISLNK(info.st_mode) - or info.st_uid != current_uid - ): - raise LiveIopError("stream_incompatible") - with _temporary_owned_permissions(path, stat.S_IXUSR, expected=info): - try: - resolved = path.resolve(strict=True) - except OSError as exc: - raise LiveIopError("stream_incompatible") from exc - return resolved - - -def _blind_controlled_roots( - blind: BlindWorkspace, -) -> tuple[Path, tuple[Path, Path, Path]]: - blind_path = Path(blind.root) - blind_root = _owned_plain_directory(blind_path) - controlled = ( - _owned_plain_directory(Path(blind.input_dir)), - _owned_plain_directory(Path(blind.session_dir)), - _owned_plain_directory(Path(blind.output_dir)), - ) - expected = tuple(blind_root / name for name in ("input", "session", "output")) - if controlled != expected: - raise LiveIopError("stream_incompatible") - return blind_root, controlled - - -def _walk_no_follow(root: Path) -> tuple[tuple[Path, os.stat_result], ...]: - entries: list[tuple[Path, os.stat_result]] = [] - - def visit( - directory: Path, depth: int, expected: os.stat_result | None = None - ) -> None: - if depth > _SECRET_SCAN_MAX_DEPTH: - raise LiveIopError("stream_incompatible") - with _temporary_owned_permissions( - directory, stat.S_IRUSR | stat.S_IXUSR, expected=expected - ): - try: - with os.scandir(directory) as iterator: - children = sorted(iterator, key=lambda item: item.name) - except OSError as exc: - raise LiveIopError("stream_incompatible") from exc - for entry in children: - try: - info = entry.stat(follow_symlinks=False) - except OSError as exc: - raise LiveIopError("stream_incompatible") from exc - path = directory / entry.name - entries.append((path, info)) - if len(entries) > _SECRET_SCAN_MAX_ENTRIES: - raise LiveIopError("stream_incompatible") - if stat.S_ISDIR(info.st_mode) and not stat.S_ISLNK(info.st_mode): - visit(path, depth + 1, info) - - visit(root, 0) - return tuple(entries) - - -def _read_bounded_regular( - root: Path, path: Path, expected: os.stat_result -) -> bytes: - if expected.st_size > _SECRET_SCAN_MAX_FILE_BYTES: - raise LiveIopError("stream_incompatible") - flags = os.O_RDONLY | os.O_CLOEXEC | os.O_NONBLOCK - if hasattr(os, "O_NOFOLLOW"): - flags |= os.O_NOFOLLOW - with _temporary_directory_chain(root, path.parent): - with _temporary_owned_permissions(path, stat.S_IRUSR, expected=expected): - try: - descriptor = os.open(path, flags) - except OSError as exc: - raise LiveIopError("stream_incompatible") from exc - try: - current = os.fstat(descriptor) - if ( - not stat.S_ISREG(current.st_mode) - or current.st_uid - != getattr(os, "geteuid", lambda: current.st_uid)() - or current.st_dev != expected.st_dev - or current.st_ino != expected.st_ino - or current.st_size != expected.st_size - or current.st_size > _SECRET_SCAN_MAX_FILE_BYTES - ): - raise LiveIopError("stream_incompatible") - data = bytearray() - while len(data) < current.st_size: - chunk = os.read(descriptor, current.st_size - len(data)) - if not chunk: - raise LiveIopError("stream_incompatible") - data.extend(chunk) - if os.read(descriptor, 1): - raise LiveIopError("stream_incompatible") - final = os.fstat(descriptor) - if ( - final.st_size != current.st_size - or final.st_mtime_ns != current.st_mtime_ns - ): - raise LiveIopError("stream_incompatible") - return bytes(data) - except OSError as exc: - raise LiveIopError("stream_incompatible") from exc - finally: - os.close(descriptor) - - -def _contains_sensitive(data: bytes, sensitive: tuple[bytes, ...]) -> bool: - return any(value in data for value in sensitive) - - -def _relative_bytes(root: Path, path: Path) -> bytes: - try: - return os.fsencode(path.relative_to(root).as_posix()) - except ValueError as exc: - raise LiveIopError("stream_incompatible") from exc - - -def _unlink_owned_entry( - root: Path, - path: Path, - expected: os.stat_result, - *, - directory: bool, -) -> None: - with _temporary_directory_chain(root, path.parent, writable_parent=True): - try: - current = os.lstat(path) - current_uid = getattr(os, "geteuid", lambda: current.st_uid)() - if ( - current.st_uid != current_uid - or (current.st_dev, current.st_ino) - != (expected.st_dev, expected.st_ino) - or (directory and not stat.S_ISDIR(current.st_mode)) - or (not directory and stat.S_ISDIR(current.st_mode)) - ): - raise LiveIopError("stream_incompatible") - if directory: - path.rmdir() - else: - path.unlink() - except OSError as exc: - raise LiveIopError("stream_incompatible") from exc - - -@dataclass -class _SanitizationOutcome: - secret: bool = False - input_invalid: bool = False - output_invalid: bool = False - - def record(self, root_kind: str, *, secret: bool, invalid: bool) -> None: - self.secret = self.secret or secret - if invalid and root_kind == "input": - self.input_invalid = True - elif invalid: - self.output_invalid = True - - def finalization(self) -> ScoringEvidenceFinalization: - if self.secret: - return ScoringEvidenceFinalization(False, "runtime_secret_leak") - if self.input_invalid: - return ScoringEvidenceFinalization(False, "input_mutated") - if self.output_invalid: - return ScoringEvidenceFinalization(False, "evaluator_output_leak") - return ScoringEvidenceFinalization(True) - - -def _remove_sensitive_blind_paths( - roots: tuple[Path, Path, Path], sensitive: tuple[bytes, ...] -) -> _SanitizationOutcome: - outcome = _SanitizationOutcome() - for root_kind, root in zip(("input", "session", "output"), roots): - entries = sorted( - _walk_no_follow(root), - key=lambda item: (len(item[0].parts), item[0].as_posix()), - reverse=True, - ) - for path, info in entries: - current_uid = getattr(os, "geteuid", lambda: info.st_uid)() - if info.st_uid != current_uid: - raise LiveIopError("stream_incompatible") - path_leak = _contains_sensitive(_relative_bytes(root, path), sensitive) - mode = info.st_mode - if stat.S_ISDIR(mode) and not stat.S_ISLNK(mode): - if path_leak: - _unlink_owned_entry(root, path, info, directory=True) - outcome.record(root_kind, secret=True, invalid=False) - continue - if stat.S_ISLNK(mode): - with _temporary_directory_chain(root, path.parent): - try: - link_value = os.fsencode(os.readlink(path)) - current = os.lstat(path) - except OSError as exc: - raise LiveIopError("stream_incompatible") from exc - if (current.st_dev, current.st_ino) != (info.st_dev, info.st_ino): - raise LiveIopError("stream_incompatible") - link_leak = path_leak or _contains_sensitive(link_value, sensitive) - _unlink_owned_entry(root, path, info, directory=False) - outcome.record(root_kind, secret=link_leak, invalid=not link_leak) - continue - if stat.S_ISREG(mode): - content_leak = False - if not path_leak: - content_leak = _contains_sensitive( - _read_bounded_regular(root, path, info), sensitive - ) - if path_leak or content_leak: - _unlink_owned_entry(root, path, info, directory=False) - outcome.record(root_kind, secret=True, invalid=False) - continue - if stat.S_ISSOCK(mode): - if path_leak: - _unlink_owned_entry(root, path, info, directory=False) - outcome.record(root_kind, secret=True, invalid=False) - continue - if path_leak: - _unlink_owned_entry(root, path, info, directory=False) - outcome.record(root_kind, secret=True, invalid=False) - continue - raise LiveIopError("stream_incompatible") - return outcome - - -def _freeze_sanitized_input(root: Path) -> None: - entries = _walk_no_follow(root) - directories = [root] - current_uid = getattr(os, "geteuid", lambda: os.lstat(root).st_uid)() - for path, info in entries: - if info.st_uid != current_uid: - raise LiveIopError("stream_incompatible") - try: - if stat.S_ISDIR(info.st_mode) and not stat.S_ISLNK(info.st_mode): - directories.append(path) - elif stat.S_ISREG(info.st_mode) and not stat.S_ISLNK(info.st_mode): - os.chmod(path, 0o400, follow_symlinks=False) - else: - raise LiveIopError("stream_incompatible") - except OSError as exc: - raise LiveIopError("stream_incompatible") from exc - for directory in sorted( - directories, key=lambda item: len(item.parts), reverse=True - ): - try: - os.chmod(directory, 0o500, follow_symlinks=False) - except OSError as exc: - raise LiveIopError("stream_incompatible") from exc - - -def _verify_sensitive_absent(root: Path, sensitive: tuple[bytes, ...]) -> None: - root = _owned_plain_directory(root) - for path, info in _walk_no_follow(root): - if _contains_sensitive(_relative_bytes(root, path), sensitive): - raise LiveIopError("stream_incompatible") - if stat.S_ISREG(info.st_mode): - if _contains_sensitive( - _read_bounded_regular(root, path, info), sensitive - ): - raise LiveIopError("stream_incompatible") - elif stat.S_ISLNK(info.st_mode): - with _temporary_directory_chain(root, path.parent): - try: - link_value = os.fsencode(os.readlink(path)) - except OSError as exc: - raise LiveIopError("stream_incompatible") from exc - if _contains_sensitive(link_value, sensitive): - raise LiveIopError("stream_incompatible") - - -def _bind_live_spec( - cell: MatrixCell, - prepared: PreparedWorkspace, - attempt: Attempt, - control_dir: str, - spec: Any, -) -> Any: - """Immutably bind one caller spec to the controller-owned attempt paths.""" - if ( - not isinstance(prepared, PreparedWorkspace) - or not isinstance(attempt, Attempt) - or cell.id != attempt.identity.cell_id - or prepared.identity != attempt.identity - or not isinstance(control_dir, str) - or not control_dir - ): - raise LiveIopError("stream_incompatible") - try: - attempt_root = Path(attempt.root).resolve(strict=True) - prepared_root = Path(prepared.attempt_root).resolve(strict=True) - workspace_root = Path(prepared.workspace_dir).resolve(strict=True) - session_root = Path(prepared.session_dir).resolve(strict=True) - evidence_root = Path(spec.evidence_dir).resolve(strict=True) - control_path = Path(control_dir) - alias = control_path.parent - if ( - not control_path.is_absolute() - or control_path.name != "control" - or control_path.exists() - or control_path.is_symlink() - or not alias.is_symlink() - or alias.resolve(strict=True) != attempt_root - or control_path.resolve(strict=False) != attempt_root / "control" - ): - raise LiveIopError("stream_incompatible") - except (OSError, RuntimeError, TypeError, ValueError) as exc: - raise LiveIopError("stream_incompatible") from exc - if ( - prepared_root != attempt_root - or evidence_root != attempt_root - or workspace_root != attempt_root / "workspace" - or session_root != attempt_root / "session" - ): - raise LiveIopError("stream_incompatible") - return replace(spec, control_dir=control_dir) - - -class _LiveAdapter: - """One caller's live observation and invocation boundary.""" - - def __init__( - self, caller: str, capability: CallerCapability, runtime: _RuntimeResolution, - observer: Callable[[_Runtime], _Observation] = _observe, - binary_resolver: Callable[[str], str] = _caller_binary, - invokers: _InvokerSeams = _DEFAULT_INVOKERS, - ) -> None: - self.caller = caller - self.capability = capability - self._runtime_resolution = runtime - self._observer = observer - self._binary_resolver = binary_resolver - self._invokers = invokers - self._agy_preflights: dict[str, AgyPreflightResult] = {} - self._admitted_bindings: dict[str, RequestedEffectiveBinding] = {} - - def preflight(self, cell: MatrixCell) -> PreflightObservation: - if cell.caller != self.caller: - raise LiveIopError("protocol_incompatible") - self._admitted_bindings.pop(cell.id, None) - if self.caller == AGY_CALLER: - self._agy_preflights.pop(cell.id, None) - resolution = self._runtime_resolution - runtime = resolution.runtime - if runtime is None: - result = make_result(cell, self.capability, _requested(cell), _issues(resolution.issue_code or "credential_missing")) - return PreflightObservation(result, _identity("missing", self.caller), _identity("missing-config", self.caller)) - try: - observed = self._observer(runtime) - except LiveIopError as exc: - result = make_result(cell, self.capability, _requested(cell), _issues(exc.issue_code)) - return PreflightObservation(result, runtime.endpoint_identity, _identity("unobserved-config", self.caller)) - if not observed.caller_ready: - result = make_result(cell, self.capability, _requested(cell), _issues("stream_incompatible")) - return PreflightObservation(result, runtime.endpoint_identity, runtime.config.identity) - binding, issues = _binding_from_config(cell, self.capability, runtime.config) - if not issues and binding.effective_model not in observed.catalog_models: - binding, issues = _requested(cell), _issues("model_missing") - if issues: - result = make_result(cell, self.capability, binding, issues) - return PreflightObservation(result, runtime.endpoint_identity, runtime.config.identity) - agy_preflight: AgyPreflightResult | None = None - if self.caller == AGY_CALLER: - capability = inspect_agy_iop_capability(observed.agy_version, observed.agy_help) - agy_endpoint = _agy_route_base(runtime.base_url, cell.iop.route_id) - agy_observation = AgyRuntimeObservation( - cell.id, cell.iop.route_kind, cell.iop.route_id, - _agy_runtime_identity("endpoint", agy_endpoint), - _agy_runtime_identity("credential", runtime.secret), runtime.config.identity, - ) - try: - agy_preflight = preflight_agy_iop( - cell, capability, - AgyRuntimeInputs(self._binary_resolver("agy"), agy_endpoint, runtime.secret), - agy_observation, - ) - except Exception: - result = make_result(cell, self.capability, _requested(cell), _issues("stream_incompatible")) - return PreflightObservation(result, runtime.endpoint_identity, runtime.config.identity) - if agy_preflight.issues: - result = make_result(cell, self.capability, _requested(cell), agy_preflight.issues) - return PreflightObservation(result, runtime.endpoint_identity, runtime.config.identity) - result = make_result(cell, self.capability, binding, issues) - if result.status == "ready": - self._admitted_bindings[cell.id] = result.binding - if agy_preflight is not None: - self._agy_preflights[cell.id] = agy_preflight - return PreflightObservation(result, runtime.endpoint_identity, runtime.config.identity) - - def invoke( - self, - cell: MatrixCell, - prepared: PreparedWorkspace, - attempt: Attempt, - control_dir: str, - task_payload: bytes, - timeout: Timeout, - on_started: Callable[..., None], - ) -> InvocationResult: - if cell.caller != self.caller: - raise LiveIopError("protocol_incompatible") - runtime = self._runtime_resolution.runtime - if runtime is None: - raise LiveIopError(self._runtime_resolution.issue_code or "protocol_incompatible") - admitted = self._admitted_bindings.get(cell.id) - if admitted is None: - raise LiveIopError("stream_incompatible") - if self.caller == "claude": - adapter = ClaudeIopAdapter(cell, prepared, ClaudeIopRuntime("claude", runtime.base_url, runtime.secret)) - spec = _bind_live_spec( - cell, - prepared, - attempt, - control_dir, - adapter.invocation(task_payload.decode("utf-8"), attempt.root, timeout), - ) - return _bound_observations( - self._invokers.claude(adapter, spec, parse_event=adapter.parser(), redact=adapter.redactor(task_payload.decode("utf-8")), on_started=lambda locator: on_started(locator, spec_digest(spec))), - admitted, - ) - if self.caller == AGY_CALLER: - agy_preflight = self._agy_preflights.get(cell.id) - if agy_preflight is None or agy_preflight.binding != _requested(cell): - raise LiveIopError("stream_incompatible") - spec = _bind_live_spec( - cell, - prepared, - attempt, - control_dir, - build_agy_invocation( - cell, prepared, task_payload, timeout, agy_preflight - ), - ) - parser = AgyEventParser(cell, admitted) - result = self._invokers.agy(spec, parser, agy_preflight, lambda locator: on_started(locator, spec_digest(spec))) - if ( - result.product.status != "succeeded" - or result.harness.status != "passed" - ): - return _bound_observations(result, admitted) - observed = parser.observed_result(agy_preflight.capability, result) - if observed.status != "ready" or observed.binding != admitted: - raise LiveIopError("stream_incompatible") - return _bound_observations(result, admitted) - if self.caller == "codex": - invocation = build_codex_invocation(cell, prepared, runtime_from_environment({BASE_URL_ENV_KEY: runtime.base_url, SECRET_ENV_KEY: runtime.secret, "PATH": os.environ.get("PATH", "/usr/bin:/bin")}), task_payload, timeout) - invocation = replace( - invocation, - spec=_bind_live_spec( - cell, - prepared, - attempt, - control_dir, - invocation.spec, - ), - ) - result = self._invokers.codex(invocation, lambda locator: on_started(locator, spec_digest(invocation.spec))) - expected = (admitted.effective_route_kind, admitted.effective_route_id, admitted.effective_model, admitted.effective_effort) - if ( - result.effective_binding is not None - and result.effective_binding != expected - ): - raise LiveIopError("stream_incompatible") - return _bound_observations(result.lifecycle, admitted) - raise LiveIopError("protocol_incompatible") - - -def build_live_adapter_registry( - environment: Mapping[str, str], *, observer: Callable[[_Runtime], _Observation] = _observe, - binary_resolver: Callable[[str], str] = _caller_binary, - invokers: _InvokerSeams = _DEFAULT_INVOKERS, -) -> dict[str, ExecutionAdapter]: - """Build the fixed caller registry without reading ambient caller settings.""" - if not isinstance(environment, Mapping): - raise LiveIopError("protocol_incompatible") - registry: dict[str, ExecutionAdapter] = { - "claude": _LiveAdapter("claude", claude_capability(), _runtime_from_environment("claude", environment), observer, binary_resolver, invokers), - "agy": _LiveAdapter(AGY_CALLER, CallerCapability(AGY_CALLER, ("direct", "execution_preset"), ("high", "low", "medium")), _runtime_from_environment("agy", environment), observer, binary_resolver, invokers), - "codex": _LiveAdapter("codex", codex_capability(), _runtime_from_environment("codex", environment), observer, binary_resolver, invokers), - } - if tuple(registry) != CALLER_ENUM: - raise LiveIopError("protocol_incompatible") - return registry - - -class _LiveScoringAdapter: - """Codex-only scoring adapter sharing the live config and secret boundary.""" - - def __init__( - self, - live: _LiveAdapter, - *, - invoker: Callable[[Any, Callable[..., None]], Any] = _default_codex_invoker, - ) -> None: - self.capability = codex_capability() - self._live = live - self._invoker = invoker - self._control_aliases: dict[str, Path] = {} - - def preflight(self, cell: MatrixCell) -> PreflightObservation: - if cell.id != "evaluator" or cell.caller != "codex": - raise LiveIopError("protocol_incompatible") - return self._live.preflight(cell) - - def invoke( - self, - cell: MatrixCell, - blind: BlindWorkspace, - task_payload: bytes, - timeout: Timeout, - on_started: Callable[..., None], - ) -> ScoringInvocationResult: - if cell.id != "evaluator" or cell.caller != "codex": - raise LiveIopError("protocol_incompatible") - runtime = self._live._runtime_resolution.runtime - admitted = self._live._admitted_bindings.get(cell.id) - if runtime is None or admitted is None: - raise LiveIopError("stream_incompatible") - blind_root = Path(blind.root).resolve(strict=True) - input_root = Path(blind.input_dir).resolve(strict=True) - session_root = Path(blind.session_dir).resolve(strict=True) - output_root = Path(blind.output_dir).resolve(strict=True) - if ( - input_root != blind_root / "input" - or session_root != blind_root / "session" - or output_root != blind_root / "output" - or any(path.is_symlink() for path in (blind_root, input_root, session_root, output_root)) - ): - raise LiveIopError("stream_incompatible") - prepared = PreparedWorkspace( - identity=AttemptIdentity("run-blind", "blind", 1, 1), - attempt_root=str(output_root), - workspace_dir=str(blind_root), - session_dir=str(session_root), - session_id=blind.session_identity, - session_is_fresh=True, - workspace_checksum=blind.input_digest, - setup_cache_policy="isolated", - testbed_provenance=TestbedProvenance( - path="opaque", branch="opaque", head="opaque", - status_digest="sha256:" + "0" * 64, clean=True, - ), - prepared_at="opaque", - ) - invocation = build_codex_invocation( - cell, - prepared, - runtime_from_environment( - { - BASE_URL_ENV_KEY: runtime.base_url, - SECRET_ENV_KEY: runtime.secret, - "PATH": os.environ.get("PATH", "/usr/bin:/bin"), - } - ), - task_payload, - timeout, - ) - alias = Path(tempfile.gettempdir()).resolve() / ( - "iop-bench-score-" - + hashlib.sha256(os.fsencode(str(output_root))).hexdigest()[:20] - ) - try: - os.symlink(str(output_root), alias, target_is_directory=True) - except FileExistsError: - try: - if not alias.is_symlink() or alias.resolve(strict=True) != output_root: - raise LiveIopError("stream_incompatible") - except OSError as exc: - raise LiveIopError("stream_incompatible") from exc - except OSError as exc: - raise LiveIopError("stream_incompatible") from exc - self._control_aliases[blind.blind_id] = alias - invocation = replace( - invocation, - spec=replace( - invocation.spec, - evidence_dir=str(output_root), - control_dir=str(alias / "codex-control"), - ), - ) - result = self._invoker( - invocation, - lambda locator: on_started(locator, spec_digest(invocation.spec)), - ) - expected = ( - admitted.effective_route_kind, - admitted.effective_route_id, - admitted.effective_model, - admitted.effective_effort, - ) - if ( - result.effective_binding is not None - and result.effective_binding != expected - ): - return ScoringInvocationResult( - result.lifecycle.product.status, - "failed", - result.lifecycle.process.status, - result.lifecycle.process.exit_code, - result.lifecycle.process.signal, - "binding_mismatch", - result.effective_binding, - ) - lifecycle = _bound_observations(result.lifecycle, admitted) - return ScoringInvocationResult( - lifecycle.product.status, - lifecycle.harness.status, - lifecycle.process.status, - lifecycle.process.exit_code, - lifecycle.process.signal, - lifecycle.harness.reason, - expected, - ) - - def finalize_evidence( - self, blind: BlindWorkspace - ) -> ScoringEvidenceFinalization: - """Remove exact evaluator runtime values before controller publication.""" - runtime = self._live._runtime_resolution.runtime - if runtime is None: - raise LiveIopError("stream_incompatible") - blind_root, controlled = _blind_controlled_roots(blind) - sensitive = tuple( - dict.fromkeys( - value.encode("utf-8") - for value in (runtime.secret, runtime.base_url) - if value - ) - ) - alias = self._control_aliases.pop(blind.blind_id, None) - if alias is not None and (alias.exists() or alias.is_symlink()): - try: - if not alias.is_symlink() or alias.resolve(strict=True) != controlled[2]: - raise LiveIopError("stream_incompatible") - alias.unlink() - except OSError as exc: - raise LiveIopError("stream_incompatible") from exc - outcome = _remove_sensitive_blind_paths(controlled, sensitive) - _freeze_sanitized_input(controlled[0]) - _verify_sensitive_absent(blind_root.parent.parent, sensitive) - return outcome.finalization() - - -def build_live_scoring_adapter( - environment: Mapping[str, str], - *, - observer: Callable[[_Runtime], _Observation] = _observe, - binary_resolver: Callable[[str], str] = _caller_binary, - invoker: Callable[[Any, Callable[..., None]], Any] = _default_codex_invoker, -) -> ScoringAdapter: - """Build the manifest-bound Codex evaluator without caller fallback.""" - if not isinstance(environment, Mapping): - raise LiveIopError("protocol_incompatible") - live = _LiveAdapter( - "codex", - codex_capability(), - _runtime_from_environment("codex", environment), - observer, - binary_resolver, - _DEFAULT_INVOKERS, - ) - return _LiveScoringAdapter(live, invoker=invoker) diff --git a/scripts/agent_benchmark/manifest.py b/scripts/agent_benchmark/manifest.py deleted file mode 100644 index 19eb8655..00000000 --- a/scripts/agent_benchmark/manifest.py +++ /dev/null @@ -1,807 +0,0 @@ -""" -Closed manifest loader, validator, and digest calculator. - -Standard-library-only. Returns frozen dataclasses; never mutable raw dicts. -""" - -from __future__ import annotations - -import hashlib -import json -import os -import posixpath -import re -import struct -from collections.abc import Iterable -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any, Optional - -# --------------------------------------------------------------------------- -# Constants -# --------------------------------------------------------------------------- - -PIPELINE_VERSION = "2" -ENVIRONMENT = "dev" -TESTBED_REQUIRED = "../iop-s2" -SESSION_POLICY = "fresh" -SETUP_CACHE_POLICY = "isolated" -DEFAULT_REPETITIONS = 1 -RUBRIC_VERSION = "landing-quality-v1" -ONE_SHOT_RUBRIC_VERSION = "one-shot-agent-comparison-v1" -RUBRIC_VERSIONS = (RUBRIC_VERSION, ONE_SHOT_RUBRIC_VERSION) - -CALLER_ENUM = ("claude", "agy", "codex") -ROUTE_KIND_ENUM = ("direct", "execution_preset") -STAGE_ENUM = ("request", "selector", "plan", "work", "review", "repair") -# Canonical rank for sorting bindings: lower rank = earlier position -STAGE_RANK: dict[str, int] = {s: i for i, s in enumerate(STAGE_ENUM)} - -CELL_ID_RE = re.compile(r"^[a-z0-9][a-z0-9_-]{0,63}$") -TOKEN_RE = re.compile(r"^[a-z0-9][a-z0-9_.+-]{0,31}$") -VIEWPORT_ID_RE = re.compile(r"^[a-z0-9][a-z0-9_.+-]{0,31}$") -CHECKSUM_RE = re.compile(r"^sha256:[0-9a-f]{64}$") - -WORKSPACE_PREFIX = b"IOP-BENCH-WORKSPACE\x00" -MANIFEST_PREFIX = b"IOP-BENCH-MANIFEST\x00" - - -# --------------------------------------------------------------------------- -# Frozen data containers -# --------------------------------------------------------------------------- - -@dataclass(frozen=True) -class Timeout: - run_seconds: int - idle_seconds: int - quiet_seconds: int - cleanup_grace_seconds: int - - -@dataclass(frozen=True) -class Viewport: - id: str - width: int - height: int - - -@dataclass(frozen=True) -class AssetMapping: - source: str - workspace_path: str - content: bytes = field(default=b"", repr=False) - - -@dataclass(frozen=True) -class ExpectedBinding: - stage: str - model: str - effort: Optional[str] = None - - -@dataclass(frozen=True) -class IopCell: - request_model: str - requested_effort: str - route_kind: str - route_id: str - expected_bindings: tuple[ExpectedBinding, ...] - - -@dataclass(frozen=True) -class MatrixCell: - id: str - caller: str - iop: IopCell - - -@dataclass(frozen=True) -class Evaluator: - caller: str - iop: IopCell - - -@dataclass(frozen=True) -class Fixture: - version: str - prompt: str - assets: tuple[AssetMapping, ...] - checksum: str - prompt_content: bytes = field(default=b"", repr=False) - - -@dataclass(frozen=True) -class Manifest: - pipeline_version: str - environment: str - testbed: str - repetitions: int - session_policy: str - setup_cache_policy: str - timeout: Timeout - viewports: tuple[Viewport, ...] - rubric_version: str - evaluator: Evaluator - output_root: str - fixture: Fixture - matrix: tuple[MatrixCell, ...] - digest: str - execution_order_seed: str | None = None - - -# --------------------------------------------------------------------------- -# Errors -# --------------------------------------------------------------------------- - -class ManifestError(Exception): - """Base error for manifest validation failures.""" - - -class ManifestValidationError(ManifestError): - """Raised when the manifest JSON fails schema validation.""" - - -class ManifestPathError(ManifestError): - """Raised when a path rule is violated (escape, symlink, collision, etc.).""" - - -class ManifestDigestError(ManifestError): - """Raised when a declared digest does not match the computed value.""" - - -# --------------------------------------------------------------------------- -# Internal helpers -# --------------------------------------------------------------------------- - -def _require_bool(cond: bool, msg: str) -> None: - if not cond: - raise ManifestValidationError(msg) - - -def _require_str(value: Any, field_name: str) -> str: - if not isinstance(value, str): - raise ManifestValidationError(f"field '{field_name}' must be a string") - return value - - -def _require_int(value: Any, field_name: str) -> int: - if isinstance(value, bool) or not isinstance(value, int): - raise ManifestValidationError(f"field '{field_name}' must be an integer") - return value - - -def _require_object(value: Any, field_name: str) -> dict[str, Any]: - if not isinstance(value, dict): - raise ManifestValidationError(f"field '{field_name}' must be an object") - return value - - -def _require_array(value: Any, field_name: str) -> list[Any]: - if not isinstance(value, list): - raise ManifestValidationError(f"field '{field_name}' must be an array") - return value - - -def _require_enum(value: Any, field_name: str, allowed: tuple[str, ...]) -> str: - s = _require_str(value, field_name) - if s not in allowed: - raise ManifestValidationError( - f"field '{field_name}' must be one of {allowed}" - ) - return s - - -def _require_pattern(value: Any, field_name: str, pattern: re.Pattern) -> str: - s = _require_str(value, field_name) - if pattern.fullmatch(s) is None: - raise ManifestValidationError( - f"field '{field_name}' does not match required pattern" - ) - return s - - -def _require_positive_int(value: Any, field_name: str) -> int: - val = _require_int(value, field_name) - if val <= 0: - raise ManifestValidationError( - f"field '{field_name}' must be a positive integer" - ) - return val - - -def _require_bounded_int(value: Any, field_name: str, lo: int, hi: int) -> int: - val = _require_int(value, field_name) - if not (lo <= val <= hi): - raise ManifestValidationError( - f"field '{field_name}' must be between {lo} and {hi}" - ) - return val - - -def _require_sha256(value: Any, field_name: str) -> str: - s = _require_str(value, field_name) - if not CHECKSUM_RE.match(s): - raise ManifestValidationError( - f"field '{field_name}' must match sha256:<64 hex chars>" - ) - return s - - -def _normalize_posix_relative_path(path_str: Any, context: str) -> str: - """Validate and return a normalized POSIX relative path. - - Rejects non-string, empty, absolute, colon, backslash, non-normal, - or escaping paths ('.', '..', starting with '../'). - """ - if not isinstance(path_str, str) or not path_str: - raise ManifestPathError(f"field '{context}' must be a non-empty string path") - if "\\" in path_str or ":" in path_str: - raise ManifestPathError(f"field '{context}' contains invalid path characters") - if path_str.startswith("/"): - raise ManifestPathError(f"field '{context}' must be a relative path") - - norm = posixpath.normpath(path_str) - if path_str != norm: - raise ManifestPathError(f"field '{context}' is not in canonical relative form") - if norm in (".", "..") or norm.startswith("../"): - raise ManifestPathError(f"field '{context}' escapes root or is empty") - return norm - - -def _require_regular_file(path: Path, context: str) -> None: - """Require that path exists and is a regular file (no symlinks).""" - if not path.exists(): - raise ManifestPathError(f"field '{context}' target file does not exist") - if path.is_symlink(): - raise ManifestPathError(f"field '{context}' target must be a regular file, not a symlink") - if not path.is_file(): - raise ManifestPathError(f"field '{context}' target is not a regular file") - - -# --------------------------------------------------------------------------- -# Digest computation -# --------------------------------------------------------------------------- - -def digest_workspace_inputs(assets: Iterable[AssetMapping]) -> str: - """Compute the declared fixture checksum from asset workspace paths and content. - - sha256(b"IOP-BENCH-WORKSPACE\x00" + length-framed assets sorted by - workspace_path, each asset framed as: - uint64 BE len(workspace_path) + workspace_path UTF-8 bytes + - uint64 BE len(file_content) + file_content bytes - """ - sorted_assets = sorted(assets, key=lambda a: a.workspace_path) - data = bytearray(WORKSPACE_PREFIX) - for asset in sorted_assets: - wp_bytes = asset.workspace_path.encode("utf-8") - data += struct.pack(">Q", len(wp_bytes)) + wp_bytes - data += struct.pack(">Q", len(asset.content)) + asset.content - return "sha256:" + hashlib.sha256(bytes(data)).hexdigest() - - -def digest_manifest_and_resolved_inputs(manifest: Manifest) -> str: - """Compute the self-contained manifest digest. - - sha256(b"IOP-BENCH-MANIFEST\x00" + canonical JSON + length-framed - prompt path/content + length-framed every asset source/destination/content). - """ - prompt_content = manifest.fixture.prompt_content - - canonical = json.dumps( - _manifest_to_dict(manifest), - sort_keys=True, - separators=(",", ":"), - ensure_ascii=False, - ).encode("utf-8") - - data = bytearray(MANIFEST_PREFIX) - data += struct.pack(">Q", len(canonical)) + canonical - - prompt_path_bytes = manifest.fixture.prompt.encode("utf-8") - data += struct.pack(">Q", len(prompt_path_bytes)) + prompt_path_bytes - data += struct.pack(">Q", len(prompt_content)) + prompt_content - - for asset in sorted(manifest.fixture.assets, key=lambda a: a.workspace_path): - src_bytes = asset.source.encode("utf-8") - data += struct.pack(">Q", len(src_bytes)) + src_bytes - wp_bytes = asset.workspace_path.encode("utf-8") - data += struct.pack(">Q", len(wp_bytes)) + wp_bytes - data += struct.pack(">Q", len(asset.content)) + asset.content - - return "sha256:" + hashlib.sha256(bytes(data)).hexdigest() - - -def _manifest_to_dict(manifest: Manifest) -> dict[str, Any]: - """Convert a Manifest to a plain dict for canonical JSON serialization.""" - canonical = { - "pipeline_version": manifest.pipeline_version, - "environment": manifest.environment, - "testbed": manifest.testbed, - "repetitions": manifest.repetitions, - "session_policy": manifest.session_policy, - "setup_cache_policy": manifest.setup_cache_policy, - "timeout": { - "run_seconds": manifest.timeout.run_seconds, - "idle_seconds": manifest.timeout.idle_seconds, - "quiet_seconds": manifest.timeout.quiet_seconds, - "cleanup_grace_seconds": manifest.timeout.cleanup_grace_seconds, - }, - "viewports": [ - {"id": v.id, "width": v.width, "height": v.height} - for v in manifest.viewports - ], - "rubric_version": manifest.rubric_version, - "evaluator": { - "caller": manifest.evaluator.caller, - "iop": { - "request_model": manifest.evaluator.iop.request_model, - "requested_effort": manifest.evaluator.iop.requested_effort, - "route_kind": manifest.evaluator.iop.route_kind, - "route_id": manifest.evaluator.iop.route_id, - "expected_bindings": [ - { - "stage": binding.stage, - "model": binding.model, - **({"effort": binding.effort} if binding.effort else {}), - } - for binding in manifest.evaluator.iop.expected_bindings - ], - }, - }, - "output_root": manifest.output_root, - "fixture": { - "version": manifest.fixture.version, - "prompt": manifest.fixture.prompt, - "assets": [ - {"source": a.source, "workspace_path": a.workspace_path} - for a in manifest.fixture.assets - ], - "checksum": manifest.fixture.checksum, - }, - "matrix": [ - { - "id": c.id, - "caller": c.caller, - "iop": { - "request_model": c.iop.request_model, - "requested_effort": c.iop.requested_effort, - "route_kind": c.iop.route_kind, - "route_id": c.iop.route_id, - "expected_bindings": [ - { - "stage": b.stage, - "model": b.model, - **({"effort": b.effort} if b.effort else {}), - } - for b in c.iop.expected_bindings - ], - }, - } - for c in manifest.matrix - ], - } - # Pipeline-v2 manifests without an explicit seed keep their legacy - # canonical bytes and digest. An explicit seed is part of the contract. - if manifest.execution_order_seed is not None: - canonical["execution_order_seed"] = manifest.execution_order_seed - return canonical - - -# --------------------------------------------------------------------------- -# Validation -# --------------------------------------------------------------------------- - -def _validate_timeout(data: dict[str, Any]) -> Timeout: - obj = _require_object(data, "timeout") - expected_keys = {"run_seconds", "idle_seconds", "quiet_seconds", "cleanup_grace_seconds"} - if set(obj.keys()) != expected_keys: - raise ManifestValidationError("timeout has invalid schema") - return Timeout( - run_seconds=_require_bounded_int(obj["run_seconds"], "timeout.run_seconds", 1, 86400), - idle_seconds=_require_bounded_int(obj["idle_seconds"], "timeout.idle_seconds", 1, 600), - quiet_seconds=_require_bounded_int(obj["quiet_seconds"], "timeout.quiet_seconds", 1, 60), - cleanup_grace_seconds=_require_bounded_int( - obj["cleanup_grace_seconds"], "timeout.cleanup_grace_seconds", 1, 60 - ), - ) - - -def _validate_viewports(data: list[Any]) -> tuple[Viewport, ...]: - arr = _require_array(data, "viewports") - if len(arr) < 1: - raise ManifestValidationError("viewports must have at least 1 item") - seen_ids: set[str] = set() - viewports: list[Viewport] = [] - for i, item in enumerate(arr): - obj = _require_object(item, f"viewports[{i}]") - expected_keys = {"id", "width", "height"} - if set(obj.keys()) != expected_keys: - raise ManifestValidationError(f"viewports[{i}] has invalid schema") - vid = _require_pattern(obj["id"], f"viewports[{i}].id", VIEWPORT_ID_RE) - if vid in seen_ids: - raise ManifestValidationError(f"viewports[{i}].id is duplicate") - seen_ids.add(vid) - w = _require_bounded_int(obj["width"], f"viewports[{i}].width", 1, 8192) - h = _require_bounded_int(obj["height"], f"viewports[{i}].height", 1, 8192) - viewports.append(Viewport(id=vid, width=w, height=h)) - return tuple(viewports) - - -def _validate_output_root(data: Any, repo_root: Path) -> str: - s = _require_str(data, "output_root") - if "\\" in s or ":" in s: - raise ManifestPathError("field 'output_root' contains invalid path characters") - norm = posixpath.normpath(s) - if s != norm: - raise ManifestPathError("field 'output_root' is not in canonical form") - if not s.startswith("agent-test/runs/"): - raise ManifestValidationError( - "field 'output_root' must be relative to agent-test/runs/" - ) - tail = s[len("agent-test/runs/"):] - if not tail or "/" in tail: - raise ManifestValidationError( - "field 'output_root' must be a single non-empty segment after agent-test/runs/" - ) - - runs_dir = (repo_root / "agent-test" / "runs").resolve() - output_path = (repo_root / s).resolve() - try: - output_path.relative_to(runs_dir) - except ValueError: - raise ManifestPathError("field 'output_root' resolves outside agent-test/runs") - return s - - -def _validate_asset_mapping( - data: dict[str, Any], i: int -) -> tuple[str, str]: - obj = _require_object(data, f"assets[{i}]") - expected_keys = {"source", "workspace_path"} - if set(obj.keys()) != expected_keys: - raise ManifestValidationError(f"assets[{i}] has invalid schema") - source = _normalize_posix_relative_path(obj["source"], f"assets[{i}].source") - workspace_path = _normalize_posix_relative_path( - obj["workspace_path"], f"assets[{i}].workspace_path" - ) - return source, workspace_path - - -def _validate_fixture( - data: dict[str, Any], repo_root: Path -) -> Fixture: - obj = _require_object(data, "fixture") - expected_keys = {"version", "prompt", "assets", "checksum"} - if set(obj.keys()) != expected_keys: - raise ManifestValidationError("fixture has invalid schema") - - version = _require_pattern(obj["version"], "fixture.version", TOKEN_RE) - prompt = _normalize_posix_relative_path(obj["prompt"], "fixture.prompt") - - assets_raw = _require_array(obj["assets"], "fixture.assets") - if len(assets_raw) < 1: - raise ManifestValidationError("fixture.assets must have at least 1 item") - - assets_list: list[AssetMapping] = [] - workspace_destinations: set[str] = set() - for i, item in enumerate(assets_raw): - source, workspace_path = _validate_asset_mapping(item, i) - if workspace_path in workspace_destinations: - raise ManifestPathError( - "fixture.assets workspace_path is duplicate" - ) - workspace_destinations.add(workspace_path) - - src_path = repo_root / source - _require_regular_file(src_path, f"fixture asset source ({i})") - try: - src_path.resolve().relative_to(repo_root.resolve()) - except ValueError: - raise ManifestPathError("fixture asset source resolves outside repo root") - - content = src_path.read_bytes() - assets_list.append(AssetMapping(source=source, workspace_path=workspace_path, content=content)) - - assets_list.sort(key=lambda a: a.workspace_path) - - # Validate prompt file - prompt_path = repo_root / prompt - _require_regular_file(prompt_path, "fixture prompt") - try: - prompt_path.resolve().relative_to(repo_root.resolve()) - except ValueError: - raise ManifestPathError("fixture prompt resolves outside repo root") - prompt_content = prompt_path.read_bytes() - - checksum = _require_sha256(obj["checksum"], "fixture.checksum") - computed = digest_workspace_inputs(assets_list) - if checksum != computed: - raise ManifestDigestError("fixture.checksum mismatch") - - return Fixture( - version=version, - prompt=prompt, - assets=tuple(assets_list), - checksum=checksum, - prompt_content=prompt_content, - ) - - -def _validate_expected_binding( - data: dict[str, Any], i: int -) -> ExpectedBinding: - obj = _require_object(data, f"expected_bindings[{i}]") - expected_keys = {"stage", "model"} - if not expected_keys.issubset(set(obj.keys())): - raise ManifestValidationError(f"expected_bindings[{i}] missing required keys") - extra = set(obj.keys()) - {"stage", "model", "effort"} - if extra: - raise ManifestValidationError(f"expected_bindings[{i}] has unexpected keys") - stage = _require_enum(obj["stage"], f"expected_bindings[{i}].stage", STAGE_ENUM) - model = _require_pattern(obj["model"], f"expected_bindings[{i}].model", TOKEN_RE) - effort = None - if "effort" in obj: - effort = _require_pattern(obj["effort"], f"expected_bindings[{i}].effort", TOKEN_RE) - return ExpectedBinding(stage=stage, model=model, effort=effort) - - -def _validate_iop_cell(data: dict[str, Any]) -> IopCell: - obj = _require_object(data, "iop") - expected_keys = {"request_model", "requested_effort", "route_kind", "route_id", "expected_bindings"} - if set(obj.keys()) != expected_keys: - raise ManifestValidationError("iop has invalid schema") - - request_model = _require_pattern(obj["request_model"], "iop.request_model", TOKEN_RE) - requested_effort = _require_pattern(obj["requested_effort"], "iop.requested_effort", TOKEN_RE) - route_kind = _require_enum(obj["route_kind"], "iop.route_kind", ROUTE_KIND_ENUM) - route_id = _require_pattern(obj["route_id"], "iop.route_id", TOKEN_RE) - - bindings_raw = _require_array(obj["expected_bindings"], "iop.expected_bindings") - if len(bindings_raw) < 1: - raise ManifestValidationError("iop.expected_bindings must have at least 1 item") - - bindings: list[ExpectedBinding] = [] - seen_stages: set[str] = set() - for i, item in enumerate(bindings_raw): - b = _validate_expected_binding(item, i) - if b.stage in seen_stages: - raise ManifestValidationError("iop.expected_bindings contains duplicate stage") - seen_stages.add(b.stage) - bindings.append(b) - - # Sort bindings by canonical stage rank - bindings.sort(key=lambda b: STAGE_RANK[b.stage]) - - # Validate direct vs execution_preset constraints - if route_kind == "direct": - if set(b.stage for b in bindings) != {"request"}: - raise ManifestValidationError( - "direct route requires exactly one binding with stage=request" - ) - elif route_kind == "execution_preset": - required_stages = {"selector", "plan", "work", "review"} - allowed_stages = required_stages | {"repair"} - actual_stages = set(b.stage for b in bindings) - if actual_stages not in (required_stages, allowed_stages): - raise ManifestValidationError( - "execution_preset has an invalid stage set" - ) - - return IopCell( - request_model=request_model, - requested_effort=requested_effort, - route_kind=route_kind, - route_id=route_id, - expected_bindings=tuple(bindings), - ) - - -def _validate_cell(data: dict[str, Any], i: int) -> MatrixCell: - obj = _require_object(data, f"matrix[{i}]") - expected_keys = {"id", "caller", "iop"} - if set(obj.keys()) != expected_keys: - raise ManifestValidationError(f"matrix[{i}] has invalid schema") - - cell_id = _require_pattern(obj["id"], f"matrix[{i}].id", CELL_ID_RE) - caller = _require_enum(obj["caller"], f"matrix[{i}].caller", CALLER_ENUM) - iop = _validate_iop_cell(obj["iop"]) - return MatrixCell(id=cell_id, caller=caller, iop=iop) - - -def _validate_evaluator(data: dict[str, Any]) -> Evaluator: - obj = _require_object(data, "evaluator") - if set(obj) != {"caller", "iop"}: - raise ManifestValidationError("evaluator has invalid schema") - caller = _require_enum(obj["caller"], "evaluator.caller", ("codex",)) - return Evaluator(caller=caller, iop=_validate_iop_cell(obj["iop"])) - - -def _execution_order_key(seed: str, cell: MatrixCell) -> tuple[bytes, str]: - rank = hashlib.sha256( - b"iop-benchmark-order-v1\0" - + seed.encode("ascii") - + b"\0" - + cell.id.encode("ascii") - ).digest() - return rank, cell.id - - -def _validate_matrix( - data: list[Any], execution_order_seed: str | None -) -> tuple[MatrixCell, ...]: - arr = _require_array(data, "matrix") - if len(arr) < 1: - raise ManifestValidationError("matrix must have at least 1 item") - - cells: list[MatrixCell] = [] - seen_ids: set[str] = set() - for i, item in enumerate(arr): - cell = _validate_cell(item, i) - if cell.id in seen_ids: - raise ManifestValidationError("matrix contains duplicate cell id") - seen_ids.add(cell.id) - cells.append(cell) - - # No seed preserves the pipeline-v2 cell-id order. An explicit seed is - # stable across JSON input permutations and platforms. - cells.sort( - key=(lambda cell: cell.id) - if execution_order_seed is None - else (lambda cell: _execution_order_key(execution_order_seed, cell)) - ) - return tuple(cells) - - -def _default_repo_root(manifest_path: Path) -> Path: - """Walk up from manifest_path to find the repository root. - - Looks for a directory containing ``Makefile`` or ``.git``. - Falls back to the manifest file's parent directory. - """ - candidate = manifest_path.resolve() - for _ in range(20): # safety limit - if (candidate / "Makefile").is_file() or (candidate / ".git").exists(): - return candidate - parent = candidate.parent - if parent == candidate: - break - candidate = parent - return manifest_path.resolve().parent - - -# --------------------------------------------------------------------------- -# Public API -# --------------------------------------------------------------------------- - -def load_manifest(path: str | Path, repo_root: str | Path | None = None) -> Manifest: - """Load and validate a benchmark manifest JSON file. - - Args: - path: Path to the manifest JSON file. - repo_root: Repository root for resolving relative paths. - - Returns: - A frozen Manifest object with computed digest. - - Raises: - ManifestValidationError: If the JSON fails schema validation. - ManifestPathError: If a path rule is violated. - ManifestDigestError: If a declared digest does not match. - """ - path = Path(path) - if repo_root is None: - repo_root = _default_repo_root(path) - repo_root = Path(repo_root).resolve() - - try: - raw = json.loads(path.read_text(encoding="utf-8")) - except FileNotFoundError: - raise ManifestValidationError("manifest file not found") - except (json.JSONDecodeError, UnicodeDecodeError) as exc: - raise ManifestValidationError("invalid JSON format") - - if not isinstance(raw, dict): - raise ManifestValidationError("manifest must be a JSON object") - - # Top-level field validation - expected_top = { - "pipeline_version", "environment", "testbed", "fixture", "matrix", - "session_policy", "setup_cache_policy", "timeout", "viewports", - "rubric_version", "evaluator", "output_root", - } - optional_top = {"repetitions", "execution_order_seed"} - declared_keys = set(raw.keys()) - required_present = expected_top.issubset(declared_keys) - extra = declared_keys - expected_top - optional_top - if not required_present: - raise ManifestValidationError("manifest missing required top-level fields") - if extra: - raise ManifestValidationError("manifest has unexpected top-level fields") - - pipeline_version = _require_enum(raw["pipeline_version"], "pipeline_version", (PIPELINE_VERSION,)) - environment = _require_enum(raw["environment"], "environment", (ENVIRONMENT,)) - testbed = _require_enum(raw["testbed"], "testbed", (TESTBED_REQUIRED,)) - - repetitions = DEFAULT_REPETITIONS - if "repetitions" in raw: - repetitions = _require_positive_int(raw["repetitions"], "repetitions") - - execution_order_seed: str | None = None - if "execution_order_seed" in raw: - execution_order_seed = _require_pattern( - raw["execution_order_seed"], "execution_order_seed", CELL_ID_RE - ) - - session_policy = _require_enum(raw["session_policy"], "session_policy", (SESSION_POLICY,)) - setup_cache_policy = _require_enum( - raw["setup_cache_policy"], "setup_cache_policy", (SETUP_CACHE_POLICY,) - ) - timeout = _validate_timeout(raw["timeout"]) - viewports = _validate_viewports(raw["viewports"]) - rubric_version = _require_enum( - raw["rubric_version"], "rubric_version", RUBRIC_VERSIONS - ) - evaluator = _validate_evaluator(raw["evaluator"]) - output_root = _validate_output_root(raw["output_root"], repo_root) - - fixture = _validate_fixture(raw["fixture"], repo_root) - matrix = _validate_matrix(raw["matrix"], execution_order_seed) - - # Compute manifest digest during load - dummy_manifest = Manifest( - pipeline_version=pipeline_version, - environment=environment, - testbed=testbed, - repetitions=repetitions, - session_policy=session_policy, - setup_cache_policy=setup_cache_policy, - timeout=timeout, - viewports=viewports, - rubric_version=rubric_version, - evaluator=evaluator, - output_root=output_root, - fixture=fixture, - matrix=matrix, - digest="", - execution_order_seed=execution_order_seed, - ) - computed_digest = digest_manifest_and_resolved_inputs(dummy_manifest) - - return Manifest( - pipeline_version=pipeline_version, - environment=environment, - testbed=testbed, - repetitions=repetitions, - session_policy=session_policy, - setup_cache_policy=setup_cache_policy, - timeout=timeout, - viewports=viewports, - rubric_version=rubric_version, - evaluator=evaluator, - output_root=output_root, - fixture=fixture, - matrix=matrix, - digest=computed_digest, - execution_order_seed=execution_order_seed, - ) - - -def validate_manifest_bytes( - data: bytes, - path_hint: str = "", - repo_root: str | Path | None = None, -) -> Manifest: - """Validate manifest JSON bytes without writing to persistent disk.""" - import tempfile - with tempfile.NamedTemporaryFile( - mode="wb", suffix=".json", delete=False - ) as tmp: - tmp.write(data) - tmp_path = tmp.name - try: - return load_manifest(tmp_path, repo_root=repo_root) - finally: - try: - os.unlink(tmp_path) - except OSError: - pass diff --git a/scripts/agent_benchmark/manifest_test.py b/scripts/agent_benchmark/manifest_test.py deleted file mode 100644 index 52dd6485..00000000 --- a/scripts/agent_benchmark/manifest_test.py +++ /dev/null @@ -1,2572 +0,0 @@ -""" -Comprehensive tests for the benchmark manifest loader, validator, and CLI. - -Covers: valid minimum/example, omitted repetitions, data-only matrix extension, -deterministic ordering, duplicate ids/stages, direct vs preset shapes, -every enum/bound, unknown members, path escape/symlink/collision, -workspace/prompt/asset/manifest digest drift, non-positive bounds, and -secret redaction in errors. -""" - -from __future__ import annotations - -import hashlib -import inspect -import json -import os -import re -import shutil -import struct -import subprocess -import sys -import tempfile -import unittest -from dataclasses import replace -from pathlib import Path - -# Ensure repo root is importable -_REPO_ROOT = Path(__file__).resolve().parent.parent.parent -if str(_REPO_ROOT) not in sys.path: - sys.path.insert(0, str(_REPO_ROOT)) - -from scripts.agent_benchmark.manifest import ( - AssetMapping, - Fixture, - IopCell, - Manifest, - ManifestDigestError, - ManifestError, - ManifestPathError, - ManifestValidationError, - MatrixCell, - ONE_SHOT_RUBRIC_VERSION, - RUBRIC_VERSION, - RUBRIC_VERSIONS, - Timeout, - Viewport, - digest_manifest_and_resolved_inputs, - digest_workspace_inputs, - load_manifest, - _manifest_to_dict, - validate_manifest_bytes, -) - -# Sentinel values used in redaction tests -_SENTINEL_SECRET = "SUPER_SECRET_API_KEY_12345" -_SENTINEL_ENDPOINT = "https://private.internal.example.com/secret-endpoint" -_SENTINEL_PROMPT = "Do not leak this prompt content" - - -def _make_minimal_manifest_dict(**overrides: object) -> dict: - """Build a minimal valid manifest dict with optional overrides.""" - d = { - "pipeline_version": "2", - "environment": "dev", - "testbed": "../iop-s2", - "session_policy": "fresh", - "setup_cache_policy": "isolated", - "timeout": { - "run_seconds": 300, - "idle_seconds": 30, - "quiet_seconds": 10, - "cleanup_grace_seconds": 5, - }, - "viewports": [{"id": "desktop", "width": 1920, "height": 1080}], - "rubric_version": "landing-quality-v1", - "evaluator": { - "caller": "codex", - "iop": { - "request_model": "gpt-5.6-luna", - "requested_effort": "xhigh", - "route_kind": "direct", - "route_id": "gpt-5.6-luna", - "expected_bindings": [ - { - "stage": "request", - "model": "gpt-5.6-luna", - "effort": "xhigh", - } - ], - }, - }, - "output_root": "agent-test/runs/bench-01", - "fixture": { - "version": "v1.0", - "prompt": "scripts/fixtures/agent-comparison-benchmark/prompt.md", - "assets": [ - { - "source": "scripts/fixtures/agent-comparison-benchmark/reference.txt", - "workspace_path": "workspace/reference.txt", - } - ], - "checksum": "sha256:placeholder", - }, - "matrix": [ - { - "id": "cell-a", - "caller": "claude", - "iop": { - "request_model": "claude-sonnet-4-20250514", - "requested_effort": "high", - "route_kind": "direct", - "route_id": "claude-direct", - "expected_bindings": [ - {"stage": "request", "model": "claude-sonnet-4-20250514", "effort": "high"} - ], - }, - } - ], - } - d.update(overrides) - return d - - -def _compute_fixture_checksum(repo_root: Path, fixture: dict) -> str: - """Compute the fixture checksum for a given fixture dict.""" - assets_raw = fixture["assets"] - assets = tuple( - AssetMapping( - source=a["source"], - workspace_path=a["workspace_path"], - content=(repo_root / a["source"]).read_bytes(), - ) - for a in assets_raw - ) - return digest_workspace_inputs(assets) - - -def _write_tmp_manifest( - tmp_dir: Path, - data: dict, - name: str = "manifest.json", -) -> Path: - """Write manifest dict to tmp_dir/name and return the path. - - Attempts to patch the fixture checksum using repo-root asset files. - If that fails (e.g. assets were modified), writes the manifest as-is. - """ - p = tmp_dir / name - if "fixture" in data and "checksum" in data["fixture"]: - try: - data["fixture"]["checksum"] = _compute_fixture_checksum( - _REPO_ROOT, data["fixture"] - ) - except (FileNotFoundError, OSError, ValueError): - # Assets were modified; leave checksum as-is (tests should - # expect validation to fail for invalid checksums). - pass - p.write_text(json.dumps(data, indent=2), encoding="utf-8") - return p - - -def _load_tmp_manifest(path: Path) -> Manifest: - """Load a manifest from a temp path with repo_root set to _REPO_ROOT.""" - return load_manifest(path, repo_root=_REPO_ROOT) - - -class ManifestValidationTest(unittest.TestCase): - """Exact tracked-manifest regressions used by benchmark readiness packets.""" - - def test_iop_one_shot_manifest_locks_benchmark_readiness(self): - """The bench-02 manifest locks identical inputs and the complete C01-C09 matrix.""" - manifest_path = ( - _REPO_ROOT - / "scripts" - / "fixtures" - / "agent-comparison-benchmark-iop-one-shot.json" - ) - raw = json.loads(manifest_path.read_text(encoding="utf-8")) - manifest = load_manifest(manifest_path, repo_root=_REPO_ROOT) - - self.assertEqual( - { - key: raw[key] - for key in ( - "pipeline_version", - "environment", - "testbed", - "execution_order_seed", - "repetitions", - "session_policy", - "setup_cache_policy", - "timeout", - "viewports", - "rubric_version", - "evaluator", - "output_root", - ) - }, - { - "pipeline_version": "2", - "environment": "dev", - "testbed": "../iop-s2", - "execution_order_seed": "bench-02-c01-c09-v1", - "repetitions": 1, - "session_policy": "fresh", - "setup_cache_policy": "isolated", - "timeout": { - "run_seconds": 300, - "idle_seconds": 30, - "quiet_seconds": 10, - "cleanup_grace_seconds": 5, - }, - "viewports": [ - {"id": "desktop_1080", "width": 1920, "height": 1080}, - {"id": "mobile_375", "width": 375, "height": 812}, - ], - "rubric_version": ONE_SHOT_RUBRIC_VERSION, - "evaluator": { - "caller": "codex", - "iop": { - "request_model": "gpt-5.6-luna", - "requested_effort": "xhigh", - "route_kind": "direct", - "route_id": "gpt-5.6-luna", - "expected_bindings": [ - { - "stage": "request", - "model": "gpt-5.6-luna", - "effort": "xhigh", - } - ], - }, - }, - "output_root": "agent-test/runs/bench-02", - }, - ) - - expected_fixture = { - "version": "product-card-v2", - "prompt": "scripts/fixtures/agent-comparison-benchmark/prompt.md", - "assets": [ - { - "source": "scripts/fixtures/agent-comparison-benchmark/reference.txt", - "workspace_path": "brief/reference.txt", - }, - { - "source": "scripts/fixtures/agent-comparison-benchmark/aurora-grid.svg", - "workspace_path": "assets/aurora-grid.svg", - }, - { - "source": "scripts/fixtures/agent-comparison-benchmark/orbit-rings.svg", - "workspace_path": "assets/orbit-rings.svg", - }, - ], - "checksum": "sha256:fb16198fd4c3576f880f047ed7de54dddc160b0f70c2ba55b435cf078c61828e", - } - self.assertEqual(raw["fixture"], expected_fixture) - self.assertEqual( - _compute_fixture_checksum(_REPO_ROOT, raw["fixture"]), - expected_fixture["checksum"], - ) - self.assertEqual( - [ - asset["workspace_path"] - for asset in raw["fixture"]["assets"] - if asset["source"].endswith(".svg") - ], - ["assets/aurora-grid.svg", "assets/orbit-rings.svg"], - ) - self.assertEqual( - manifest.viewports, - ( - Viewport(id="desktop_1080", width=1920, height=1080), - Viewport(id="mobile_375", width=375, height=812), - ), - ) - self.assertEqual( - manifest.timeout, - Timeout( - run_seconds=300, - idle_seconds=30, - quiet_seconds=10, - cleanup_grace_seconds=5, - ), - ) - - def direct(cell_id: str, caller: str, model: str, effort: str) -> dict: - return { - "id": cell_id, - "caller": caller, - "iop": { - "request_model": model, - "requested_effort": effort, - "route_kind": "direct", - "route_id": model, - "expected_bindings": [ - {"stage": "request", "model": model, "effort": effort} - ], - }, - } - - def hybrid( - cell_id: str, - caller: str, - route_id: str, - request_effort: str, - cloud_model: str, - ) -> dict: - return { - "id": cell_id, - "caller": caller, - "iop": { - "request_model": route_id, - "requested_effort": request_effort, - "route_kind": "execution_preset", - "route_id": route_id, - "expected_bindings": [ - {"stage": "selector", "model": cloud_model, "effort": "high"}, - {"stage": "plan", "model": cloud_model, "effort": "high"}, - {"stage": "work", "model": "ornith-fast"}, - {"stage": "review", "model": cloud_model, "effort": "high"}, - {"stage": "repair", "model": cloud_model, "effort": "high"}, - ], - }, - } - - expected_cells = [ - direct("c01-claude-sonnet-direct", "claude", "claude-sonnet-5", "max"), - direct("c02-claude-gemini-direct", "claude", "gemini-3.6-flash", "high"), - direct("c03-agy-gemini-direct", "agy", "gemini-3.6-flash", "high"), - direct("c04-claude-gpt-direct", "claude", "gpt-5.6-luna", "xhigh"), - direct("c05-codex-gpt-direct", "codex", "gpt-5.6-luna", "xhigh"), - hybrid( - "c06-claude-gemini-hybrid", - "claude", - "gemini-hybrid", - "high", - "gemini-3.6-flash", - ), - hybrid( - "c07-agy-gemini-hybrid", - "agy", - "gemini-hybrid", - "high", - "gemini-3.6-flash", - ), - hybrid( - "c08-claude-gpt-hybrid", - "claude", - "gpt-hybrid", - "xhigh", - "gpt-5.6-terra", - ), - hybrid( - "c09-codex-gpt-hybrid", - "codex", - "gpt-hybrid", - "xhigh", - "gpt-5.6-terra", - ), - ] - self.assertEqual(raw["matrix"], expected_cells) - - expected_order = [ - "c02-claude-gemini-direct", - "c05-codex-gpt-direct", - "c03-agy-gemini-direct", - "c06-claude-gemini-hybrid", - "c08-claude-gpt-hybrid", - "c09-codex-gpt-hybrid", - "c01-claude-sonnet-direct", - "c07-agy-gemini-hybrid", - "c04-claude-gpt-direct", - ] - expected_by_id = {cell["id"]: cell for cell in expected_cells} - self.assertEqual( - [cell.id for cell in manifest.matrix], - expected_order, - ) - self.assertEqual( - _manifest_to_dict(manifest)["matrix"], - [expected_by_id[cell_id] for cell_id in expected_order], - ) - self.assertEqual(manifest.execution_order_seed, "bench-02-c01-c09-v1") - self.assertEqual(manifest.rubric_version, ONE_SHOT_RUBRIC_VERSION) - self.assertEqual(manifest.fixture.checksum, expected_fixture["checksum"]) - self.assertEqual(manifest.digest, digest_manifest_and_resolved_inputs(manifest)) - self.assertRegex(manifest.digest, r"^sha256:[0-9a-f]{64}$") - - def test_recovery_qualification_manifest_locks_non_scored_six_path_matrix(self): - full_path = ( - _REPO_ROOT - / "scripts" - / "fixtures" - / "agent-comparison-benchmark-iop-one-shot.json" - ) - recovery_path = ( - _REPO_ROOT - / "scripts" - / "fixtures" - / "agent-comparison-benchmark-recovery-qualification.json" - ) - full_raw = json.loads(full_path.read_text(encoding="utf-8")) - recovery_raw = json.loads(recovery_path.read_text(encoding="utf-8")) - recovery = load_manifest(recovery_path, repo_root=_REPO_ROOT) - - shared_keys = ( - "pipeline_version", - "environment", - "testbed", - "repetitions", - "session_policy", - "setup_cache_policy", - "timeout", - "viewports", - "rubric_version", - "evaluator", - "fixture", - ) - self.assertEqual( - {key: recovery_raw[key] for key in shared_keys}, - {key: full_raw[key] for key in shared_keys}, - ) - self.assertEqual(recovery_raw["timeout"]["run_seconds"], 300) - self.assertEqual( - recovery_raw["execution_order_seed"], - "bench-02-recovery-qualification-v1", - ) - self.assertNotEqual( - recovery_raw["execution_order_seed"], full_raw["execution_order_seed"] - ) - self.assertEqual( - recovery_raw["output_root"], - "agent-test/runs/bench-02-recovery", - ) - self.assertNotEqual(recovery_raw["output_root"], full_raw["output_root"]) - - expected_ids = [ - "c01-claude-sonnet-direct", - "c03-agy-gemini-direct", - "c04-claude-gpt-direct", - "c06-claude-gemini-hybrid", - "c07-agy-gemini-hybrid", - "c08-claude-gpt-hybrid", - ] - full_by_id = {cell["id"]: cell for cell in full_raw["matrix"]} - self.assertEqual(recovery_raw["matrix"], [full_by_id[item] for item in expected_ids]) - self.assertEqual( - {cell.id for cell in recovery.matrix}, set(expected_ids) - ) - self.assertEqual(len(recovery.matrix), 6) - self.assertEqual(recovery.digest, digest_manifest_and_resolved_inputs(recovery)) - - -class TestLoadManifestValid(unittest.TestCase): - """Valid manifest loading tests.""" - - def test_minimal_valid_manifest(self): - """Minimal manifest with explicit repetitions=1 loads.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - path = _write_tmp_manifest(tmp_dir, _make_minimal_manifest_dict()) - m = _load_tmp_manifest(path) - self.assertEqual(m.pipeline_version, "2") - self.assertEqual(m.environment, "dev") - self.assertEqual(m.repetitions, 1) - self.assertEqual(m.session_policy, "fresh") - self.assertEqual(m.setup_cache_policy, "isolated") - self.assertEqual(len(m.viewports), 1) - self.assertEqual(len(m.matrix), 1) - self.assertIsInstance(m, Manifest) - # Frozen - with self.assertRaises(AttributeError): - m.repetitions = 99 - - def test_omitted_repetitions_defaults_to_one(self): - """Omitted repetitions defaults to 1.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict() - # repetitions is not included (optional field) - path = _write_tmp_manifest(tmp_dir, d) - m = _load_tmp_manifest(path) - self.assertEqual(m.repetitions, 1) - - def test_omitted_equals_explicit_one(self): - """Omitted repetitions and explicit repetitions=1 produce identical manifests.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - # d_no_rep: repetitions is not included (optional field) - d_no_rep = _make_minimal_manifest_dict() - d_explicit = _make_minimal_manifest_dict(repetitions=1) - p1 = _write_tmp_manifest(tmp_dir, d_no_rep, "no_rep.json") - p2 = _write_tmp_manifest(tmp_dir, d_explicit, "explicit_rep.json") - m1 = _load_tmp_manifest(p1) - m2 = _load_tmp_manifest(p2) - self.assertEqual(m1.repetitions, m2.repetitions) - self.assertEqual(m1, m2) - - def test_explicit_repetitions_greater_than_one(self): - """Explicit repetitions > 1 is preserved.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - path = _write_tmp_manifest( - tmp_dir, _make_minimal_manifest_dict(repetitions=5) - ) - m = _load_tmp_manifest(path) - self.assertEqual(m.repetitions, 5) - - def test_example_manifest_loads(self): - """The shipped example manifest loads successfully.""" - example_path = ( - _REPO_ROOT - / "scripts" - / "fixtures" - / "agent-comparison-benchmark-manifest.example.json" - ) - if example_path.exists(): - m = _load_tmp_manifest(example_path) - self.assertEqual(m.pipeline_version, "2") - self.assertEqual(len(m.matrix), 3) - # Cells sorted by id - ids = [c.id for c in m.matrix] - self.assertEqual(ids, sorted(ids)) - - def test_multiple_viewports_unique(self): - """Multiple viewports with unique ids load.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict( - viewports=[ - {"id": "desktop", "width": 1920, "height": 1080}, - {"id": "mobile", "width": 375, "height": 812}, - ] - ) - path = _write_tmp_manifest(tmp_dir, d) - m = _load_tmp_manifest(path) - self.assertEqual(len(m.viewports), 2) - - def test_execution_preset_cell_loads(self): - """Execution-preset cell with all required stages loads.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict( - matrix=[ - { - "id": "preset-cell", - "caller": "agy", - "iop": { - "request_model": "gemini-2.0-flash", - "requested_effort": "high", - "route_kind": "execution_preset", - "route_id": "agy-generic", - "expected_bindings": [ - {"stage": "plan", "model": "gemini-2.0-flash"}, - {"stage": "work", "model": "gemini-2.0-flash"}, - {"stage": "review", "model": "gemini-2.0-flash"}, - {"stage": "selector", "model": "gemini-2.0-flash"}, - ], - }, - } - ] - ) - path = _write_tmp_manifest(tmp_dir, d) - m = _load_tmp_manifest(path) - cell = m.matrix[0] - # Sorted by canonical stage rank - stages = [b.stage for b in cell.iop.expected_bindings] - self.assertEqual(stages, ["selector", "plan", "work", "review"]) - - def test_execution_preset_with_repair(self): - """Execution-preset cell with optional repair stage loads.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict( - matrix=[ - { - "id": "preset-repair", - "caller": "codex", - "iop": { - "request_model": "gpt-4.1", - "requested_effort": "xhigh", - "route_kind": "execution_preset", - "route_id": "codex-generic", - "expected_bindings": [ - {"stage": "selector", "model": "gpt-4.1"}, - {"stage": "plan", "model": "gpt-4.1"}, - {"stage": "work", "model": "gpt-4.1"}, - {"stage": "review", "model": "gpt-4.1"}, - {"stage": "repair", "model": "gpt-4.1"}, - ], - }, - } - ] - ) - path = _write_tmp_manifest(tmp_dir, d) - m = _load_tmp_manifest(path) - stages = [b.stage for b in m.matrix[0].iop.expected_bindings] - self.assertEqual(stages, ["selector", "plan", "work", "review", "repair"]) - - -class TestMatrixExtension(unittest.TestCase): - """Data-only matrix extension tests.""" - - def test_data_only_matrix_extension(self): - """Adding a new cell to the matrix does not require code changes.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict( - matrix=[ - { - "id": "cell-a", - "caller": "claude", - "iop": { - "request_model": "claude-sonnet-4-20250514", - "requested_effort": "high", - "route_kind": "direct", - "route_id": "claude-direct", - "expected_bindings": [ - {"stage": "request", "model": "claude-sonnet-4-20250514"} - ], - }, - }, - { - "id": "cell-b", - "caller": "agy", - "iop": { - "request_model": "gemini-2.0-flash", - "requested_effort": "high", - "route_kind": "execution_preset", - "route_id": "agy-generic", - "expected_bindings": [ - {"stage": "selector", "model": "gemini-2.0-flash"}, - {"stage": "plan", "model": "gemini-2.0-flash"}, - {"stage": "work", "model": "gemini-2.0-flash"}, - {"stage": "review", "model": "gemini-2.0-flash"}, - ], - }, - }, - { - "id": "cell-c", - "caller": "codex", - "iop": { - "request_model": "gpt-4.1", - "requested_effort": "xhigh", - "route_kind": "execution_preset", - "route_id": "codex-generic", - "expected_bindings": [ - {"stage": "selector", "model": "gpt-4.1"}, - {"stage": "plan", "model": "gpt-4.1"}, - {"stage": "work", "model": "gpt-4.1"}, - {"stage": "review", "model": "gpt-4.1"}, - ], - }, - }, - ] - ) - path = _write_tmp_manifest(tmp_dir, d) - m = _load_tmp_manifest(path) - self.assertEqual(len(m.matrix), 3) - # Sorted by id - ids = [c.id for c in m.matrix] - self.assertEqual(ids, sorted(ids)) - self.assertEqual(ids, ["cell-a", "cell-b", "cell-c"]) - - -class TestDeterministicOrdering(unittest.TestCase): - """Deterministic cell and binding ordering tests.""" - - @staticmethod - def _direct_cells(*cell_ids: str) -> list[dict]: - return [ - { - "id": cell_id, - "caller": "claude", - "iop": { - "request_model": "claude-sonnet-4-20250514", - "requested_effort": "high", - "route_kind": "direct", - "route_id": "claude-direct", - "expected_bindings": [ - { - "stage": "request", - "model": "claude-sonnet-4-20250514", - } - ], - }, - } - for cell_id in cell_ids - ] - - def test_execution_order_seed_is_canonical_and_permutation_stable(self): - """An explicit seed produces one repeatable order for every input permutation.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - cells = self._direct_cells("cell-a", "cell-b", "cell-c") - seeded = _make_minimal_manifest_dict( - execution_order_seed="seed-a", matrix=cells - ) - permuted = _make_minimal_manifest_dict( - execution_order_seed="seed-a", matrix=list(reversed(cells)) - ) - different_seed = _make_minimal_manifest_dict( - execution_order_seed="seed-b", matrix=cells - ) - - m1 = _load_tmp_manifest( - _write_tmp_manifest(tmp_dir, seeded, "seeded.json") - ) - m2 = _load_tmp_manifest( - _write_tmp_manifest(tmp_dir, permuted, "permuted.json") - ) - m3 = _load_tmp_manifest( - _write_tmp_manifest(tmp_dir, different_seed, "different.json") - ) - - self.assertEqual(m1.execution_order_seed, "seed-a") - self.assertEqual( - _manifest_to_dict(m1)["execution_order_seed"], - "seed-a", - ) - self.assertEqual( - [cell.id for cell in m1.matrix], - ["cell-c", "cell-a", "cell-b"], - ) - self.assertEqual(m1.matrix, m2.matrix) - self.assertEqual(m1.digest, m2.digest) - self.assertEqual( - [cell.id for cell in m3.matrix], - ["cell-a", "cell-c", "cell-b"], - ) - self.assertNotEqual(m1.matrix, m3.matrix) - self.assertNotEqual(m1.digest, m3.digest) - - def test_omitted_execution_order_seed_preserves_legacy_order_and_digest_contract(self): - """Omitting the seed keeps id order and excludes a synthetic default from canonical JSON.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - data = _make_minimal_manifest_dict( - matrix=self._direct_cells("cell-c", "cell-a", "cell-b") - ) - manifest = _load_tmp_manifest(_write_tmp_manifest(tmp_dir, data)) - - self.assertIsNone(manifest.execution_order_seed) - self.assertEqual( - [cell.id for cell in manifest.matrix], - ["cell-a", "cell-b", "cell-c"], - ) - self.assertNotIn("execution_order_seed", _manifest_to_dict(manifest)) - self.assertEqual( - manifest.digest, digest_manifest_and_resolved_inputs(manifest) - ) - - def test_cells_sorted_by_id(self): - """Cells are sorted by id regardless of input order.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict( - matrix=[ - { - "id": "z-cell", - "caller": "claude", - "iop": { - "request_model": "claude-sonnet-4-20250514", - "requested_effort": "high", - "route_kind": "direct", - "route_id": "claude-direct", - "expected_bindings": [ - {"stage": "request", "model": "claude-sonnet-4-20250514"} - ], - }, - }, - { - "id": "a-cell", - "caller": "agy", - "iop": { - "request_model": "gemini-2.0-flash", - "requested_effort": "high", - "route_kind": "execution_preset", - "route_id": "agy-generic", - "expected_bindings": [ - {"stage": "work", "model": "gemini-2.0-flash"}, - {"stage": "plan", "model": "gemini-2.0-flash"}, - {"stage": "review", "model": "gemini-2.0-flash"}, - {"stage": "selector", "model": "gemini-2.0-flash"}, - ], - }, - }, - ] - ) - path = _write_tmp_manifest(tmp_dir, d) - m = _load_tmp_manifest(path) - ids = [c.id for c in m.matrix] - self.assertEqual(ids, ["a-cell", "z-cell"]) - - def test_bindings_sorted_by_canonical_rank(self): - """Bindings are sorted by fixed stage rank, not lexical order.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict( - matrix=[ - { - "id": "rank-test", - "caller": "agy", - "iop": { - "request_model": "gemini-2.0-flash", - "requested_effort": "high", - "route_kind": "execution_preset", - "route_id": "agy-generic", - "expected_bindings": [ - {"stage": "review", "model": "gemini-2.0-flash"}, - {"stage": "work", "model": "gemini-2.0-flash"}, - {"stage": "selector", "model": "gemini-2.0-flash"}, - {"stage": "plan", "model": "gemini-2.0-flash"}, - ], - }, - } - ] - ) - path = _write_tmp_manifest(tmp_dir, d) - m = _load_tmp_manifest(path) - stages = [b.stage for b in m.matrix[0].iop.expected_bindings] - self.assertEqual(stages, ["selector", "plan", "work", "review"]) - - def test_canonical_rank_full_order(self): - """Full canonical rank order for preset: selector, plan, work, review, repair.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict( - matrix=[ - { - "id": "full-rank", - "caller": "agy", - "iop": { - "request_model": "gemini-2.0-flash", - "requested_effort": "high", - "route_kind": "execution_preset", - "route_id": "agy-generic", - "expected_bindings": [ - {"stage": "repair", "model": "gemini-2.0-flash"}, - {"stage": "review", "model": "gemini-2.0-flash"}, - {"stage": "plan", "model": "gemini-2.0-flash"}, - {"stage": "work", "model": "gemini-2.0-flash"}, - {"stage": "selector", "model": "gemini-2.0-flash"}, - ], - }, - } - ] - ) - path = _write_tmp_manifest(tmp_dir, d) - m = _load_tmp_manifest(path) - stages = [b.stage for b in m.matrix[0].iop.expected_bindings] - self.assertEqual( - stages, ["selector", "plan", "work", "review", "repair"] - ) - - -class TestDuplicateDetection(unittest.TestCase): - """Duplicate id and stage detection tests.""" - - def test_duplicate_cell_ids_rejected(self): - """Two cells with the same id are rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict( - matrix=[ - { - "id": "dup", - "caller": "claude", - "iop": { - "request_model": "claude-sonnet-4-20250514", - "requested_effort": "high", - "route_kind": "direct", - "route_id": "r1", - "expected_bindings": [ - {"stage": "request", "model": "claude-sonnet-4-20250514"} - ], - }, - }, - { - "id": "dup", - "caller": "agy", - "iop": { - "request_model": "gemini-2.0-flash", - "requested_effort": "high", - "route_kind": "execution_preset", - "route_id": "r2", - "expected_bindings": [ - {"stage": "selector", "model": "gemini-2.0-flash"}, - {"stage": "plan", "model": "gemini-2.0-flash"}, - {"stage": "work", "model": "gemini-2.0-flash"}, - {"stage": "review", "model": "gemini-2.0-flash"}, - ], - }, - }, - ] - ) - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestValidationError): - _load_tmp_manifest(path) - - def test_duplicate_binding_stages_rejected(self): - """Two bindings with the same stage in one cell are rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict( - matrix=[ - { - "id": "dup-stage", - "caller": "agy", - "iop": { - "request_model": "gemini-2.0-flash", - "requested_effort": "high", - "route_kind": "execution_preset", - "route_id": "r1", - "expected_bindings": [ - {"stage": "work", "model": "gemini-2.0-flash"}, - {"stage": "work", "model": "gemini-2.0-flash"}, - ], - }, - } - ] - ) - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestValidationError): - _load_tmp_manifest(path) - - def test_duplicate_viewport_ids_rejected(self): - """Two viewports with the same id are rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict( - viewports=[ - {"id": "dup", "width": 1920, "height": 1080}, - {"id": "dup", "width": 375, "height": 812}, - ] - ) - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestValidationError): - _load_tmp_manifest(path) - - -class TestDirectVsPresetShapes(unittest.TestCase): - """Direct vs execution-preset shape validation tests.""" - - def test_direct_requires_exactly_one_request_binding(self): - """Direct route with no bindings is rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict( - matrix=[ - { - "id": "direct-empty", - "caller": "claude", - "iop": { - "request_model": "claude-sonnet-4-20250514", - "requested_effort": "high", - "route_kind": "direct", - "route_id": "r1", - "expected_bindings": [], - }, - } - ] - ) - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestValidationError): - _load_tmp_manifest(path) - - def test_direct_with_non_request_binding_rejected(self): - """Direct route with a non-request binding is rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict( - matrix=[ - { - "id": "direct-wrong", - "caller": "claude", - "iop": { - "request_model": "claude-sonnet-4-20250514", - "requested_effort": "high", - "route_kind": "direct", - "route_id": "r1", - "expected_bindings": [ - {"stage": "plan", "model": "claude-sonnet-4-20250514"} - ], - }, - } - ] - ) - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestValidationError): - _load_tmp_manifest(path) - - def test_preset_missing_required_stages_rejected(self): - """Execution-preset missing selector/plan/work/review is rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict( - matrix=[ - { - "id": "preset-incomplete", - "caller": "agy", - "iop": { - "request_model": "gemini-2.0-flash", - "requested_effort": "high", - "route_kind": "execution_preset", - "route_id": "r1", - "expected_bindings": [ - {"stage": "work", "model": "gemini-2.0-flash"}, - ], - }, - } - ] - ) - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestValidationError): - _load_tmp_manifest(path) - - def test_preset_two_repair_bindings_rejected(self): - """Execution-preset with two repair bindings is rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict( - matrix=[ - { - "id": "preset-double-repair", - "caller": "agy", - "iop": { - "request_model": "gemini-2.0-flash", - "requested_effort": "high", - "route_kind": "execution_preset", - "route_id": "r1", - "expected_bindings": [ - {"stage": "selector", "model": "gemini-2.0-flash"}, - {"stage": "plan", "model": "gemini-2.0-flash"}, - {"stage": "work", "model": "gemini-2.0-flash"}, - {"stage": "review", "model": "gemini-2.0-flash"}, - {"stage": "repair", "model": "gemini-2.0-flash"}, - {"stage": "repair", "model": "gemini-2.0-flash"}, - ], - }, - } - ] - ) - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestValidationError): - _load_tmp_manifest(path) - - -class TestEnumsAndBounds(unittest.TestCase): - """Enum and numeric bound validation tests.""" - - def test_invalid_caller_rejected(self): - """Invalid caller value is rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict( - matrix=[ - { - "id": "bad-caller", - "caller": "invalid_caller", - "iop": { - "request_model": "claude-sonnet-4-20250514", - "requested_effort": "high", - "route_kind": "direct", - "route_id": "r1", - "expected_bindings": [ - {"stage": "request", "model": "claude-sonnet-4-20250514"} - ], - }, - } - ] - ) - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestValidationError): - _load_tmp_manifest(path) - - def test_invalid_route_kind_rejected(self): - """Invalid route_kind is rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict( - matrix=[ - { - "id": "bad-route", - "caller": "claude", - "iop": { - "request_model": "claude-sonnet-4-20250514", - "requested_effort": "high", - "route_kind": "invalid_route", - "route_id": "r1", - "expected_bindings": [ - {"stage": "request", "model": "claude-sonnet-4-20250514"} - ], - }, - } - ] - ) - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestValidationError): - _load_tmp_manifest(path) - - def test_invalid_cell_id_pattern_rejected(self): - """Cell id with uppercase is rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict( - matrix=[ - { - "id": "Bad-Id", - "caller": "claude", - "iop": { - "request_model": "claude-sonnet-4-20250514", - "requested_effort": "high", - "route_kind": "direct", - "route_id": "r1", - "expected_bindings": [ - {"stage": "request", "model": "claude-sonnet-4-20250514"} - ], - }, - } - ] - ) - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestValidationError): - _load_tmp_manifest(path) - - def test_cell_id_too_long_rejected(self): - """Cell id exceeding 64 chars is rejected.""" - long_id = "a" * 65 - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict( - matrix=[ - { - "id": long_id, - "caller": "claude", - "iop": { - "request_model": "claude-sonnet-4-20250514", - "requested_effort": "high", - "route_kind": "direct", - "route_id": "r1", - "expected_bindings": [ - {"stage": "request", "model": "claude-sonnet-4-20250514"} - ], - }, - } - ] - ) - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestValidationError): - _load_tmp_manifest(path) - - def test_run_seconds_zero_rejected(self): - """timeout.run_seconds of 0 is rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict( - timeout={ - "run_seconds": 0, - "idle_seconds": 30, - "quiet_seconds": 10, - "cleanup_grace_seconds": 5, - } - ) - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestValidationError): - _load_tmp_manifest(path) - - def test_run_seconds_too_large_rejected(self): - """timeout.run_seconds > 86400 is rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict( - timeout={ - "run_seconds": 86401, - "idle_seconds": 30, - "quiet_seconds": 10, - "cleanup_grace_seconds": 5, - } - ) - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestValidationError): - _load_tmp_manifest(path) - - def test_idle_seconds_zero_rejected(self): - """timeout.idle_seconds of 0 is rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict( - timeout={ - "run_seconds": 300, - "idle_seconds": 0, - "quiet_seconds": 10, - "cleanup_grace_seconds": 5, - } - ) - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestValidationError): - _load_tmp_manifest(path) - - def test_quiet_seconds_zero_rejected(self): - """timeout.quiet_seconds of 0 is rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict( - timeout={ - "run_seconds": 300, - "idle_seconds": 30, - "quiet_seconds": 0, - "cleanup_grace_seconds": 5, - } - ) - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestValidationError): - _load_tmp_manifest(path) - - def test_cleanup_grace_seconds_zero_rejected(self): - """timeout.cleanup_grace_seconds of 0 is rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict( - timeout={ - "run_seconds": 300, - "idle_seconds": 30, - "quiet_seconds": 10, - "cleanup_grace_seconds": 0, - } - ) - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestValidationError): - _load_tmp_manifest(path) - - def test_repetitions_zero_rejected(self): - """repetitions of 0 is rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict(repetitions=0) - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestValidationError): - _load_tmp_manifest(path) - - def test_repetitions_negative_rejected(self): - """Negative repetitions is rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict(repetitions=-1) - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestValidationError): - _load_tmp_manifest(path) - - def test_viewport_width_zero_rejected(self): - """Viewport width of 0 is rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict( - viewports=[{"id": "vp", "width": 0, "height": 1080}] - ) - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestValidationError): - _load_tmp_manifest(path) - - def test_viewport_width_too_large_rejected(self): - """Viewport width > 8192 is rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict( - viewports=[{"id": "vp", "width": 8193, "height": 1080}] - ) - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestValidationError): - _load_tmp_manifest(path) - - def test_viewport_height_too_large_rejected(self): - """Viewport height > 8192 is rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict( - viewports=[{"id": "vp", "width": 1920, "height": 8193}] - ) - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestValidationError): - _load_tmp_manifest(path) - - def test_empty_viewports_rejected(self): - """Empty viewports array is rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict(viewports=[]) - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestValidationError): - _load_tmp_manifest(path) - - def test_invalid_pipeline_version_rejected(self): - """Invalid pipeline_version is rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict(pipeline_version="1") - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestValidationError): - _load_tmp_manifest(path) - - def test_invalid_environment_rejected(self): - """Invalid environment is rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict(environment="prod") - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestValidationError): - _load_tmp_manifest(path) - - def test_invalid_session_policy_rejected(self): - """Invalid session_policy is rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict(session_policy="persistent") - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestValidationError): - _load_tmp_manifest(path) - - def test_invalid_setup_cache_policy_rejected(self): - """Invalid setup_cache_policy is rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict(setup_cache_policy="shared") - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestValidationError): - _load_tmp_manifest(path) - - def test_invalid_rubric_version_rejected(self): - """A rubric version outside the closed catalog is rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict(rubric_version="Invalid Version!") - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestValidationError): - _load_tmp_manifest(path) - - -class TestUnknownMembers(unittest.TestCase): - """Unknown member rejection tests.""" - - def test_unknown_top_level_field_rejected(self): - """Unknown top-level field is rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict() - d["unknown_field"] = "should_fail" - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestValidationError): - _load_tmp_manifest(path) - - def test_unknown_timeout_field_rejected(self): - """Unknown timeout field is rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict( - timeout={ - "run_seconds": 300, - "idle_seconds": 30, - "quiet_seconds": 10, - "cleanup_grace_seconds": 5, - "extra_field": 42, - } - ) - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestValidationError): - _load_tmp_manifest(path) - - def test_unknown_fixture_field_rejected(self): - """Unknown fixture field is rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict() - d["fixture"]["extra_fixture_field"] = "should_fail" - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestValidationError): - _load_tmp_manifest(path) - - def test_unknown_cell_field_rejected(self): - """Unknown cell field is rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict() - d["matrix"][0]["extra_cell_field"] = "should_fail" - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestValidationError): - _load_tmp_manifest(path) - - def test_unknown_iop_field_rejected(self): - """Unknown iop field is rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict() - d["matrix"][0]["iop"]["extra_iop_field"] = "should_fail" - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestValidationError): - _load_tmp_manifest(path) - - def test_unknown_binding_field_rejected(self): - """Unknown binding field is rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict() - d["matrix"][0]["iop"]["expected_bindings"][0]["extra_binding"] = "should_fail" - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestValidationError): - _load_tmp_manifest(path) - - def test_unknown_viewport_field_rejected(self): - """Unknown viewport field is rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict( - viewports=[{"id": "vp", "width": 1920, "height": 1080, "extra": True}] - ) - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestValidationError): - _load_tmp_manifest(path) - - def test_unknown_asset_field_rejected(self): - """Unknown asset field is rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict() - d["fixture"]["assets"][0]["extra_asset"] = "should_fail" - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestValidationError): - _load_tmp_manifest(path) - - -class TestPathRules(unittest.TestCase): - """Path escape, symlink, collision, and containment tests.""" - - def test_absolute_prompt_path_rejected(self): - """Absolute prompt path is rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict() - d["fixture"]["prompt"] = "/absolute/path/prompt.md" - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestPathError): - _load_tmp_manifest(path) - - def test_absolute_asset_source_rejected(self): - """Absolute asset source path is rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict() - d["fixture"]["assets"][0]["source"] = "/absolute/source.txt" - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestPathError): - _load_tmp_manifest(path) - - def test_absolute_workspace_path_rejected(self): - """Absolute workspace_path is rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict() - d["fixture"]["assets"][0]["workspace_path"] = "/absolute/workspace.txt" - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestPathError): - _load_tmp_manifest(path) - - def test_dotdot_escape_in_prompt_rejected(self): - """Prompt path with .. escape is rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict() - d["fixture"]["prompt"] = "../escape/prompt.md" - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestPathError): - _load_tmp_manifest(path) - - def test_dotdot_escape_in_asset_source_rejected(self): - """Asset source with .. escape is rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict() - d["fixture"]["assets"][0]["source"] = "../escape/source.txt" - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestPathError): - _load_tmp_manifest(path) - - def test_dotdot_escape_in_workspace_path_rejected(self): - """Asset workspace_path with .. escape is rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict() - d["fixture"]["assets"][0]["workspace_path"] = "../escape/workspace.txt" - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestPathError): - _load_tmp_manifest(path) - - def test_colon_in_path_rejected(self): - """Path with colon is rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict() - d["fixture"]["prompt"] = "path:with:colons.md" - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestPathError): - _load_tmp_manifest(path) - - def test_symlink_source_rejected(self): - """Symlink as asset source is rejected.""" - with tempfile.TemporaryDirectory(dir=_REPO_ROOT) as tmp_sub: - tmp_sub_dir = Path(tmp_sub) - rel_sub = tmp_sub_dir.relative_to(_REPO_ROOT) - real_file = tmp_sub_dir / "real.txt" - real_file.write_text("content") - link_file = tmp_sub_dir / "link.txt" - link_file.symlink_to(real_file) - - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict() - d["fixture"]["assets"] = [ - { - "source": str(rel_sub / "link.txt"), - "workspace_path": "workspace/link_dest.txt", - } - ] - manifest_path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestPathError): - _load_tmp_manifest(manifest_path) - - def test_destination_collision_rejected(self): - """Two assets with the same workspace_path are rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict() - d["fixture"]["assets"] = [ - { - "source": "scripts/fixtures/agent-comparison-benchmark/prompt.md", - "workspace_path": "workspace/dup.txt", - }, - { - "source": "scripts/fixtures/agent-comparison-benchmark/reference.txt", - "workspace_path": "workspace/dup.txt", - }, - ] - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestPathError): - _load_tmp_manifest(path) - - def test_output_root_not_under_runs_rejected(self): - """output_root not under agent-test/runs/ is rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict() - d["output_root"] = "other/path/bench-01" - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestValidationError): - _load_tmp_manifest(path) - - def test_output_root_with_subpath_rejected(self): - """output_root with sub-path segments is rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict() - d["output_root"] = "agent-test/runs/sub/path" - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestValidationError): - _load_tmp_manifest(path) - - def test_testbed_pattern_rejected(self): - r"""testbed not matching ^\.\./[^/]+$ is rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict() - d["testbed"] = "../../double-escape" - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestValidationError): - _load_tmp_manifest(path) - - -class TestChecksumAndDigest(unittest.TestCase): - """Checksum and digest drift tests.""" - - def test_wrong_fixture_checksum_rejected(self): - """Wrong fixture checksum is rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict() - d["fixture"]["checksum"] = "sha256:" + "0" * 64 - p = tmp_dir / "manifest.json" - p.write_text(json.dumps(d, indent=2), encoding="utf-8") - with self.assertRaises(ManifestDigestError): - _load_tmp_manifest(p) - - def test_computed_checksum_matches(self): - """Computed checksum equals declared checksum for valid manifest.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict() - path = _write_tmp_manifest(tmp_dir, d) - m = _load_tmp_manifest(path) - expected = digest_workspace_inputs(m.fixture.assets) - self.assertEqual(m.fixture.checksum, expected) - - def test_manifest_digest_computed(self): - """Manifest digest is computed deterministically.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict() - path = _write_tmp_manifest(tmp_dir, d) - m = _load_tmp_manifest(path) - digest = digest_manifest_and_resolved_inputs(m) - self.assertTrue(digest.startswith("sha256:")) - self.assertEqual(len(digest), 7 + 64) - - def test_manifest_digest_deterministic(self): - """Same manifest produces the same digest on repeated calls.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict() - path = _write_tmp_manifest(tmp_dir, d) - m = _load_tmp_manifest(path) - d1 = digest_manifest_and_resolved_inputs(m) - d2 = digest_manifest_and_resolved_inputs(m) - self.assertEqual(d1, d2) - - -class TestSecretRedaction(unittest.TestCase): - """Secret and private-endpoint redaction in error messages.""" - - def test_secret_not_in_validation_error(self): - """Secret values do not appear in validation errors.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict() - d["pipeline_version"] = _SENTINEL_SECRET - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestValidationError) as caught: - _load_tmp_manifest(path) - self.assertNotIn(_SENTINEL_SECRET, str(caught.exception)) - - def test_secret_not_in_path_error(self): - """Secret values do not appear in path errors.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict() - d["fixture"]["prompt"] = f"nonexistent/{_SENTINEL_SECRET}/prompt.md" - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestPathError) as caught: - _load_tmp_manifest(path) - self.assertNotIn(_SENTINEL_SECRET, str(caught.exception)) - - def test_secret_not_in_digest_error(self): - """Secret values do not appear in digest errors.""" - with tempfile.TemporaryDirectory(dir=_REPO_ROOT) as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict() - d["output_root"] = f"agent-test/runs/{_SENTINEL_SECRET}" - d["fixture"]["checksum"] = "sha256:" + "0" * 64 - p = tmp_dir / "manifest.json" - p.write_text(json.dumps(d, indent=2), encoding="utf-8") - with self.assertRaises(ManifestDigestError) as caught: - _load_tmp_manifest(p) - self.assertNotIn(_SENTINEL_SECRET, str(caught.exception)) - - def test_prompt_content_not_in_any_error(self): - """Prompt content does not appear in any error.""" - with tempfile.TemporaryDirectory(dir=_REPO_ROOT) as tmp: - tmp_dir = Path(tmp) - prompt_path = tmp_dir / "prompt.md" - prompt_path.write_text(_SENTINEL_PROMPT, encoding="utf-8") - d = _make_minimal_manifest_dict() - d["fixture"]["prompt"] = prompt_path.relative_to(_REPO_ROOT).as_posix() - d["fixture"]["checksum"] = "sha256:" + "0" * 64 - manifest_path = tmp_dir / "manifest.json" - manifest_path.write_text(json.dumps(d, indent=2), encoding="utf-8") - with self.assertRaises(ManifestDigestError) as caught: - _load_tmp_manifest(manifest_path) - self.assertNotIn(_SENTINEL_PROMPT, str(caught.exception)) - - -class TestSchemaLoaderParity(unittest.TestCase): - """Schema and loader parity tests for types, bounds, enums, and grammar.""" - - def test_tracked_example_parity(self): - """Tracked example loads cleanly.""" - example_path = ( - _REPO_ROOT - / "scripts" - / "fixtures" - / "agent-comparison-benchmark-manifest.example.json" - ) - m = load_manifest(example_path) - self.assertEqual(m.pipeline_version, "2") - self.assertEqual(m.testbed, "../iop-s2") - self.assertEqual(m.rubric_version, RUBRIC_VERSION) - - def test_schema_and_loader_accept_exact_rubric_version_catalog(self): - """The schema and loader accept both immutable rubric versions only.""" - schema_path = ( - _REPO_ROOT - / "scripts" - / "fixtures" - / "agent-comparison-benchmark-manifest.schema.json" - ) - schema = json.loads(schema_path.read_text(encoding="utf-8")) - rubric_schema = schema["properties"]["rubric_version"] - self.assertEqual(tuple(rubric_schema["enum"]), RUBRIC_VERSIONS) - self.assertEqual( - RUBRIC_VERSIONS, - (RUBRIC_VERSION, ONE_SHOT_RUBRIC_VERSION), - ) - - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - for index, version in enumerate(RUBRIC_VERSIONS): - with self.subTest(version=version): - path = _write_tmp_manifest( - tmp_dir, - _make_minimal_manifest_dict(rubric_version=version), - f"rubric-{index}.json", - ) - self.assertEqual(_load_tmp_manifest(path).rubric_version, version) - - unknown = "one-shot-agent-comparison-v2" - self.assertNotIn(unknown, rubric_schema["enum"]) - path = _write_tmp_manifest( - tmp_dir, - _make_minimal_manifest_dict(rubric_version=unknown), - "rubric-unknown.json", - ) - with self.assertRaises(ManifestValidationError): - _load_tmp_manifest(path) - - def test_tracked_fixtures_separate_generic_contract_from_direct_preflight(self): - fixtures = _REPO_ROOT / "scripts" / "fixtures" - generic = load_manifest( - fixtures / "agent-comparison-benchmark-manifest.example.json" - ) - direct = load_manifest( - fixtures / "agent-comparison-benchmark-direct-preflight.example.json" - ) - - self.assertEqual( - [(cell.caller, cell.iop.route_kind) for cell in generic.matrix], - [ - ("agy", "execution_preset"), - ("claude", "execution_preset"), - ("codex", "execution_preset"), - ], - ) - self.assertEqual(len(direct.matrix), 5) - self.assertTrue(all(cell.iop.route_kind == "direct" for cell in direct.matrix)) - self.assertEqual( - [ - (cell.caller, cell.iop.request_model, cell.iop.requested_effort) - for cell in direct.matrix - ], - [ - ("agy", "gemini-3.6-flash", "high"), - ("claude", "gemini-3.6-flash", "high"), - ("claude", "gpt-5.6-luna", "xhigh"), - ("claude", "claude-sonnet-5", "max"), - ("codex", "gpt-5.6-luna", "xhigh"), - ], - ) - public_aliases = { - cell.iop.request_model for cell in generic.matrix + direct.matrix - } - self.assertEqual( - public_aliases, - {"claude-sonnet-5", "gemini-3.6-flash", "gpt-5.6-luna"}, - ) - - def test_booleans_rejected_in_numeric_fields(self): - """Booleans in numeric fields raise ManifestValidationError.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - for field_name in ("run_seconds", "idle_seconds", "quiet_seconds", "cleanup_grace_seconds"): - d = _make_minimal_manifest_dict() - d["timeout"][field_name] = True - p = _write_tmp_manifest(tmp_dir, d, f"{field_name}_bool.json") - with self.assertRaises(ManifestValidationError): - _load_tmp_manifest(p) - - d = _make_minimal_manifest_dict(repetitions=True) - p = _write_tmp_manifest(tmp_dir, d, "repetitions_bool.json") - with self.assertRaises(ManifestValidationError): - _load_tmp_manifest(p) - - d = _make_minimal_manifest_dict( - viewports=[{"id": "vp", "width": True, "height": 1080}] - ) - p = _write_tmp_manifest(tmp_dir, d, "width_bool.json") - with self.assertRaises(ManifestValidationError): - _load_tmp_manifest(p) - - def test_schema_and_loader_reject_same_invalid_execution_order_seeds(self): - """The tracked schema and loader enforce the same bounded seed token.""" - schema_path = ( - _REPO_ROOT - / "scripts" - / "fixtures" - / "agent-comparison-benchmark-manifest.schema.json" - ) - schema = json.loads(schema_path.read_text(encoding="utf-8")) - seed_schema = schema["properties"]["execution_order_seed"] - self.assertEqual(seed_schema["type"], "string") - self.assertEqual(seed_schema["pattern"], "^[a-z0-9][a-z0-9_-]{0,63}$") - - boundary_seed = "a" * 64 - self.assertIsNotNone(re.fullmatch(seed_schema["pattern"], boundary_seed)) - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - data = _make_minimal_manifest_dict( - execution_order_seed=boundary_seed - ) - path = _write_tmp_manifest(tmp_dir, data, "boundary-seed.json") - self.assertEqual( - _load_tmp_manifest(path).execution_order_seed, - boundary_seed, - ) - - invalid_seeds: tuple[object, ...] = ( - "", - "Uppercase", - "contains.dot", - "seed-a\n", - "a" * 65, - True, - None, - ) - pattern = re.compile(seed_schema["pattern"]) - for index, seed in enumerate(invalid_seeds): - schema_accepts = isinstance(seed, str) and pattern.fullmatch(seed) is not None - self.assertFalse(schema_accepts) - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - data = _make_minimal_manifest_dict(execution_order_seed=seed) - path = _write_tmp_manifest(tmp_dir, data, f"invalid-seed-{index}.json") - with self.assertRaises(ManifestValidationError): - _load_tmp_manifest(path) - - def test_preset_with_request_stage_rejected(self): - """Execution preset cell with extra request stage raises ManifestValidationError.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict( - matrix=[ - { - "id": "bad-preset", - "caller": "agy", - "iop": { - "request_model": "gemini-2.0-flash", - "requested_effort": "high", - "route_kind": "execution_preset", - "route_id": "agy-generic", - "expected_bindings": [ - {"stage": "request", "model": "gemini-2.0-flash"}, - {"stage": "selector", "model": "gemini-2.0-flash"}, - {"stage": "plan", "model": "gemini-2.0-flash"}, - {"stage": "work", "model": "gemini-2.0-flash"}, - {"stage": "review", "model": "gemini-2.0-flash"}, - ], - }, - } - ] - ) - p = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestValidationError): - _load_tmp_manifest(p) - - def test_testbed_must_be_exact(self): - """Testbed other than ../iop-s2 raises ManifestValidationError.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict(testbed="../another-repo") - p = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestValidationError): - _load_tmp_manifest(p) - - def test_dotted_tokens_accepted(self): - """Tokens with dots like v1.0 and gemini-2.0-flash load without error.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict(rubric_version="landing-quality-v1") - d["matrix"][0]["iop"]["request_model"] = "claude-sonnet-4-20250514" - p = _write_tmp_manifest(tmp_dir, d) - m = _load_tmp_manifest(p) - self.assertEqual(m.rubric_version, "landing-quality-v1") - self.assertEqual(m.matrix[0].iop.request_model, "claude-sonnet-4-20250514") - - def _evaluate_schema_execution_preset_bindings( - self, schema_path: Path, expected_bindings: list[dict] - ) -> bool: - """Evaluate candidate expected_bindings against declared execution_preset schema constraints.""" - raw_schema = json.loads(schema_path.read_text(encoding="utf-8")) - iop_cell_schema = raw_schema["$defs"]["iop_cell"] - preset_branch = None - for cond in iop_cell_schema.get("allOf", []): - if cond.get("if", {}).get("properties", {}).get("route_kind", {}).get("const") == "execution_preset": - preset_branch = cond.get("then", {}).get("properties", {}).get("expected_bindings", {}) - break - if not preset_branch: - return False - - min_items = preset_branch.get("minItems", 0) - max_items = preset_branch.get("maxItems", float("inf")) - if not (min_items <= len(expected_bindings) <= max_items): - return False - - allowed_enum = preset_branch.get("items", {}).get("properties", {}).get("stage", {}).get("enum", []) - for binding in expected_bindings: - if not isinstance(binding, dict) or "stage" not in binding or binding["stage"] not in allowed_enum: - return False - - all_of = preset_branch.get("allOf", []) - for sub in all_of: - contains = sub.get("contains", {}) - target_stage = contains.get("properties", {}).get("stage", {}).get("const") - min_c = sub.get("minContains", 0) - max_c = sub.get("maxContains", float("inf")) - matches = sum(1 for b in expected_bindings if isinstance(b, dict) and b.get("stage") == target_stage) - if not (min_c <= matches <= max_c): - return False - - return True - - def test_schema_and_loader_share_route_shape_corpus(self): - """Schema-backed evaluator and loader agree on all valid and malformed route shapes.""" - schema_path = ( - _REPO_ROOT - / "scripts" - / "fixtures" - / "agent-comparison-benchmark-manifest.schema.json" - ) - example_path = ( - _REPO_ROOT - / "scripts" - / "fixtures" - / "agent-comparison-benchmark-manifest.example.json" - ) - - # 1. Tracked example preset cells - example_raw = json.loads(example_path.read_text(encoding="utf-8")) - for cell in example_raw["matrix"]: - iop = cell["iop"] - if iop["route_kind"] == "execution_preset": - bindings = iop["expected_bindings"] - self.assertTrue(self._evaluate_schema_execution_preset_bindings(schema_path, bindings)) - - # 2. Valid four-stage and five-stage presets - valid_4 = [ - {"stage": "selector", "model": "m1"}, - {"stage": "plan", "model": "m1"}, - {"stage": "work", "model": "m1"}, - {"stage": "review", "model": "m1"}, - ] - valid_5 = [ - {"stage": "selector", "model": "m1"}, - {"stage": "plan", "model": "m1"}, - {"stage": "work", "model": "m1"}, - {"stage": "review", "model": "m1"}, - {"stage": "repair", "model": "m1"}, - ] - - for valid_b in (valid_4, valid_5): - self.assertTrue(self._evaluate_schema_execution_preset_bindings(schema_path, valid_b)) - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict( - matrix=[ - { - "id": "valid-preset", - "caller": "agy", - "iop": { - "request_model": "gemini-2.0-flash", - "requested_effort": "high", - "route_kind": "execution_preset", - "route_id": "r1", - "expected_bindings": valid_b, - }, - } - ] - ) - path = _write_tmp_manifest(tmp_dir, d) - m = _load_tmp_manifest(path) - self.assertEqual(len(m.matrix[0].iop.expected_bindings), len(valid_b)) - - # 3. Malformed preset stage variants - malformed_corpus = [ - # missing selector - [{"stage": "plan", "model": "m1"}, {"stage": "work", "model": "m1"}, {"stage": "review", "model": "m1"}, {"stage": "repair", "model": "m1"}], - # missing plan - [{"stage": "selector", "model": "m1"}, {"stage": "work", "model": "m1"}, {"stage": "review", "model": "m1"}, {"stage": "repair", "model": "m1"}], - # missing work - [{"stage": "selector", "model": "m1"}, {"stage": "plan", "model": "m1"}, {"stage": "review", "model": "m1"}, {"stage": "repair", "model": "m1"}], - # missing review - [{"stage": "selector", "model": "m1"}, {"stage": "plan", "model": "m1"}, {"stage": "work", "model": "m1"}, {"stage": "repair", "model": "m1"}], - # duplicate repair - [{"stage": "selector", "model": "m1"}, {"stage": "plan", "model": "m1"}, {"stage": "work", "model": "m1"}, {"stage": "review", "model": "m1"}, {"stage": "repair", "model": "m1"}, {"stage": "repair", "model": "m2"}], - # duplicate work - [{"stage": "selector", "model": "m1"}, {"stage": "plan", "model": "m1"}, {"stage": "work", "model": "m1"}, {"stage": "work", "model": "m2"}], - # extra request stage - [{"stage": "request", "model": "m1"}, {"stage": "selector", "model": "m1"}, {"stage": "plan", "model": "m1"}, {"stage": "work", "model": "m1"}, {"stage": "review", "model": "m1"}], - ] - - for bad_b in malformed_corpus: - self.assertFalse(self._evaluate_schema_execution_preset_bindings(schema_path, bad_b)) - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict( - matrix=[ - { - "id": "bad-preset", - "caller": "agy", - "iop": { - "request_model": "gemini-2.0-flash", - "requested_effort": "high", - "route_kind": "execution_preset", - "route_id": "r1", - "expected_bindings": bad_b, - }, - } - ] - ) - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestValidationError): - _load_tmp_manifest(path) - - -class TestCanonicalPaths(unittest.TestCase): - """Canonical path normalization, containment, and collision tests.""" - - def test_non_normal_asset_source_rejected(self): - """Asset source with ./ is rejected as non-canonical.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict() - d["fixture"]["assets"][0]["source"] = ( - "scripts/fixtures/agent-comparison-benchmark/./reference.txt" - ) - p = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestPathError): - _load_tmp_manifest(p) - - def test_non_normal_workspace_path_rejected(self): - """Asset workspace_path with ./ is rejected as non-canonical.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict() - d["fixture"]["assets"][0]["workspace_path"] = "workspace/./reference.txt" - p = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestPathError): - _load_tmp_manifest(p) - - def test_output_root_containment_and_normalization(self): - """output_root escaping agent-test/runs via .. or non-normal segment is rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict() - d["output_root"] = "agent-test/runs/../escape" - p = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises((ManifestValidationError, ManifestPathError)): - _load_tmp_manifest(p) - - -class TestCanonicalDigestAPI(unittest.TestCase): - """Public digest exposure, content immutability, and drift tests.""" - - def test_loaded_manifest_digest_property(self): - """Manifest object exposes digest property matching sha256: format.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - p = _write_tmp_manifest(tmp_dir, _make_minimal_manifest_dict()) - m = _load_tmp_manifest(p) - self.assertTrue(hasattr(m, "digest")) - self.assertTrue(m.digest.startswith("sha256:")) - self.assertEqual(len(m.digest), 7 + 64) - - def test_digest_helpers_match_loaded_manifest(self): - """digest helpers reproduce loaded checksum and digest.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - p = _write_tmp_manifest(tmp_dir, _make_minimal_manifest_dict()) - m = _load_tmp_manifest(p) - self.assertEqual(digest_workspace_inputs(m.fixture.assets), m.fixture.checksum) - self.assertEqual(digest_manifest_and_resolved_inputs(m), m.digest) - - def test_repr_omits_content_bytes(self): - """repr of Manifest, Fixture, AssetMapping does not include raw prompt/asset bytes.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - p = _write_tmp_manifest(tmp_dir, _make_minimal_manifest_dict()) - m = _load_tmp_manifest(p) - r_manifest = repr(m) - r_fixture = repr(m.fixture) - r_asset = repr(m.fixture.assets[0]) - self.assertNotIn(_SENTINEL_PROMPT, r_manifest) - self.assertNotIn(_SENTINEL_PROMPT, r_fixture) - self.assertNotIn(_SENTINEL_PROMPT, r_asset) - self.assertNotIn("content=", r_asset) - - def test_digest_signatures_exact(self): - """digest helpers reject legacy override arguments.""" - sig_ws = inspect.signature(digest_workspace_inputs) - self.assertEqual(list(sig_ws.parameters.keys()), ["assets"]) - sig_manifest = inspect.signature(digest_manifest_and_resolved_inputs) - self.assertEqual(list(sig_manifest.parameters.keys()), ["manifest"]) - - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - p = _write_tmp_manifest(tmp_dir, _make_minimal_manifest_dict()) - m = _load_tmp_manifest(p) - with self.assertRaises(TypeError): - digest_workspace_inputs(m.fixture.assets, read_content=True) # type: ignore - with self.assertRaises(TypeError): - digest_manifest_and_resolved_inputs(m, prompt_content=b"test") # type: ignore - - def test_asset_input_order_equivalence_and_canonicalization(self): - """Assets passed in different order produce identical sorted assets, checksum, and digest.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d_order1 = _make_minimal_manifest_dict() - d_order1["fixture"]["assets"] = [ - {"source": "scripts/fixtures/agent-comparison-benchmark/reference.txt", "workspace_path": "workspace/z_ref.txt"}, - {"source": "scripts/fixtures/agent-comparison-benchmark/prompt.md", "workspace_path": "workspace/a_prompt.md"}, - ] - - d_order2 = _make_minimal_manifest_dict() - d_order2["fixture"]["assets"] = [ - {"source": "scripts/fixtures/agent-comparison-benchmark/prompt.md", "workspace_path": "workspace/a_prompt.md"}, - {"source": "scripts/fixtures/agent-comparison-benchmark/reference.txt", "workspace_path": "workspace/z_ref.txt"}, - ] - - p1 = _write_tmp_manifest(tmp_dir, d_order1, "order1.json") - p2 = _write_tmp_manifest(tmp_dir, d_order2, "order2.json") - m1 = _load_tmp_manifest(p1) - m2 = _load_tmp_manifest(p2) - - self.assertEqual(m1.fixture.assets[0].workspace_path, "workspace/a_prompt.md") - self.assertEqual(m1.fixture.assets[1].workspace_path, "workspace/z_ref.txt") - self.assertEqual(m1.fixture.assets, m2.fixture.assets) - self.assertEqual(m1.fixture.checksum, m2.fixture.checksum) - self.assertEqual(m1.digest, m2.digest) - - def test_input_drift_changes_digest(self): - """Altering manifest, prompt content, asset path, or asset content changes m.digest.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d1 = _make_minimal_manifest_dict() - p1 = _write_tmp_manifest(tmp_dir, d1, "m1.json") - m1 = _load_tmp_manifest(p1) - - # A valid evaluator binding drift remains loadable and changes the - # digest even though the rubric revision itself is closed. - d2 = _make_minimal_manifest_dict() - d2["evaluator"]["iop"]["route_id"] = "gpt-5.6-luna-alt" - p2 = _write_tmp_manifest(tmp_dir, d2, "m2.json") - m2 = _load_tmp_manifest(p2) - self.assertNotEqual(m1.digest, m2.digest) - - # Prompt content drift - prompt_changed = replace( - m1, - fixture=replace(m1.fixture, prompt_content=m1.fixture.prompt_content + b" changed"), - ) - self.assertNotEqual(digest_manifest_and_resolved_inputs(prompt_changed), m1.digest) - - # Asset source drift - asset0 = m1.fixture.assets[0] - source_changed = replace( - m1, - fixture=replace( - m1.fixture, - assets=(replace(asset0, source="changed/source.txt"),) + m1.fixture.assets[1:], - ), - ) - self.assertNotEqual(digest_manifest_and_resolved_inputs(source_changed), m1.digest) - - # Asset workspace_path drift - d3 = _make_minimal_manifest_dict() - d3["fixture"]["assets"][0]["workspace_path"] = "workspace/other_ref.txt" - p3 = _write_tmp_manifest(tmp_dir, d3, "m3.json") - m3 = _load_tmp_manifest(p3) - self.assertNotEqual(m1.fixture.checksum, m3.fixture.checksum) - self.assertNotEqual(m1.digest, m3.digest) - - dest_changed = replace( - m1, - fixture=replace( - m1.fixture, - assets=(replace(asset0, workspace_path="workspace/other_ref.txt"),) + m1.fixture.assets[1:], - ), - ) - self.assertNotEqual(digest_manifest_and_resolved_inputs(dest_changed), m1.digest) - - # Asset content drift - content_changed = replace( - m1, - fixture=replace( - m1.fixture, - assets=(replace(asset0, content=b"changed asset bytes"),) + m1.fixture.assets[1:], - ), - ) - self.assertNotEqual(digest_manifest_and_resolved_inputs(content_changed), m1.digest) - - asset_a = AssetMapping(source="src.txt", workspace_path="w.txt", content=b"content A") - asset_b = AssetMapping(source="src.txt", workspace_path="w.txt", content=b"content B") - self.assertNotEqual(digest_workspace_inputs([asset_a]), digest_workspace_inputs([asset_b])) - - -class TestCLI(unittest.TestCase): - """Public CLI tests.""" - - def _run_cli(self, *args: str) -> subprocess.CompletedProcess: - cli = str(_REPO_ROOT / "scripts" / "agent_comparison_benchmark.py") - return subprocess.run( - [sys.executable, cli, *args], - capture_output=True, - text=True, - cwd=str(_REPO_ROOT), - ) - - def test_cli_validate_valid_manifest(self): - """Valid manifest exits 0 with sanitized single success line.""" - example = str( - _REPO_ROOT - / "scripts" - / "fixtures" - / "agent-comparison-benchmark-manifest.example.json" - ) - if not Path(example).exists(): - self.skipTest("example manifest not found") - result = self._run_cli("validate", "--manifest", example) - self.assertEqual(result.returncode, 0) - stdout_lines = [line for line in result.stdout.splitlines() if line.strip()] - stderr_lines = [line for line in result.stderr.splitlines() if line.strip()] - self.assertEqual(len(stdout_lines), 1) - self.assertEqual(stdout_lines[0], "ok: manifest is valid") - self.assertEqual(len(stderr_lines), 0) - - def test_cli_validate_missing_file(self): - """Missing manifest file exits 69 with single sanitized line.""" - result = self._run_cli("validate", "--manifest", "/nonexistent/path.json") - self.assertEqual(result.returncode, 69) - stdout_lines = [line for line in result.stdout.splitlines() if line.strip()] - stderr_lines = [line for line in result.stderr.splitlines() if line.strip()] - self.assertEqual(len(stdout_lines), 0) - self.assertEqual(len(stderr_lines), 1) - self.assertIn("error:", stderr_lines[0]) - self.assertNotIn("Traceback", result.stderr) - - def test_cli_validate_malformed_json(self): - """Malformed JSON exits 69 with single sanitized line.""" - with tempfile.NamedTemporaryFile( - mode="w", suffix=".json", delete=False - ) as f: - f.write("{invalid json") - tmp_path = f.name - try: - result = self._run_cli("validate", "--manifest", tmp_path) - self.assertEqual(result.returncode, 69) - stdout_lines = [line for line in result.stdout.splitlines() if line.strip()] - stderr_lines = [line for line in result.stderr.splitlines() if line.strip()] - self.assertEqual(len(stdout_lines), 0) - self.assertEqual(len(stderr_lines), 1) - self.assertIn("error:", stderr_lines[0]) - self.assertNotIn("Traceback", result.stderr) - finally: - os.unlink(tmp_path) - - def test_cli_validate_secret_manifest(self): - """Manifest with secret values exits 69 without echoing secrets.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict() - d["pipeline_version"] = _SENTINEL_SECRET - path = _write_tmp_manifest(tmp_dir, d) - result = self._run_cli("validate", "--manifest", str(path)) - self.assertEqual(result.returncode, 69) - self.assertNotIn(_SENTINEL_SECRET, result.stdout) - self.assertNotIn(_SENTINEL_SECRET, result.stderr) - self.assertNotIn("Traceback", result.stderr) - - def test_cli_usage_error(self): - """Missing subcommand exits 64 with single sanitized line.""" - result = self._run_cli() - self.assertEqual(result.returncode, 64) - stdout_lines = [line for line in result.stdout.splitlines() if line.strip()] - stderr_lines = [line for line in result.stderr.splitlines() if line.strip()] - self.assertEqual(len(stdout_lines), 0) - self.assertEqual(len(stderr_lines), 1) - self.assertEqual(stderr_lines[0], "error: invalid usage") - - def test_cli_validate_no_manifest_flag(self): - """Missing --manifest flag exits 64 with single sanitized line.""" - result = self._run_cli("validate") - self.assertEqual(result.returncode, 64) - stdout_lines = [line for line in result.stdout.splitlines() if line.strip()] - stderr_lines = [line for line in result.stderr.splitlines() if line.strip()] - self.assertEqual(len(stdout_lines), 0) - self.assertEqual(len(stderr_lines), 1) - self.assertEqual(stderr_lines[0], "error: invalid usage") - - def test_cli_validate_invalid_utf8(self): - """Invalid UTF-8 manifest file exits 69 with single sanitized error line.""" - with tempfile.NamedTemporaryFile(mode="wb", suffix=".json", delete=False) as f: - f.write(b"\x80\xff\xfe") - tmp_path = f.name - try: - result = self._run_cli("validate", "--manifest", tmp_path) - self.assertEqual(result.returncode, 69) - stdout_lines = [line for line in result.stdout.splitlines() if line.strip()] - stderr_lines = [line for line in result.stderr.splitlines() if line.strip()] - self.assertEqual(len(stdout_lines), 0) - self.assertEqual(len(stderr_lines), 1) - self.assertEqual(stderr_lines[0], "error: invalid JSON format") - self.assertNotIn("Traceback", result.stderr) - finally: - os.unlink(tmp_path) - - def test_cli_validate_checksum_mismatch(self): - """Manifest with checksum mismatch exits 69 with single sanitized error line.""" - with tempfile.TemporaryDirectory(dir=_REPO_ROOT) as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict() - d["fixture"]["checksum"] = "sha256:" + "0" * 64 - p = tmp_dir / "manifest.json" - p.write_text(json.dumps(d, indent=2), encoding="utf-8") - result = self._run_cli("validate", "--manifest", str(p)) - self.assertEqual(result.returncode, 69) - stdout_lines = [line for line in result.stdout.splitlines() if line.strip()] - stderr_lines = [line for line in result.stderr.splitlines() if line.strip()] - self.assertEqual(len(stdout_lines), 0) - self.assertEqual(len(stderr_lines), 1) - self.assertEqual(stderr_lines[0], "error: fixture.checksum mismatch") - self.assertNotIn("Traceback", result.stderr) - - def test_cli_validate_secret_missing_path(self): - """Secret in missing path exits 69 with single sanitized error line without echoing secret.""" - with tempfile.TemporaryDirectory(dir=_REPO_ROOT) as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict() - d["fixture"]["prompt"] = f"nonexistent/{_SENTINEL_SECRET}/prompt.md" - p = tmp_dir / "manifest.json" - p.write_text(json.dumps(d, indent=2), encoding="utf-8") - result = self._run_cli("validate", "--manifest", str(p)) - self.assertEqual(result.returncode, 69) - stdout_lines = [line for line in result.stdout.splitlines() if line.strip()] - stderr_lines = [line for line in result.stderr.splitlines() if line.strip()] - self.assertEqual(stdout_lines, []) - self.assertEqual(len(stderr_lines), 1) - self.assertNotIn(_SENTINEL_SECRET, result.stdout) - self.assertNotIn(_SENTINEL_SECRET, result.stderr) - self.assertNotIn("Traceback", result.stderr) - - def test_cli_validate_secret_unknown_argument(self): - """Secret in unknown CLI flag exits 64 without echoing secret.""" - example = str( - _REPO_ROOT - / "scripts" - / "fixtures" - / "agent-comparison-benchmark-manifest.example.json" - ) - result = self._run_cli("validate", "--manifest", example, f"--secret={_SENTINEL_SECRET}") - self.assertEqual(result.returncode, 64) - stdout_lines = [line for line in result.stdout.splitlines() if line.strip()] - stderr_lines = [line for line in result.stderr.splitlines() if line.strip()] - self.assertEqual(len(stdout_lines), 0) - self.assertEqual(len(stderr_lines), 1) - self.assertEqual(stderr_lines[0], "error: invalid usage") - self.assertNotIn(_SENTINEL_SECRET, result.stdout) - self.assertNotIn(_SENTINEL_SECRET, result.stderr) - self.assertNotIn("Traceback", result.stderr) - - -class TestValidateManifestBytes(unittest.TestCase): - """validate_manifest_bytes tests.""" - - def test_validate_bytes_valid(self): - """Valid bytes validate without disk write.""" - d = _make_minimal_manifest_dict() - # Compute correct checksum for the fixture - d["fixture"]["checksum"] = _compute_fixture_checksum(_REPO_ROOT, d["fixture"]) - data = json.dumps(d).encode("utf-8") - m = validate_manifest_bytes(data, repo_root=_REPO_ROOT) - self.assertEqual(m.pipeline_version, "2") - - def test_validate_bytes_invalid(self): - """Invalid bytes raise error.""" - data = b"{invalid" - with self.assertRaises(ManifestValidationError): - validate_manifest_bytes(data) - - -class TestFrozenReturnTypes(unittest.TestCase): - """Return type immutability tests.""" - - def test_manifest_is_frozen(self): - """Manifest is a frozen dataclass.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - path = _write_tmp_manifest(tmp_dir, _make_minimal_manifest_dict()) - m = _load_tmp_manifest(path) - self.assertTrue(hasattr(m, "__dataclass_fields__")) - with self.assertRaises(AttributeError): - m.repetitions = 99 - - def test_timeout_is_frozen(self): - """Timeout is frozen.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - path = _write_tmp_manifest(tmp_dir, _make_minimal_manifest_dict()) - m = _load_tmp_manifest(path) - with self.assertRaises(AttributeError): - m.timeout.run_seconds = 999 - - def test_viewport_is_frozen(self): - """Viewport is frozen.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - path = _write_tmp_manifest(tmp_dir, _make_minimal_manifest_dict()) - m = _load_tmp_manifest(path) - with self.assertRaises(AttributeError): - m.viewports[0].width = 9999 - - def test_cell_is_frozen(self): - """MatrixCell is frozen.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - path = _write_tmp_manifest(tmp_dir, _make_minimal_manifest_dict()) - m = _load_tmp_manifest(path) - with self.assertRaises(AttributeError): - m.matrix[0].id = "changed" - - def test_tuple_fields_are_tuples(self): - """tuple fields are actual tuples, not lists.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - path = _write_tmp_manifest(tmp_dir, _make_minimal_manifest_dict()) - m = _load_tmp_manifest(path) - self.assertIsInstance(m.viewports, tuple) - self.assertIsInstance(m.matrix, tuple) - self.assertIsInstance(m.fixture.assets, tuple) - self.assertIsInstance(m.matrix[0].iop.expected_bindings, tuple) - - -class TestEdgeCases(unittest.TestCase): - """Additional edge cases.""" - - def test_non_object_top_level_rejected(self): - """Top-level JSON array is rejected.""" - with tempfile.NamedTemporaryFile( - mode="w", suffix=".json", delete=False - ) as f: - f.write("[]") - tmp_path = f.name - try: - with self.assertRaises(ManifestValidationError): - _load_tmp_manifest(tmp_path) - finally: - os.unlink(tmp_path) - - def test_missing_required_field_rejected(self): - """Missing required top-level field is rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict() - del d["timeout"] - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestValidationError): - _load_tmp_manifest(path) - - def test_fixture_missing_required_field_rejected(self): - """Missing fixture.version is rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict() - del d["fixture"]["version"] - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestValidationError): - _load_tmp_manifest(path) - - def test_fixture_missing_prompt_file_rejected(self): - """Prompt file that does not exist is rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict() - d["fixture"]["prompt"] = "nonexistent_prompt.md" - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestPathError): - _load_tmp_manifest(path) - - def test_fixture_missing_asset_file_rejected(self): - """Asset source file that does not exist is rejected.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict() - d["fixture"]["assets"] = [ - { - "source": "nonexistent_source.txt", - "workspace_path": "workspace/dst.txt", - } - ] - path = _write_tmp_manifest(tmp_dir, d) - with self.assertRaises(ManifestPathError): - _load_tmp_manifest(path) - - def test_multiple_assets_loaded(self): - """Manifest with multiple assets loads correctly.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict() - d["fixture"]["assets"] = [ - { - "source": "scripts/fixtures/agent-comparison-benchmark/prompt.md", - "workspace_path": "workspace/prompt.md", - }, - { - "source": "scripts/fixtures/agent-comparison-benchmark/reference.txt", - "workspace_path": "workspace/reference.txt", - }, - ] - path = _write_tmp_manifest(tmp_dir, d) - m = _load_tmp_manifest(path) - self.assertEqual(len(m.fixture.assets), 2) - - def test_file_not_found(self): - """Non-existent manifest file raises ManifestValidationError.""" - with self.assertRaises(ManifestValidationError): - load_manifest("/nonexistent/manifest.json") - - def test_caller_request_vs_evidence_separation(self): - """request_model/requested_effort are separate from route/binding evidence.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_dir = Path(tmp) - d = _make_minimal_manifest_dict( - matrix=[ - { - "id": "separation-test", - "caller": "claude", - "iop": { - "request_model": "claude-sonnet-4-20250514", - "requested_effort": "high", - "route_kind": "direct", - "route_id": "claude-direct", - "expected_bindings": [ - { - "stage": "request", - "model": "claude-sonnet-4-20250514", - "effort": "high", - } - ], - }, - } - ] - ) - path = _write_tmp_manifest(tmp_dir, d) - m = _load_tmp_manifest(path) - cell = m.matrix[0] - # request_model and requested_effort are on iop, not in bindings - self.assertEqual(cell.iop.request_model, "claude-sonnet-4-20250514") - self.assertEqual(cell.iop.requested_effort, "high") - # route_kind/route_id are preflight evidence, not adapter inputs - self.assertEqual(cell.iop.route_kind, "direct") - self.assertEqual(cell.iop.route_id, "claude-direct") - # bindings contain the expected evidence - binding = cell.iop.expected_bindings[0] - self.assertEqual(binding.stage, "request") - self.assertEqual(binding.model, "claude-sonnet-4-20250514") - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/agent_benchmark/measurement.py b/scripts/agent_benchmark/measurement.py deleted file mode 100644 index c79a8ef9..00000000 --- a/scripts/agent_benchmark/measurement.py +++ /dev/null @@ -1,926 +0,0 @@ -"""Source-aware timing and usage evidence for exactly one benchmark attempt. - -This module owns three things and nothing else: the closed measurement schema, -a bounded workspace write observer, and the no-clobber sidecar publisher and -strict loader for ``attempt-measurement.json``. - -Every required value is either an ``observed`` value carrying its unit, clock -and source, or an explicit ``unavailable`` value carrying the reason it could -not be observed. Nothing here decomposes, sums, subtracts or reconstructs a -value the caller or the harness did not report: a missing provider total stays -unavailable, overlapping intervals stay overlapping, and a filesystem -modification time is never presented as proof of the first write. -""" - -from __future__ import annotations - -import hashlib -import json -import os -import stat -import threading -import time -from collections import deque -from dataclasses import dataclass -from pathlib import Path -from typing import Any, Callable, Mapping, Optional - -from scripts.agent_benchmark.lifecycle import ( - CLOCK_FILESYSTEM_MTIME, - CLOCK_HARNESS_MONOTONIC, - CLOCK_NONE, - EVENT_FIRST_OUTPUT, - EVENT_SUBMITTED, - METRIC_CLOCKS, - METRIC_NAMES, - METRIC_PREFIX, - METRIC_SOURCES, - METRIC_UNITS, - SOURCE_HARNESS, - SOURCE_WORKSPACE_POLL, - UNIT_NANOSECONDS, - InvocationResult, - HarnessOutcome, - LifecycleMetricError, - LifecycleValidationError, - ParsedMetric, - ProcessOutcome, - ProductOutcome, - metric_record, - publish_bytes_no_replace, - validate_metric, -) - -MEASUREMENT_FILENAME = "attempt-measurement.json" -MEASUREMENT_VERSION = 2 -MEASUREMENT_RECORD = "attempt_measurement" - -STATUS_OBSERVED = "observed" -STATUS_UNAVAILABLE = "unavailable" - -REASON_NOT_REPORTED = "not_reported" -REASON_NOT_OBSERVED = "not_observed" -REASON_AMBIGUOUS_TOTAL = "ambiguous_total" -REASON_OBSERVER_UNAVAILABLE = "observer_unavailable" -UNAVAILABLE_REASONS = ( - REASON_NOT_REPORTED, REASON_NOT_OBSERVED, REASON_AMBIGUOUS_TOTAL, - REASON_OBSERVER_UNAVAILABLE, -) - -TIMELINE_NAMES = ( - "submitted_at", "first_output_at", "first_write_observed_at", - "first_write_mtime", "total_duration", -) -OBSERVATION_UNITS = tuple(sorted(set(METRIC_UNITS.values()))) - -DURATION_NS_PER_SECOND = 10 ** 9 -# The sampling cadence matches the lifecycle controller's own poll interval and -# is published as the observation precision. Each sample is bounded by the -# entry and depth caps below, and sampling ends at the first observed write. -OBSERVER_INTERVAL_SECONDS = 0.02 -OBSERVER_JOIN_SECONDS = 10.0 -OBSERVER_MAX_ENTRIES = 4096 -OBSERVER_MAX_DEPTH = 16 -MAX_OBSERVATION_RECORDS = 1000 -DIGEST_PREFIX = "sha256:" -_PATH_DIGEST_DOMAIN = b"iop-benchmark-workspace-path-v1\0" - - -class MeasurementError(Exception): - """Raised when measurement evidence cannot be produced or trusted.""" - - -@dataclass(frozen=True) -class Observation: - """One required value that is either observed or explicitly unavailable.""" - - status: str - value: Optional[int] - unit: str - clock: str - source: str - reason: str - - -@dataclass(frozen=True) -class WorkspaceWriteObservation: - """The bounded observer's report about the first observed workspace write.""" - - observed: bool - monotonic_ns: Optional[int] - mtime_ns: Optional[int] - path_digest: str - precision_ns: int - samples: int - reason: str = "" - - -@dataclass(frozen=True) -class WorkspaceScan: - """One closed workspace snapshot; incomplete snapshots are never compared.""" - - files: dict[str, tuple[int, int, int]] - status: str - - @property - def complete(self) -> bool: - return self.status == "complete" - - -@dataclass(frozen=True) -class AttemptMeasurement: - """One immutable measurement record bound to exactly one attempt.""" - - run_id: str - cell_id: str - repetition: int - attempt: int - caller: str - spec_digest: str - product: ProductOutcome - harness: HarnessOutcome - process: ProcessOutcome - timeline: dict[str, Observation] - usage: dict[str, Observation] - observer: WorkspaceWriteObservation - observations: tuple[ParsedMetric, ...] - - -def observed(value: int, unit: str, clock: str, source: str) -> Observation: - """Build one observed value with its exact unit, clock and source.""" - if isinstance(value, bool) or not isinstance(value, int) or value < 0: - raise MeasurementError("observed value must be a non-negative integer") - return Observation(STATUS_OBSERVED, value, unit, clock, source, "") - - -def unavailable(reason: str, source: str) -> Observation: - """Build one explicitly unavailable value; never substitute a zero.""" - if reason not in UNAVAILABLE_REASONS: - raise MeasurementError("unavailable reason is not a closed value") - return Observation(STATUS_UNAVAILABLE, None, "", "", source, reason) - - -def observation_record(observation: Observation) -> dict[str, Any]: - """Return the canonical projection of one observed/unavailable value.""" - if observation.status == STATUS_OBSERVED: - return { - "status": STATUS_OBSERVED, - "value": observation.value, - "unit": observation.unit, - "clock": observation.clock, - "source": observation.source, - } - if observation.status != STATUS_UNAVAILABLE: - raise MeasurementError("observation status is not a closed value") - return { - "status": STATUS_UNAVAILABLE, - "value": None, - "reason": observation.reason, - "source": observation.source, - } - - -def _observation_from_record(raw: Any) -> Observation: - if not isinstance(raw, dict): - raise MeasurementError("observation is invalid") - status = raw.get("status") - if status == STATUS_OBSERVED: - if set(raw) != {"status", "value", "unit", "clock", "source"}: - raise MeasurementError("observed value schema is invalid") - value, unit, clock = raw["value"], raw["unit"], raw["clock"] - if ( - isinstance(value, bool) or not isinstance(value, int) or value < 0 - or unit not in OBSERVATION_UNITS - or clock not in METRIC_CLOCKS or raw["source"] not in METRIC_SOURCES - ): - raise MeasurementError("observed value is invalid") - return Observation(STATUS_OBSERVED, value, unit, clock, raw["source"], "") - if status == STATUS_UNAVAILABLE: - if set(raw) != {"status", "value", "reason", "source"}: - raise MeasurementError("unavailable value schema is invalid") - if ( - raw["value"] is not None - or raw["reason"] not in UNAVAILABLE_REASONS - or raw["source"] not in METRIC_SOURCES - ): - raise MeasurementError("unavailable value is invalid") - return Observation(STATUS_UNAVAILABLE, None, "", "", raw["source"], raw["reason"]) - raise MeasurementError("observation status is not a closed value") - - -def _is_digest(value: Any) -> bool: - """True for one exact lowercase sha256 identity string.""" - if not isinstance(value, str) or not value.startswith(DIGEST_PREFIX): - return False - body = value[len(DIGEST_PREFIX):] - return len(body) == 64 and all(char in "0123456789abcdef" for char in body) - - -def path_digest(relative_path: str) -> str: - """Digest one workspace-relative path so no caller-chosen name persists.""" - return DIGEST_PREFIX + hashlib.sha256( - _PATH_DIGEST_DOMAIN + os.fsencode(relative_path) - ).hexdigest() - - -# --------------------------------------------------------------------------- -# Bounded workspace write observer -# --------------------------------------------------------------------------- - -def _scan_workspace(root: Path) -> WorkspaceScan: - """Snapshot contained regular files without following any link. - - Every directory entry consumes one shared budget, including directories, - links and non-regular files. A cap, depth or I/O boundary returns an - incomplete result rather than a partial snapshot that could be compared. - """ - found: dict[str, tuple[int, int, int]] = {} - pending: deque[tuple[Path, int]] = deque([(root, 0)]) - consumed = 0 - while pending: - current, depth = pending.popleft() - try: - with os.scandir(current) as scan: - for entry in scan: - # Conservatively report exhaustion as soon as the bounded - # budget has been consumed. This avoids reading one more - # entry merely to distinguish an exactly-full directory. - if consumed >= OBSERVER_MAX_ENTRIES: - return WorkspaceScan(found, "exhausted") - consumed += 1 - try: - if entry.is_symlink(): - continue - if entry.is_dir(follow_symlinks=False): - if depth >= OBSERVER_MAX_DEPTH: - return WorkspaceScan(found, "exhausted") - pending.append((Path(entry.path), depth + 1)) - continue - info = entry.stat(follow_symlinks=False) - if not stat.S_ISREG(info.st_mode): - continue - relative = os.path.relpath(entry.path, root) - except (OSError, ValueError): - return WorkspaceScan(found, "unavailable") - found[relative] = (info.st_mtime_ns, info.st_size, info.st_ino) - except OSError: - return WorkspaceScan(found, "unavailable") - return WorkspaceScan(found, "complete") - - -class WorkspaceWriteObserver: - """Sample one workspace at a bounded interval and keep the first write seen. - - The observer starts before the caller is invoked so that its baseline is - older than any caller write. It reports its own harness observation time, - the filesystem modification time it read, its source and its polling - precision. It never claims that a terminal snapshot proves the first write. - """ - - def __init__( - self, - root: str | Path, - *, - interval_seconds: float = OBSERVER_INTERVAL_SECONDS, - clock: Callable[[], int] = time.monotonic_ns, - ) -> None: - self.root = Path(root) - if interval_seconds <= 0: - raise MeasurementError("observer interval must be positive") - self.interval_seconds = float(interval_seconds) - self._clock = clock - self._stop = threading.Event() - self._thread: Optional[threading.Thread] = None - self._baseline: dict[str, tuple[int, int, int]] = {} - self._baseline_complete = False - self._samples = 0 - self._first: Optional[tuple[int, int, str]] = None - self._unavailable_reason = "" - self._started = False - self._stopped = False - self._final_observation: Optional[WorkspaceWriteObservation] = None - - @property - def precision_ns(self) -> int: - return int(self.interval_seconds * DURATION_NS_PER_SECOND) - - @property - def stopped(self) -> bool: - """True once the sampling thread has been joined and is gone.""" - return self._stopped - - def start(self) -> None: - """Take the immutable baseline, then start the sampling thread.""" - if self._started: - raise MeasurementError("observer has already started") - if not self.root.is_dir() or self.root.is_symlink(): - raise MeasurementError("observer root must be an existing directory") - self._started = True - baseline = _scan_workspace(self.root) - if not baseline.complete: - self._unavailable_reason = REASON_OBSERVER_UNAVAILABLE - return - self._baseline = baseline.files - self._baseline_complete = True - self._thread = threading.Thread(target=self._sample_until_stopped, daemon=True) - self._thread.start() - - def _sample_until_stopped(self) -> None: - while not self._stop.is_set(): - if self._sample_once(): - return - self._stop.wait(self.interval_seconds) - - def _sample_once(self) -> bool: - """Return True once the first created or changed file has been seen.""" - current = _scan_workspace(self.root) - self._samples += 1 - if not current.complete: - self._unavailable_reason = REASON_OBSERVER_UNAVAILABLE - return True - changed = sorted( - (relative, info) for relative, info in current.files.items() - if self._baseline.get(relative) != info - ) - if not changed: - return False - relative, info = changed[0] - # A detection instant is sampled only after the complete snapshot has - # found the change; it never labels scan work as observation time. - now = self._clock() - self._first = (now, info[0], relative) - return True - - def stop(self) -> WorkspaceWriteObservation: - """Stop and join the sampling thread, then freeze the observation. - - Cleanup never raises, so it is safe on every terminal path; a thread - that refuses to leave is reported through :attr:`stopped` instead. A - fully stopped result is frozen and returned unchanged by every later - call, so a file written after shutdown can never become the - invocation's first write. A join that times out is not frozen so a - later call can retry cleanup once the sampler has exited. - """ - if self._final_observation is not None: - return self._final_observation - self._stop.set() - thread = self._thread - if thread is None: - self._stopped = self._started - else: - thread.join(OBSERVER_JOIN_SECONDS) - self._stopped = not thread.is_alive() - if self._stopped: - self._thread = None - if ( - self._stopped - and self._baseline_complete - and self._first is None - and not self._unavailable_reason - ): - # The joined sampler cannot race this final bounded scan. It closes - # the interval between its final poll and caller cleanup. - self._sample_once() - if self._first is None: - observation = WorkspaceWriteObservation( - False, None, None, "", self.precision_ns, self._samples, - self._unavailable_reason or REASON_NOT_OBSERVED, - ) - else: - monotonic_ns, mtime_ns, relative = self._first - observation = WorkspaceWriteObservation( - True, monotonic_ns, mtime_ns, path_digest(relative), - self.precision_ns, self._samples, - ) - if self._stopped: - self._final_observation = observation - return observation - - -# --------------------------------------------------------------------------- -# Measurement construction -# --------------------------------------------------------------------------- - -def _event_instant(result: InvocationResult, kind: str) -> Observation: - for event in result.events: - if event.kind == kind: - return observed( - event.monotonic_ns, UNIT_NANOSECONDS, - CLOCK_HARNESS_MONOTONIC, SOURCE_HARNESS, - ) - return unavailable(REASON_NOT_OBSERVED, SOURCE_HARNESS) - - -def _write_instant(value: Any, clock: str, observation: WorkspaceWriteObservation) -> Observation: - """Report one observer value, or say plainly that it was not observed.""" - if (not observation.observed or isinstance(value, bool) - or not isinstance(value, int) or value < 0): - return unavailable( - observation.reason or REASON_NOT_OBSERVED, SOURCE_WORKSPACE_POLL - ) - return observed(value, UNIT_NANOSECONDS, clock, SOURCE_WORKSPACE_POLL) - - -def _timeline( - result: InvocationResult, observation: WorkspaceWriteObservation -) -> dict[str, Observation]: - """Build the timeline without comparing values across clock domains.""" - duration = result.duration_ns - return { - "submitted_at": _event_instant(result, EVENT_SUBMITTED), - "first_output_at": _event_instant(result, EVENT_FIRST_OUTPUT), - "first_write_observed_at": _write_instant( - observation.monotonic_ns, CLOCK_HARNESS_MONOTONIC, observation - ), - # The filesystem clock is reported beside the harness clock and never - # subtracted from it; a caller of this record cannot mix the two. - "first_write_mtime": _write_instant( - observation.mtime_ns, CLOCK_FILESYSTEM_MTIME, observation - ), - "total_duration": ( - observed(duration, UNIT_NANOSECONDS, CLOCK_HARNESS_MONOTONIC, SOURCE_HARNESS) - if isinstance(duration, int) and not isinstance(duration, bool) and duration >= 0 - else unavailable(REASON_NOT_OBSERVED, SOURCE_HARNESS) - ), - } - - -def _usage(metrics: tuple[ParsedMetric, ...]) -> dict[str, Observation]: - """Project only whole caller totals; never sum or split labelled intervals. - - A stage or call label marks one part of a larger report, so only unstaged, - uncalled observations can be a total. Two totals with identical labels are - a contradiction and fail closed; two totals bound to different models are - ambiguous and stay unavailable rather than being merged. - """ - totals: dict[str, list[ParsedMetric]] = {} - for metric in metrics: - if metric.stage or metric.call_id: - continue - candidates = totals.setdefault(metric.name, []) - if any(item.model == metric.model for item in candidates): - raise MeasurementError("caller reported a duplicate total") - candidates.append(metric) - usage: dict[str, Observation] = {} - for name in METRIC_NAMES: - candidates = totals.get(name, []) - if len(candidates) == 1: - metric = candidates[0] - usage[name] = observed(metric.value, metric.unit, metric.clock, metric.source) - elif candidates: - usage[name] = unavailable(REASON_AMBIGUOUS_TOTAL, candidates[0].source) - else: - usage[name] = unavailable(REASON_NOT_REPORTED, SOURCE_HARNESS) - return usage - - -def build_measurement( - *, - run_id: str, - cell_id: str, - repetition: int, - attempt: int, - caller: str, - result: InvocationResult, - observation: WorkspaceWriteObservation, -) -> AttemptMeasurement: - """Join lifecycle events, caller observations and the observer into one record.""" - if not isinstance(result, InvocationResult): - raise MeasurementError("invocation result is invalid") - if not isinstance(observation, WorkspaceWriteObservation): - raise MeasurementError("workspace observation is invalid") - if any(not isinstance(text, str) or not text for text in (run_id, cell_id, caller)): - raise MeasurementError("measurement identity is invalid") - if any( - isinstance(number, bool) or not isinstance(number, int) or number < 1 - for number in (repetition, attempt) - ): - raise MeasurementError("measurement identity is invalid") - if not _is_digest(result.spec_digest): - raise MeasurementError("measurement invocation identity is invalid") - metrics = tuple(result.metrics) - if len(metrics) > MAX_OBSERVATION_RECORDS: - raise MeasurementError("observation count exceeds the bounded record") - try: - for metric in metrics: - validate_metric(metric) - except LifecycleMetricError as exc: - raise MeasurementError("caller observation is invalid") from exc - published = [ - event.kind for event in result.events - if event.kind.startswith(METRIC_PREFIX) - and event.kind[len(METRIC_PREFIX):] in METRIC_UNITS - ] - if published != [METRIC_PREFIX + metric.name for metric in metrics]: - raise MeasurementError("observation set does not match published events") - return AttemptMeasurement( - run_id=run_id, - cell_id=cell_id, - repetition=repetition, - attempt=attempt, - caller=caller, - spec_digest=result.spec_digest, - product=result.product, - harness=result.harness, - process=result.process, - timeline=_timeline(result, observation), - usage=_usage(metrics), - observer=observation, - observations=metrics, - ) - - -def build_recovery_measurement( - *, - run_id: str, - cell_id: str, - repetition: int, - attempt: int, - caller: str, - spec_digest: str, - product: ProductOutcome, - harness: HarnessOutcome, - process: ProcessOutcome, -) -> AttemptMeasurement: - """Build closed unavailable evidence for an authenticated receipt-only terminal. - - A controller can disappear after its supervisor has durably cleaned up the - caller but before it publishes lifecycle, observer, or measurement output. - This constructor records only that absence; it never infers values from the - cleanup receipt or the workspace. - """ - if any(not isinstance(text, str) or not text for text in ( - run_id, cell_id, caller, spec_digest, - )) or not _is_digest(spec_digest): - raise MeasurementError("measurement identity is invalid") - if any( - isinstance(number, bool) or not isinstance(number, int) or number < 1 - for number in (repetition, attempt) - ): - raise MeasurementError("measurement identity is invalid") - if not all(isinstance(value, expected) for value, expected in ( - (product, ProductOutcome), - (harness, HarnessOutcome), - (process, ProcessOutcome), - )): - raise MeasurementError("recovery outcomes are invalid") - timeline = { - name: unavailable( - REASON_NOT_OBSERVED, - SOURCE_WORKSPACE_POLL if name.startswith("first_write") else SOURCE_HARNESS, - ) - for name in TIMELINE_NAMES - } - return AttemptMeasurement( - run_id=run_id, - cell_id=cell_id, - repetition=repetition, - attempt=attempt, - caller=caller, - spec_digest=spec_digest, - product=product, - harness=harness, - process=process, - timeline=timeline, - usage={ - name: unavailable(REASON_NOT_REPORTED, SOURCE_HARNESS) - for name in METRIC_NAMES - }, - observer=WorkspaceWriteObservation( - observed=False, - monotonic_ns=None, - mtime_ns=None, - path_digest="", - precision_ns=int(OBSERVER_INTERVAL_SECONDS * DURATION_NS_PER_SECOND), - samples=0, - reason=REASON_OBSERVER_UNAVAILABLE, - ), - observations=(), - ) - - -def measurement_record(measurement: AttemptMeasurement) -> dict[str, Any]: - """Return the canonical durable projection of one measurement.""" - if not isinstance(measurement, AttemptMeasurement): - raise MeasurementError("measurement is invalid") - observer = measurement.observer - return { - "record": MEASUREMENT_RECORD, - "measurement_version": MEASUREMENT_VERSION, - "attempt": { - "run_id": measurement.run_id, - "cell_id": measurement.cell_id, - "repetition": measurement.repetition, - "attempt": measurement.attempt, - }, - "caller": measurement.caller, - "spec_digest": measurement.spec_digest, - "product": { - "status": measurement.product.status, - "reason": measurement.product.reason, - }, - "harness": { - "status": measurement.harness.status, - "reason": measurement.harness.reason, - "ordered_terminal": measurement.harness.ordered_terminal, - "cleanup_complete": measurement.harness.cleanup_complete, - }, - "process": { - "status": measurement.process.status, - "exit_code": measurement.process.exit_code, - "signal": measurement.process.signal, - }, - "timeline": { - name: observation_record(measurement.timeline[name]) - for name in TIMELINE_NAMES - }, - "usage": { - name: observation_record(measurement.usage[name]) for name in METRIC_NAMES - }, - "observer": { - "source": SOURCE_WORKSPACE_POLL, - "status": STATUS_OBSERVED if observer.observed else STATUS_UNAVAILABLE, - "path_digest": observer.path_digest, - "precision_ns": observer.precision_ns, - "samples": observer.samples, - "reason": "" if observer.observed else ( - observer.reason or REASON_NOT_OBSERVED - ), - }, - "observations": [metric_record(metric) for metric in measurement.observations], - } - - -def measurement_bytes(measurement: AttemptMeasurement) -> bytes: - """Serialize one measurement into canonical, sorted, ASCII bytes.""" - return json.dumps( - measurement_record(measurement), sort_keys=True, separators=(",", ":"), - ensure_ascii=True, - ).encode("ascii") + b"\n" - - -# --------------------------------------------------------------------------- -# Durable publication and strict loading -# --------------------------------------------------------------------------- - -def measurement_path(attempt_root: str | Path) -> Path: - return Path(attempt_root) / MEASUREMENT_FILENAME - - -def publish_measurement( - attempt_root: str | Path, measurement: AttemptMeasurement -) -> Path: - """Publish the sidecar once; a collision never mutates the prior bytes.""" - path = measurement_path(attempt_root) - data = measurement_bytes(measurement) - try: - publish_bytes_no_replace(path, data) - except OSError as exc: - raise MeasurementError("measurement publication refused an existing target") from exc - except Exception as exc: # lifecycle publication failure is never silent - raise MeasurementError("measurement publication failed") from exc - return path - - -def _read_regular_bytes(path: Path) -> bytes: - """Read one durable file without following links or trusting its type.""" - try: - fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) - except OSError as exc: - raise MeasurementError("measurement is unavailable") from exc - try: - if not stat.S_ISREG(os.fstat(fd).st_mode): - raise MeasurementError("measurement must be a regular file") - chunks: list[bytes] = [] - while True: - chunk = os.read(fd, 1 << 20) - if not chunk: - return b"".join(chunks) - chunks.append(chunk) - finally: - os.close(fd) - - -def _metric_from_record(raw: Any) -> ParsedMetric: - fields = { - "name", "value", "unit", "clock", "source", "stage", "model", "call_id", - "overlap", - } - if not isinstance(raw, dict) or set(raw) != fields: - raise MeasurementError("observation schema is invalid") - try: - return validate_metric(ParsedMetric( - raw["name"], raw["value"], raw["unit"], raw["clock"], raw["source"], - raw["stage"], raw["model"], raw["call_id"], raw["overlap"], - )) - except (LifecycleMetricError, TypeError) as exc: - raise MeasurementError("observation is invalid") from exc - - -def _observer_from_record(raw: Any) -> WorkspaceWriteObservation: - fields = {"source", "status", "path_digest", "precision_ns", "samples", "reason"} - if not isinstance(raw, dict) or set(raw) != fields: - raise MeasurementError("observer schema is invalid") - precision, samples = raw["precision_ns"], raw["samples"] - if ( - raw["source"] != SOURCE_WORKSPACE_POLL - or raw["status"] not in (STATUS_OBSERVED, STATUS_UNAVAILABLE) - or isinstance(precision, bool) or not isinstance(precision, int) or precision <= 0 - or isinstance(samples, bool) or not isinstance(samples, int) or samples < 0 - or not isinstance(raw["path_digest"], str) - or not isinstance(raw["reason"], str) - ): - raise MeasurementError("observer record is invalid") - seen = raw["status"] == STATUS_OBSERVED - digest = raw["path_digest"] - if (seen != bool(digest) or (digest and not _is_digest(digest)) - or (seen and raw["reason"]) - or (not seen and raw["reason"] not in UNAVAILABLE_REASONS)): - raise MeasurementError("observer record is invalid") - return WorkspaceWriteObservation( - seen, None, None, digest, precision, samples, raw["reason"] - ) - - -def _identity_from_record(raw: Any) -> tuple[str, str, int, int]: - if not isinstance(raw, dict) or set(raw) != { - "run_id", "cell_id", "repetition", "attempt" - }: - raise MeasurementError("attempt identity schema is invalid") - repetition, attempt = raw["repetition"], raw["attempt"] - if ( - not isinstance(raw["run_id"], str) or not raw["run_id"] - or not isinstance(raw["cell_id"], str) or not raw["cell_id"] - or isinstance(repetition, bool) or not isinstance(repetition, int) or repetition < 1 - or isinstance(attempt, bool) or not isinstance(attempt, int) or attempt < 1 - ): - raise MeasurementError("attempt identity is invalid") - return raw["run_id"], raw["cell_id"], repetition, attempt - - -def _observation_map(raw: Any, names: tuple[str, ...], label: str) -> dict[str, Observation]: - if not isinstance(raw, dict) or set(raw) != set(names): - raise MeasurementError(f"{label} schema is invalid") - return {name: _observation_from_record(raw[name]) for name in names} - - -def load_measurement(attempt_root: str | Path) -> AttemptMeasurement: - """Load one sidecar and revalidate every closed field before use.""" - raw_bytes = _read_regular_bytes(measurement_path(attempt_root)) - try: - record = json.loads(raw_bytes.decode("ascii")) - except (UnicodeDecodeError, json.JSONDecodeError) as exc: - raise MeasurementError("measurement is not canonical JSON") from exc - fields = { - "record", "measurement_version", "attempt", "caller", "spec_digest", - "product", "harness", "process", "timeline", "usage", "observer", - "observations", - } - if not isinstance(record, dict) or set(record) != fields: - raise MeasurementError("measurement schema is invalid") - if ( - record["record"] != MEASUREMENT_RECORD - or record["measurement_version"] != MEASUREMENT_VERSION - or not isinstance(record["caller"], str) or not record["caller"] - or not isinstance(record["spec_digest"], str) or not _is_digest(record["spec_digest"]) - or not isinstance(record["observations"], list) - ): - raise MeasurementError("measurement identity is invalid") - run_id, cell_id, repetition, attempt = _identity_from_record(record["attempt"]) - try: - product = ProductOutcome(**record["product"]) - harness = HarnessOutcome(**record["harness"]) - process = ProcessOutcome(**record["process"]) - except (TypeError, LifecycleValidationError, ValueError) as exc: - raise MeasurementError("measurement outcomes are invalid") from exc - measurement = AttemptMeasurement( - run_id=run_id, - cell_id=cell_id, - repetition=repetition, - attempt=attempt, - caller=record["caller"], - spec_digest=record["spec_digest"], - product=product, - harness=harness, - process=process, - timeline=_observation_map(record["timeline"], TIMELINE_NAMES, "timeline"), - usage=_observation_map(record["usage"], METRIC_NAMES, "usage"), - observer=_observer_from_record(record["observer"]), - observations=tuple(_metric_from_record(item) for item in record["observations"]), - ) - if len(measurement.observations) > MAX_OBSERVATION_RECORDS: - raise MeasurementError("observation count exceeds the bounded record") - if measurement_record(measurement) != record or measurement_bytes(measurement) != raw_bytes: - raise MeasurementError("measurement is non-canonical") - _require_derived_coherence(measurement) - return measurement - - -def _require_derived_coherence(measurement: AttemptMeasurement) -> None: - """Refuse any usage or write value that its own observations do not support.""" - if measurement.usage != _usage(measurement.observations): - raise MeasurementError("usage does not match the recorded observations") - semantics = { - "submitted_at": (CLOCK_HARNESS_MONOTONIC, SOURCE_HARNESS), - "first_output_at": (CLOCK_HARNESS_MONOTONIC, SOURCE_HARNESS), - "first_write_observed_at": (CLOCK_HARNESS_MONOTONIC, SOURCE_WORKSPACE_POLL), - "first_write_mtime": (CLOCK_FILESYSTEM_MTIME, SOURCE_WORKSPACE_POLL), - "total_duration": (CLOCK_HARNESS_MONOTONIC, SOURCE_HARNESS), - } - for name, value in measurement.timeline.items(): - clock, source = semantics[name] - if value.source != source: - raise MeasurementError("timeline source does not match its evidence") - if value.status == STATUS_OBSERVED and ( - value.unit != UNIT_NANOSECONDS or value.clock != clock - ): - raise MeasurementError("timeline clock does not match its evidence") - seen = measurement.observer.observed - write = measurement.timeline["first_write_observed_at"].status - mtime = measurement.timeline["first_write_mtime"].status - if ((not seen and STATUS_OBSERVED in (write, mtime)) - or (seen and write != STATUS_OBSERVED) - or (seen and mtime != STATUS_OBSERVED)): - raise MeasurementError("workspace write does not match the observer") - - -def validate_measurement_lifecycle_binding( - measurement: AttemptMeasurement, lifecycle: Mapping[str, Any] -) -> None: - """Bind a sidecar to the immutable lifecycle metric and timeline evidence. - - The lifecycle result/journal validation belongs to ``RunStore``. This - helper consumes that already-validated terminal projection and refuses a - coherent sidecar rewrite whose observations or derived values no longer - correspond to its immutable event stream. - """ - if not isinstance(lifecycle, Mapping): - raise MeasurementError("lifecycle evidence is invalid") - expected_outcomes = { - "product": { - "status": measurement.product.status, - "reason": measurement.product.reason, - }, - "harness": { - "status": measurement.harness.status, - "reason": measurement.harness.reason, - "ordered_terminal": measurement.harness.ordered_terminal, - "cleanup_complete": measurement.harness.cleanup_complete, - }, - "process": { - "status": measurement.process.status, - "exit_code": measurement.process.exit_code, - "signal": measurement.process.signal, - }, - } - if any(lifecycle.get(name) != value for name, value in expected_outcomes.items()): - raise MeasurementError("measurement outcomes do not match lifecycle evidence") - events = lifecycle.get("events") - if not isinstance(events, list): - raise MeasurementError("lifecycle events are invalid") - - metric_events: list[ParsedMetric] = [] - instants: dict[str, int] = {} - for event in events: - if not isinstance(event, Mapping): - raise MeasurementError("lifecycle event is invalid") - kind, source = event.get("kind"), event.get("source") - if kind in (EVENT_SUBMITTED, EVENT_FIRST_OUTPUT): - value = event.get("monotonic_ns") - if source != SOURCE_HARNESS or isinstance(value, bool) or not isinstance(value, int): - raise MeasurementError("lifecycle timeline evidence is invalid") - if kind in instants: - raise MeasurementError("lifecycle timeline evidence is ambiguous") - instants[kind] = value - if not isinstance(kind, str) or not kind.startswith(METRIC_PREFIX): - continue - name = kind[len(METRIC_PREFIX):] - try: - raw = json.loads(str(event.get("detail", ""))) - metric = _metric_from_record(raw) - except (json.JSONDecodeError, MeasurementError) as exc: - raise MeasurementError("lifecycle metric evidence is invalid") from exc - if metric.name != name or source != metric.source: - raise MeasurementError("lifecycle metric evidence is invalid") - metric_events.append(metric) - - if tuple(metric_events) != measurement.observations: - raise MeasurementError("measurement observations do not match lifecycle evidence") - required_instants = { - "submitted_at": EVENT_SUBMITTED, - "first_output_at": EVENT_FIRST_OUTPUT, - } - for timeline_name, event_kind in required_instants.items(): - timeline = measurement.timeline[timeline_name] - event_value = instants.get(event_kind) - if event_value is None: - if timeline.status != STATUS_UNAVAILABLE: - raise MeasurementError("timeline claims a missing lifecycle event") - elif timeline.status != STATUS_OBSERVED or timeline.value != event_value: - raise MeasurementError("timeline does not match lifecycle evidence") - duration = lifecycle.get("duration_ns") - total = measurement.timeline["total_duration"] - if isinstance(duration, bool) or not isinstance(duration, int) or duration < 0: - raise MeasurementError("lifecycle duration is invalid") - if total.status != STATUS_OBSERVED or total.value != duration: - raise MeasurementError("timeline duration does not match lifecycle evidence") diff --git a/scripts/agent_benchmark/measurement_test.py b/scripts/agent_benchmark/measurement_test.py deleted file mode 100644 index f3485cfa..00000000 --- a/scripts/agent_benchmark/measurement_test.py +++ /dev/null @@ -1,686 +0,0 @@ -"""Credential-free tests for source-aware timing and usage evidence.""" - -from __future__ import annotations - -import json -import os -import tempfile -import threading -import time -import unittest -from decimal import Decimal -from pathlib import Path -from typing import Any - -import scripts.agent_benchmark.measurement as measurement_module - -from scripts.agent_benchmark.lifecycle import ( - CLOCK_CALLER_REPORTED, - CLOCK_HARNESS_MONOTONIC, - CLOCK_NONE, - EVENT_FIRST_OUTPUT, - EVENT_SUBMITTED, - METRIC_PREFIX, - SOURCE_CALLER_OUTPUT, - SOURCE_HARNESS, - SOURCE_WORKSPACE_POLL, - UNIT_NANOSECONDS, - CaptureStream, - ProductOutcome, - HarnessOutcome, - ProcessOutcome, - InvocationResult, - LifecycleMetricError, - LifecycleEvent, - ParsedMetric, - count_metric, - duration_metric, - normalize_count, - normalize_duration_ns, - validate_metric, -) -from scripts.agent_benchmark.measurement import ( - MEASUREMENT_FILENAME, - MeasurementError, - WorkspaceWriteObservation, - WorkspaceWriteObserver, - WorkspaceScan, - OBSERVER_MAX_ENTRIES, - REASON_NOT_OBSERVED, - REASON_OBSERVER_UNAVAILABLE, - _scan_workspace, - build_measurement, - load_measurement, - measurement_bytes, - measurement_record, - observation_record, - observed, - path_digest, - publish_measurement, - unavailable, -) - - -def _event(kind: str, monotonic_ns: int, source: str = SOURCE_HARNESS) -> LifecycleEvent: - return LifecycleEvent(kind, source, "", monotonic_ns, 0, "2026-08-11T00:00:00+00:00", "") - - -def _capture(stream: str) -> CaptureStream: - return CaptureStream(stream, "", 0, 0, False) - - -def _result( - metrics: tuple[ParsedMetric, ...] = (), - *, - events: tuple[LifecycleEvent, ...] | None = None, - duration_ns: int = 5_000, - terminal_reason: str = "success", -) -> InvocationResult: - """Build one frozen lifecycle projection with matching metric events.""" - if events is None: - events = (_event(EVENT_SUBMITTED, 1_000), _event(EVENT_FIRST_OUTPUT, 2_000)) - published = tuple( - _event(METRIC_PREFIX + metric.name, 3_000, metric.source) for metric in metrics - ) - return InvocationResult( - product=ProductOutcome( - "succeeded" if terminal_reason == "success" else "unknown", - "caller_success" if terminal_reason == "success" else "unavailable", - ), - harness=HarnessOutcome( - "passed" if terminal_reason == "success" else "failed", - terminal_reason, - True, - True, - ), - process=ProcessOutcome("exited", 0, None), - submitted=True, - process_group_alive=False, - events=events + published, - stdout=_capture("stdout"), - stderr=_capture("stderr"), - journal_path="", - result_path="", - locator=None, - spec_digest="sha256:" + "a" * 64, - started_at="2026-08-11T00:00:00+00:00", - ended_at="2026-08-11T00:00:01+00:00", - duration_ns=duration_ns, - metrics=metrics, - ) - - -def _observation() -> WorkspaceWriteObservation: - """One frozen observer report used by the record-level tests.""" - return WorkspaceWriteObservation( - True, 4_000, 1_700_000_000_000_000_000, path_digest("out.txt"), 10_000_000, 3 - ) - - -class MetricContractTest(unittest.TestCase): - def test_duration_decimals_normalize_losslessly_to_nanoseconds(self) -> None: - for value, unit, expected in ( - (12, "ms", 12_000_000), - (12.5, "ms", 12_500_000), - ("0.000001", "ms", 1), - (Decimal("1.5"), "s", 1_500_000_000), - (7, "us", 7_000), - (0, "ms", 0), - (9, "ns", 9), - ): - with self.subTest(value=value, unit=unit): - self.assertEqual(normalize_duration_ns(value, unit), expected) - - def test_unrepresentable_and_non_numeric_durations_fail_closed(self) -> None: - for value, unit in ( - (0.0000001, "ms"), # 0.1 ns cannot be represented without invention - (Decimal("0.5"), "ns"), - (-1, "ms"), - (True, "ms"), - (float("inf"), "ms"), - ("nan", "ms"), - ("not-a-number", "ms"), - (None, "ms"), - (12, "minutes"), - ): - with self.subTest(value=value, unit=unit): - with self.assertRaises(LifecycleMetricError): - normalize_duration_ns(value, unit) - - def test_counts_admit_only_non_negative_integers(self) -> None: - self.assertEqual(normalize_count(0), 0) - self.assertEqual(normalize_count(41), 41) - for value in (1.5, 2.0, True, -1, "3", None, Decimal("4")): - with self.subTest(value=value): - with self.assertRaises(LifecycleMetricError): - normalize_count(value) - - def test_metric_vocabulary_clock_and_labels_are_closed(self) -> None: - rejected = ( - ParsedMetric("unknown_metric", 1, UNIT_NANOSECONDS, CLOCK_CALLER_REPORTED, SOURCE_CALLER_OUTPUT), - ParsedMetric("total_duration", 1, "tokens", CLOCK_CALLER_REPORTED, SOURCE_CALLER_OUTPUT), - ParsedMetric("total_duration", 1, UNIT_NANOSECONDS, CLOCK_NONE, SOURCE_CALLER_OUTPUT), - ParsedMetric("input_tokens", 1, "tokens", CLOCK_CALLER_REPORTED, SOURCE_CALLER_OUTPUT), - ParsedMetric("input_tokens", 1, "tokens", CLOCK_NONE, SOURCE_CALLER_OUTPUT, overlap=True), - ParsedMetric("input_tokens", 1, "tokens", CLOCK_NONE, "invented_source"), - ParsedMetric("input_tokens", -1, "tokens", CLOCK_NONE, SOURCE_CALLER_OUTPUT), - ParsedMetric("input_tokens", 1, "tokens", CLOCK_NONE, SOURCE_CALLER_OUTPUT, model="two words"), - ParsedMetric("input_tokens", 1, "tokens", CLOCK_NONE, SOURCE_CALLER_OUTPUT, call_id="sk-abcdefgh12345"), - "metric:total_duration", - ) - for metric in rejected: - with self.subTest(metric=metric): - with self.assertRaises(LifecycleMetricError): - validate_metric(metric) - accepted = duration_metric( - "tool_duration", 3, model="claude-sonnet", call_id="call-1", overlap=True - ) - self.assertEqual(accepted.value, 3_000_000) - self.assertEqual(accepted.clock, CLOCK_CALLER_REPORTED) - - def test_observed_and_unavailable_projections_stay_distinct(self) -> None: - self.assertEqual( - observation_record(observed(7, UNIT_NANOSECONDS, CLOCK_HARNESS_MONOTONIC, SOURCE_HARNESS)), - { - "status": "observed", "value": 7, "unit": UNIT_NANOSECONDS, - "clock": CLOCK_HARNESS_MONOTONIC, "source": SOURCE_HARNESS, - }, - ) - # An unavailable value is explicitly null; it is never a zero. - self.assertEqual( - observation_record(unavailable("not_reported", SOURCE_CALLER_OUTPUT)), - { - "status": "unavailable", "value": None, "reason": "not_reported", - "source": SOURCE_CALLER_OUTPUT, - }, - ) - with self.assertRaises(MeasurementError): - unavailable("because", SOURCE_HARNESS) - with self.assertRaises(MeasurementError): - observed(-1, UNIT_NANOSECONDS, CLOCK_HARNESS_MONOTONIC, SOURCE_HARNESS) - - -class MeasurementRecordTest(unittest.TestCase): - def _measurement(self, metrics: tuple[ParsedMetric, ...], **kwargs: Any): - return build_measurement( - run_id="run-20260811T000000Z-0123456789ab", - cell_id="claude-direct", - repetition=1, - attempt=1, - caller="claude", - result=_result(metrics, **kwargs), - observation=_observation(), - ) - - def test_reported_totals_are_preserved_without_any_synthesis(self) -> None: - metrics = ( - duration_metric("total_duration", 1000, model="claude-sonnet"), - duration_metric("model_duration", 400, model="claude-sonnet", overlap=True), - count_metric("input_tokens", 11, model="claude-sonnet"), - count_metric("output_tokens", 22, model="claude-sonnet"), - ) - usage = self._measurement(metrics).usage - self.assertEqual(usage["total_duration"].value, 1_000_000_000) - self.assertEqual(usage["model_duration"].value, 400_000_000) - self.assertEqual(usage["input_tokens"].value, 11) - # Nothing is added and nothing is subtracted: the unreported provider - # total and the unreported queue time both stay unavailable. - for name in ("total_tokens", "queue_duration", "tool_duration", "model_calls"): - self.assertEqual(usage[name].status, "unavailable") - self.assertIsNone(usage[name].value) - self.assertEqual(usage[name].reason, "not_reported") - - def test_overlapping_and_labelled_intervals_never_become_totals(self) -> None: - metrics = ( - duration_metric("tool_duration", 30, call_id="call-1", overlap=True), - duration_metric("tool_duration", 70, call_id="call-2", overlap=True), - ) - measurement = self._measurement(metrics) - self.assertEqual(measurement.usage["tool_duration"].status, "unavailable") - self.assertEqual( - [(item.call_id, item.value, item.overlap) for item in measurement.observations], - [("call-1", 30_000_000, True), ("call-2", 70_000_000, True)], - ) - - def test_duplicate_total_fails_closed(self) -> None: - metrics = ( - duration_metric("total_duration", 10, model="claude-sonnet"), - duration_metric("total_duration", 20, model="claude-sonnet"), - ) - with self.assertRaises(MeasurementError): - self._measurement(metrics) - - def test_two_model_totals_stay_ambiguous_instead_of_merging(self) -> None: - metrics = ( - duration_metric("total_duration", 10, model="claude-sonnet"), - duration_metric("total_duration", 20, model="gemini-2.0-flash"), - ) - measurement = self._measurement(metrics) - self.assertEqual(measurement.usage["total_duration"].status, "unavailable") - self.assertEqual(measurement.usage["total_duration"].reason, "ambiguous_total") - self.assertEqual( - [(item.model, item.value) for item in measurement.observations], - [("claude-sonnet", 10_000_000), ("gemini-2.0-flash", 20_000_000)], - ) - - def test_timeline_keeps_every_clock_domain_separate(self) -> None: - measurement = self._measurement(()) - timeline = measurement.timeline - self.assertEqual(timeline["submitted_at"].value, 1_000) - self.assertEqual(timeline["submitted_at"].clock, CLOCK_HARNESS_MONOTONIC) - self.assertEqual(timeline["first_output_at"].value, 2_000) - self.assertEqual(timeline["first_output_at"].source, SOURCE_HARNESS) - # The observer's own clock and the filesystem clock are reported as two - # separate values, so nothing can subtract one from the other. - self.assertEqual(timeline["first_write_observed_at"].clock, CLOCK_HARNESS_MONOTONIC) - self.assertEqual(timeline["first_write_observed_at"].source, SOURCE_WORKSPACE_POLL) - self.assertEqual(timeline["first_write_mtime"].clock, "filesystem_mtime") - self.assertEqual(timeline["total_duration"].value, 5_000) - - def test_missing_first_output_is_unavailable_rather_than_zero(self) -> None: - measurement = self._measurement((), events=(_event(EVENT_SUBMITTED, 1_000),)) - self.assertEqual(measurement.timeline["first_output_at"].status, "unavailable") - self.assertIsNone(measurement.timeline["first_output_at"].value) - self.assertEqual(measurement.timeline["first_output_at"].reason, "not_observed") - - def test_incomplete_identity_is_refused_before_any_publication(self) -> None: - base = { - "run_id": "run-20260811T000000Z-0123456789ab", "cell_id": "claude-direct", - "repetition": 1, "attempt": 1, "caller": "claude", - "result": _result(), "observation": _observation(), - } - for override in ( - {"run_id": ""}, {"cell_id": None}, {"caller": ""}, - {"repetition": 0}, {"attempt": True}, - {"result": object()}, - {"observation": None}, - ): - with self.subTest(override=tuple(override)): - with self.assertRaises(MeasurementError): - build_measurement(**{**base, **override}) - - def test_observation_set_must_match_published_events(self) -> None: - result = _result((count_metric("input_tokens", 1),)) - forged = ParsedMetric("output_tokens", 5, "tokens", CLOCK_NONE, SOURCE_CALLER_OUTPUT) - with self.assertRaises(MeasurementError): - build_measurement( - run_id="run-20260811T000000Z-0123456789ab", cell_id="c", repetition=1, - attempt=1, caller="claude", - result=InvocationResult(**{**result.__dict__, "metrics": (*result.metrics, forged)}), - observation=_observation(), - ) - - -class WorkspaceObserverTest(unittest.TestCase): - def setUp(self) -> None: - self.temp = tempfile.TemporaryDirectory(dir="/tmp", prefix="measurement-") - self.root = Path(self.temp.name) - self.workspace = self.root / "workspace" - self.workspace.mkdir() - - def tearDown(self) -> None: - self.temp.cleanup() - - def _write(self, name: str, text: str) -> Path: - path = self.workspace / name - path.write_text(text, encoding="utf-8") - return path - - @staticmethod - def _wait_observed(observer: WorkspaceWriteObserver) -> None: - deadline = time.monotonic() + 10 - while observer._first is None and time.monotonic() < deadline: - time.sleep(0.005) - - def test_first_observed_write_is_reported_with_clock_source_and_precision(self) -> None: - observer = WorkspaceWriteObserver( - self.workspace, interval_seconds=0.005, - ) - observer.start() - self._write("out.txt", "x") - self._wait_observed(observer) - observation = observer.stop() - self.assertTrue(observer.stopped) - self.assertTrue(observation.observed) - self.assertEqual(observation.path_digest, path_digest("out.txt")) - self.assertEqual(observation.precision_ns, 5_000_000) - self.assertGreaterEqual(observation.samples, 2) - self.assertEqual( - observation.mtime_ns, (self.workspace / "out.txt").stat().st_mtime_ns - ) - - def test_first_observed_order_survives_a_later_final_mtime(self) -> None: - observer = WorkspaceWriteObserver( - self.workspace, interval_seconds=0.005, - ) - observer.start() - self._write("b.txt", "1") - self._wait_observed(observer) - observation = observer.stop() - # After the observation is frozen, a second file is created and the - # first file is rewritten, so the final snapshot now orders a.txt - # before b.txt. The observer still reports the write it actually saw. - later = self._write("a.txt", "2") - os.utime(later, ns=(2_000_000_000_000_000_000, 2_000_000_000_000_000_000)) - rewritten = self._write("b.txt", "3") - os.utime(rewritten, ns=(3_000_000_000_000_000_000, 3_000_000_000_000_000_000)) - snapshot = sorted( - (item.stat().st_mtime_ns, item.name) for item in self.workspace.iterdir() - ) - self.assertEqual(snapshot[0][1], "a.txt") - self.assertEqual(observation.path_digest, path_digest("b.txt")) - - def test_exhausted_scan_is_unavailable_and_never_compared(self) -> None: - for index in range(OBSERVER_MAX_ENTRIES + 1): - (self.workspace / f"empty-{index:04d}").mkdir() - baseline = _scan_workspace(self.workspace) - self.assertEqual(baseline.status, "exhausted") - self.assertLessEqual(len(baseline.files), OBSERVER_MAX_ENTRIES) - observer = WorkspaceWriteObserver(self.workspace, interval_seconds=0.005) - observer.start() - observation = observer.stop() - self.assertFalse(observation.observed) - self.assertEqual(observation.reason, REASON_OBSERVER_UNAVAILABLE) - self.assertTrue(observer.stopped) - - def test_detection_clock_runs_after_the_complete_scan(self) -> None: - order: list[str] = [] - observer = WorkspaceWriteObserver( - self.workspace, interval_seconds=0.005, - clock=lambda: (order.append("clock") or 123), - ) - observer._baseline = {} - original = _scan_workspace - - def scan(_root: Path) -> WorkspaceScan: - order.append("scan") - return WorkspaceScan({"out.txt": (1, 1, 1)}, "complete") - - try: - import scripts.agent_benchmark.measurement as measurement_module - measurement_module._scan_workspace = scan - self.assertTrue(observer._sample_once()) - finally: - measurement_module._scan_workspace = original - self.assertEqual(order, ["scan", "clock"]) - - def test_symlink_and_non_regular_entries_are_never_observed(self) -> None: - outside = self.root / "outside.txt" - outside.write_text("outside", encoding="utf-8") - os.symlink(outside, self.workspace / "link.txt") - os.mkfifo(self.workspace / "pipe") - observer = WorkspaceWriteObserver(self.workspace, interval_seconds=0.005) - observer.start() - outside.write_text("changed outside", encoding="utf-8") - observation = observer.stop() - self.assertFalse(observation.observed) - self.assertEqual(observation.path_digest, "") - - def test_no_write_is_unavailable_and_leaves_no_thread(self) -> None: - before = set(threading.enumerate()) - observer = WorkspaceWriteObserver(self.workspace, interval_seconds=0.005) - observer.start() - observation = observer.stop() - self.assertFalse(observation.observed) - self.assertIsNone(observation.monotonic_ns) - self.assertTrue(observer.stopped) - self.assertEqual(set(threading.enumerate()) - before, set()) - with self.assertRaises(MeasurementError): - observer.start() - - def test_stop_closes_the_final_interval_with_one_scan_after_sampler_exit(self) -> None: - calls: list[str] = [] - sampled = threading.Event() - snapshots = iter(( - WorkspaceScan({}, "complete"), - WorkspaceScan({}, "complete"), - WorkspaceScan({"out.txt": (1, 1, 1)}, "complete"), - )) - original = measurement_module._scan_workspace - - def scan(_root: Path) -> WorkspaceScan: - calls.append("scan") - result = next(snapshots) - if len(calls) == 2: - sampled.set() - return result - - try: - measurement_module._scan_workspace = scan - observer = WorkspaceWriteObserver( - self.workspace, interval_seconds=60, - clock=lambda: (calls.append("clock") or 123), - ) - observer.start() - self.assertTrue(sampled.wait(5)) - observation = observer.stop() - finally: - measurement_module._scan_workspace = original - self.assertTrue(observer.stopped) - self.assertTrue(observation.observed) - self.assertEqual(observation.samples, 2) - self.assertEqual(calls, ["scan", "scan", "scan", "clock"]) - - def test_final_exhausted_scan_is_unavailable_and_thread_is_cleaned_up(self) -> None: - sampled = threading.Event() - calls = 0 - snapshots = iter(( - WorkspaceScan({}, "complete"), - WorkspaceScan({}, "complete"), - WorkspaceScan({}, "exhausted"), - )) - original = measurement_module._scan_workspace - - def scan(_root: Path) -> WorkspaceScan: - nonlocal calls - calls += 1 - result = next(snapshots) - if calls == 2: - sampled.set() - return result - - try: - measurement_module._scan_workspace = scan - observer = WorkspaceWriteObserver(self.workspace, interval_seconds=60) - observer.start() - self.assertTrue(sampled.wait(5)) - observation = observer.stop() - finally: - measurement_module._scan_workspace = original - self.assertTrue(observer.stopped) - self.assertFalse(observation.observed) - self.assertEqual(observation.reason, REASON_OBSERVER_UNAVAILABLE) - self.assertEqual(observation.samples, 2) - self.assertEqual(calls, 3) - - def test_successful_stop_is_idempotent_and_excludes_post_stop_writes(self) -> None: - sampled = threading.Event() - calls = 0 - original = measurement_module._scan_workspace - - def scan(root: Path) -> WorkspaceScan: - nonlocal calls - calls += 1 - result = original(root) - # Signal once the background sampler has completed its own empty - # scan so the main thread stops a started, no-write observer. - if calls == 2: - sampled.set() - return result - - try: - measurement_module._scan_workspace = scan - observer = WorkspaceWriteObserver(self.workspace, interval_seconds=60) - observer.start() - self.assertTrue(sampled.wait(5)) - first = observer.stop() - self._write("post-stop.txt", "after shutdown") - second = observer.stop() - finally: - measurement_module._scan_workspace = original - self.assertTrue(observer.stopped) - self.assertIs(second, first) - self.assertFalse(first.observed) - self.assertEqual(first.reason, REASON_NOT_OBSERVED) - self.assertEqual(second.samples, first.samples) - # Baseline, the background sampler's empty scan, and the one final scan - # account for every scan; the cached second stop performs no fourth scan. - self.assertEqual(calls, 3) - self.assertIsNone(observer._thread) - - def test_observer_requires_a_real_directory_and_positive_interval(self) -> None: - with self.assertRaises(MeasurementError): - WorkspaceWriteObserver(self.workspace, interval_seconds=0) - with self.assertRaises(MeasurementError): - WorkspaceWriteObserver(self.root / "missing").start() - os.symlink(self.workspace, self.root / "alias") - with self.assertRaises(MeasurementError): - WorkspaceWriteObserver(self.root / "alias").start() - - -class MeasurementSidecarTest(unittest.TestCase): - def setUp(self) -> None: - self.temp = tempfile.TemporaryDirectory(dir="/tmp", prefix="measurement-") - self.root = Path(self.temp.name) - self.measurement = build_measurement( - run_id="run-20260811T000000Z-0123456789ab", - cell_id="claude-direct", - repetition=1, - attempt=1, - caller="claude", - result=_result(( - duration_metric("total_duration", 1000, model="claude-sonnet"), - count_metric("input_tokens", 11, model="claude-sonnet"), - )), - observation=_observation(), - ) - - def tearDown(self) -> None: - self.temp.cleanup() - - def test_publish_then_load_round_trips_every_closed_field(self) -> None: - path = publish_measurement(self.root, self.measurement) - self.assertEqual(path.name, MEASUREMENT_FILENAME) - loaded = load_measurement(self.root) - self.assertEqual(measurement_record(loaded), measurement_record(self.measurement)) - self.assertEqual(loaded.caller, "claude") - self.assertEqual(loaded.usage["input_tokens"].value, 11) - self.assertEqual(loaded.observations[0].name, "total_duration") - self.assertEqual(loaded.observer.path_digest, path_digest("out.txt")) - self.assertEqual(path.stat().st_mode & 0o777, 0o600) - - def test_publication_is_no_clobber_and_preserves_prior_bytes(self) -> None: - prior = b'{"record":"prior"}\n' - (self.root / MEASUREMENT_FILENAME).write_bytes(prior) - with self.assertRaises(MeasurementError): - publish_measurement(self.root, self.measurement) - self.assertEqual((self.root / MEASUREMENT_FILENAME).read_bytes(), prior) - - def test_tampered_and_non_canonical_records_fail_closed(self) -> None: - record = measurement_record(self.measurement) - cases = { - "unknown-field": {**record, "extra": 1}, - "wrong-version": {**record, "measurement_version": 1}, - "forged-digest": {**record, "spec_digest": "sha256:not-a-digest"}, - "zeroed-unavailable": { - **record, - "usage": { - **record["usage"], - "total_tokens": { - "status": "unavailable", "value": 0, - "reason": "not_reported", "source": "harness", - }, - }, - }, - "invented-clock": { - **record, - "timeline": { - **record["timeline"], - "submitted_at": { - **record["timeline"]["submitted_at"], "clock": "wall_clock", - }, - }, - }, - "non-temporal-instant": { - **record, - "timeline": { - **record["timeline"], - "submitted_at": { - **record["timeline"]["submitted_at"], "clock": "none", - }, - }, - }, - "wrong-timeline-source": { - **record, - "timeline": { - **record["timeline"], - "first_write_observed_at": { - **record["timeline"]["first_write_observed_at"], - "source": SOURCE_HARNESS, - }, - }, - }, - "invented-usage-total": { - **record, - "usage": { - **record["usage"], - "total_tokens": { - "status": "observed", "value": 11, "unit": "tokens", - "clock": "none", "source": "caller_output", - }, - }, - }, - "observer-contradiction": { - **record, - "observer": { - **record["observer"], "status": "unavailable", "path_digest": "", - }, - }, - "unbound-observation": { - **record, - "observations": [ - {**record["observations"][0], "name": "unknown_metric"} - ], - }, - } - for name, payload in cases.items(): - with self.subTest(name=name): - target = self.root / name - target.mkdir() - (target / MEASUREMENT_FILENAME).write_bytes( - json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() + b"\n" - ) - with self.assertRaises(MeasurementError): - load_measurement(target) - - def test_reordered_bytes_are_rejected_as_non_canonical(self) -> None: - (self.root / MEASUREMENT_FILENAME).write_bytes( - json.dumps(measurement_record(self.measurement), indent=2).encode() + b"\n" - ) - with self.assertRaises(MeasurementError): - load_measurement(self.root) - - def test_missing_symlinked_and_non_regular_sidecars_fail_closed(self) -> None: - with self.assertRaises(MeasurementError): - load_measurement(self.root) - publish_measurement(self.root, self.measurement) - aliased = self.root / "aliased" - aliased.mkdir() - os.symlink(self.root / MEASUREMENT_FILENAME, aliased / MEASUREMENT_FILENAME) - with self.assertRaises(MeasurementError): - load_measurement(aliased) - piped = self.root / "piped" - piped.mkdir() - os.mkfifo(piped / MEASUREMENT_FILENAME) - with self.assertRaises(MeasurementError): - load_measurement(piped) - - def test_canonical_bytes_are_stable_and_ascii(self) -> None: - first = measurement_bytes(self.measurement) - self.assertEqual(first, measurement_bytes(self.measurement)) - first.decode("ascii") - self.assertTrue(first.endswith(b"\n")) - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/agent_benchmark/reporting.py b/scripts/agent_benchmark/reporting.py deleted file mode 100644 index 650e6e88..00000000 --- a/scripts/agent_benchmark/reporting.py +++ /dev/null @@ -1,544 +0,0 @@ -"""Deterministic, fail-closed Markdown reporting for benchmark evidence. - -This module is intentionally a reader of the immutable run tree. It does not -retry execution or scoring, derive values that producers did not record, or -write anything except the idempotent ``report.md`` artifact after every input -has passed its owning strict loader. -""" - -from __future__ import annotations - -import json -import os -import stat -from dataclasses import dataclass -from pathlib import Path, PurePosixPath -from typing import Any - -from scripts.agent_benchmark import scoring as _scoring -from scripts.agent_benchmark.attempts import ( - Attempt, - AttemptStateError, - RunIdentity, - RunStore, - _connectivity_result_from_payload, -) -from scripts.agent_benchmark.manifest import Manifest, MatrixCell -from scripts.agent_benchmark.measurement import ( - METRIC_NAMES, - TIMELINE_NAMES, - AttemptMeasurement, - MeasurementError, - Observation, - load_measurement, -) -from scripts.agent_benchmark.web_validation import ( - WebValidation, - WebValidationError, - load_web_validation, -) -from scripts.agent_benchmark.rubric import validate_worksheet - -REPORT_FILENAME = "report.md" -_MAX_REPORT_BYTES = 2 * 1024 * 1024 - - -class ReportError(Exception): - """The immutable evidence cannot safely be projected into a report.""" - - -@dataclass(frozen=True) -class CategoryProjection: - id: str - score: int - max_score: int - - -@dataclass(frozen=True) -class ScoreProjection: - status: str - categories: tuple[CategoryProjection, ...] - total: int | None - rank: int | None - reasons: tuple[str, ...] - score_id: str | None - evaluator: tuple[str, str, str, str] | None - raw_paths: tuple[str, ...] - - -@dataclass(frozen=True) -class AttemptProjection: - cell: MatrixCell - attempt: Attempt - measurement: AttemptMeasurement | None - web: WebValidation | None - score: ScoreProjection - raw_paths: tuple[str, ...] - - -@dataclass(frozen=True) -class ReportProjection: - run: RunIdentity - manifest: Manifest - preflights: tuple[dict[str, Any], ...] - attempts: tuple[AttemptProjection, ...] - - -def _markdown(value: object) -> str: - """Escape one table cell without allowing a value to alter Markdown shape.""" - return str(value).replace("\\", "\\\\").replace("|", "\\|").replace( - "\r", " ").replace("\n", " ") - - -def _relative_path(value: str) -> str: - if not isinstance(value, str) or not value or "\\" in value or ":" in value: - raise ReportError("report evidence path is invalid") - path = PurePosixPath(value) - if path.is_absolute() or str(path) != value or any( - part in ("", ".", "..") for part in path.parts - ): - raise ReportError("report evidence path is invalid") - return value - - -def _regular_under(root: Path, relative: str) -> None: - """Require a contained regular file and reject every symlink component.""" - relative = _relative_path(relative) - current = root - try: - root_info = os.lstat(root) - except OSError as exc: - raise ReportError("report run root is unavailable") from exc - if not stat.S_ISDIR(root_info.st_mode) or stat.S_ISLNK(root_info.st_mode): - raise ReportError("report run root is invalid") - for component in PurePosixPath(relative).parts: - current = current / component - try: - info = os.lstat(current) - except OSError as exc: - raise ReportError("report evidence is unavailable") from exc - if stat.S_ISLNK(info.st_mode): - raise ReportError("report evidence path is invalid") - if not stat.S_ISREG(info.st_mode): - raise ReportError("report evidence must be a regular file") - try: - current.resolve().relative_to(root.resolve()) - except ValueError as exc: - raise ReportError("report evidence escapes run root") from exc - - -def _raw_link(path: str) -> str: - path = _relative_path(path) - return f"[raw](<{path.replace('>', '%3E')}>)" - - -def _attempt_label(item: AttemptProjection) -> str: - """Render the complete durable identity used by detailed report rows.""" - identity = item.attempt.identity - return f"{item.cell.id}/r{identity.repetition}/a{identity.attempt}" - - -def _observation(value: Observation) -> str: - if value.status == "observed": - return ( - f"{value.value} {value.unit}; clock={value.clock}; " - f"source={value.source}" - ) - return f"unavailable; reason={value.reason}; source={value.source}" - - -def _observation_group(values: dict[str, Observation]) -> str: - """Keep every named value while coalescing identical unavailable causes.""" - groups: dict[tuple[str, str], list[str]] = {} - rendered: list[str] = [] - for name, value in values.items(): - if value.status == "observed": - rendered.append(f"{name}={_observation(value)}") - else: - groups.setdefault((value.reason, value.source), []).append(name) - for (reason, source), names in groups.items(): - rendered.append( - f"{','.join(names)}=unavailable; reason={reason}; source={source}" - ) - return "; ".join(rendered) - - -def _read_canonical_json(path: Path, label: str) -> dict[str, Any]: - """Use scoring's no-follow reader and reject non-canonical JSON records.""" - try: - value = _scoring._load_canonical(path, label) - except Exception as exc: - raise ReportError("report scoring evidence is unavailable") from exc - if not isinstance(value, dict): - raise ReportError("report scoring evidence is invalid") - return value - - -def _score_projection( - run: RunIdentity, manifest: Manifest, attempt: Attempt, *, blocked: bool -) -> ScoreProjection: - """Strictly load the terminal score state without allocating or recovering.""" - root = Path(attempt.root) - raw_paths: list[str] = [] - try: - score_root = _scoring._score_root(attempt, create=False) - score_dirs = _scoring._score_dirs(score_root) - is_unscored = _scoring._validate_unscored(run, manifest, attempt) - if is_unscored: - if score_dirs: - raise ReportError("report scoring state is invalid") - raw_paths.append( - f"cells/{attempt.identity.cell_id}/repetition-" - f"{attempt.identity.repetition:04d}/attempt-" - f"{attempt.identity.attempt:06d}/scoring/unscored.json" - ) - record = _read_canonical_json(root / "scoring" / "unscored.json", "unscored evidence") - reasons = record.get("reasons") - if not isinstance(reasons, list) or not all(isinstance(item, str) for item in reasons): - raise ReportError("report unscored evidence is invalid") - return ScoreProjection( - "unscored", (), None, None, tuple(reasons), None, None, - tuple(raw_paths), - ) - - if not score_dirs: - return ScoreProjection( - "blocked" if blocked else "unavailable", (), None, None, - ("evaluator_preflight_blocked",) if blocked else ("not_recorded",), - None, None, (), - ) - - statuses: list[str] = [] - allocations: list[dict[str, Any]] = [] - for score_root in score_dirs: - status = _scoring._result_status(score_root, run, manifest, attempt) - if status is None: - raise ReportError("report scoring result is incomplete") - statuses.append(status) - allocations.append( - _scoring._validate_allocation( - score_root / "allocation.json", run, manifest, attempt, score_root.name - ) - ) - prefix = ( - f"cells/{attempt.identity.cell_id}/repetition-" - f"{attempt.identity.repetition:04d}/attempt-" - f"{attempt.identity.attempt:06d}/scoring/{score_root.name}" - ) - raw_paths.extend((f"{prefix}/allocation.json", f"{prefix}/result.json")) - - if "scored" in statuses: - if statuses[-1] != "scored" or statuses.count("scored") != 1: - raise ReportError("report scoring state is invalid") - score_root = score_dirs[-1] - result = _read_canonical_json(score_root / "result.json", "scoring result") - try: - worksheet = validate_worksheet(result.get("worksheet")) - except Exception as exc: - raise ReportError("report worksheet is invalid") from exc - categories = tuple( - CategoryProjection(item.id, item.score, item.max_score) - for item in worksheet.categories - ) - if not categories: - raise ReportError("report worksheet is invalid") - evaluator = allocations[-1]["evaluator"] - binding = ( - evaluator["caller"], evaluator["route_id"], - evaluator["request_model"], evaluator["requested_effort"], - ) - return ScoreProjection( - "scored", categories, worksheet.total, None, (), score_root.name, - binding, tuple(raw_paths), - ) - - if any(status != "scoring_failed" for status in statuses): - raise ReportError("report scoring state is invalid") - result = _read_canonical_json(score_dirs[-1] / "result.json", "scoring result") - reason = result.get("reason") - if not isinstance(reason, str) or not reason: - raise ReportError("report scoring failure is invalid") - evaluator = allocations[-1]["evaluator"] - binding = ( - evaluator["caller"], evaluator["route_id"], - evaluator["request_model"], evaluator["requested_effort"], - ) - return ScoreProjection( - "scoring_failed", (), None, None, (reason,), score_dirs[-1].name, - binding, tuple(raw_paths), - ) - except ReportError: - raise - except Exception as exc: - raise ReportError("report scoring evidence is invalid") from exc - - -def _scoring_preflight_blocked(run: RunIdentity, manifest: Manifest) -> bool: - """Validate evaluator preflights and report whether the latest one blocked.""" - root = Path(run.root) / "scoring-preflight" - if not root.exists() and not root.is_symlink(): - return False - try: - if root.is_symlink() or not root.is_dir(): - raise ReportError("report scoring preflight is invalid") - evaluator = _scoring._evaluator_cell(manifest) - statuses: list[str] = [] - for expected, path in enumerate(sorted(root.iterdir()), start=1): - if path.name != f"preflight-{expected:06d}.json": - raise ReportError("report scoring preflight sequence is invalid") - payload = _read_canonical_json(path, "evaluator preflight") - result, _, _ = _connectivity_result_from_payload(payload, evaluator) - statuses.append(result.status) - return bool(statuses and statuses[-1] != "ready") - except ReportError: - raise - except Exception as exc: - raise ReportError("report scoring preflight is invalid") from exc - - -def _rank(attempts: list[AttemptProjection]) -> tuple[AttemptProjection, ...]: - """Assign competition ranks without using display order as a tie-breaker.""" - scored = sorted( - (item for item in attempts if item.score.status == "scored"), - key=lambda item: -int(item.score.total), - ) - ranks: dict[tuple[str, int, int], int] = {} - previous: int | None = None - for index, item in enumerate(scored, start=1): - total = int(item.score.total) - if total != previous: - rank = index - previous = total - ranks[(item.attempt.identity.cell_id, item.attempt.identity.repetition, item.attempt.identity.attempt)] = rank - result: list[AttemptProjection] = [] - for item in attempts: - key = (item.attempt.identity.cell_id, item.attempt.identity.repetition, item.attempt.identity.attempt) - score = item.score - result.append(AttemptProjection( - item.cell, item.attempt, item.measurement, item.web, - ScoreProjection( - score.status, score.categories, score.total, ranks.get(key), - score.reasons, score.score_id, score.evaluator, score.raw_paths, - ), - item.raw_paths, - )) - return tuple(result) - - -def project_report(store: RunStore, run: RunIdentity, manifest: Manifest) -> ReportProjection: - """Read every report input in stable manifest/slot/attempt order.""" - try: - bound = store.open(manifest, run.run_id) - if bound != run: - raise ReportError("report run identity is invalid") - preflights = store.preflights(bound, manifest) - blocked = _scoring_preflight_blocked(bound, manifest) - cells = {cell.id: cell for cell in manifest.matrix} - rows: list[AttemptProjection] = [] - for attempt in store.execution_attempts(bound, manifest): - cell = cells.get(attempt.identity.cell_id) - if cell is None: - raise ReportError("report attempt cell is invalid") - measurement = None - web = None - raw_paths = [ - f"cells/{attempt.identity.cell_id}/repetition-" - f"{attempt.identity.repetition:04d}/attempt-" - f"{attempt.identity.attempt:06d}/attempt.json", - ] - if attempt.state != "running": - measurement = load_measurement(attempt.root) - if ( - measurement.run_id != bound.run_id - or measurement.cell_id != attempt.identity.cell_id - or measurement.repetition != attempt.identity.repetition - or measurement.attempt != attempt.identity.attempt - or measurement.caller != cell.caller - ): - raise ReportError("report measurement identity is invalid") - web = load_web_validation(attempt.root, manifest=manifest) - if web.record["attempt"] != { - "run_id": bound.run_id, "cell_id": attempt.identity.cell_id, - "repetition": attempt.identity.repetition, "attempt": attempt.identity.attempt, - }: - raise ReportError("report web evidence identity is invalid") - raw_paths.extend(( - f"cells/{attempt.identity.cell_id}/repetition-{attempt.identity.repetition:04d}/attempt-{attempt.identity.attempt:06d}/attempt-measurement.json", - f"cells/{attempt.identity.cell_id}/repetition-{attempt.identity.repetition:04d}/attempt-{attempt.identity.attempt:06d}/web-validation.json", - )) - score = _score_projection(bound, manifest, attempt, blocked=blocked) - rows.append(AttemptProjection(cell, attempt, measurement, web, score, tuple(raw_paths))) - return ReportProjection(bound, manifest, preflights, _rank(rows)) - except ReportError: - raise - except (AttemptStateError, MeasurementError, WebValidationError) as exc: - raise ReportError("report evidence is invalid") from exc - except Exception as exc: - raise ReportError("report evidence is unavailable") from exc - - -def render_report(projection: ReportProjection) -> bytes: - """Render one fixed-order UTF-8/LF Markdown projection.""" - root = Path(projection.run.root) - raw_paths = {"manifest.json", "run.json"} - scoring_preflight = root / "scoring-preflight" - if scoring_preflight.exists() or scoring_preflight.is_symlink(): - # project_report already performed the schema validation; retain the - # exact immutable preflight bytes as the blocked/ready score pointer. - for expected, path in enumerate(sorted(scoring_preflight.iterdir()), start=1): - if path.name != f"preflight-{expected:06d}.json": - raise ReportError("report scoring preflight sequence is invalid") - raw_paths.add(f"scoring-preflight/{path.name}") - lines = [ - "# Agent comparison benchmark report", - "", - "## Run identity", - "", - "| field | value |", - "|---|---|", - f"| run_id | {_markdown(projection.run.run_id)} |", - f"| manifest_digest | {_markdown(projection.run.manifest_digest)} |", - f"| pipeline_version | {_markdown(projection.manifest.pipeline_version)} |", - "", - "## Immutable conditions", - "", - "| field | value |", - "|---|---|", - f"| environment | {_markdown(projection.manifest.environment)} |", - f"| fixture | {_markdown(projection.manifest.fixture.version)} ({_markdown(projection.manifest.fixture.checksum)}) |", - f"| rubric | {_markdown(projection.manifest.rubric_version)} |", - f"| session_policy | {_markdown(projection.manifest.session_policy)} |", - f"| setup_cache_policy | {_markdown(projection.manifest.setup_cache_policy)} |", - f"| evaluator | {_markdown(projection.manifest.evaluator.caller)}/{_markdown(projection.manifest.evaluator.iop.request_model)}/{_markdown(projection.manifest.evaluator.iop.requested_effort)} |", - "", - "## Execution preflight", - "", - "| sequence | status | results |", - "|---:|---|---:|", - ] - if projection.preflights: - for item in projection.preflights: - path = f"preflight/preflight-{item['sequence']:06d}.json" - raw_paths.add(path) - lines.append(f"| {item['sequence']} | {_markdown(item['status'])} | {len(item['results'])} |") - else: - lines.append("| — | unavailable | 0 |") - - lines.extend(("", "## Attempt outcomes", "", "| cell | repetition | attempt | controller | product | harness | process | artifact | scoring | total | rank |", "|---|---:|---:|---|---|---|---|---|---|---:|---:|")) - if not projection.attempts: - lines.append("| — | — | — | blocked | unavailable | unavailable | unavailable | unavailable | unavailable | — | — |") - for item in projection.attempts: - product = "unavailable" if item.measurement is None else item.measurement.product.status - harness = "unavailable" if item.measurement is None else item.measurement.harness.status - process = "unavailable" if item.measurement is None else item.measurement.process.status - artifact = "unavailable" if item.web is None else item.web.status - total = "—" if item.score.total is None else str(item.score.total) - rank = "—" if item.score.rank is None else str(item.score.rank) - lines.append( - f"| {_markdown(item.cell.id)} | {item.attempt.identity.repetition} | {item.attempt.identity.attempt} | " - f"{_markdown(item.attempt.state)} | {_markdown(product)} | {_markdown(harness)} | " - f"{_markdown(process)} | {_markdown(artifact)} | " - f"{_markdown(item.score.status)} | {total} | {rank} |" - ) - raw_paths.update(item.raw_paths) - raw_paths.update(item.score.raw_paths) - - lines.extend(( - "", "## Quality score breakdown", "", - "| cell/repetition/attempt | category | score | max |", - "|---|---|---:|---:|", - )) - wrote_categories = False - for item in projection.attempts: - label = _attempt_label(item) - for category in item.score.categories: - lines.append( - f"| {_markdown(label)} | {_markdown(category.id)} | " - f"{category.score} | {category.max_score} |" - ) - wrote_categories = True - if not wrote_categories: - lines.append("| — | unavailable | — | — |") - - lines.extend(("", "## Timing and token evidence", "", "| cell/repetition/attempt | time observations | token observations |", "|---|---|---|")) - wrote_metrics = False - for item in projection.attempts: - if item.measurement is None: - continue - label = _attempt_label(item) - timeline = _observation_group(item.measurement.timeline) - usage = _observation_group(item.measurement.usage) - lines.append(f"| {_markdown(label)} | {_markdown(timeline)} | {_markdown(usage)} |") - wrote_metrics = True - if not wrote_metrics: - lines.append("| — | unavailable; reason=not_recorded | unavailable; reason=not_recorded |") - - lines.extend(("", "## Web validation and scoring provenance", "", "| cell/repetition/attempt | web gates | screenshots | score_id | evaluator | scoring condition |", "|---|---|---|---|---|---|")) - if not projection.attempts: - lines.append("| — | unavailable | unavailable | — | — | blocked |") - for item in projection.attempts: - gates, shots = "unavailable", "unavailable" - if item.web is not None: - gates = ", ".join( - f"{gate['id']}={'pass' if gate['passed'] else 'fail'}" - for gate in item.web.record["gates"] - ) - shots = ", ".join( - str(view["screenshot"]["file"]) - for view in item.web.record["viewports"] - ) or "unavailable" - evaluator = "unavailable" if item.score.evaluator is None else "/".join(item.score.evaluator) - condition = ", ".join(item.score.reasons) or "recorded" - lines.append( - f"| {_markdown(_attempt_label(item))} | {_markdown(gates)} | {_markdown(shots)} | " - f"{_markdown(item.score.score_id or '—')} | {_markdown(evaluator)} | {_markdown(condition)} |" - ) - - lines.extend(("", "## Limitations", "", "- Values marked `unavailable` retain the producing source and reason; they are not inferred as zero.", "- Automatic web gates establish eligibility only and contribute no quality points.", "- Equal scored totals share a competition rank; unscored and scoring-failed attempts do not receive a rank.", "", "## Raw evidence index", "", "| contained pointer |", "|---|")) - for path in sorted(raw_paths): - _regular_under(root, path) - lines.append(f"| {_raw_link(path)} |") - lines.append("") - return "\n".join(lines).encode("utf-8") - - -def publish_report(store: RunStore, run: RunIdentity, manifest: Manifest) -> Path: - """Render and idempotently publish the sole run-owned report artifact.""" - data = render_report(project_report(store, run, manifest)) - if len(data) > _MAX_REPORT_BYTES: - raise ReportError("report exceeds the bounded artifact size") - root = Path(run.root) - path = root / REPORT_FILENAME - try: - info = os.lstat(path) - except FileNotFoundError: - info = None - except OSError as exc: - raise ReportError("report publication is unavailable") from exc - if info is not None: - if stat.S_ISLNK(info.st_mode) or not stat.S_ISREG(info.st_mode): - raise ReportError("report publication target is invalid") - try: - existing = path.read_bytes() - except OSError as exc: - raise ReportError("report publication is unavailable") from exc - if existing != data: - raise ReportError("report publication refused an existing target") - return path - flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_CLOEXEC - if hasattr(os, "O_NOFOLLOW"): - flags |= os.O_NOFOLLOW - try: - descriptor = os.open(path, flags, 0o600) - with os.fdopen(descriptor, "wb") as handle: - handle.write(data) - handle.flush() - os.fsync(handle.fileno()) - directory = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC) - try: - os.fsync(directory) - finally: - os.close(directory) - except OSError as exc: - raise ReportError("report publication failed") from exc - return path diff --git a/scripts/agent_benchmark/reporting_test.py b/scripts/agent_benchmark/reporting_test.py deleted file mode 100644 index 235414ff..00000000 --- a/scripts/agent_benchmark/reporting_test.py +++ /dev/null @@ -1,303 +0,0 @@ -"""Regression coverage for the strict benchmark report projection.""" - -from __future__ import annotations - -import argparse -import contextlib -import io -import tempfile -import unittest -from dataclasses import replace -from pathlib import Path -from unittest import mock - -import scripts.agent_comparison_benchmark as cli_module -from scripts.agent_benchmark import scoring as scoring_module -from scripts.agent_benchmark.reporting import ReportError, project_report, publish_report, render_report -from scripts.agent_benchmark import scoring_test - - -class ReportingTest(unittest.TestCase): - """Use the S13 synthetic run builder to cover all S14 terminal states.""" - - def setUp(self) -> None: - self.fixture = Path( - "scripts/fixtures/agent-comparison-benchmark-report.expected.md" - ) - self.harness = scoring_test.ScoringTest() - self.harness.setUp() - self.addCleanup(self.harness.doCleanups) - - def _all_status_run(self) -> None: - # The RunStore retains every immutable attempt. Scoring sees the latest - # slot attempt on each invocation, leaving a representative S14 history. - self.harness._attempt() - scoring_module.score_run( - self.harness.store, self.harness.run, self.harness.manifest, - adapter=scoring_test.FakeScoringAdapter(), - ) - self.harness._attempt() - scoring_module.score_run( - self.harness.store, self.harness.run, self.harness.manifest, - adapter=scoring_test.FakeScoringAdapter(), - ) - self.harness._attempt("failed") - scoring_module.score_run( - self.harness.store, self.harness.run, self.harness.manifest, - adapter=scoring_test.FakeScoringAdapter(), - ) - self.harness._attempt() - scoring_module.score_run( - self.harness.store, self.harness.run, self.harness.manifest, - adapter=scoring_test.FakeScoringAdapter(modes=["malformed"]), - ) - self.harness._attempt() - summary = scoring_module.score_run( - self.harness.store, self.harness.run, self.harness.manifest, - adapter=scoring_test.FakeScoringAdapter(blocked=True), - ) - self.assertEqual(summary.blocked, 1) - - def test_all_status_tie_projection_matches_golden(self) -> None: - self._all_status_run() - projection = project_report( - self.harness.store, self.harness.run, self.harness.manifest - ) - self.assertEqual( - [item.score.status for item in projection.attempts], - ["scored", "scored", "unscored", "scoring_failed", "blocked"], - ) - self.assertEqual( - [item.score.rank for item in projection.attempts], [1, 1, None, None, None] - ) - self.assertEqual( - [ - tuple((category.id, category.score, category.max_score) - for category in item.score.categories) - for item in projection.attempts - ], - [ - ( - ("task_fidelity", 24, 25), - ("visual_hierarchy", 25, 25), - ("responsive_composition", 20, 20), - ("typography_readability", 15, 15), - ("polish_consistency", 15, 15), - ), - ( - ("task_fidelity", 24, 25), - ("visual_hierarchy", 25, 25), - ("responsive_composition", 20, 20), - ("typography_readability", 15, 15), - ("polish_consistency", 15, 15), - ), - (), - (), - (), - ], - ) - self.assertEqual( - render_report(projection), self.fixture.read_bytes() - ) - - def test_detail_tables_include_repetition_in_attempt_label(self) -> None: - self._all_status_run() - projection = project_report( - self.harness.store, self.harness.run, self.harness.manifest - ) - first = projection.attempts[0] - repeated = replace( - first, - attempt=replace( - first.attempt, - identity=replace(first.attempt.identity, repetition=2), - ), - ) - rendered = render_report( - replace(projection, attempts=(first, repeated)) - ).decode("utf-8") - - for section, next_section in ( - ("## Quality score breakdown", "## Timing and token evidence"), - ("## Timing and token evidence", "## Web validation and scoring provenance"), - ("## Web validation and scoring provenance", "## Limitations"), - ): - table = rendered.split(section, 1)[1].split(next_section, 1)[0] - self.assertIn("| cell-sentinel/r1/a1 |", table) - self.assertIn("| cell-sentinel/r2/a1 |", table) - - def test_unscored_and_score_directory_conflict_is_rejected(self) -> None: - attempt = self.harness._attempt("failed") - scoring_module.score_run( - self.harness.store, self.harness.run, self.harness.manifest, - adapter=scoring_test.FakeScoringAdapter(), - ) - (Path(attempt.root) / "scoring" / "score-000001").mkdir() - with self.assertRaisesRegex(ReportError, "report scoring state is invalid"): - publish_report(self.harness.store, self.harness.run, self.harness.manifest) - self.assertFalse((Path(self.harness.run.root) / "report.md").exists()) - - def test_publication_is_idempotent_and_refuses_replacement(self) -> None: - self._all_status_run() - path = publish_report( - self.harness.store, self.harness.run, self.harness.manifest - ) - expected = self.fixture.read_bytes() - self.assertEqual(path.read_bytes(), expected) - self.assertEqual( - publish_report(self.harness.store, self.harness.run, self.harness.manifest), - path, - ) - path.write_bytes(expected + b"changed\n") - with self.assertRaisesRegex(ReportError, "refused an existing target"): - publish_report(self.harness.store, self.harness.run, self.harness.manifest) - self.assertEqual(path.read_bytes(), expected + b"changed\n") - - def test_corrupt_required_measurement_creates_no_report(self) -> None: - attempt = self.harness._attempt() - measurement = Path(attempt.root) / "attempt-measurement.json" - measurement.write_bytes(b"{}\n") - with self.assertRaises(ReportError): - publish_report(self.harness.store, self.harness.run, self.harness.manifest) - self.assertFalse((Path(self.harness.run.root) / "report.md").exists()) - - def test_symlinked_required_evidence_is_not_a_contained_raw_link(self) -> None: - attempt = self.harness._attempt() - measurement = Path(attempt.root) / "attempt-measurement.json" - measurement.unlink() - measurement.symlink_to(Path(self.harness.run.root) / "manifest.json") - with self.assertRaises(ReportError): - publish_report(self.harness.store, self.harness.run, self.harness.manifest) - self.assertFalse((Path(self.harness.run.root) / "report.md").exists()) - - def test_report_target_symlink_is_rejected(self) -> None: - self.harness._attempt() - path = Path(self.harness.run.root) / "report.md" - path.symlink_to("manifest.json") - with self.assertRaisesRegex(ReportError, "publication target is invalid"): - publish_report(self.harness.store, self.harness.run, self.harness.manifest) - - def test_markdown_cell_escaping_is_stable(self) -> None: - from scripts.agent_benchmark.reporting import _markdown - - self.assertEqual(_markdown("line|next\\tail\nlast"), "line\\|next\\\\tail last") - - -class ReportCliTest(unittest.TestCase): - """Deterministic boundary coverage for the public ``report`` CLI handler.""" - - def setUp(self) -> None: - self._tmp = tempfile.TemporaryDirectory() - self.addCleanup(self._tmp.cleanup) - self.manifest_path = Path(self._tmp.name) / "manifest.json" - self.manifest_bytes = b"{\"version\": \"example\"}\n" - self.manifest_path.write_bytes(self.manifest_bytes) - self.report_path = cli_module._REPO_ROOT / "runs" / "example" / "report.md" - self.report_rel = "runs/example/report.md" - - def _namespace(self, run_id: str = "run-1") -> argparse.Namespace: - return argparse.Namespace(manifest=str(self.manifest_path), run_id=run_id) - - def _patch_boundaries(self) -> dict[str, mock.Mock]: - stack = contextlib.ExitStack() - self.addCleanup(stack.close) - load_manifest = stack.enter_context( - mock.patch.object(cli_module, "load_manifest", return_value=mock.sentinel.manifest) - ) - run_store_cls = stack.enter_context(mock.patch.object(cli_module, "RunStore")) - store = run_store_cls.return_value - store.open.return_value = mock.sentinel.run - publish_report_mock = stack.enter_context( - mock.patch.object( - cli_module, "publish_report", return_value=self.report_path - ) - ) - build_registry = stack.enter_context( - mock.patch.object(cli_module, "build_adapter_registry") - ) - return { - "load_manifest": load_manifest, - "RunStore": run_store_cls, - "store": store, - "publish_report": publish_report_mock, - "build_adapter_registry": build_registry, - } - - def _invoke(self, run_id: str = "run-1") -> tuple[int, str, str]: - out = io.StringIO() - err = io.StringIO() - with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): - code = cli_module._cmd_report(self._namespace(run_id=run_id)) - return code, out.getvalue(), err.getvalue() - - def test_success_prints_repo_relative_path_and_exits_zero(self) -> None: - mocks = self._patch_boundaries() - code, out, err = self._invoke(run_id="run-1") - - self.assertEqual(code, cli_module.EXIT_VALID) - mocks["load_manifest"].assert_called_once_with( - self.manifest_path, repo_root=cli_module._REPO_ROOT - ) - mocks["RunStore"].assert_called_once_with(cli_module._REPO_ROOT) - mocks["store"].open.assert_called_once_with( - mock.sentinel.manifest, "run-1", self.manifest_bytes - ) - mocks["publish_report"].assert_called_once_with( - mocks["store"], mock.sentinel.run, mock.sentinel.manifest - ) - self.assertEqual(out, f"ok: report run_id=run-1 path={self.report_rel}\n") - self.assertEqual(err, "") - mocks["build_adapter_registry"].assert_not_called() - - def test_typed_failure_emits_only_closed_line_and_exits_invalid(self) -> None: - mocks = self._patch_boundaries() - mocks["publish_report"].side_effect = ReportError("projection failed") - - code, out, err = self._invoke(run_id="run-9") - - self.assertEqual(code, cli_module.EXIT_INVALID) - self.assertEqual(out, "") - self.assertEqual(err, "error: benchmark report is unavailable\n") - mocks["build_adapter_registry"].assert_not_called() - - def test_generic_failure_emits_only_closed_line_and_exits_invalid(self) -> None: - mocks = self._patch_boundaries() - mocks["publish_report"].side_effect = ValueError("unexpected boundary") - - code, out, err = self._invoke(run_id="run-9") - - self.assertEqual(code, cli_module.EXIT_INVALID) - self.assertEqual(out, "") - self.assertEqual(err, "error: benchmark report is unavailable\n") - mocks["build_adapter_registry"].assert_not_called() - - def test_two_successful_calls_delegate_idempotently_to_reporter(self) -> None: - mocks = self._patch_boundaries() - - for _ in range(2): - code, out, err = self._invoke(run_id="run-1") - self.assertEqual(code, cli_module.EXIT_VALID) - self.assertEqual(out, f"ok: report run_id=run-1 path={self.report_rel}\n") - self.assertEqual(err, "") - - # Each CLI call delegates exactly once to the strict reporter boundary; - # the reporter itself remains the sole idempotent publication surface. - self.assertEqual(mocks["publish_report"].call_count, 2) - self.assertEqual(mocks["store"].open.call_count, 2) - mocks["build_adapter_registry"].assert_not_called() - - def test_adapter_registry_is_never_constructed(self) -> None: - mocks = self._patch_boundaries() - - code, _, _ = self._invoke(run_id="run-1") - self.assertEqual(code, cli_module.EXIT_VALID) - - mocks["publish_report"].side_effect = ReportError("publication failed") - code, _, _ = self._invoke(run_id="run-1") - self.assertEqual(code, cli_module.EXIT_INVALID) - - mocks["build_adapter_registry"].assert_not_called() - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/agent_benchmark/rubric.py b/scripts/agent_benchmark/rubric.py deleted file mode 100644 index 5288b26d..00000000 --- a/scripts/agent_benchmark/rubric.py +++ /dev/null @@ -1,197 +0,0 @@ -"""Strict versioned benchmark worksheet contract. - -Automatic web gates establish scoring eligibility; they are intentionally not -represented in this 100-point worksheet and can never contribute points. -""" - -from __future__ import annotations - -import json -import os -import stat -from dataclasses import dataclass -from pathlib import Path -from types import MappingProxyType -from typing import Any, Mapping - -from scripts.agent_benchmark.manifest import ( - ONE_SHOT_RUBRIC_VERSION, - RUBRIC_VERSION, -) - - -MAX_WORKSHEET_BYTES = 64 * 1024 -MAX_EVIDENCE_CHARS = 4096 -RUBRIC_CATEGORIES = ( - ("task_fidelity", 25), - ("visual_hierarchy", 25), - ("responsive_composition", 20), - ("typography_readability", 15), - ("polish_consistency", 15), -) -ONE_SHOT_RUBRIC_CATEGORIES = ( - ("requirements_fidelity", 25), - ("visual_completeness", 25), - ("responsive_accessibility", 15), - ("image_detail_usage", 10), - ("behavior_stability", 10), - ("code_quality", 10), - ("self_verification", 5), -) -RUBRIC_CATEGORIES_BY_VERSION: Mapping[str, tuple[tuple[str, int], ...]] = ( - MappingProxyType( - { - RUBRIC_VERSION: RUBRIC_CATEGORIES, - ONE_SHOT_RUBRIC_VERSION: ONE_SHOT_RUBRIC_CATEGORIES, - } - ) -) - - -class RubricError(Exception): - """A worksheet is missing, malformed, non-canonical, or out of bounds.""" - - -def rubric_categories(version: str) -> tuple[tuple[str, int], ...]: - """Return the immutable ordered category table for a known rubric version.""" - if not isinstance(version, str): - raise RubricError("rubric version is invalid") - try: - return RUBRIC_CATEGORIES_BY_VERSION[version] - except KeyError as exc: - raise RubricError("rubric version is invalid") from exc - - -@dataclass(frozen=True) -class CategoryScore: - id: str - max_score: int - score: int - evidence: str - - -@dataclass(frozen=True) -class Worksheet: - rubric_version: str - categories: tuple[CategoryScore, ...] - total: int - - def as_dict(self) -> dict[str, Any]: - return { - "rubric_version": self.rubric_version, - "categories": [ - { - "id": item.id, - "max_score": item.max_score, - "score": item.score, - "evidence": item.evidence, - } - for item in self.categories - ], - "total": self.total, - } - - -def canonical_worksheet_bytes(worksheet: Worksheet) -> bytes: - if not isinstance(worksheet, Worksheet): - raise RubricError("worksheet object is invalid") - return ( - json.dumps( - worksheet.as_dict(), - sort_keys=True, - separators=(",", ":"), - ensure_ascii=True, - ).encode("ascii") - + b"\n" - ) - - -def validate_worksheet( - value: Any, expected_version: str | None = None -) -> Worksheet: - if not isinstance(value, dict) or set(value) != { - "rubric_version", "categories", "total", - }: - raise RubricError("worksheet schema is invalid") - rubric_version = value["rubric_version"] - expected_categories = rubric_categories(rubric_version) - if expected_version is not None: - rubric_categories(expected_version) - if expected_version is not None and rubric_version != expected_version: - raise RubricError("worksheet rubric version is invalid") - raw_categories = value["categories"] - if not isinstance(raw_categories, list) or len(raw_categories) != len( - expected_categories - ): - raise RubricError("worksheet categories are invalid") - - categories: list[CategoryScore] = [] - for raw, (expected_id, expected_max) in zip( - raw_categories, expected_categories - ): - if not isinstance(raw, dict) or set(raw) != { - "id", "max_score", "score", "evidence", - }: - raise RubricError("worksheet category schema is invalid") - score = raw["score"] - evidence = raw["evidence"] - if ( - raw["id"] != expected_id - or raw["max_score"] != expected_max - or isinstance(score, bool) - or not isinstance(score, int) - or not 0 <= score <= expected_max - or not isinstance(evidence, str) - or not evidence.strip() - or len(evidence) > MAX_EVIDENCE_CHARS - or any(ord(char) < 0x20 and char not in "\n\t" for char in evidence) - ): - raise RubricError("worksheet category is invalid") - categories.append( - CategoryScore(expected_id, expected_max, score, evidence) - ) - - total = value["total"] - expected_total = sum(item.score for item in categories) - if ( - isinstance(total, bool) - or not isinstance(total, int) - or total != expected_total - or not 0 <= total <= 100 - ): - raise RubricError("worksheet total is invalid") - return Worksheet(rubric_version, tuple(categories), total) - - -def load_worksheet( - path: str | Path, expected_version: str | None = None -) -> Worksheet: - target = Path(path) - flags = os.O_RDONLY | os.O_CLOEXEC | os.O_NONBLOCK - if hasattr(os, "O_NOFOLLOW"): - flags |= os.O_NOFOLLOW - try: - fd = os.open(target, flags) - except OSError as exc: - raise RubricError("worksheet is unavailable") from exc - try: - info = os.fstat(fd) - if not stat.S_ISREG(info.st_mode) or info.st_size > MAX_WORKSHEET_BYTES: - raise RubricError("worksheet must be a bounded regular file") - raw = bytearray() - while len(raw) < info.st_size: - chunk = os.read(fd, info.st_size - len(raw)) - if not chunk: - raise RubricError("worksheet changed while reading") - raw.extend(chunk) - if os.read(fd, 1): - raise RubricError("worksheet changed while reading") - except OSError as exc: - raise RubricError("worksheet is unavailable") from exc - finally: - os.close(fd) - try: - value = json.loads(bytes(raw).decode("utf-8")) - except (UnicodeDecodeError, json.JSONDecodeError) as exc: - raise RubricError("worksheet JSON is invalid") from exc - return validate_worksheet(value, expected_version=expected_version) diff --git a/scripts/agent_benchmark/rubric_test.py b/scripts/agent_benchmark/rubric_test.py deleted file mode 100644 index 70399843..00000000 --- a/scripts/agent_benchmark/rubric_test.py +++ /dev/null @@ -1,157 +0,0 @@ -from __future__ import annotations - -import hashlib -import json -import os -import tempfile -import unittest -from pathlib import Path - -from scripts.agent_benchmark.manifest import ( - ONE_SHOT_RUBRIC_VERSION, - RUBRIC_VERSION, -) -from scripts.agent_benchmark.rubric import ( - ONE_SHOT_RUBRIC_CATEGORIES, - RUBRIC_CATEGORIES, - RUBRIC_CATEGORIES_BY_VERSION, - RubricError, - canonical_worksheet_bytes, - load_worksheet, - rubric_categories, - validate_worksheet, -) - - -def _worksheet(rubric_version: str = RUBRIC_VERSION) -> dict: - categories = [ - { - "id": ident, - "max_score": maximum, - "score": maximum, - "evidence": f"Evidence for {ident}.", - } - for ident, maximum in rubric_categories(rubric_version) - ] - return { - "rubric_version": rubric_version, - "categories": categories, - "total": 100, - } - - -class RubricTest(unittest.TestCase): - def test_exact_categories_and_total_are_accepted(self): - worksheet = validate_worksheet(_worksheet()) - self.assertEqual(worksheet.total, 100) - self.assertEqual( - [(item.id, item.max_score) for item in worksheet.categories], - list(RUBRIC_CATEGORIES), - ) - self.assertEqual( - validate_worksheet(json.loads(canonical_worksheet_bytes(worksheet))), - worksheet, - ) - self.assertEqual( - hashlib.sha256(canonical_worksheet_bytes(worksheet)).hexdigest(), - "233727170226ab30409657007751c13616a16c83ca75700e15a2c8dce56a96ee", - ) - - def test_one_shot_rubric_exact_categories_and_total_are_accepted(self): - expected = ( - ("requirements_fidelity", 25), - ("visual_completeness", 25), - ("responsive_accessibility", 15), - ("image_detail_usage", 10), - ("behavior_stability", 10), - ("code_quality", 10), - ("self_verification", 5), - ) - self.assertEqual(ONE_SHOT_RUBRIC_CATEGORIES, expected) - self.assertEqual(rubric_categories(ONE_SHOT_RUBRIC_VERSION), expected) - self.assertEqual(tuple(RUBRIC_CATEGORIES_BY_VERSION), ( - RUBRIC_VERSION, - ONE_SHOT_RUBRIC_VERSION, - )) - self.assertEqual(sum(maximum for _, maximum in RUBRIC_CATEGORIES), 100) - self.assertEqual(sum(maximum for _, maximum in expected), 100) - - worksheet = validate_worksheet( - _worksheet(ONE_SHOT_RUBRIC_VERSION), - expected_version=ONE_SHOT_RUBRIC_VERSION, - ) - self.assertEqual(worksheet.rubric_version, ONE_SHOT_RUBRIC_VERSION) - self.assertEqual(worksheet.total, 100) - self.assertEqual( - [(item.id, item.max_score) for item in worksheet.categories], - list(expected), - ) - - def test_unknown_and_cross_version_worksheets_are_rejected(self): - with self.assertRaises(RubricError): - rubric_categories("unknown-rubric-v1") - with self.assertRaises(RubricError): - validate_worksheet(_worksheet(), expected_version=ONE_SHOT_RUBRIC_VERSION) - with self.assertRaises(RubricError): - validate_worksheet( - _worksheet(ONE_SHOT_RUBRIC_VERSION), - expected_version=RUBRIC_VERSION, - ) - unknown = _worksheet() - unknown["rubric_version"] = "unknown-rubric-v1" - with self.assertRaises(RubricError): - validate_worksheet(unknown) - - def test_cross_version_worksheet_is_rejected(self): - with tempfile.TemporaryDirectory() as temp: - path = Path(temp) / "worksheet.json" - path.write_text(json.dumps(_worksheet()), encoding="utf-8") - with self.assertRaises(RubricError): - load_worksheet(path, expected_version=ONE_SHOT_RUBRIC_VERSION) - - def test_missing_extra_reordered_and_out_of_range_values_fail(self): - cases = [] - missing = _worksheet() - missing["categories"] = missing["categories"][:-1] - cases.append(missing) - extra = _worksheet() - extra["automatic_gate_points"] = 1 - cases.append(extra) - reordered = _worksheet() - reordered["categories"] = list(reversed(reordered["categories"])) - cases.append(reordered) - too_high = _worksheet() - too_high["categories"][0]["score"] = 26 - too_high["total"] = 101 - cases.append(too_high) - bad_total = _worksheet() - bad_total["total"] = 0 - cases.append(bad_total) - empty_evidence = _worksheet() - empty_evidence["categories"][0]["evidence"] = " " - cases.append(empty_evidence) - for value in cases: - with self.subTest(value=value): - with self.assertRaises(RubricError): - validate_worksheet(value) - - def test_loader_refuses_nonregular_and_malformed_files(self): - with tempfile.TemporaryDirectory() as temp: - root = Path(temp) - valid = root / "worksheet.json" - valid.write_bytes(json.dumps(_worksheet()).encode()) - self.assertEqual(load_worksheet(valid).total, 100) - - malformed = root / "malformed.json" - malformed.write_bytes(b"{}") - with self.assertRaises(RubricError): - load_worksheet(malformed) - - link = root / "link.json" - os.symlink(valid, link) - with self.assertRaises(RubricError): - load_worksheet(link) - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/agent_benchmark/scoring.py b/scripts/agent_benchmark/scoring.py deleted file mode 100644 index 93a140b1..00000000 --- a/scripts/agent_benchmark/scoring.py +++ /dev/null @@ -1,2235 +0,0 @@ -"""Blind, append-only S13 scoring over immutable execution attempts. - -The original cell identity remains under ``cells/`` and in a run-owned mapping -that is never copied into the evaluator tree. A scorer receives only one -opaque ``blind/`` directory, anonymous input bytes, the manifest-selected -rubric, and a fresh session/output pair. -""" - -from __future__ import annotations - -import hashlib -import json -import os -import re -import secrets -import stat -import time -from dataclasses import dataclass -from pathlib import Path, PurePosixPath -from typing import Any, Callable, Mapping, Protocol - -from scripts.agent_benchmark.attempts import ( - Attempt, - AttemptStateError, - PreflightObservation, - RunIdentity, - RunStore, - TERMINAL_STATES, -) -from scripts.agent_benchmark.connectivity import ( - CallerCapability, - canonical_evidence_bytes, - validate_result, -) -from scripts.agent_benchmark.manifest import Manifest, MatrixCell, Timeout -from scripts.agent_benchmark.lifecycle import ( - JOURNAL_VERSION, - LifecycleRecoveryError, - REASON_CONTROLLER_LOST, - REASON_RECOVERED_STOP, - SupervisorLocator, - TERMINAL_REASONS, - recover_invocation, -) -from scripts.agent_benchmark.rubric import ( - RubricError, - Worksheet, - canonical_worksheet_bytes, - load_worksheet, - rubric_categories, -) -from scripts.agent_benchmark.web_validation import ( - GENERATED_FILES, - WEB_GATES, - WEB_VALIDATION_FILENAME, - WebValidationError, - load_web_validation, -) - - -SCORING_VERSION = 2 -SCORE_RE = re.compile(r"^score-([0-9]{6})$") -BLIND_ID_RE = re.compile(r"^blind-[0-9a-f]{32}$") -DIGEST_RE = re.compile(r"^sha256:[0-9a-f]{64}$") -IMAGE_SUFFIXES = frozenset((".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg")) -MAX_INPUT_FILE_BYTES = 32 * 1024 * 1024 -MAX_RECORD_BYTES = 256 * 1024 -UNSCORED_FILENAME = "unscored.json" -ALLOCATION_FILENAME = "allocation.json" -INPUT_FILENAME = "input.json" -RESULT_FILENAME = "result.json" -RUNNER_FILENAME = "runner.json" -SCORING_STATUSES = ("scored", "unscored", "scoring_failed", "blocked") -HARD_ELIGIBILITY_GATES = frozenset( - ("generated_files", "static_safety", "images", "network") -) -QUALITY_SCORING_GATES = frozenset(("console", "responsive", "accessibility")) -SCORING_INVOCATION_REASONS = TERMINAL_REASONS + ( - "binding_mismatch", "evaluator_failed", -) -_POST_CLEANUP_TIMEOUT_SECONDS = 300.0 -_POST_CLEANUP_QUIET_SECONDS = 0.2 -_POST_CLEANUP_POLL_SECONDS = 0.01 - -if ( - HARD_ELIGIBILITY_GATES & QUALITY_SCORING_GATES - or HARD_ELIGIBILITY_GATES | QUALITY_SCORING_GATES != frozenset(WEB_GATES) -): - raise RuntimeError("scoring web gate partition is invalid") - - -class ScoringError(Exception): - """Scoring state or evaluator evidence cannot be trusted.""" - - -@dataclass(frozen=True) -class BlindWorkspace: - blind_id: str - root: str - input_dir: str - session_dir: str - output_dir: str - input_digest: str - session_identity: str - - -@dataclass(frozen=True) -class ScoringInvocationResult: - product: str - harness: str - process: str - process_exit_code: int | None - process_signal: int | None - reason: str - effective_binding: tuple[str, str, str, str] | None - - def __post_init__(self) -> None: - if ( - self.product not in {"succeeded", "failed", "unknown"} - or self.harness not in {"passed", "failed"} - or self.process not in { - "exited", "signalled", "timed_out", "cancelled", "not_started" - } - or ( - self.process_exit_code is not None - and ( - not isinstance(self.process_exit_code, int) - or isinstance(self.process_exit_code, bool) - ) - ) - or ( - self.process_signal is not None - and ( - not isinstance(self.process_signal, int) - or isinstance(self.process_signal, bool) - ) - ) - or (self.process == "signalled" and self.process_signal is None) - or ( - self.process in {"exited", "not_started"} - and self.process_signal is not None - ) - or (self.process == "not_started" and self.process_exit_code is not None) - or self.reason not in SCORING_INVOCATION_REASONS - or (self.harness == "passed") != (self.reason == "success") - ): - raise ScoringError("scoring invocation result is invalid") - - -@dataclass(frozen=True) -class ScoringEvidenceFinalization: - """Closed post-invocation projection from the secret-owning adapter.""" - - safe: bool - reason: str = "" - - -@dataclass(frozen=True) -class ProducerIdentity: - """Producer-only identity values, separated from evaluator evidence.""" - - exact_tokens: tuple[str, ...] - path_tokens: tuple[str, ...] - producer_tokens: tuple[str, ...] - evaluator_shared_tokens: tuple[str, ...] - - -@dataclass(frozen=True) -class ScoringSummary: - run_id: str - scored: int - unscored: int - scoring_failed: int - blocked: int - - -class ScoringAdapter(Protocol): - capability: CallerCapability - - def preflight(self, cell: MatrixCell) -> PreflightObservation: - """Return one manifest-bound, secret-free evaluator observation.""" - - def invoke( - self, - cell: MatrixCell, - blind: BlindWorkspace, - task_payload: bytes, - timeout: Timeout, - on_started: Callable[[SupervisorLocator, str], None], - ) -> ScoringInvocationResult: - """Run exactly one fresh evaluator session for this score id.""" - - def finalize_evidence( - self, blind: BlindWorkspace - ) -> ScoringEvidenceFinalization: - """Scrub secret-owned output after cleanup and return a closed status.""" - - -def _digest(data: bytes) -> str: - return "sha256:" + hashlib.sha256(data).hexdigest() - - -def _json_bytes(value: Any) -> bytes: - return ( - json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True) - .encode("ascii") - + b"\n" - ) - - -def _fsync_dir(path: Path) -> None: - fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY) - try: - os.fsync(fd) - finally: - os.close(fd) - - -def _mkdir_new(path: Path) -> None: - try: - path.mkdir(mode=0o700) - _fsync_dir(path.parent) - except FileExistsError as exc: - raise ScoringError("scoring allocation collision") from exc - except OSError as exc: - raise ScoringError("scoring directory is unavailable") from exc - - -def _ensure_directory(path: Path) -> None: - try: - mode = os.lstat(path).st_mode - except OSError as exc: - raise ScoringError("scoring directory is unavailable") from exc - if not stat.S_ISDIR(mode) or path.is_symlink(): - raise ScoringError("scoring directory is invalid") - - -def _write_new(path: Path, data: bytes) -> None: - flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_CLOEXEC - if hasattr(os, "O_NOFOLLOW"): - flags |= os.O_NOFOLLOW - try: - fd = os.open(path, flags, 0o600) - except OSError as exc: - raise ScoringError("scoring evidence collision") from exc - try: - os.write(fd, data) - os.fsync(fd) - except OSError as exc: - raise ScoringError("scoring evidence write failed") from exc - finally: - os.close(fd) - _fsync_dir(path.parent) - - -def _read_regular(path: Path, label: str, *, maximum: int = MAX_RECORD_BYTES) -> bytes: - flags = os.O_RDONLY | os.O_CLOEXEC | os.O_NONBLOCK - if hasattr(os, "O_NOFOLLOW"): - flags |= os.O_NOFOLLOW - try: - fd = os.open(path, flags) - except OSError as exc: - raise ScoringError(f"{label} is unavailable") from exc - try: - info = os.fstat(fd) - if not stat.S_ISREG(info.st_mode) or info.st_size > maximum: - raise ScoringError(f"{label} is invalid") - data = bytearray() - while len(data) < info.st_size: - chunk = os.read(fd, info.st_size - len(data)) - if not chunk: - raise ScoringError(f"{label} changed while reading") - data.extend(chunk) - if os.read(fd, 1): - raise ScoringError(f"{label} changed while reading") - return bytes(data) - except OSError as exc: - raise ScoringError(f"{label} is unavailable") from exc - finally: - os.close(fd) - - -def _load_canonical(path: Path, label: str) -> dict[str, Any]: - raw = _read_regular(path, label) - try: - value = json.loads(raw.decode("ascii")) - except (UnicodeDecodeError, json.JSONDecodeError) as exc: - raise ScoringError(f"{label} is invalid") from exc - if not isinstance(value, dict) or _json_bytes(value) != raw: - raise ScoringError(f"{label} is not canonical") - return value - - -def _relative(value: str) -> str: - if not isinstance(value, str) or not value or "\\" in value or ":" in value: - raise ScoringError("scoring path is invalid") - path = PurePosixPath(value) - if path.is_absolute() or str(path) != value or any( - part in ("", ".", "..") for part in path.parts - ): - raise ScoringError("scoring path is invalid") - return value - - -def _safe_source(root: Path, relative: str) -> bytes: - """Read one bounded regular file without following any component.""" - parts = PurePosixPath(_relative(relative)).parts - directory_flags = os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC - file_flags = os.O_RDONLY | os.O_CLOEXEC | os.O_NONBLOCK - if hasattr(os, "O_NOFOLLOW"): - directory_flags |= os.O_NOFOLLOW - file_flags |= os.O_NOFOLLOW - descriptors: list[int] = [] - try: - current = os.open(root, directory_flags) - descriptors.append(current) - for component in parts[:-1]: - current = os.open(component, directory_flags, dir_fd=current) - descriptors.append(current) - fd = os.open(parts[-1], file_flags, dir_fd=current) - descriptors.append(fd) - info = os.fstat(fd) - if not stat.S_ISREG(info.st_mode) or info.st_size > MAX_INPUT_FILE_BYTES: - raise OSError("not a bounded regular file") - data = bytearray() - while len(data) < info.st_size: - chunk = os.read(fd, info.st_size - len(data)) - if not chunk: - raise OSError("short read") - data.extend(chunk) - if os.read(fd, 1): - raise OSError("file grew while reading") - return bytes(data) - except OSError as exc: - raise ScoringError("blind input source is invalid") from exc - finally: - for fd in reversed(descriptors): - os.close(fd) - - -def _write_relative(root: Path, relative: str, data: bytes) -> None: - target = root / _relative(relative) - try: - target.parent.mkdir(mode=0o700, parents=True, exist_ok=True) - except OSError as exc: - raise ScoringError("blind input path is unavailable") from exc - current = root - for part in target.relative_to(root).parts[:-1]: - current = current / part - _ensure_directory(current) - _write_new(target, data) - - -def _identity_values(manifest: Manifest, attempt: Attempt) -> ProducerIdentity: - cell = next( - (item for item in manifest.matrix if item.id == attempt.identity.cell_id), - None, - ) - if cell is None: - raise ScoringError("execution attempt cell is unavailable") - evaluator = manifest.evaluator.iop - shared = { - evaluator.route_kind, - evaluator.route_id, - evaluator.request_model, - evaluator.requested_effort, - } - shared.update(item.model for item in evaluator.expected_bindings) - shared.update( - item.effort for item in evaluator.expected_bindings if item.effort - ) - producer = { - cell.iop.route_kind, - cell.iop.route_id, - cell.iop.request_model, - cell.iop.requested_effort, - } - producer.update(item.model for item in cell.iop.expected_bindings) - producer.update( - item.effort for item in cell.iop.expected_bindings if item.effort - ) - exact = {attempt.identity.cell_id} - if cell.caller != manifest.evaluator.caller: - exact.add(cell.caller) - return ProducerIdentity( - exact_tokens=tuple(sorted(value for value in exact if value)), - path_tokens=(str(Path(attempt.root).resolve()),), - producer_tokens=tuple(sorted(value for value in producer if value)), - evaluator_shared_tokens=tuple(sorted(value for value in shared if value)), - ) - - -def _ascii_identity_bytes(value: str) -> bytes: - try: - return value.encode("ascii") - except UnicodeEncodeError: - return b"" - - -def _route_token_present_bytes(data: bytes, value: str) -> bool: - candidate = _ascii_identity_bytes(value) - if not candidate: - return False - pattern = rb"(? bool: - """Match an ASCII caller/cell identity on the original evidence bytes.""" - candidate = _ascii_identity_bytes(value) - if not candidate: - return False - pattern = rb"(? bool: - """Detect producer identity without rejecting legitimate evaluator binding.""" - if any( - _exact_identity_present_bytes(data, value) - for value in identity.exact_tokens - ): - return True - shared = {value.casefold() for value in identity.evaluator_shared_tokens} - if any( - value.casefold() not in shared - and _route_token_present_bytes(data, value) - for value in identity.producer_tokens - ): - return True - - # Paths can contain non-ASCII values, so retain decoded comparison only for - # that typed field. ASCII identities above are matched before lossy decode - # can join byte runs across invalid image/screenshot bytes. - text = data.decode("utf-8", errors="ignore") - lowered = text.casefold() - for value in identity.path_tokens: - if value.casefold() in lowered: - return True - return False - - -def _path_bytes(value: str) -> bytes: - # Frame paths explicitly instead of through the process filesystem codec so - # ordinary Unicode keeps canonical UTF-8 bytes under any locale while - # surrogateescaped raw POSIX filename bytes are restored exactly. - try: - return value.encode("utf-8", errors="surrogateescape") - except UnicodeEncodeError as exc: - raise ScoringError("scoring path is invalid") from exc - - -def _input_digest(files: list[tuple[str, bytes]]) -> str: - framed = bytearray(b"IOP-BENCH-BLIND-INPUT-V1\0") - for relative, data in sorted(files): - path_bytes = _path_bytes(relative) - framed += len(path_bytes).to_bytes(8, "big") + path_bytes - framed += len(data).to_bytes(8, "big") + data - return _digest(bytes(framed)) - - -def _evaluator_cell(manifest: Manifest) -> MatrixCell: - return MatrixCell("evaluator", manifest.evaluator.caller, manifest.evaluator.iop) - - -def _evaluator_payload(manifest: Manifest) -> dict[str, Any]: - evaluator = manifest.evaluator - return { - "caller": evaluator.caller, - "route_kind": evaluator.iop.route_kind, - "route_id": evaluator.iop.route_id, - "request_model": evaluator.iop.request_model, - "requested_effort": evaluator.iop.requested_effort, - "expected_bindings": [ - { - "stage": item.stage, - "model": item.model, - "effort": item.effort, - } - for item in evaluator.iop.expected_bindings - ], - } - - -def _score_root(attempt: Attempt, *, create: bool) -> Path: - root = Path(attempt.root) / "scoring" - if root.exists() or root.is_symlink(): - _ensure_directory(root) - elif create: - _mkdir_new(root) - return root - - -def _score_dirs(root: Path) -> tuple[Path, ...]: - if not root.exists() and not root.is_symlink(): - return () - _ensure_directory(root) - found: list[Path] = [] - for expected, child in enumerate(sorted(root.iterdir()), start=1): - if child.name == UNSCORED_FILENAME: - continue - match = SCORE_RE.fullmatch(child.name) - if match is None or int(match.group(1)) != expected: - raise ScoringError("scoring attempt sequence is invalid") - _ensure_directory(child) - found.append(child) - return tuple(found) - - -def _record_digest(path: Path, label: str) -> str: - return _digest(_read_regular(path, label, maximum=MAX_INPUT_FILE_BYTES)) - - -def _load_json(path: Path, label: str) -> dict[str, Any]: - try: - value = json.loads( - _read_regular(path, label, maximum=MAX_INPUT_FILE_BYTES).decode("utf-8") - ) - except (UnicodeDecodeError, json.JSONDecodeError) as exc: - raise ScoringError(f"{label} is invalid") from exc - if not isinstance(value, dict): - raise ScoringError(f"{label} is invalid") - return value - - -def _contained(path: Path, root: Path) -> bool: - try: - path.resolve(strict=False).relative_to(root.resolve(strict=True)) - except (OSError, RuntimeError, ValueError): - return False - return True - - -def _locator_payload(locator: SupervisorLocator) -> dict[str, Any]: - return { - "supervisor_pid": locator.supervisor_pid, - "start_identity": locator.start_identity, - "socket_path": locator.socket_path, - "challenge": locator.challenge, - "control_dir": locator.control_dir, - "created_at": locator.created_at, - } - - -def _locator_public(locator: SupervisorLocator) -> dict[str, Any]: - return { - "supervisor_pid": locator.supervisor_pid, - "start_identity": locator.start_identity, - "socket_path": locator.socket_path, - "control_dir": locator.control_dir, - "challenge_digest": hashlib.sha256( - locator.challenge.encode("utf-8") - ).hexdigest(), - "created_at": locator.created_at, - } - - -def _validate_locator( - locator: SupervisorLocator, - blind_root: Path, - *, - control_target: Path | None = None, -) -> Path: - if ( - not isinstance(locator, SupervisorLocator) - or isinstance(locator.supervisor_pid, bool) - or locator.supervisor_pid < 1 - or any( - not isinstance(value, str) or not value - for value in ( - locator.start_identity, - locator.socket_path, - locator.challenge, - locator.control_dir, - locator.created_at, - ) - ) - ): - raise ScoringError("evaluator locator is invalid") - control = Path(locator.control_dir) - socket = Path(locator.socket_path) - if control_target is None: - try: - resolved_control = control.resolve(strict=True) - except OSError as exc: - raise ScoringError("evaluator control directory is unavailable") from exc - else: - try: - resolved_control = control_target.resolve(strict=True) - except OSError as exc: - raise ScoringError("evaluator control directory is unavailable") from exc - if ( - not control.is_absolute() - or not socket.is_absolute() - or socket.parent != control - or not _contained(resolved_control, blind_root) - or not _contained(resolved_control / socket.name, resolved_control) - ): - raise ScoringError("evaluator locator escapes blind workspace") - if control.exists() or control.is_symlink(): - try: - if control.resolve(strict=True) != resolved_control: - raise ScoringError("evaluator control alias is invalid") - except OSError as exc: - raise ScoringError("evaluator control alias is invalid") from exc - registered = _load_json( - resolved_control / "locator.json", "registered evaluator locator" - ) - if registered != _locator_payload(locator): - raise ScoringError("registered evaluator locator is invalid") - return resolved_control - - -def _publish_runner( - score_root: Path, - blind_root: Path, - blind: BlindWorkspace, - run: RunIdentity, - attempt: Attempt, - locator: SupervisorLocator, - invocation_digest: str, -) -> None: - if not isinstance(invocation_digest, str) or not DIGEST_RE.fullmatch( - invocation_digest - ): - raise ScoringError("evaluator invocation digest is invalid") - control_target = _validate_locator(locator, blind_root) - control = Path(locator.control_dir) - alias = control.parent if control != control_target else None - if alias is not None: - try: - if not alias.is_symlink() or alias.resolve(strict=True) != control_target.parent: - raise ScoringError("evaluator control alias is invalid") - except OSError as exc: - raise ScoringError("evaluator control alias is invalid") from exc - record = { - "record": "scoring-runner", - "scoring_version": SCORING_VERSION, - "run_id": run.run_id, - "cell_id": attempt.identity.cell_id, - "repetition": attempt.identity.repetition, - "attempt": attempt.identity.attempt, - "score_id": score_root.name, - "blind_id": blind.blind_id, - "session_identity": blind.session_identity, - "spec_digest": invocation_digest, - "control_target": str(control_target), - "control_alias": "" if alias is None else str(alias), - "locator": _locator_payload(locator), - } - _write_new(score_root / RUNNER_FILENAME, _json_bytes(record)) - - -def _validate_runner( - score_root: Path, - blind_root: Path, - allocation: Mapping[str, Any], - run: RunIdentity, - attempt: Attempt, -) -> tuple[dict[str, Any], SupervisorLocator, str] | None: - path = score_root / RUNNER_FILENAME - if not path.exists() and not path.is_symlink(): - return None - value = _load_canonical(path, "evaluator runner") - fields = { - "record", "scoring_version", "run_id", "cell_id", "repetition", - "attempt", "score_id", "blind_id", "session_identity", "spec_digest", - "control_target", "control_alias", "locator", - } - raw_locator = value.get("locator") - locator_fields = { - "supervisor_pid", "start_identity", "socket_path", "challenge", - "control_dir", "created_at", - } - if not isinstance(raw_locator, dict) or set(raw_locator) != locator_fields: - raise ScoringError("evaluator runner is invalid") - try: - locator = SupervisorLocator(**raw_locator) - except TypeError as exc: - raise ScoringError("evaluator runner is invalid") from exc - if ( - set(value) != fields - or value["record"] != "scoring-runner" - or value["scoring_version"] != SCORING_VERSION - or value["run_id"] != run.run_id - or value["cell_id"] != attempt.identity.cell_id - or value["repetition"] != attempt.identity.repetition - or value["attempt"] != attempt.identity.attempt - or value["score_id"] != score_root.name - or value["blind_id"] != allocation["blind_id"] - or value["session_identity"] != allocation["session_identity"] - or not isinstance(value["spec_digest"], str) - or not DIGEST_RE.fullmatch(value["spec_digest"]) - or not isinstance(value["control_target"], str) - or not value["control_target"] - or not isinstance(value["control_alias"], str) - ): - raise ScoringError("evaluator runner is invalid") - control_target = Path(value["control_target"]) - resolved_control = _validate_locator( - locator, blind_root, control_target=control_target - ) - if resolved_control != control_target.resolve(strict=True): - raise ScoringError("evaluator runner control target is invalid") - alias = value["control_alias"] - if alias: - alias_path = Path(alias) - if Path(locator.control_dir).parent != alias_path: - raise ScoringError("evaluator runner control alias is invalid") - if alias_path.exists() or alias_path.is_symlink(): - try: - if ( - not alias_path.is_symlink() - or alias_path.resolve(strict=True) != resolved_control.parent - ): - raise ScoringError("evaluator runner control alias is invalid") - except OSError as exc: - raise ScoringError("evaluator runner control alias is invalid") from exc - elif Path(locator.control_dir) != resolved_control: - raise ScoringError("evaluator runner control alias is invalid") - return value, locator, _digest(_read_regular(path, "evaluator runner")) - - -def _validate_cleanup_receipt( - locator: SupervisorLocator, - *, - expected_reason: str | None = None, - control_target: Path | None = None, -) -> tuple[dict[str, Any], str]: - control = control_target or Path(locator.control_dir) - path = control / "cleanup-receipt.json" - receipt = _load_json(path, "evaluator cleanup receipt") - required = { - "receipt_version", "supervisor_pid", "challenge_digest", "reason", - "exit_code", "signal", "caller_launched", "cleanup_complete", - "process_group_alive", "completed_at", - } - if ( - set(receipt) != required - or receipt["receipt_version"] != 1 - or receipt["supervisor_pid"] != locator.supervisor_pid - or receipt["challenge_digest"] - != hashlib.sha256(locator.challenge.encode("utf-8")).hexdigest() - or receipt["reason"] not in TERMINAL_REASONS - or (expected_reason is not None and receipt["reason"] != expected_reason) - or not isinstance(receipt["caller_launched"], bool) - or not isinstance(receipt["cleanup_complete"], bool) - or (not receipt["cleanup_complete"]) - != (receipt["reason"] == "cleanup_failed") - or receipt["process_group_alive"] is not False - or not isinstance(receipt["completed_at"], str) - ): - raise ScoringError("evaluator cleanup receipt is invalid") - for name in ("exit_code", "signal"): - if receipt[name] is not None and ( - isinstance(receipt[name], bool) or not isinstance(receipt[name], int) - ): - raise ScoringError("evaluator cleanup receipt is invalid") - return receipt, _digest(_read_regular(path, "evaluator cleanup receipt")) - - -def _validate_lifecycle_binding( - blind_root: Path, - locator: SupervisorLocator, - invocation_digest: str, - *, - control_target: Path | None = None, -) -> str | None: - path = blind_root / "output" / "lifecycle-result.json" - if not path.exists() and not path.is_symlink(): - return None - value = _load_json(path, "evaluator lifecycle") - product = value.get("product") - harness = value.get("harness") - process = value.get("process") - if ( - value.get("record") != "result" - or value.get("spec_digest") != invocation_digest - or value.get("locator") != _locator_public(locator) - or not isinstance(product, dict) - or not isinstance(harness, dict) - or not isinstance(process, dict) - or set(product) != {"status", "reason"} - or product.get("status") not in {"succeeded", "failed", "unknown"} - or product.get("reason") - != { - "succeeded": "caller_success", - "failed": "caller_error", - "unknown": "unavailable", - }.get(product.get("status")) - or set(harness) - != {"status", "reason", "ordered_terminal", "cleanup_complete"} - or harness.get("status") not in {"passed", "failed"} - or harness.get("reason") not in TERMINAL_REASONS - or (harness.get("status") == "passed") - != (harness.get("reason") == "success") - or not isinstance(harness.get("ordered_terminal"), bool) - or (harness.get("status") == "passed") - and not harness.get("ordered_terminal") - or not isinstance(harness.get("cleanup_complete"), bool) - or (not harness.get("cleanup_complete")) - != (harness.get("reason") == "cleanup_failed") - or set(process) != {"status", "exit_code", "signal"} - or process.get("status") - not in {"exited", "signalled", "timed_out", "cancelled", "not_started"} - or any( - value is not None - and (not isinstance(value, int) or isinstance(value, bool)) - for value in (process.get("exit_code"), process.get("signal")) - ) - or (process.get("status") == "signalled" and process.get("signal") is None) - or ( - process.get("status") in {"exited", "not_started"} - and process.get("signal") is not None - ) - or ( - process.get("status") == "not_started" - and process.get("exit_code") is not None - ) - or value.get("process_group_alive") is not False - ): - raise ScoringError("evaluator lifecycle binding is invalid") - receipt, _ = _validate_cleanup_receipt( - locator, - expected_reason=str(harness["reason"]), - control_target=control_target, - ) - if ( - receipt["exit_code"] != process["exit_code"] - or receipt["signal"] != process["signal"] - ): - raise ScoringError("evaluator lifecycle binding is invalid") - journal = blind_root / "output" / "lifecycle-journal.jsonl" - try: - lines = _read_regular( - journal, "evaluator lifecycle journal", maximum=MAX_INPUT_FILE_BYTES - ).decode("utf-8").splitlines() - header = json.loads(lines[0]) - terminal = json.loads(lines[-1]) - except (IndexError, UnicodeDecodeError, json.JSONDecodeError) as exc: - raise ScoringError("evaluator lifecycle journal is invalid") from exc - if ( - not isinstance(header, dict) - or header.get("record") != "header" - or header.get("journal_version") != JOURNAL_VERSION - or header.get("spec_digest") != invocation_digest - or not isinstance(terminal, dict) - or terminal.get("record") != "terminal" - or terminal.get("product") != product - or terminal.get("harness") != harness - or terminal.get("process") != process - or terminal.get("process_group_alive") is not False - ): - raise ScoringError("evaluator lifecycle journal is invalid") - return _digest(_read_regular(path, "evaluator lifecycle")) - - -def _wait_post_cleanup_quiet( - root: Path, - *, - lifecycle_validator: Callable[[], str | None] | None = None, -) -> str | None: - """Wait for required publication and then one stable quiet interval.""" - deadline = time.monotonic() + _POST_CLEANUP_TIMEOUT_SECONDS - quiet_since = time.monotonic() - previous: tuple[tuple[str, int, int], ...] | None = None - while True: - snapshot: list[tuple[str, int, int]] = [] - for path in sorted(root.rglob("*")): - try: - info = os.lstat(path) - except FileNotFoundError: - # Atomic lifecycle publication uses short-lived staging files; - # disappearance is itself a change and the next poll observes - # the stable post-cleanup tree. - continue - except OSError as exc: - raise ScoringError("evaluator post-cleanup state is unavailable") from exc - if stat.S_ISREG(info.st_mode): - snapshot.append( - (path.relative_to(root).as_posix(), info.st_size, info.st_mtime_ns) - ) - current = tuple(snapshot) - now = time.monotonic() - if current != previous: - previous = current - quiet_since = now - - lifecycle_digest: str | None = None - if lifecycle_validator is not None: - journal = root / "lifecycle-journal.jsonl" - result = root / "lifecycle-result.json" - journal_published = journal.exists() or journal.is_symlink() - result_published = result.exists() or result.is_symlink() - if journal_published and result_published: - lifecycle_digest = lifecycle_validator() - if lifecycle_digest is None: - raise ScoringError( - "evaluator lifecycle publication is incomplete" - ) - if ( - now - quiet_since >= _POST_CLEANUP_QUIET_SECONDS - and (lifecycle_validator is None or lifecycle_digest is not None) - ): - if lifecycle_validator is None: - return None - final_digest = lifecycle_validator() - if final_digest != lifecycle_digest: - raise ScoringError("evaluator lifecycle publication changed") - return final_digest - if now >= deadline: - if lifecycle_validator is not None: - raise ScoringError("evaluator lifecycle publication is incomplete") - raise ScoringError("evaluator post-cleanup state did not quiesce") - time.sleep(_POST_CLEANUP_POLL_SECONDS) - - -def _remove_cleaned_socket( - locator: SupervisorLocator, control_target: Path -) -> None: - socket = control_target / Path(locator.socket_path).name - try: - mode = os.lstat(socket).st_mode - except FileNotFoundError: - return - except OSError as exc: - raise ScoringError("cleaned evaluator control socket is unavailable") from exc - if not stat.S_ISSOCK(mode): - raise ScoringError("cleaned evaluator control socket is invalid") - try: - socket.unlink() - _fsync_dir(control_target) - except OSError as exc: - raise ScoringError("cleaned evaluator control socket cleanup failed") from exc - - -def _recover_runner( - blind_root: Path, - locator: SupervisorLocator, - invocation_digest: str, - *, - control_target: Path, -) -> tuple[str | None, str]: - lifecycle = _validate_lifecycle_binding( - blind_root, - locator, - invocation_digest, - control_target=control_target, - ) - if lifecycle is not None: - _, receipt_digest = _validate_cleanup_receipt( - locator, control_target=control_target - ) - stable_lifecycle = _wait_post_cleanup_quiet( - blind_root / "output", - lifecycle_validator=lambda: _validate_lifecycle_binding( - blind_root, - locator, - invocation_digest, - control_target=control_target, - ), - ) - if stable_lifecycle != lifecycle: - raise ScoringError("evaluator lifecycle publication changed") - _remove_cleaned_socket(locator, control_target) - return stable_lifecycle, receipt_digest - receipt_path = control_target / "cleanup-receipt.json" - if receipt_path.exists() or receipt_path.is_symlink(): - _, receipt_digest = _validate_cleanup_receipt( - locator, control_target=control_target - ) - lifecycle = _wait_post_cleanup_quiet( - blind_root / "output", - lifecycle_validator=lambda: _validate_lifecycle_binding( - blind_root, - locator, - invocation_digest, - control_target=control_target, - ), - ) - if lifecycle is None: - raise ScoringError("evaluator lifecycle publication is incomplete") - _remove_cleaned_socket(locator, control_target) - return lifecycle, receipt_digest - try: - outcome = recover_invocation(locator, stop=True) - except LifecycleRecoveryError as exc: - try: - receipt, receipt_digest = _validate_cleanup_receipt( - locator, control_target=control_target - ) - except ScoringError: - raise ScoringError("evaluator recovery is unverified") from exc - if receipt["reason"] not in { - REASON_CONTROLLER_LOST, - REASON_RECOVERED_STOP, - }: - raise ScoringError("evaluator recovery is unverified") from exc - else: - if not outcome.cleanup_complete or outcome.process_group_alive: - raise ScoringError("evaluator cleanup is unverified") - _, receipt_digest = _validate_cleanup_receipt( - locator, - expected_reason=outcome.reason, - control_target=control_target, - ) - _wait_post_cleanup_quiet(blind_root / "output") - _remove_cleaned_socket(locator, control_target) - lifecycle = _validate_lifecycle_binding( - blind_root, - locator, - invocation_digest, - control_target=control_target, - ) - return lifecycle, receipt_digest - - -def _release_runner_alias(runner: Mapping[str, Any]) -> None: - raw = runner.get("control_alias") - if not isinstance(raw, str) or not raw: - return - alias = Path(raw) - target = Path(str(runner.get("control_target", ""))).parent - try: - mode = os.lstat(alias).st_mode - except FileNotFoundError: - return - except OSError as exc: - raise ScoringError("evaluator control alias is unavailable") from exc - if not stat.S_ISLNK(mode): - raise ScoringError("evaluator control alias is invalid") - try: - if alias.resolve(strict=True) != target.resolve(strict=True): - raise ScoringError("evaluator control alias is invalid") - alias.unlink() - _fsync_dir(alias.parent) - except OSError as exc: - raise ScoringError("evaluator control alias cleanup failed") from exc - - -def _eligibility(manifest: Manifest, attempt: Attempt) -> tuple[bool, tuple[str, ...]]: - if attempt.state not in TERMINAL_STATES: - return False, ("lifecycle_running",) - try: - execution = _load_json(Path(attempt.root) / "attempt.json", "execution attempt") - except ScoringError: - raise - lifecycle = execution.get("lifecycle") - if not isinstance(lifecycle, dict): - raise ScoringError("execution eligibility evidence is invalid") - reasons: list[str] = [] - product = lifecycle.get("product") - harness = lifecycle.get("harness") - process = lifecycle.get("process") - if not isinstance(product, dict) or product.get("status") != "succeeded": - reasons.append("product_" + str((product or {}).get("status", "unknown"))) - if not isinstance(harness, dict) or harness.get("status") != "passed": - reasons.append("harness_" + str((harness or {}).get("status", "failed"))) - if not isinstance(process, dict) or process.get("status") != "exited": - reasons.append("process_" + str((process or {}).get("status", "not_started"))) - elif process.get("exit_code") != 0 or process.get("signal") is not None: - reasons.append("process_nonzero_exit") - try: - web = load_web_validation(attempt.root, manifest=manifest) - except WebValidationError as exc: - raise ScoringError("web eligibility evidence is invalid") from exc - gates = web.record["gates"] - if [item["id"] for item in gates] != list(WEB_GATES): - raise ScoringError("web eligibility gates are invalid") - reasons.extend( - f"gate_{item['id']}" - for item in gates - if item["id"] in HARD_ELIGIBILITY_GATES and not item["passed"] - ) - return not reasons, tuple(reasons) - - -def _publish_unscored( - run: RunIdentity, manifest: Manifest, attempt: Attempt, reasons: tuple[str, ...] -) -> None: - root = _score_root(attempt, create=True) - if _score_dirs(root): - raise ScoringError("eligible scoring and unscored evidence conflict") - path = root / UNSCORED_FILENAME - record = { - "record": "unscored", - "scoring_version": SCORING_VERSION, - "status": "unscored", - "reasons": list(reasons), - "manifest_digest": run.manifest_digest, - "attempt_digest": _record_digest( - Path(attempt.root) / "attempt.json", "execution attempt" - ), - "web_validation_digest": _record_digest( - Path(attempt.root) / WEB_VALIDATION_FILENAME, "web validation" - ), - } - raw = _json_bytes(record) - if path.exists() or path.is_symlink(): - if _read_regular(path, "unscored evidence") != raw: - raise ScoringError("unscored evidence is immutable") - return - _write_new(path, raw) - - -def _validate_unscored( - run: RunIdentity, manifest: Manifest, attempt: Attempt -) -> bool: - path = _score_root(attempt, create=False) / UNSCORED_FILENAME - if not path.exists() and not path.is_symlink(): - return False - eligible, reasons = _eligibility(manifest, attempt) - if eligible: - raise ScoringError("unscored evidence conflicts with eligibility") - expected = { - "record": "unscored", - "scoring_version": SCORING_VERSION, - "status": "unscored", - "reasons": list(reasons), - "manifest_digest": run.manifest_digest, - "attempt_digest": _record_digest( - Path(attempt.root) / "attempt.json", "execution attempt" - ), - "web_validation_digest": _record_digest( - Path(attempt.root) / WEB_VALIDATION_FILENAME, "web validation" - ), - } - if _load_canonical(path, "unscored evidence") != expected: - raise ScoringError("unscored evidence is invalid") - if any(key in expected for key in ("score", "total", "worksheet")): - raise ScoringError("unscored evidence contains a score") - return True - - -def _append_preflight( - run: RunIdentity, - cell: MatrixCell, - observation: PreflightObservation, -) -> tuple[int, str, str]: - if not isinstance(observation, PreflightObservation): - raise ScoringError("evaluator preflight is invalid") - try: - validate_result(cell, observation.result) - evidence = canonical_evidence_bytes( - cell, - observation.result, - observation.endpoint_identity, - observation.config_identity, - ) - except Exception as exc: - raise ScoringError("evaluator preflight is invalid") from exc - root = Path(run.root) / "scoring-preflight" - if root.exists() or root.is_symlink(): - _ensure_directory(root) - else: - _mkdir_new(root) - children = sorted(root.iterdir()) - for index, child in enumerate(children, start=1): - if child.name != f"preflight-{index:06d}.json": - raise ScoringError("evaluator preflight sequence is invalid") - _read_regular(child, "evaluator preflight") - sequence = len(children) + 1 - path = root / f"preflight-{sequence:06d}.json" - _write_new(path, evidence) - return sequence, _digest(evidence), observation.result.status - - -def _blind_id( - manifest_digest: str, ordinal: int, score_number: int, nonce: bytes -) -> str: - material = ( - b"IOP-BENCH-BLIND-ID-V1\0" - + manifest_digest.encode("ascii") - + ordinal.to_bytes(8, "big") - + score_number.to_bytes(8, "big") - + nonce - ) - return "blind-" + hashlib.sha256(material).hexdigest()[:32] - - -def _allocate_score( - run: RunIdentity, - manifest: Manifest, - attempt: Attempt, - ordinal: int, - score_number: int, - preflight_sequence: int, - preflight_digest: str, -) -> tuple[Path, str, Path, str]: - root = _score_root(attempt, create=True) - score_id = f"score-{score_number:06d}" - score_root = root / score_id - _mkdir_new(score_root) - nonce = secrets.token_bytes(32) - blind_id = _blind_id(manifest.digest, ordinal, score_number, nonce) - if not BLIND_ID_RE.fullmatch(blind_id): - raise ScoringError("blind id allocation failed") - blind_root = Path(run.root) / "blind" / blind_id - blind_parent = blind_root.parent - if blind_parent.exists() or blind_parent.is_symlink(): - _ensure_directory(blind_parent) - else: - _mkdir_new(blind_parent) - _mkdir_new(blind_root) - for name in ("input", "session", "output"): - _mkdir_new(blind_root / name) - - relative_blind = f"blind/{blind_id}" - session_identity = _digest( - b"IOP-BENCH-SCORING-SESSION-V1\0" + nonce - ) - allocation = { - "record": "scoring-allocation", - "scoring_version": SCORING_VERSION, - "score_id": score_id, - "rubric_version": manifest.rubric_version, - "blind_id": blind_id, - "blind_path": relative_blind, - "session_identity": session_identity, - "manifest_digest": manifest.digest, - "evaluator": _evaluator_payload(manifest), - "preflight_sequence": preflight_sequence, - "preflight_digest": preflight_digest, - } - _write_new(score_root / ALLOCATION_FILENAME, _json_bytes(allocation)) - - mappings = Path(run.root) / "blind-mappings" - if mappings.exists() or mappings.is_symlink(): - _ensure_directory(mappings) - else: - _mkdir_new(mappings) - mapping = { - "record": "blind-mapping", - "scoring_version": SCORING_VERSION, - "blind_id": blind_id, - "blind_path": relative_blind, - "score_id": score_id, - "attempt_ordinal": ordinal, - "attempt": { - "run_id": attempt.identity.run_id, - "cell_id": attempt.identity.cell_id, - "repetition": attempt.identity.repetition, - "attempt": attempt.identity.attempt, - }, - "nonce_digest": _digest(nonce), - } - _write_new(mappings / f"{blind_id}.json", _json_bytes(mapping)) - return score_root, blind_id, blind_root, session_identity - - -def _materialize_blind( - manifest: Manifest, - attempt: Attempt, - blind_id: str, - blind_root: Path, - session_identity: str, -) -> BlindWorkspace: - web = load_web_validation(attempt.root, manifest=manifest) - gates = web.record["gates"] - if [item["id"] for item in gates] != list(WEB_GATES) or any( - item["id"] in HARD_ELIGIBILITY_GATES and not item["passed"] - for item in gates - ): - raise ScoringError("execution attempt is not eligible") - identities = _identity_values(manifest, attempt) - if _contains_identity(str(blind_root).encode("utf-8"), identities): - raise ScoringError("blind path leaks execution identity") - - generated = {item["path"]: item for item in web.record["workspace"]["generated"]} - files: list[tuple[str, bytes]] = [] - workspace_root = Path(attempt.root) / "workspace" - for name in GENERATED_FILES: - fact = generated.get(name) - if not isinstance(fact, dict) or fact.get("state") != "regular": - raise ScoringError("blind generated input is unavailable") - data = _safe_source(workspace_root, name) - if _digest(data) != fact["digest"] or len(data) != fact["size"]: - raise ScoringError("blind generated input digest is invalid") - files.append((f"input/{name}", data)) - - image_assets = [ - item.workspace_path - for item in manifest.fixture.assets - if Path(item.workspace_path).suffix.lower() in IMAGE_SUFFIXES - ] - if len(image_assets) != 2: - raise ScoringError("blind input requires exactly two local images") - input_facts = {item["path"]: item for item in web.record["workspace"]["inputs"]} - for relative in sorted(image_assets): - fact = input_facts.get(relative) - data = _safe_source(workspace_root, relative) - if ( - not isinstance(fact, dict) - or fact.get("state") != "regular" - or _digest(data) != fact.get("digest") - or len(data) != fact.get("size") - ): - raise ScoringError("blind image input digest is invalid") - files.append((f"input/{relative}", data)) - - for screenshot in web.record["screenshots"]: - source = _relative(screenshot["file"]) - data = _safe_source(Path(attempt.root), source) - if _digest(data) != screenshot["digest"] or len(data) != screenshot["size"]: - raise ScoringError("blind screenshot digest is invalid") - files.append((f"input/screenshots/{Path(source).name}", data)) - - for relative, data in files: - if _contains_identity(relative.encode("utf-8"), identities) or _contains_identity(data, identities): - raise ScoringError("blind input leaks execution identity") - _write_relative(blind_root, relative, data) - digest = _input_digest(files) - return BlindWorkspace( - blind_id=blind_id, - root=str(blind_root), - input_dir=str(blind_root / "input"), - session_dir=str(blind_root / "session"), - output_dir=str(blind_root / "output"), - input_digest=digest, - session_identity=session_identity, - ) - - -def _prompt(blind: BlindWorkspace, rubric_version: str) -> bytes: - categories = ", ".join( - f"{ident} ({maximum})" - for ident, maximum in rubric_categories(rubric_version) - ) - value = ( - "Evaluate the anonymous landing page using only the files under input/. " - "Inspect index.html, styles.css, script.js, the two local images, and all " - "screenshots. Do not use network access or infer the producing system. " - f"Use exactly these categories: {categories}. " - "Write exactly one JSON object to output/worksheet.json with fields " - "rubric_version, categories, and total. Each category must have exactly " - "id, max_score, score, and non-empty evidence. Use rubric_version " - f"{rubric_version}, integer scores within each maximum, and total equal " - "to their sum. Do not add automatic gate points or any extra field." - ) - return value.encode("utf-8") - - -def _publish_input_record(score_root: Path, blind: BlindWorkspace) -> None: - record = { - "record": "scoring-input", - "scoring_version": SCORING_VERSION, - "blind_id": blind.blind_id, - "input_digest": blind.input_digest, - "session_identity": blind.session_identity, - } - _write_new(score_root / INPUT_FILENAME, _json_bytes(record)) - - -def _lifecycle_digest(blind_root: Path) -> str | None: - path = blind_root / "output" / "lifecycle-result.json" - if not path.exists() and not path.is_symlink(): - return None - return _record_digest(path, "evaluator lifecycle") - - -def _publish_failure( - score_root: Path, - blind_id: str, - reason: str, - *, - lifecycle_digest: str | None = None, - runner_digest: str | None = None, - cleanup_receipt_digest: str | None = None, - post_tree_digest: str, -) -> None: - record = { - "record": "scoring-result", - "scoring_version": SCORING_VERSION, - "status": "scoring_failed", - "blind_id": blind_id, - "reason": reason, - "lifecycle_digest": lifecycle_digest, - "runner_digest": runner_digest, - "cleanup_receipt_digest": cleanup_receipt_digest, - "post_tree_digest": post_tree_digest, - } - _write_new(score_root / RESULT_FILENAME, _json_bytes(record)) - - -def _publish_success( - score_root: Path, - blind: BlindWorkspace, - worksheet: Worksheet, - lifecycle_digest: str, - runner_digest: str, - cleanup_receipt_digest: str, - post_tree_digest: str, -) -> None: - canonical = canonical_worksheet_bytes(worksheet) - record = { - "record": "scoring-result", - "scoring_version": SCORING_VERSION, - "status": "scored", - "blind_id": blind.blind_id, - "reason": "", - "lifecycle_digest": lifecycle_digest, - "runner_digest": runner_digest, - "cleanup_receipt_digest": cleanup_receipt_digest, - "post_tree_digest": post_tree_digest, - "input_digest": blind.input_digest, - "worksheet_digest": _digest(canonical), - "worksheet": worksheet.as_dict(), - } - _write_new(score_root / RESULT_FILENAME, _json_bytes(record)) - - -def _validate_allocation( - path: Path, - run: RunIdentity, - manifest: Manifest, - attempt: Attempt, - score_id: str, -) -> dict[str, Any]: - value = _load_canonical(path, "scoring allocation") - if not isinstance(value.get("evaluator"), dict): - raise ScoringError("scoring allocation is invalid") - expected_fields = { - "record", "scoring_version", "score_id", "rubric_version", "blind_id", - "blind_path", "session_identity", "manifest_digest", "evaluator", - "preflight_sequence", "preflight_digest", - } - if ( - set(value) != expected_fields - or value["record"] != "scoring-allocation" - or value["scoring_version"] != SCORING_VERSION - or value["score_id"] != score_id - or value["rubric_version"] != manifest.rubric_version - or not isinstance(value["blind_id"], str) - or not BLIND_ID_RE.fullmatch(value["blind_id"]) - or value["blind_path"] != f"blind/{value['blind_id']}" - or not isinstance(value["session_identity"], str) - or not DIGEST_RE.fullmatch(value["session_identity"]) - or value["manifest_digest"] != run.manifest_digest - or value["evaluator"] != _evaluator_payload(manifest) - or isinstance(value["preflight_sequence"], bool) - or not isinstance(value["preflight_sequence"], int) - or value["preflight_sequence"] < 1 - or not isinstance(value["preflight_digest"], str) - or not DIGEST_RE.fullmatch(value["preflight_digest"]) - ): - raise ScoringError("scoring allocation is invalid") - blind = Path(run.root) / value["blind_path"] - _ensure_directory(blind) - for name in ("input", "session", "output"): - _ensure_directory(blind / name) - preflight_path = ( - Path(run.root) - / "scoring-preflight" - / f"preflight-{value['preflight_sequence']:06d}.json" - ) - if _record_digest(preflight_path, "evaluator preflight") != value["preflight_digest"]: - raise ScoringError("scoring preflight binding is invalid") - mapping = _load_canonical( - Path(run.root) / "blind-mappings" / f"{value['blind_id']}.json", - "blind mapping", - ) - if ( - set(mapping) != { - "record", "scoring_version", "blind_id", "blind_path", "score_id", - "attempt_ordinal", "attempt", "nonce_digest", - } - or mapping["record"] != "blind-mapping" - or mapping["scoring_version"] != SCORING_VERSION - or mapping["blind_id"] != value["blind_id"] - or mapping["blind_path"] != value["blind_path"] - or mapping["score_id"] != score_id - or isinstance(mapping["attempt_ordinal"], bool) - or not isinstance(mapping["attempt_ordinal"], int) - or mapping["attempt_ordinal"] < 1 - or mapping["attempt"] != { - "run_id": attempt.identity.run_id, - "cell_id": attempt.identity.cell_id, - "repetition": attempt.identity.repetition, - "attempt": attempt.identity.attempt, - } - or not isinstance(mapping["nonce_digest"], str) - or not DIGEST_RE.fullmatch(mapping["nonce_digest"]) - ): - raise ScoringError("blind mapping is invalid") - return value - - -def _blind_tree_digest(root: Path) -> str: - _ensure_directory(root) - files: list[tuple[str, bytes]] = [] - - def restore_mode( - path: Path, expected: os.stat_result, original_mode: int - ) -> None: - try: - current = os.lstat(path) - if ( - stat.S_ISLNK(current.st_mode) - or (current.st_dev, current.st_ino) - != (expected.st_dev, expected.st_ino) - ): - raise ScoringError("blind input path changed") - os.chmod(path, original_mode, follow_symlinks=False) - restored = os.lstat(path) - if ( - (restored.st_dev, restored.st_ino) - != (expected.st_dev, expected.st_ino) - or stat.S_IMODE(restored.st_mode) != original_mode - ): - raise ScoringError("blind input mode restoration failed") - except OSError as exc: - raise ScoringError("blind input is unavailable") from exc - - def visit( - directory: Path, - expected: os.stat_result | None = None, - depth: int = 0, - ) -> None: - if depth > 64: - raise ScoringError("blind input tree is too deep") - try: - info = os.lstat(directory) - except OSError as exc: - raise ScoringError("blind input is unavailable") from exc - current_uid = getattr(os, "geteuid", lambda: info.st_uid)() - if ( - not stat.S_ISDIR(info.st_mode) - or stat.S_ISLNK(info.st_mode) - or info.st_uid != current_uid - or ( - expected is not None - and (info.st_dev, info.st_ino) - != (expected.st_dev, expected.st_ino) - ) - ): - raise ScoringError("blind input path is invalid") - original_mode = stat.S_IMODE(info.st_mode) - temporary_mode = original_mode | stat.S_IRUSR | stat.S_IXUSR - changed = temporary_mode != original_mode - try: - if changed: - os.chmod(directory, temporary_mode, follow_symlinks=False) - current = os.lstat(directory) - if ( - (current.st_dev, current.st_ino) != (info.st_dev, info.st_ino) - or stat.S_ISLNK(current.st_mode) - ): - raise ScoringError("blind input path changed") - with os.scandir(directory) as iterator: - children = sorted(iterator, key=lambda item: item.name) - for entry in children: - try: - child_info = entry.stat(follow_symlinks=False) - except OSError as exc: - raise ScoringError("blind input is unavailable") from exc - path = directory / entry.name - if stat.S_ISDIR(child_info.st_mode) and not stat.S_ISLNK( - child_info.st_mode - ): - visit(path, child_info, depth + 1) - continue - if ( - not stat.S_ISREG(child_info.st_mode) - or stat.S_ISLNK(child_info.st_mode) - or child_info.st_uid != current_uid - ): - raise ScoringError("blind input path is invalid") - file_mode = stat.S_IMODE(child_info.st_mode) - readable_mode = file_mode | stat.S_IRUSR - file_changed = readable_mode != file_mode - try: - if file_changed: - os.chmod(path, readable_mode, follow_symlinks=False) - current_file = os.lstat(path) - if ( - (current_file.st_dev, current_file.st_ino) - != (child_info.st_dev, child_info.st_ino) - or stat.S_ISLNK(current_file.st_mode) - ): - raise ScoringError("blind input path changed") - relative = path.relative_to(root.parent).as_posix() - files.append( - ( - relative, - _read_regular( - path, - "blind input", - maximum=MAX_INPUT_FILE_BYTES, - ), - ) - ) - if len(files) > 100_000: - raise ScoringError("blind input tree has too many files") - finally: - if file_changed: - restore_mode(path, child_info, file_mode) - except OSError as exc: - raise ScoringError("blind input is unavailable") from exc - finally: - if changed: - restore_mode(directory, info, original_mode) - - visit(root) - for relative, _data in files: - if not relative: - # Defensive only: every collected entry must be below ``root``. - raise ScoringError("blind input path is invalid") - return _input_digest(files) - - -def _freeze_input_tree(root: Path) -> None: - _ensure_directory(root) - directories: list[Path] = [root] - for path in sorted(root.rglob("*")): - try: - mode = os.lstat(path).st_mode - except OSError as exc: - raise ScoringError("blind input is unavailable") from exc - if stat.S_ISDIR(mode): - if path.is_symlink(): - raise ScoringError("blind input path is invalid") - directories.append(path) - elif stat.S_ISREG(mode) and not path.is_symlink(): - try: - os.chmod(path, 0o400, follow_symlinks=False) - except OSError as exc: - raise ScoringError("blind input could not be frozen") from exc - else: - raise ScoringError("blind input path is invalid") - for directory in reversed(directories): - try: - os.chmod(directory, 0o500, follow_symlinks=False) - except OSError as exc: - raise ScoringError("blind input could not be frozen") from exc - - -def _validate_input_record( - score_root: Path, - allocation: Mapping[str, Any], - run: RunIdentity, - *, - verify_tree: bool = True, -) -> str | None: - path = score_root / INPUT_FILENAME - if not path.exists() and not path.is_symlink(): - return None - value = _load_canonical(path, "scoring input") - if ( - set(value) != { - "record", "scoring_version", "blind_id", "input_digest", - "session_identity", - } - or value["record"] != "scoring-input" - or value["scoring_version"] != SCORING_VERSION - or value["blind_id"] != allocation["blind_id"] - or value["session_identity"] != allocation["session_identity"] - or not isinstance(value["input_digest"], str) - or not DIGEST_RE.fullmatch(value["input_digest"]) - ): - raise ScoringError("scoring input is invalid") - if verify_tree: - input_root = Path(run.root) / allocation["blind_path"] / "input" - actual = _blind_tree_digest(input_root) - if actual != value["input_digest"]: - raise ScoringError("blind input changed after allocation") - for candidate in (input_root, *sorted(input_root.rglob("*"))): - try: - mode = os.lstat(candidate).st_mode - except OSError as exc: - raise ScoringError("blind input is unavailable") from exc - if mode & 0o222: - raise ScoringError("blind input changed after allocation") - return str(value["input_digest"]) - - -def _result_status( - score_root: Path, - run: RunIdentity, - manifest: Manifest, - attempt: Attempt, -) -> str | None: - allocation = _validate_allocation( - score_root / ALLOCATION_FILENAME, - run, - manifest, - attempt, - score_root.name, - ) - input_digest = _validate_input_record( - score_root, allocation, run, verify_tree=False - ) - blind_root = Path(run.root) / allocation["blind_path"] - runner = _validate_runner( - score_root, blind_root, allocation, run, attempt - ) - path = score_root / RESULT_FILENAME - if not path.exists() and not path.is_symlink(): - return None - value = _load_canonical(path, "scoring result") - common = { - "record", "scoring_version", "status", "blind_id", "reason", - "lifecycle_digest", "runner_digest", "cleanup_receipt_digest", - "post_tree_digest", - } - status = value.get("status") - if ( - status not in {"scored", "scoring_failed"} - or value.get("record") != "scoring-result" - or value.get("scoring_version") != SCORING_VERSION - or value.get("blind_id") != allocation["blind_id"] - or not isinstance(value.get("reason"), str) - or ( - value.get("lifecycle_digest") is not None - and ( - not isinstance(value["lifecycle_digest"], str) - or not DIGEST_RE.fullmatch(value["lifecycle_digest"]) - ) - ) - or ( - value.get("runner_digest") is not None - and ( - not isinstance(value["runner_digest"], str) - or not DIGEST_RE.fullmatch(value["runner_digest"]) - ) - ) - or ( - value.get("cleanup_receipt_digest") is not None - and ( - not isinstance(value["cleanup_receipt_digest"], str) - or not DIGEST_RE.fullmatch(value["cleanup_receipt_digest"]) - ) - ) - or not isinstance(value.get("post_tree_digest"), str) - or not DIGEST_RE.fullmatch(value["post_tree_digest"]) - ): - raise ScoringError("scoring result is invalid") - actual_post_tree = _blind_tree_digest(blind_root) - if actual_post_tree != value["post_tree_digest"]: - raise ScoringError("scoring post-tree changed") - actual_runner = None if runner is None else runner[2] - if value["runner_digest"] != actual_runner: - raise ScoringError("scoring runner binding is invalid") - actual_receipt: str | None = None - actual_lifecycle: str | None = None - if runner is not None: - runner_record, locator, _ = runner - control_target = Path(runner_record["control_target"]) - receipt_path = control_target / "cleanup-receipt.json" - if receipt_path.exists() or receipt_path.is_symlink(): - _, actual_receipt = _validate_cleanup_receipt( - locator, control_target=control_target - ) - lifecycle_path = blind_root / "output" / "lifecycle-result.json" - if lifecycle_path.exists() or lifecycle_path.is_symlink(): - actual_lifecycle = _validate_lifecycle_binding( - blind_root, - locator, - str(runner_record["spec_digest"]), - control_target=control_target, - ) - if value["cleanup_receipt_digest"] != actual_receipt: - raise ScoringError("scoring cleanup binding is invalid") - if value["lifecycle_digest"] != actual_lifecycle: - raise ScoringError("scoring lifecycle binding is invalid") - if status == "scoring_failed": - if set(value) != common or not value["reason"]: - raise ScoringError("scoring failure is invalid") - return status - if set(value) != common | { - "input_digest", "worksheet_digest", "worksheet", - } or value["reason"]: - raise ScoringError("scored result is invalid") - try: - worksheet = load_worksheet( - Path(run.root) / allocation["blind_path"] / "output" / "worksheet.json", - expected_version=manifest.rubric_version, - ) - except RubricError as exc: - raise ScoringError("scored worksheet is invalid") from exc - canonical = canonical_worksheet_bytes(worksheet) - if ( - value["worksheet"] != worksheet.as_dict() - or value["worksheet_digest"] != _digest(canonical) - or input_digest is None - or value["input_digest"] != input_digest - or value["lifecycle_digest"] is None - or value["runner_digest"] is None - or value["cleanup_receipt_digest"] is None - ): - raise ScoringError("scored worksheet binding is invalid") - _validate_input_record(score_root, allocation, run, verify_tree=True) - return status - - -def _blind_from_allocation( - run: RunIdentity, - allocation: Mapping[str, Any], - input_digest: str | None, -) -> BlindWorkspace: - root = Path(run.root) / str(allocation["blind_path"]) - return BlindWorkspace( - blind_id=str(allocation["blind_id"]), - root=str(root), - input_dir=str(root / "input"), - session_dir=str(root / "session"), - output_dir=str(root / "output"), - input_digest=input_digest or _blind_tree_digest(root / "input"), - session_identity=str(allocation["session_identity"]), - ) - - -def _finalize_adapter_evidence( - adapter: ScoringAdapter, blind: BlindWorkspace -) -> ScoringEvidenceFinalization: - try: - finalized = adapter.finalize_evidence(blind) - except Exception as exc: - raise ScoringError("evaluator evidence finalization failed") from exc - if ( - not isinstance(finalized, ScoringEvidenceFinalization) - or not isinstance(finalized.safe, bool) - or not isinstance(finalized.reason, str) - or (finalized.safe and finalized.reason) - or ( - not finalized.safe - and finalized.reason - not in { - "runtime_secret_leak", - "input_mutated", - "evaluator_output_leak", - } - ) - ): - raise ScoringError("evaluator evidence finalization is invalid") - return finalized - - -def _evidence_digests( - score_root: Path, - blind_root: Path, - allocation: Mapping[str, Any], - run: RunIdentity, - attempt: Attempt, -) -> tuple[str | None, str | None, str | None, str]: - runner = _validate_runner( - score_root, blind_root, allocation, run, attempt - ) - if runner is None: - runner_digest = lifecycle_digest = receipt_digest = None - else: - runner_record, locator, runner_digest = runner - control_target = Path(runner_record["control_target"]) - lifecycle_path = blind_root / "output" / "lifecycle-result.json" - lifecycle_digest = ( - _validate_lifecycle_binding( - blind_root, - locator, - str(runner_record["spec_digest"]), - control_target=control_target, - ) - if lifecycle_path.exists() or lifecycle_path.is_symlink() - else None - ) - receipt_path = control_target / "cleanup-receipt.json" - receipt_digest = ( - _validate_cleanup_receipt( - locator, control_target=control_target - )[1] - if receipt_path.exists() or receipt_path.is_symlink() - else None - ) - return ( - lifecycle_digest, - runner_digest, - receipt_digest, - _blind_tree_digest(blind_root), - ) - - -def _publish_current_failure( - score_root: Path, - blind: BlindWorkspace, - allocation: Mapping[str, Any], - run: RunIdentity, - attempt: Attempt, - reason: str, -) -> None: - lifecycle, runner, receipt, post_tree = _evidence_digests( - score_root, Path(blind.root), allocation, run, attempt - ) - _publish_failure( - score_root, - blind.blind_id, - reason, - lifecycle_digest=lifecycle, - runner_digest=runner, - cleanup_receipt_digest=receipt, - post_tree_digest=post_tree, - ) - - -def _complete_interrupted( - adapter: ScoringAdapter, - score_root: Path, - run: RunIdentity, - manifest: Manifest, - attempt: Attempt, -) -> str: - allocation = _validate_allocation( - score_root / ALLOCATION_FILENAME, - run, - manifest, - attempt, - score_root.name, - ) - status = _result_status(score_root, run, manifest, attempt) - if status is not None: - return status - blind_root = Path(run.root) / allocation["blind_path"] - input_digest = _validate_input_record( - score_root, allocation, run, verify_tree=False - ) - blind = _blind_from_allocation(run, allocation, input_digest) - runner = _validate_runner( - score_root, blind_root, allocation, run, attempt - ) - if runner is not None: - runner_record, locator, _ = runner - _recover_runner( - blind_root, - locator, - str(runner_record["spec_digest"]), - control_target=Path(runner_record["control_target"]), - ) - _release_runner_alias(runner_record) - finalized = _finalize_adapter_evidence(adapter, blind) - reason = "interrupted" if finalized.safe else finalized.reason - _publish_current_failure( - score_root, blind, allocation, run, attempt, reason - ) - return "scoring_failed" - - -def _scan_visible_tree(root: Path, identities: ProducerIdentity) -> None: - _ensure_directory(root) - for path in sorted(root.rglob("*")): - try: - mode = os.lstat(path).st_mode - except OSError as exc: - raise ScoringError("evaluator-visible state is unavailable") from exc - if stat.S_ISDIR(mode): - if path.is_symlink(): - raise ScoringError("evaluator-visible path is invalid") - continue - if not stat.S_ISREG(mode) or path.is_symlink(): - raise ScoringError("evaluator-visible path is invalid") - relative = _path_bytes(path.relative_to(root).as_posix()) - data = _read_regular( - path, "evaluator-visible evidence", maximum=MAX_INPUT_FILE_BYTES - ) - if _contains_identity(relative, identities) or _contains_identity( - data, identities - ): - raise ScoringError("evaluator-visible evidence leaks execution identity") - - -def _score_one( - adapter: ScoringAdapter, - run: RunIdentity, - manifest: Manifest, - attempt: Attempt, - ordinal: int, - score_number: int, - preflight_sequence: int, - preflight_digest: str, -) -> str: - score_root, blind_id, blind_root, session_identity = _allocate_score( - run, - manifest, - attempt, - ordinal, - score_number, - preflight_sequence, - preflight_digest, - ) - allocation = _validate_allocation( - score_root / ALLOCATION_FILENAME, - run, - manifest, - attempt, - score_root.name, - ) - try: - blind = _materialize_blind( - manifest, attempt, blind_id, blind_root, session_identity - ) - _publish_input_record(score_root, blind) - _freeze_input_tree(Path(blind.input_dir)) - except Exception: - _publish_failure( - score_root, - blind_id, - "blind_preparation_failed", - post_tree_digest=_blind_tree_digest(blind_root), - ) - return "scoring_failed" - - prompt = _prompt(blind, manifest.rubric_version) - identities = _identity_values(manifest, attempt) - if _contains_identity(prompt, identities): - _publish_current_failure( - score_root, blind, allocation, run, attempt, "blind_prompt_leak" - ) - return "scoring_failed" - cell = _evaluator_cell(manifest) - invocation: ScoringInvocationResult | None = None - invocation_failed = False - - def on_started(locator: SupervisorLocator, invocation_digest: str) -> None: - _publish_runner( - score_root, - blind_root, - blind, - run, - attempt, - locator, - invocation_digest, - ) - - try: - invocation = adapter.invoke( - cell, blind, prompt, manifest.timeout, on_started - ) - except Exception: - invocation_failed = True - - runner = _validate_runner( - score_root, blind_root, allocation, run, attempt - ) - if runner is not None: - runner_record, locator, _ = runner - _recover_runner( - blind_root, - locator, - str(runner_record["spec_digest"]), - control_target=Path(runner_record["control_target"]), - ) - _release_runner_alias(runner_record) - finalized = _finalize_adapter_evidence(adapter, blind) - if not finalized.safe: - _publish_current_failure( - score_root, blind, allocation, run, attempt, finalized.reason - ) - return "scoring_failed" - try: - _validate_input_record(score_root, allocation, run, verify_tree=True) - except ScoringError: - _publish_current_failure( - score_root, blind, allocation, run, attempt, "input_mutated" - ) - return "scoring_failed" - if invocation_failed: - _publish_current_failure( - score_root, blind, allocation, run, attempt, "evaluator_failed" - ) - return "scoring_failed" - if runner is None: - _publish_current_failure( - score_root, blind, allocation, run, attempt, "evaluator_owner_missing" - ) - return "scoring_failed" - if not isinstance(invocation, ScoringInvocationResult): - _publish_current_failure( - score_root, blind, allocation, run, attempt, "evaluator_protocol_failed" - ) - return "scoring_failed" - expected_binding = ( - manifest.evaluator.iop.route_kind, - manifest.evaluator.iop.route_id, - manifest.evaluator.iop.request_model, - manifest.evaluator.iop.requested_effort, - ) - lifecycle, runner_digest, receipt_digest, _ = _evidence_digests( - score_root, blind_root, allocation, run, attempt - ) - lifecycle_record = ( - None - if lifecycle is None - else _load_json( - blind_root / "output" / "lifecycle-result.json", - "evaluator lifecycle", - ) - ) - if ( - invocation.product != "succeeded" - or invocation.harness != "passed" - or invocation.process != "exited" - or invocation.process_exit_code != 0 - or invocation.process_signal is not None - or invocation.reason != "success" - or invocation.effective_binding != expected_binding - or lifecycle_record is None - or lifecycle_record["product"]["status"] != invocation.product - or lifecycle_record["harness"]["status"] != invocation.harness - or lifecycle_record["process"]["status"] != invocation.process - or lifecycle_record["process"]["exit_code"] - != invocation.process_exit_code - or lifecycle_record["process"]["signal"] != invocation.process_signal - or lifecycle_record["harness"]["reason"] != invocation.reason - or runner_digest is None - or receipt_digest is None - ): - _publish_current_failure( - score_root, blind, allocation, run, attempt, "evaluator_failed" - ) - return "scoring_failed" - try: - # Producer identity belongs only to the frozen anonymous input and the - # evaluator's publication surface. The fresh session tree is owned by - # the evaluator itself and remains covered by secret scrubbing and the - # whole-tree digest, but is not producer provenance. - for evidence_root in ( - Path(blind.input_dir), - Path(blind.output_dir), - ): - _scan_visible_tree(evidence_root, identities) - except ScoringError: - _publish_current_failure( - score_root, blind, allocation, run, attempt, "evaluator_output_leak" - ) - return "scoring_failed" - try: - worksheet = load_worksheet( - blind_root / "output" / "worksheet.json", - expected_version=manifest.rubric_version, - ) - post_tree = _blind_tree_digest(blind_root) - _publish_success( - score_root, - blind, - worksheet, - lifecycle, - runner_digest, - receipt_digest, - post_tree, - ) - except (RubricError, ScoringError): - _publish_current_failure( - score_root, blind, allocation, run, attempt, "invalid_worksheet" - ) - return "scoring_failed" - return "scored" - - -def score_run( - store: RunStore, - run: RunIdentity, - manifest: Manifest, - *, - adapter: ScoringAdapter, - retry_scoring_failed: bool = False, -) -> ScoringSummary: - """Classify every execution attempt and append only explicitly allowed work.""" - if not isinstance(store, RunStore) or not isinstance(manifest, Manifest): - raise ScoringError("scoring inputs are invalid") - if not callable(getattr(adapter, "preflight", None)) or not callable( - getattr(adapter, "invoke", None) - ) or not callable(getattr(adapter, "finalize_evidence", None)): - raise ScoringError("scoring adapter is unavailable") - bound = store.open(manifest, run.run_id) - if bound != run: - raise ScoringError("run identity is invalid") - - counts = {status: 0 for status in SCORING_STATUSES} - with store.writer(bound): - retained = store.execution_attempts(bound, manifest) - retained_by_slot = { - (item.identity.cell_id, item.identity.repetition) for item in retained - } - counts["blocked"] += len(store.slots(manifest)) - len(retained_by_slot) - - pending: list[tuple[int, Attempt, int]] = [] - for ordinal, attempt in enumerate(retained, start=1): - eligible, reasons = _eligibility(manifest, attempt) - if not eligible: - if reasons == ("lifecycle_running",): - counts["blocked"] += 1 - continue - _publish_unscored(bound, manifest, attempt, reasons) - _validate_unscored(bound, manifest, attempt) - counts["unscored"] += 1 - continue - if _validate_unscored(bound, manifest, attempt): - raise ScoringError("eligible attempt is marked unscored") - score_root = _score_root(attempt, create=False) - score_dirs = _score_dirs(score_root) - statuses = [ - _complete_interrupted(adapter, item, bound, manifest, attempt) - for item in score_dirs - ] - if "scored" in statuses: - if statuses[-1] != "scored" or statuses.count("scored") != 1: - raise ScoringError("successful scoring is not terminal") - counts["scored"] += 1 - continue - if statuses and not retry_scoring_failed: - counts["scoring_failed"] += 1 - continue - pending.append((ordinal, attempt, len(score_dirs) + 1)) - - if not pending: - return ScoringSummary(bound.run_id, **counts) - - cell = _evaluator_cell(manifest) - observation = adapter.preflight(cell) - sequence, preflight_digest, status = _append_preflight( - bound, cell, observation - ) - if status != "ready": - counts["blocked"] += len(pending) - return ScoringSummary(bound.run_id, **counts) - - for ordinal, attempt, score_number in pending: - outcome = _score_one( - adapter, - bound, - manifest, - attempt, - ordinal, - score_number, - sequence, - preflight_digest, - ) - counts[outcome] += 1 - return ScoringSummary(bound.run_id, **counts) diff --git a/scripts/agent_benchmark/scoring_test.py b/scripts/agent_benchmark/scoring_test.py deleted file mode 100644 index f11431c7..00000000 --- a/scripts/agent_benchmark/scoring_test.py +++ /dev/null @@ -1,2137 +0,0 @@ -from __future__ import annotations - -import datetime -import hashlib -import json -import os -import socket -import sys -import tempfile -import threading -import time -import unittest -from pathlib import Path -from unittest import mock - -from scripts.agent_benchmark.attempts import ( - AttemptStateError, - PreflightObservation, - RunStore, - Slot, -) -from scripts.agent_benchmark.browser_cdp import RenderObservation, ViewportObservation -from scripts.agent_benchmark.connectivity import ( - CallerCapability, - ConnectivityIssue, - EffectiveBinding, - ISSUE_RESUME_CODES, - RequestedEffectiveBinding, - make_result, -) -from scripts.agent_benchmark.lifecycle import ( - CALLER_REASON_ERROR, - CALLER_REASON_SUCCESS, - COMPLETION_EXIT_AFTER_IDLE, - CLOCK_HARNESS_MONOTONIC, - METRIC_NAMES, - SOURCE_HARNESS, - SOURCE_WORKSPACE_POLL, - SUBMISSION_STDIN_ONCE, - UNIT_NANOSECONDS, - InvocationSpec, - JOURNAL_VERSION, - HarnessOutcome, - ProcessOutcome, - ProductOutcome, - SupervisorLocator, - env_pairs, - recover_invocation, - run_invocation, - spec_digest, -) -from scripts.agent_benchmark.manifest import ( - AssetMapping, - ONE_SHOT_RUBRIC_VERSION, - RUBRIC_VERSION, - digest_workspace_inputs, - load_manifest, -) -from scripts.agent_benchmark.measurement import ( - AttemptMeasurement, - REASON_NOT_OBSERVED, - REASON_NOT_REPORTED, - WorkspaceWriteObservation, - observed, - publish_measurement, - unavailable, -) -from scripts.agent_benchmark.rubric import rubric_categories -from scripts.agent_benchmark import scoring as scoring_module -from scripts.agent_benchmark.scoring import ( - BlindWorkspace, - ScoringEvidenceFinalization, - ScoringError, - ScoringInvocationResult, - score_run, -) -from scripts.agent_benchmark.web_validation import ( - WEB_GATES, - build_web_validation, - publish_web_validation, -) - - -def _digest(data: bytes) -> str: - return "sha256:" + hashlib.sha256(data).hexdigest() - - -def _worksheet( - total_delta: int = 0, rubric_version: str = RUBRIC_VERSION -) -> dict: - categories = [] - for index, (ident, maximum) in enumerate(rubric_categories(rubric_version)): - score = maximum - (1 if index == 0 else 0) - categories.append( - { - "id": ident, - "max_score": maximum, - "score": score, - "evidence": f"Anonymous evidence for {ident}.", - } - ) - return { - "rubric_version": rubric_version, - "categories": categories, - "total": sum(item["score"] for item in categories) + total_delta, - } - - -class FakeScoringAdapter: - capability = CallerCapability( - "codex", ("direct", "execution_preset"), ("xhigh",) - ) - - def __init__( - self, - *, - blocked: bool = False, - modes: list[str] | None = None, - sensitive_value: str = "", - rubric_version: str = RUBRIC_VERSION, - ): - self.blocked = blocked - self.modes = list(modes or ["success"]) - self.preflights = 0 - self.invocations: list[tuple[BlindWorkspace, bytes]] = [] - self.sensitive_value = sensitive_value - self.rubric_version = rubric_version - self.last_mode = "" - - def preflight(self, cell): - self.preflights += 1 - iop = cell.iop - requested = RequestedEffectiveBinding( - cell.id, - cell.caller, - iop.route_kind, - iop.route_id, - iop.request_model, - iop.requested_effort, - ) - issues = () - if self.blocked: - issues = ( - ConnectivityIssue( - "credential_missing", - ISSUE_RESUME_CODES["credential_missing"], - ), - ) - else: - requested = RequestedEffectiveBinding( - cell.id, - cell.caller, - iop.route_kind, - iop.route_id, - iop.request_model, - iop.requested_effort, - iop.route_kind, - iop.route_id, - iop.request_model, - iop.requested_effort, - tuple( - EffectiveBinding(item.stage, item.model, item.effort) - for item in iop.expected_bindings - ), - ) - result = make_result(cell, self.capability, requested, issues) - return PreflightObservation( - result, "sha256:" + "1" * 64, "sha256:" + "2" * 64 - ) - - def _publish_lifecycle(self, cell, blind, mode, on_started): - output = Path(blind.output_dir) - control = output / "fake-control" - control.mkdir() - locator = SupervisorLocator( - os.getpid(), - "fake-start-identity", - str(control / "control.sock"), - "fake-challenge-" + blind.blind_id, - str(control), - "2026-08-11T00:00:00+00:00", - ) - locator_payload = { - "supervisor_pid": locator.supervisor_pid, - "start_identity": locator.start_identity, - "socket_path": locator.socket_path, - "challenge": locator.challenge, - "control_dir": locator.control_dir, - "created_at": locator.created_at, - } - (control / "locator.json").write_text( - json.dumps(locator_payload), encoding="utf-8" - ) - invocation_digest = "sha256:" + "4" * 64 - on_started(locator, invocation_digest) - terminal_reason = "nonzero_exit" if mode == "raise" else "success" - exit_code = 7 if mode == "process_nonzero" else (1 if mode == "raise" else 0) - receipt = { - "receipt_version": 1, - "supervisor_pid": locator.supervisor_pid, - "challenge_digest": hashlib.sha256( - locator.challenge.encode("utf-8") - ).hexdigest(), - "reason": terminal_reason, - "exit_code": exit_code, - "signal": None, - "caller_launched": True, - "cleanup_complete": True, - "process_group_alive": False, - "completed_at": "2026-08-11T00:00:01+00:00", - } - (control / "cleanup-receipt.json").write_text( - json.dumps(receipt), encoding="utf-8" - ) - public_locator = { - key: value - for key, value in locator_payload.items() - if key != "challenge" - } - public_locator["challenge_digest"] = receipt["challenge_digest"] - product = ( - {"status": "unknown", "reason": "unavailable"} - if mode == "raise" - else {"status": "succeeded", "reason": CALLER_REASON_SUCCESS} - ) - harness = { - "status": "failed" if mode == "raise" else "passed", - "reason": terminal_reason, - "ordered_terminal": mode != "raise", - "cleanup_complete": True, - } - process = { - "status": "exited", - "exit_code": exit_code, - "signal": None, - } - lifecycle = { - "record": "result", - "product": product, - "harness": harness, - "process": process, - "process_group_alive": False, - "spec_digest": invocation_digest, - "locator": public_locator, - "effective_binding": { - "route_kind": cell.iop.route_kind, - "route_id": cell.iop.route_id, - "model": cell.iop.request_model, - "effort": cell.iop.requested_effort, - }, - } - (output / "lifecycle-result.json").write_text( - json.dumps(lifecycle), encoding="utf-8" - ) - journal = ( - json.dumps( - { - "record": "header", - "journal_version": JOURNAL_VERSION, - "spec_digest": invocation_digest, - } - ) - + "\n" - + json.dumps( - { - "record": "terminal", - "product": product, - "harness": harness, - "process": process, - "process_group_alive": False, - } - ) - + "\n" - ) - (output / "lifecycle-journal.jsonl").write_text( - journal, encoding="utf-8" - ) - - def invoke(self, cell, blind, task_payload, timeout, on_started): - self.invocations.append((blind, task_payload)) - mode = self.modes.pop(0) if self.modes else "success" - self.last_mode = mode - output = Path(blind.output_dir) - self._publish_lifecycle(cell, blind, mode, on_started) - if mode == "mutate": - target = Path(blind.input_dir) / "index.html" - target.chmod(0o600) - target.write_text("
mutated
", encoding="utf-8") - if mode == "raise": - raise RuntimeError("fake evaluator failed") - if mode == "malformed": - (output / "worksheet.json").write_text("{}", encoding="utf-8") - else: - worksheet = _worksheet(rubric_version=self.rubric_version) - if mode == "secret": - worksheet["categories"][0]["evidence"] = self.sensitive_value - (output / "worksheet.json").write_text( - json.dumps(worksheet, sort_keys=True, separators=(",", ":")) - + "\n", - encoding="ascii", - ) - binding = ( - cell.iop.route_kind, - cell.iop.route_id, - cell.iop.request_model, - cell.iop.requested_effort, - ) - if mode == "binding": - binding = (binding[0], "substituted", binding[2], binding[3]) - passed = mode not in {"failed", "binding"} - return ScoringInvocationResult( - "succeeded" if passed else "failed", - "passed" if passed else "failed", - "exited", - 7 if mode == "process_nonzero" else 0, - None, - "success" if passed else "evaluator_failed", - binding, - ) - - def finalize_evidence(self, blind): - leaked = False - if self.sensitive_value: - sensitive = self.sensitive_value.encode("utf-8") - for path in Path(blind.root).rglob("*"): - if path.is_file() and sensitive in path.read_bytes(): - path.unlink() - leaked = True - return ScoringEvidenceFinalization( - not leaked, "" if not leaked else "runtime_secret_leak" - ) - - -class ScoringTest(unittest.TestCase): - def setUp(self): - temporary = tempfile.TemporaryDirectory(dir="/tmp", prefix="iop-score-") - self.addCleanup(temporary.cleanup) - self.root = Path(temporary.name) - (self.root / "Makefile").write_text("test:\n\t@true\n") - fixture_root = self.root / "scripts" / "fixtures" / "bench" - fixture_root.mkdir(parents=True) - (fixture_root / "prompt.md").write_text("Build the page.", encoding="utf-8") - self.asset_bytes = { - "scripts/fixtures/bench/reference.txt": b"anonymous reference\n", - "scripts/fixtures/bench/a.svg": b"", - "scripts/fixtures/bench/b.svg": b"", - } - for relative, data in self.asset_bytes.items(): - path = self.root / relative - path.parent.mkdir(parents=True, exist_ok=True) - path.write_bytes(data) - assets = ( - AssetMapping( - "scripts/fixtures/bench/reference.txt", - "brief/reference.txt", - self.asset_bytes["scripts/fixtures/bench/reference.txt"], - ), - AssetMapping( - "scripts/fixtures/bench/a.svg", - "assets/a.svg", - self.asset_bytes["scripts/fixtures/bench/a.svg"], - ), - AssetMapping( - "scripts/fixtures/bench/b.svg", - "assets/b.svg", - self.asset_bytes["scripts/fixtures/bench/b.svg"], - ), - ) - raw = { - "pipeline_version": "2", - "environment": "dev", - "testbed": "../iop-s2", - "repetitions": 1, - "session_policy": "fresh", - "setup_cache_policy": "isolated", - "timeout": { - "run_seconds": 5, - "idle_seconds": 1, - "quiet_seconds": 1, - "cleanup_grace_seconds": 1, - }, - "viewports": [ - {"id": "desktop", "width": 800, "height": 600}, - {"id": "mobile", "width": 375, "height": 600}, - ], - "rubric_version": "landing-quality-v1", - "evaluator": { - "caller": "codex", - "iop": { - "request_model": "judge-model", - "requested_effort": "xhigh", - "route_kind": "direct", - "route_id": "judge-route", - "expected_bindings": [ - { - "stage": "request", - "model": "judge-model", - "effort": "xhigh", - } - ], - }, - }, - "output_root": "agent-test/runs/anonymous-bench", - "fixture": { - "version": "landing-v1", - "prompt": "scripts/fixtures/bench/prompt.md", - "assets": [ - { - "source": item.source, - "workspace_path": item.workspace_path, - } - for item in assets - ], - "checksum": digest_workspace_inputs(assets), - }, - "matrix": [ - { - "id": "cell-sentinel", - "caller": "claude", - "iop": { - "request_model": "source-model", - "requested_effort": "high", - "route_kind": "direct", - "route_id": "source-route", - "expected_bindings": [ - { - "stage": "request", - "model": "source-model", - "effort": "high", - } - ], - }, - } - ], - } - self.manifest_path = self.root / "manifest.json" - self.manifest_path.write_text(json.dumps(raw), encoding="utf-8") - self.manifest = load_manifest(self.manifest_path, repo_root=self.root) - tokens = iter( - ( - "123456abcdef", - "234567abcdef", - "345678abcdef", - *(f"{index:012x}" for index in range(4, 100)), - ) - ) - self.store = RunStore( - self.root, - clock=lambda: datetime.datetime( - 2026, 8, 11, 1, 2, 3, tzinfo=datetime.timezone.utc - ), - token_hex=lambda _n: next(tokens), - ) - self.run = self.store.create(self.manifest, self.manifest_path.read_bytes()) - - @staticmethod - def _measurement(attempt, terminal_reason: str) -> AttemptMeasurement: - timeline = { - "submitted_at": unavailable(REASON_NOT_OBSERVED, SOURCE_HARNESS), - "first_output_at": unavailable(REASON_NOT_OBSERVED, SOURCE_HARNESS), - "first_write_observed_at": unavailable( - REASON_NOT_OBSERVED, SOURCE_WORKSPACE_POLL - ), - "first_write_mtime": unavailable( - REASON_NOT_OBSERVED, SOURCE_WORKSPACE_POLL - ), - "total_duration": observed( - 1, UNIT_NANOSECONDS, CLOCK_HARNESS_MONOTONIC, SOURCE_HARNESS - ), - } - usage = { - name: unavailable(REASON_NOT_REPORTED, SOURCE_HARNESS) - for name in METRIC_NAMES - } - succeeded = terminal_reason == "success" - return AttemptMeasurement( - attempt.identity.run_id, - attempt.identity.cell_id, - attempt.identity.repetition, - attempt.identity.attempt, - "claude", - "sha256:" + "3" * 64, - ProductOutcome( - "succeeded" if succeeded else "failed", - CALLER_REASON_SUCCESS if succeeded else CALLER_REASON_ERROR, - ), - HarnessOutcome("passed", "success", True, True), - ProcessOutcome("exited", 0 if succeeded else 1, None), - timeline, - usage, - WorkspaceWriteObservation( - False, None, None, "", 1, 0, REASON_NOT_OBSERVED - ), - (), - ) - - def _view( - self, - root: Path, - ident: str, - width: int, - web_gate_failures: frozenset[str] = frozenset(), - ) -> ViewportObservation: - screenshot = f"screenshot-{ident}.png" - png = b"\x89PNG\r\n\x1a\n" + ident.encode("ascii") - (root / screenshot).write_bytes(png) - images = tuple( - { - "src": path, - "alt": path, - "complete": True, - "natural_width": 20, - "natural_height": 20, - "visible": "images" not in web_gate_failures, - "rect": { - "x": 0, - "y": 0, - "width": 20, - "height": 20, - "right": 20, - "bottom": 20, - }, - } - for path in ("assets/a.svg", "assets/b.svg") - ) - return ViewportObservation( - ident, - width, - 600, - screenshot, - _digest(png), - len(png), - images, - { - "scroll_width": width, - "client_width": width, - "clipped": 1 if "responsive" in web_gate_failures else 0, - "overlaps": 0, - }, - { - "h1_count": 1, - "headings": [1], - "heading_progression": True, - "main_count": 1, - "landmarks": 1, - "controls": [ - { - "name": "accessibility" not in web_gate_failures, - "tab_index": 0, - "focused": True, - "focus_visible": True, - "contrast": 7.0, - } - ], - "ax": {"nodes": 4, "non_ignored": 3, "named": 2}, - }, - ) - - def _attempt( - self, state: str = "success", *, leaked: bool = False, - rendered: bool = True, leaked_identity: str | None = None, - web_gate_failures: frozenset[str] = frozenset(), - ): - with self.store.writer(self.run): - attempt = self.store.allocate(self.run, Slot("cell-sentinel", 1)) - workspace = Path(attempt.root) / "workspace" - (workspace / "assets").mkdir(parents=True) - (workspace / "brief").mkdir() - for asset in self.manifest.fixture.assets: - (workspace / asset.workspace_path).write_bytes(asset.content) - heading = leaked_identity or ("source-route" if leaked else "Ready") - (workspace / "index.html").write_text( - f"

{heading}

A" - "Bgo" - "
", - encoding="utf-8", - ) - (workspace / "styles.css").write_text( - "body{color:#111;background:#fff}img{width:20px}" - "a:focus{outline:2px solid #05f}", - encoding="utf-8", - ) - (workspace / "script.js").write_text( - "document.body.dataset.ready='1';", encoding="utf-8" - ) - if "generated_files" in web_gate_failures: - (workspace / "styles.css").unlink() - elif "static_safety" in web_gate_failures: - (workspace / "index.html").write_text( - "

Ready

", - encoding="utf-8", - ) - terminal_reason = "success" if state == "success" else "caller_error" - measurement = self._measurement(attempt, terminal_reason) - publish_measurement(attempt.root, measurement) - if state == "success" and rendered: - render = RenderObservation( - "Chromium/Test", - "http://127.0.0.1:12345", - ( - {"kind": "local", "path": "/index.html", "allowed": True, "status": 200}, - {"kind": "local", "path": "/assets/a.svg", "allowed": True, "status": 200}, - {"kind": "local", "path": "/assets/b.svg", "allowed": True, "status": 200}, - *( - ({"kind": "external", "url_digest": _digest(b"blocked"), "allowed": False, "status": 0},) - if "network" in web_gate_failures - else () - ), - ), - ( - ({"kind": "console", "level": "error"},) - if "console" in web_gate_failures - else () - ), - ( - self._view( - Path(attempt.root), "desktop", 800, web_gate_failures - ), - self._view( - Path(attempt.root), "mobile", 375, web_gate_failures - ), - ), - ) - else: - render = None - web = build_web_validation( - self.manifest, workspace, measurement, render - ) - publish_web_validation(attempt.root, web) - succeeded = state == "success" - lifecycle = { - "product": { - "status": "succeeded" if succeeded else "failed", - "reason": ( - CALLER_REASON_SUCCESS if succeeded else CALLER_REASON_ERROR - ), - }, - "harness": { - "status": "passed", - "reason": "success", - "ordered_terminal": True, - "cleanup_complete": True, - }, - "process": { - "status": "exited", - "exit_code": 0 if succeeded else 1, - "signal": None, - }, - } - terminal = self.store.publish_terminal( - attempt, "completed", result=lifecycle - ) - return terminal - - def test_scored_attempt_is_blind_exactly_once_and_strict(self): - attempt = self._attempt() - adapter = FakeScoringAdapter() - summary = score_run( - self.store, self.run, self.manifest, adapter=adapter - ) - self.assertEqual( - (summary.scored, summary.unscored, summary.scoring_failed, summary.blocked), - (1, 0, 0, 0), - ) - self.assertEqual(adapter.preflights, 1) - self.assertEqual(len(adapter.invocations), 1) - blind, prompt = adapter.invocations[0] - self.assertEqual( - prompt, - ( - "Evaluate the anonymous landing page using only the files under input/. " - "Inspect index.html, styles.css, script.js, the two local images, and all " - "screenshots. Do not use network access or infer the producing system. " - "Use exactly these categories: task_fidelity (25), visual_hierarchy " - "(25), responsive_composition (20), typography_readability (15), " - "polish_consistency (15). Write exactly one JSON object to " - "output/worksheet.json with fields rubric_version, categories, and " - "total. Each category must have exactly id, max_score, score, and " - "non-empty evidence. Use rubric_version landing-quality-v1, integer " - "scores within each maximum, and total equal to their sum. Do not add " - "automatic gate points or any extra field." - ).encode("utf-8"), - ) - self.assertNotIn("cell-sentinel", blind.root) - self.assertNotIn("source-route", blind.root) - self.assertNotIn(str(Path(attempt.root).resolve()), blind.root) - visible = prompt - for path in Path(blind.root).rglob("*"): - if path.is_file(): - visible += b"\n" + str(path).encode() + b"\n" + path.read_bytes() - for sentinel in ( - b"cell-sentinel", - b"source-route", - b"source-model", - str(Path(attempt.root).resolve()).encode(), - ): - self.assertNotIn(sentinel, visible) - self.assertEqual( - sorted(path.relative_to(blind.input_dir).as_posix() for path in Path(blind.input_dir).rglob("*") if path.is_file()), - [ - "assets/a.svg", - "assets/b.svg", - "index.html", - "screenshots/screenshot-desktop.png", - "screenshots/screenshot-mobile.png", - "script.js", - "styles.css", - ], - ) - result_path = Path(attempt.root) / "scoring" / "score-000001" / "result.json" - result = json.loads(result_path.read_text(encoding="ascii")) - self.assertEqual(result["status"], "scored") - self.assertEqual(result["worksheet"]["total"], 99) - self.assertNotIn("gates", result["worksheet"]) - mappings = tuple((Path(self.run.root) / "blind-mappings").glob("*.json")) - self.assertEqual(len(mappings), 1) - mapping = json.loads(mappings[0].read_text(encoding="ascii")) - self.assertEqual(mapping["attempt"]["cell_id"], "cell-sentinel") - self.assertFalse(mappings[0].is_relative_to(Path(blind.root))) - - before = {path: path.read_bytes() for path in Path(attempt.root).rglob("*") if path.is_file()} - second = score_run(self.store, self.run, self.manifest, adapter=adapter) - self.assertEqual(second.scored, 1) - self.assertEqual(len(adapter.invocations), 1) - self.assertEqual(before, {path: path.read_bytes() for path in Path(attempt.root).rglob("*") if path.is_file()}) - - def test_scoring_invocation_result_rejects_open_or_contradictory_reasons(self): - with self.assertRaises(ScoringError): - ScoringInvocationResult( - "succeeded", "passed", "exited", 0, None, - "arbitrary_reason", None, - ) - with self.assertRaises(ScoringError): - ScoringInvocationResult( - "succeeded", "passed", "exited", 0, None, - "evaluator_failed", None, - ) - - def test_manifest_selected_rubric_drives_prompt_and_worksheet_validation(self): - raw = json.loads(self.manifest_path.read_text(encoding="utf-8")) - raw["rubric_version"] = ONE_SHOT_RUBRIC_VERSION - raw["output_root"] = "agent-test/runs/new-rubric-bench" - path = self.root / "new-rubric.json" - path.write_text(json.dumps(raw), encoding="utf-8") - self.manifest_path = path - self.manifest = load_manifest(path, repo_root=self.root) - - self.run = self.store.create(self.manifest, path.read_bytes()) - attempt = self._attempt() - selected = FakeScoringAdapter(rubric_version=ONE_SHOT_RUBRIC_VERSION) - summary = score_run( - self.store, self.run, self.manifest, adapter=selected - ) - self.assertEqual((summary.scored, summary.scoring_failed), (1, 0)) - self.assertEqual(len(selected.invocations), 1) - _, prompt = selected.invocations[0] - self.assertIn( - b"Use exactly these categories: requirements_fidelity (25), " - b"visual_completeness (25), responsive_accessibility (15), " - b"image_detail_usage (10), behavior_stability (10), code_quality (10), " - b"self_verification (5).", - prompt, - ) - self.assertIn( - b"Use rubric_version one-shot-agent-comparison-v1, integer scores", - prompt, - ) - result_path = ( - Path(attempt.root) / "scoring" / "score-000001" / "result.json" - ) - result = json.loads(result_path.read_text(encoding="ascii")) - self.assertEqual(result["status"], "scored") - self.assertEqual( - result["worksheet"]["rubric_version"], ONE_SHOT_RUBRIC_VERSION - ) - self.assertEqual( - [item["id"] for item in result["worksheet"]["categories"]], - [ident for ident, _ in rubric_categories(ONE_SHOT_RUBRIC_VERSION)], - ) - - retained = score_run( - self.store, - self.run, - self.manifest, - adapter=FakeScoringAdapter(rubric_version=ONE_SHOT_RUBRIC_VERSION), - ) - self.assertEqual(retained.scored, 1) - - self.run = self.store.create(self.manifest, path.read_bytes()) - mismatched_attempt = self._attempt() - legacy = FakeScoringAdapter() - rejected = score_run( - self.store, self.run, self.manifest, adapter=legacy - ) - self.assertEqual((rejected.scored, rejected.scoring_failed), (0, 1)) - self.assertEqual(len(legacy.invocations), 1) - mismatch_result = json.loads( - ( - Path(mismatched_attempt.root) - / "scoring" - / "score-000001" - / "result.json" - ).read_text(encoding="ascii") - ) - self.assertEqual(mismatch_result["status"], "scoring_failed") - self.assertEqual(mismatch_result["reason"], "invalid_worksheet") - self.assertNotIn("worksheet", mismatch_result) - - def test_ineligible_attempt_is_unscored_without_preflight_or_zero(self): - attempt = self._attempt("failed") - adapter = FakeScoringAdapter() - summary = score_run(self.store, self.run, self.manifest, adapter=adapter) - self.assertEqual((summary.unscored, summary.scored), (1, 0)) - self.assertEqual((adapter.preflights, adapter.invocations), (0, [])) - path = Path(attempt.root) / "scoring" / "unscored.json" - before = path.read_bytes() - record = json.loads(before) - self.assertEqual(record["status"], "unscored") - self.assertEqual( - record["reasons"], - [ - "product_failed", - "process_nonzero_exit", - "gate_images", - "gate_network", - ], - ) - self.assertFalse(set(record) & {"score", "total", "worksheet"}) - score_run(self.store, self.run, self.manifest, adapter=adapter) - self.assertEqual(path.read_bytes(), before) - - def test_each_hard_web_gate_failure_is_unscored_without_evaluator(self): - for gate in sorted(scoring_module.HARD_ELIGIBILITY_GATES): - with self.subTest(gate=gate): - self.run = self.store.create( - self.manifest, self.manifest_path.read_bytes() - ) - attempt = self._attempt(web_gate_failures=frozenset((gate,))) - adapter = FakeScoringAdapter() - summary = score_run( - self.store, self.run, self.manifest, adapter=adapter - ) - self.assertEqual((summary.unscored, summary.scored), (1, 0)) - self.assertEqual((adapter.preflights, adapter.invocations), (0, [])) - record = json.loads( - (Path(attempt.root) / "scoring" / "unscored.json").read_text() - ) - self.assertIn(f"gate_{gate}", record["reasons"]) - self.assertFalse( - any( - f"gate_{quality}" in record["reasons"] - for quality in scoring_module.QUALITY_SCORING_GATES - ) - ) - - def test_each_quality_only_web_failure_remains_score_eligible(self): - for gate in sorted(scoring_module.QUALITY_SCORING_GATES): - with self.subTest(gate=gate): - self.run = self.store.create( - self.manifest, self.manifest_path.read_bytes() - ) - attempt = self._attempt(web_gate_failures=frozenset((gate,))) - adapter = FakeScoringAdapter() - summary = score_run( - self.store, self.run, self.manifest, adapter=adapter - ) - self.assertEqual((summary.scored, summary.unscored), (1, 0)) - self.assertEqual(adapter.preflights, 1) - self.assertEqual(len(adapter.invocations), 1) - self.assertFalse((Path(attempt.root) / "scoring" / "unscored.json").exists()) - - def test_mixed_hard_and_quality_web_failures_remain_unscored(self): - attempt = self._attempt( - web_gate_failures=frozenset(("network", "accessibility")) - ) - adapter = FakeScoringAdapter() - summary = score_run(self.store, self.run, self.manifest, adapter=adapter) - self.assertEqual((summary.unscored, summary.scored), (1, 0)) - self.assertEqual((adapter.preflights, adapter.invocations), (0, [])) - record = json.loads( - (Path(attempt.root) / "scoring" / "unscored.json").read_text() - ) - self.assertIn("gate_network", record["reasons"]) - self.assertNotIn("gate_accessibility", record["reasons"]) - - def test_missing_or_malformed_web_evidence_fails_closed(self): - for mode in ("missing", "malformed"): - with self.subTest(mode=mode): - self.run = self.store.create( - self.manifest, self.manifest_path.read_bytes() - ) - attempt = self._attempt() - path = Path(attempt.root) / "web-validation.json" - if mode == "missing": - path.unlink() - else: - path.write_bytes(b"not-json\n") - adapter = FakeScoringAdapter() - with self.assertRaises((ScoringError, AttemptStateError)): - score_run( - self.store, self.run, self.manifest, adapter=adapter - ) - self.assertEqual((adapter.preflights, adapter.invocations), (0, [])) - - def test_not_run_web_gates_are_all_unscored_without_evaluator(self): - attempt = self._attempt(rendered=False) - adapter = FakeScoringAdapter() - summary = score_run(self.store, self.run, self.manifest, adapter=adapter) - self.assertEqual((summary.unscored, summary.scored), (1, 0)) - self.assertEqual((adapter.preflights, adapter.invocations), (0, [])) - record = json.loads( - (Path(attempt.root) / "scoring" / "unscored.json").read_text() - ) - self.assertEqual(record["status"], "unscored") - web = json.loads( - (Path(attempt.root) / "web-validation.json").read_text() - ) - failed_hard_gates = { - f"gate_{item['id']}" for item in web["gates"] if not item["passed"] - and item["id"] in scoring_module.HARD_ELIGIBILITY_GATES - } - self.assertTrue(failed_hard_gates) - self.assertTrue(failed_hard_gates.issubset(record["reasons"])) - self.assertEqual( - [item["id"] for item in web["gates"]], list(WEB_GATES) - ) - self.assertFalse(set(record) & {"score", "total", "worksheet"}) - - def test_symlinked_generated_input_fails_before_evaluator_invocation(self): - attempt = self._attempt() - workspace = Path(attempt.root) / "workspace" - (workspace / "index.html").unlink() - (workspace / "index.html").symlink_to("styles.css") - adapter = FakeScoringAdapter() - with self.assertRaises(AttemptStateError): - score_run(self.store, self.run, self.manifest, adapter=adapter) - self.assertEqual(adapter.invocations, []) - self.assertFalse((Path(attempt.root) / "scoring").exists()) - - def test_preflight_blocker_allocates_no_score(self): - attempt = self._attempt() - adapter = FakeScoringAdapter(blocked=True) - summary = score_run(self.store, self.run, self.manifest, adapter=adapter) - self.assertEqual(summary.blocked, 1) - self.assertEqual(adapter.invocations, []) - self.assertFalse((Path(attempt.root) / "scoring").exists()) - self.assertEqual( - len(list((Path(self.run.root) / "scoring-preflight").iterdir())), 1 - ) - - def test_failed_score_retries_only_with_new_id_and_fresh_session(self): - attempt = self._attempt() - adapter = FakeScoringAdapter(modes=["malformed", "success"]) - first = score_run(self.store, self.run, self.manifest, adapter=adapter) - self.assertEqual(first.scoring_failed, 1) - failed = Path(attempt.root) / "scoring" / "score-000001" - failed_bytes = {path: path.read_bytes() for path in failed.rglob("*") if path.is_file()} - - second = score_run(self.store, self.run, self.manifest, adapter=adapter) - self.assertEqual(second.scoring_failed, 1) - self.assertEqual(len(adapter.invocations), 1) - self.assertFalse((Path(attempt.root) / "scoring" / "score-000002").exists()) - - third = score_run( - self.store, - self.run, - self.manifest, - adapter=adapter, - retry_scoring_failed=True, - ) - self.assertEqual(third.scored, 1) - self.assertEqual(len(adapter.invocations), 2) - self.assertEqual( - failed_bytes, - {path: path.read_bytes() for path in failed.rglob("*") if path.is_file()}, - ) - allocations = [ - json.loads((Path(attempt.root) / "scoring" / f"score-{index:06d}" / "allocation.json").read_text()) - for index in (1, 2) - ] - self.assertNotEqual(allocations[0]["blind_id"], allocations[1]["blind_id"]) - self.assertNotEqual( - allocations[0]["session_identity"], allocations[1]["session_identity"] - ) - - def test_invalid_binding_and_identity_leak_fail_without_fallback(self): - attempt = self._attempt() - binding = FakeScoringAdapter(modes=["binding"]) - summary = score_run(self.store, self.run, self.manifest, adapter=binding) - self.assertEqual(summary.scoring_failed, 1) - result = json.loads( - (Path(attempt.root) / "scoring" / "score-000001" / "result.json").read_text() - ) - self.assertEqual(result["status"], "scoring_failed") - self.assertNotIn("worksheet", result) - - self.run = self.store.create(self.manifest, self.manifest_path.read_bytes()) - process_attempt = self._attempt() - process = FakeScoringAdapter(modes=["process_nonzero"]) - summary = score_run(self.store, self.run, self.manifest, adapter=process) - self.assertEqual(summary.scoring_failed, 1) - result = json.loads( - ( - Path(process_attempt.root) - / "scoring" - / "score-000001" - / "result.json" - ).read_text() - ) - self.assertEqual((result["status"], result["reason"]), ( - "scoring_failed", "evaluator_failed", - )) - - # A separate run proves an identity sentinel in retained page bytes is - # rejected before the evaluator is invoked. - self.run = self.store.create(self.manifest, self.manifest_path.read_bytes()) - leaked = self._attempt(leaked=True) - adapter = FakeScoringAdapter() - leaked_summary = score_run( - self.store, self.run, self.manifest, adapter=adapter - ) - self.assertEqual(leaked_summary.scoring_failed, 1) - self.assertEqual(adapter.invocations, []) - leaked_result = json.loads( - (Path(leaked.root) / "scoring" / "score-000001" / "result.json").read_text() - ) - self.assertEqual(leaked_result["reason"], "blind_preparation_failed") - - def test_shared_evaluator_binding_is_allowed_but_short_caller_leak_fails(self): - raw = json.loads(self.manifest_path.read_text()) - raw["matrix"][0]["caller"] = "agy" - raw["matrix"][0]["iop"] = { - "request_model": "judge-model", - "requested_effort": "xhigh", - "route_kind": "direct", - "route_id": "judge-route", - "expected_bindings": [ - { - "stage": "request", - "model": "judge-model", - "effort": "xhigh", - } - ], - } - raw["output_root"] = "agent-test/runs/shared-binding" - path = self.root / "shared.json" - path.write_text(json.dumps(raw), encoding="utf-8") - self.manifest_path = path - self.manifest = load_manifest(path, repo_root=self.root) - self.run = self.store.create(self.manifest, path.read_bytes()) - self._attempt() - shared = FakeScoringAdapter() - summary = score_run( - self.store, self.run, self.manifest, adapter=shared - ) - self.assertEqual((summary.scored, summary.scoring_failed), (1, 0)) - - self.run = self.store.create(self.manifest, path.read_bytes()) - leaked = self._attempt(leaked_identity="agy") - rejected = FakeScoringAdapter() - summary = score_run( - self.store, self.run, self.manifest, adapter=rejected - ) - self.assertEqual((summary.scored, summary.scoring_failed), (0, 1)) - self.assertEqual(rejected.invocations, []) - result = json.loads( - ( - Path(leaked.root) - / "scoring" - / "score-000001" - / "result.json" - ).read_text(encoding="ascii") - ) - self.assertEqual(result["reason"], "blind_preparation_failed") - - def test_evaluator_session_identity_tokens_are_accepted_but_anonymous_evidence_rejects_them(self): - raw = json.loads(self.manifest_path.read_text(encoding="utf-8")) - raw["matrix"][0]["iop"]["request_model"] = "gpt-5.6-terra" - raw["matrix"][0]["iop"]["expected_bindings"][0]["model"] = ( - "gpt-5.6-terra" - ) - raw["output_root"] = "agent-test/runs/evaluator-session-provenance" - path = self.root / "evaluator-session-provenance.json" - path.write_text(json.dumps(raw), encoding="utf-8") - self.manifest_path = path - self.manifest = load_manifest(path, repo_root=self.root) - tokens = iter(f"{index:012x}" for index in range(1, 9)) - self.store = RunStore( - self.root, - clock=lambda: datetime.datetime( - 2026, 8, 11, 1, 2, 4, tzinfo=datetime.timezone.utc - ), - token_hex=lambda _n: next(tokens), - ) - self.run = self.store.create(self.manifest, path.read_bytes()) - - class IdentityEvidenceAdapter(FakeScoringAdapter): - def __init__(self, location): - super().__init__() - self.location = location - - def invoke(self, cell, blind, task_payload, timeout, on_started): - result = super().invoke( - cell, blind, task_payload, timeout, on_started - ) - if self.location == "session": - state = Path(blind.session_dir) / "gpt-5.6-terra" - state.mkdir() - (state / "high").write_text( - "gpt-5.6-terra high", encoding="utf-8" - ) - elif self.location == "output-path": - (Path(blind.output_dir) / "gpt-5.6-terra").write_text( - "anonymous", encoding="utf-8" - ) - elif self.location == "output-content": - (Path(blind.output_dir) / "identity.txt").write_text( - "gpt-5.6-terra high", encoding="utf-8" - ) - elif self.location == "worksheet": - worksheet_path = Path(blind.output_dir) / "worksheet.json" - worksheet = json.loads(worksheet_path.read_text(encoding="ascii")) - worksheet["categories"][0]["evidence"] = ( - "gpt-5.6-terra high" - ) - worksheet_path.write_text( - json.dumps( - worksheet, - sort_keys=True, - separators=(",", ":"), - ) - + "\n", - encoding="ascii", - ) - return result - - accepted_attempt = self._attempt() - identities = scoring_module._identity_values( - self.manifest, accepted_attempt - ) - self.assertIn("gpt-5.6-terra", identities.producer_tokens) - self.assertIn("high", identities.producer_tokens) - accepted = IdentityEvidenceAdapter("session") - accepted_summary = score_run( - self.store, self.run, self.manifest, adapter=accepted - ) - self.assertEqual( - (accepted_summary.scored, accepted_summary.scoring_failed), - (1, 0), - ) - - self.run = self.store.create( - self.manifest, self.manifest_path.read_bytes() - ) - leaked_input = self._attempt(leaked_identity="gpt-5.6-terra high") - input_adapter = FakeScoringAdapter() - input_summary = score_run( - self.store, self.run, self.manifest, adapter=input_adapter - ) - self.assertEqual( - (input_summary.scored, input_summary.scoring_failed), (0, 1) - ) - self.assertEqual(input_adapter.invocations, []) - input_result = json.loads( - ( - Path(leaked_input.root) - / "scoring" - / "score-000001" - / "result.json" - ).read_text(encoding="ascii") - ) - self.assertEqual(input_result["reason"], "blind_preparation_failed") - - for location in ("output-path", "output-content", "worksheet"): - with self.subTest(location=location): - self.run = self.store.create( - self.manifest, self.manifest_path.read_bytes() - ) - leaked_output = self._attempt() - output_adapter = IdentityEvidenceAdapter(location) - output_summary = score_run( - self.store, - self.run, - self.manifest, - adapter=output_adapter, - ) - self.assertEqual( - (output_summary.scored, output_summary.scoring_failed), - (0, 1), - ) - output_result = json.loads( - ( - Path(leaked_output.root) - / "scoring" - / "score-000001" - / "result.json" - ).read_text(encoding="ascii") - ) - self.assertEqual( - output_result["reason"], "evaluator_output_leak" - ) - - def test_delimited_short_caller_and_cell_identity_leaks_fail(self): - identity = scoring_module.ProducerIdentity( - exact_tokens=("agy", "cell-sentinel"), - path_tokens=(), - producer_tokens=(), - evaluator_shared_tokens=(), - ) - for value in ( - b"agy", - b"caller=agy", - b"agy-output", - b"agy_output", - b"cell-sentinel-output", - b"cell-sentinel_output", - ): - with self.subTest(value=value): - self.assertTrue(scoring_module._contains_identity(value, identity)) - for value in ( - b"strategy", - b"agyextended", - b"mycell-sentinel", - ): - with self.subTest(value=value): - self.assertFalse(scoring_module._contains_identity(value, identity)) - - def test_binary_identity_boundaries_do_not_disappear(self): - identity = scoring_module.ProducerIdentity( - exact_tokens=("agy", "cell-sentinel"), - path_tokens=(), - producer_tokens=("producer-model", "shared-model"), - evaluator_shared_tokens=("shared-model",), - ) - for value in ( - b"\x89PNG\r\n\x1a\nx\xffagy\x00", - b"\x89PNG\r\n\x1a\nagy\xffx", - b"\x89PNG\r\n\x1a\nx\xffcell-sentinel\x00", - b"\x89PNG\r\n\x1a\ncell-sentinel\xffx", - b"\x89PNG\r\n\x1a\nx\xffproducer-model\x00", - b"\x89PNG\r\n\x1a\nproducer-model\xffx", - ): - with self.subTest(value=value): - self.assertTrue(scoring_module._contains_identity(value, identity)) - for value in ( - b"strategy", - b"xagy", - b"agyx", - b"xproducer-model", - b"producer-modelx", - b"x\xffshared-model\x00", - ): - with self.subTest(value=value): - self.assertFalse(scoring_module._contains_identity(value, identity)) - - def test_invalid_filesystem_bytes_do_not_bypass_identity_scan(self): - if os.name != "posix": - self.skipTest("raw invalid filesystem bytes require POSIX paths") - - identity = scoring_module.ProducerIdentity( - exact_tokens=("agy", "cell-sentinel"), - path_tokens=(), - producer_tokens=("source-route", "source-model"), - evaluator_shared_tokens=(), - ) - - def write_raw(directory: Path, name: bytes, data: bytes = b"safe") -> None: - descriptor = os.open( - os.fsencode(directory) + b"/" + name, - os.O_WRONLY | os.O_CREAT | os.O_EXCL, - 0o600, - ) - try: - os.write(descriptor, data) - finally: - os.close(descriptor) - - for index, name in enumerate( - ( - b"x\xffagy\xfe.png", - b"x\xffcell-sentinel\xfe.png", - b"x\xffsource-route\xfe.png", - b"x\xffsource-model\xfe.png", - ) - ): - with self.subTest(name=name): - visible_root = self.root / f"invalid-visible-{index}" - visible_root.mkdir() - write_raw(visible_root, name) - with self.assertRaisesRegex( - ScoringError, "evaluator-visible evidence leaks execution identity" - ): - scoring_module._scan_visible_tree(visible_root, identity) - - class RawFilenameAdapter(FakeScoringAdapter): - def __init__(self, raw_name: bytes): - super().__init__() - self.raw_name = raw_name - - def invoke(self, cell, blind, task_payload, timeout, on_started): - result = super().invoke( - cell, blind, task_payload, timeout, on_started - ) - write_raw(Path(blind.output_dir), self.raw_name) - return result - - leaked_attempt = self._attempt() - leaked_adapter = RawFilenameAdapter( - b"x\xffcell-sentinel\xfe-output.bin" - ) - leaked_summary = score_run( - self.store, self.run, self.manifest, adapter=leaked_adapter - ) - self.assertEqual( - (leaked_summary.scored, leaked_summary.scoring_failed), (0, 1) - ) - leaked_root = ( - Path(leaked_attempt.root) / "scoring" / "score-000001" - ) - leaked_result = json.loads( - (leaked_root / "result.json").read_text(encoding="ascii") - ) - self.assertEqual(leaked_result["status"], "scoring_failed") - self.assertEqual(leaked_result["reason"], "evaluator_output_leak") - leaked_bytes = { - path: path.read_bytes() - for path in leaked_root.rglob("*") - if path.is_file() - } - retained = score_run( - self.store, self.run, self.manifest, adapter=leaked_adapter - ) - self.assertEqual(retained.scoring_failed, 1) - self.assertEqual(len(leaked_adapter.invocations), 1) - self.assertEqual( - leaked_bytes, - { - path: path.read_bytes() - for path in leaked_root.rglob("*") - if path.is_file() - }, - ) - - self.run = self.store.create( - self.manifest, self.manifest_path.read_bytes() - ) - safe_attempt = self._attempt() - safe_adapter = RawFilenameAdapter(b"x\xffanonymous\xfe-output.bin") - safe_summary = score_run( - self.store, self.run, self.manifest, adapter=safe_adapter - ) - self.assertEqual( - (safe_summary.scored, safe_summary.scoring_failed), (1, 0) - ) - safe_blind = Path(safe_adapter.invocations[0][0].root) - first_digest = scoring_module._blind_tree_digest(safe_blind) - self.assertEqual( - first_digest, scoring_module._blind_tree_digest(safe_blind) - ) - safe_result = json.loads( - ( - Path(safe_attempt.root) - / "scoring" - / "score-000001" - / "result.json" - ).read_text(encoding="ascii") - ) - self.assertEqual(safe_result["status"], "scored") - self.assertEqual(safe_result["post_tree_digest"], first_digest) - - ordinary_path = "input/caf\N{LATIN SMALL LETTER E WITH ACUTE}.txt" - ordinary_data = b"ordinary" - framed = bytearray(b"IOP-BENCH-BLIND-INPUT-V1\0") - ordinary_bytes = ordinary_path.encode("utf-8") - framed += len(ordinary_bytes).to_bytes(8, "big") + ordinary_bytes - framed += len(ordinary_data).to_bytes(8, "big") + ordinary_data - self.assertEqual( - scoring_module._input_digest([(ordinary_path, ordinary_data)]), - _digest(bytes(framed)), - ) - - def unusable_fsencode(value): - raise AssertionError("path framing must not use the filesystem codec") - - with mock.patch.object(scoring_module.os, "fsencode", unusable_fsencode): - self.assertEqual( - scoring_module._input_digest([(ordinary_path, ordinary_data)]), - _digest(bytes(framed)), - ) - self.assertEqual( - scoring_module._path_bytes(ordinary_path), - ordinary_path.encode("utf-8"), - ) - self.assertEqual( - scoring_module._path_bytes("input/x\udcff.txt"), - b"input/x\xff.txt", - ) - with self.assertRaisesRegex(ScoringError, "scoring path is invalid"): - scoring_module._path_bytes("input/x\ud800.txt") - - surrogate_path = "input/x\udcff.txt" - surrogate_data = b"raw" - surrogate_framed = bytearray(b"IOP-BENCH-BLIND-INPUT-V1\0") - surrogate_bytes = b"input/x\xff.txt" - surrogate_framed += len(surrogate_bytes).to_bytes(8, "big") + surrogate_bytes - surrogate_framed += len(surrogate_data).to_bytes(8, "big") + surrogate_data - self.assertEqual( - scoring_module._input_digest([(surrogate_path, surrogate_data)]), - _digest(bytes(surrogate_framed)), - ) - - def test_receipt_only_recovery_waits_for_lifecycle_quiescence(self): - blind_root = self.root / "receipt-only" / "blind" / "blind-paused" - for name in ("input", "session", "output"): - (blind_root / name).mkdir(parents=True, exist_ok=True) - output = blind_root / "output" - control = output / "paused-control" - control.mkdir() - alias = Path(tempfile.gettempdir()) / ( - "iop-score-paused-" - + hashlib.sha256(str(output).encode("utf-8")).hexdigest()[:16] - ) - os.symlink(output, alias, target_is_directory=True) - self.addCleanup( - lambda: alias.unlink() - if alias.exists() or alias.is_symlink() - else None - ) - locator = SupervisorLocator( - os.getpid(), - "paused-start-identity", - str(alias / "paused-control" / "control.sock"), - "paused-challenge", - str(alias / "paused-control"), - "2026-08-11T00:00:00+00:00", - ) - locator_payload = { - "supervisor_pid": locator.supervisor_pid, - "start_identity": locator.start_identity, - "socket_path": locator.socket_path, - "challenge": locator.challenge, - "control_dir": locator.control_dir, - "created_at": locator.created_at, - } - (control / "locator.json").write_text( - json.dumps(locator_payload), encoding="utf-8" - ) - receipt = { - "receipt_version": 1, - "supervisor_pid": locator.supervisor_pid, - "challenge_digest": hashlib.sha256( - locator.challenge.encode("utf-8") - ).hexdigest(), - "reason": "success", - "exit_code": 0, - "signal": None, - "caller_launched": True, - "cleanup_complete": True, - "process_group_alive": False, - "completed_at": "2026-08-11T00:00:01+00:00", - } - (control / "cleanup-receipt.json").write_text( - json.dumps(receipt), encoding="utf-8" - ) - invocation_digest = "sha256:" + "9" * 64 - public_locator = { - key: value for key, value in locator_payload.items() if key != "challenge" - } - public_locator["challenge_digest"] = receipt["challenge_digest"] - product = {"status": "succeeded", "reason": CALLER_REASON_SUCCESS} - harness = { - "status": "passed", - "reason": "success", - "ordered_terminal": True, - "cleanup_complete": True, - } - process = {"status": "exited", "exit_code": 0, "signal": None} - lifecycle = { - "record": "result", - "product": product, - "harness": harness, - "process": process, - "process_group_alive": False, - "spec_digest": invocation_digest, - "locator": public_locator, - } - journal = ( - json.dumps( - { - "record": "header", - "journal_version": JOURNAL_VERSION, - "spec_digest": invocation_digest, - } - ) - + "\n" - + json.dumps( - { - "record": "terminal", - "product": product, - "harness": harness, - "process": process, - "process_group_alive": False, - } - ) - + "\n" - ) - prior = { - path: path.read_bytes() - for path in (control / "locator.json", control / "cleanup-receipt.json") - } - recovered: list[tuple[str | None, str]] = [] - errors: list[BaseException] = [] - successor_started = threading.Event() - - def recover() -> None: - try: - recovered.append( - scoring_module._recover_runner( - blind_root, - locator, - invocation_digest, - control_target=control, - ) - ) - scoring_module._release_runner_alias( - { - "control_alias": str(alias), - "control_target": str(control), - } - ) - successor_started.set() - except BaseException as exc: - errors.append(exc) - - worker = threading.Thread(target=recover) - wait_started = threading.Event() - original_wait = scoring_module._wait_post_cleanup_quiet - - def observe_wait(*args, **kwargs): - wait_started.set() - return original_wait(*args, **kwargs) - - started = time.monotonic() - with mock.patch.object( - scoring_module, - "_wait_post_cleanup_quiet", - side_effect=observe_wait, - ): - worker.start() - self.assertTrue(wait_started.wait(1)) - self.assertTrue(worker.is_alive()) - self.assertFalse(successor_started.is_set()) - self.assertTrue(alias.is_symlink()) - self.assertFalse((output / "lifecycle-result.json").exists()) - - time.sleep(0.35) - (output / "lifecycle-journal.jsonl").write_text( - journal, encoding="utf-8" - ) - (output / "lifecycle-result.json").write_text( - json.dumps(lifecycle), encoding="utf-8" - ) - time.sleep(0.05) - self.assertTrue(worker.is_alive()) - worker.join(5) - self.assertFalse(worker.is_alive()) - self.assertGreaterEqual(time.monotonic() - started, 0.5) - self.assertEqual(errors, []) - self.assertTrue(successor_started.is_set()) - self.assertEqual(len(recovered), 1) - self.assertRegex(recovered[0][0] or "", r"^sha256:[0-9a-f]{64}$") - self.assertRegex(recovered[0][1], r"^sha256:[0-9a-f]{64}$") - self.assertFalse(alias.exists() or alias.is_symlink()) - self.assertTrue((output / "lifecycle-result.json").is_file()) - self.assertTrue((output / "lifecycle-journal.jsonl").is_file()) - for path, data in prior.items(): - self.assertEqual(path.read_bytes(), data) - - def prepare_prepublished(name: str): - root = self.root / "receipt-only" / "blind" / name - for directory in ("input", "session", "output"): - (root / directory).mkdir(parents=True, exist_ok=True) - local_output = root / "output" - local_control = self.root / f"{name}-control" - local_control.mkdir() - local_alias = Path(tempfile.gettempdir()) / ( - f"iop-score-{name}-" - + hashlib.sha256(str(local_output).encode("utf-8")).hexdigest()[:16] - ) - os.symlink( - local_control.parent, local_alias, target_is_directory=True - ) - self.addCleanup( - lambda: local_alias.unlink() - if local_alias.exists() or local_alias.is_symlink() - else None - ) - local_locator = SupervisorLocator( - os.getpid(), - f"{name}-start-identity", - str(local_alias / local_control.name / "control.sock"), - f"{name}-challenge", - str(local_alias / local_control.name), - "2026-08-11T00:00:00+00:00", - ) - local_receipt = dict(receipt) - local_receipt["challenge_digest"] = hashlib.sha256( - local_locator.challenge.encode("utf-8") - ).hexdigest() - (local_control / "cleanup-receipt.json").write_text( - json.dumps(local_receipt), encoding="utf-8" - ) - local_public_locator = { - "supervisor_pid": local_locator.supervisor_pid, - "start_identity": local_locator.start_identity, - "socket_path": local_locator.socket_path, - "control_dir": local_locator.control_dir, - "created_at": local_locator.created_at, - "challenge_digest": local_receipt["challenge_digest"], - } - local_lifecycle = dict(lifecycle) - local_lifecycle["locator"] = local_public_locator - (local_output / "lifecycle-journal.jsonl").write_text( - journal, encoding="utf-8" - ) - (local_output / "lifecycle-result.json").write_text( - json.dumps(local_lifecycle), encoding="utf-8" - ) - control_socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - control_socket.bind(str(local_control / "control.sock")) - self.addCleanup(control_socket.close) - return ( - root, - local_output, - local_control, - local_alias, - local_locator, - local_lifecycle, - ) - - def start_prepublished_recovery(root, control, alias, local_locator): - local_recovered: list[tuple[str | None, str]] = [] - local_errors: list[BaseException] = [] - local_successor_started = threading.Event() - - def local_recover() -> None: - try: - local_recovered.append( - scoring_module._recover_runner( - root, - local_locator, - invocation_digest, - control_target=control, - ) - ) - scoring_module._release_runner_alias( - { - "control_alias": str(alias), - "control_target": str(control), - } - ) - local_successor_started.set() - except BaseException as exc: - local_errors.append(exc) - - local_worker = threading.Thread(target=local_recover) - local_worker.start() - return ( - local_worker, - local_recovered, - local_errors, - local_successor_started, - ) - - ( - stable_root, - _stable_output, - stable_control, - stable_alias, - stable_locator, - _stable_lifecycle, - ) = prepare_prepublished("prepublished-stable") - stable_quiet_started = threading.Event() - original_wait = scoring_module._wait_post_cleanup_quiet - - def observe_stable_quiet(*args, **kwargs): - stable_quiet_started.set() - return original_wait(*args, **kwargs) - - with mock.patch.object( - scoring_module, "_POST_CLEANUP_TIMEOUT_SECONDS", 0.5 - ), mock.patch.object( - scoring_module, "_POST_CLEANUP_QUIET_SECONDS", 0.1 - ), mock.patch.object( - scoring_module, "_POST_CLEANUP_POLL_SECONDS", 0.01 - ), mock.patch.object( - scoring_module, - "_wait_post_cleanup_quiet", - side_effect=observe_stable_quiet, - ): - ( - stable_worker, - stable_recovered, - stable_errors, - stable_successor_started, - ) = start_prepublished_recovery( - stable_root, stable_control, stable_alias, stable_locator - ) - self.assertTrue(stable_quiet_started.wait(1)) - time.sleep(0.03) - self.assertTrue(stable_worker.is_alive()) - self.assertTrue(stable_alias.is_symlink()) - self.assertTrue((stable_control / "control.sock").exists()) - self.assertFalse(stable_successor_started.is_set()) - stable_worker.join(2) - self.assertFalse(stable_worker.is_alive()) - self.assertEqual(stable_errors, []) - self.assertEqual(len(stable_recovered), 1) - self.assertFalse(stable_alias.exists() or stable_alias.is_symlink()) - self.assertFalse((stable_control / "control.sock").exists()) - self.assertTrue(stable_successor_started.is_set()) - - ( - churn_root, - churn_output, - churn_control, - churn_alias, - churn_locator, - _churn_lifecycle, - ) = prepare_prepublished("prepublished-session-churn") - churn_session = churn_root / "session" - for index in range(2048): - (churn_session / f"state-{index:04d}.txt").write_text( - "evaluator session state", encoding="utf-8" - ) - protected = { - path: path.read_bytes() - for path in ( - churn_control / "cleanup-receipt.json", - churn_output / "lifecycle-journal.jsonl", - churn_output / "lifecycle-result.json", - ) - } - stop_churn = threading.Event() - churn_count = 0 - - def churn_session_state() -> None: - nonlocal churn_count - changing = churn_session / "changing-state.txt" - while not stop_churn.is_set(): - churn_count += 1 - changing.write_text(str(churn_count), encoding="ascii") - time.sleep(0.005) - - churn_writer = threading.Thread(target=churn_session_state) - churn_writer.start() - try: - with mock.patch.object( - scoring_module, "_POST_CLEANUP_TIMEOUT_SECONDS", 0.4 - ), mock.patch.object( - scoring_module, "_POST_CLEANUP_QUIET_SECONDS", 0.1 - ), mock.patch.object( - scoring_module, "_POST_CLEANUP_POLL_SECONDS", 0.005 - ): - churn_recovered = scoring_module._recover_runner( - churn_root, - churn_locator, - invocation_digest, - control_target=churn_control, - ) - finally: - stop_churn.set() - churn_writer.join(2) - self.assertFalse(churn_writer.is_alive()) - self.assertGreater(churn_count, 1) - self.assertRegex(churn_recovered[0] or "", r"^sha256:[0-9a-f]{64}$") - self.assertRegex(churn_recovered[1], r"^sha256:[0-9a-f]{64}$") - self.assertFalse((churn_control / "control.sock").exists()) - self.assertTrue(churn_alias.is_symlink()) - for path, data in protected.items(): - self.assertEqual(path.read_bytes(), data) - - ( - changed_root, - changed_output, - changed_control, - changed_alias, - changed_locator, - changed_lifecycle, - ) = prepare_prepublished("prepublished-changed") - changed_quiet_started = threading.Event() - - def observe_changed_quiet(*args, **kwargs): - changed_quiet_started.set() - return original_wait(*args, **kwargs) - - with mock.patch.object( - scoring_module, "_POST_CLEANUP_TIMEOUT_SECONDS", 0.5 - ), mock.patch.object( - scoring_module, "_POST_CLEANUP_QUIET_SECONDS", 0.1 - ), mock.patch.object( - scoring_module, "_POST_CLEANUP_POLL_SECONDS", 0.01 - ), mock.patch.object( - scoring_module, - "_wait_post_cleanup_quiet", - side_effect=observe_changed_quiet, - ): - ( - changed_worker, - changed_recovered, - changed_errors, - changed_successor_started, - ) = start_prepublished_recovery( - changed_root, changed_control, changed_alias, changed_locator - ) - self.assertTrue(changed_quiet_started.wait(1)) - mutated_lifecycle = dict(changed_lifecycle) - mutated_lifecycle["publication_revision"] = 2 - staged_lifecycle = changed_output / "lifecycle-result.changed.tmp" - staged_lifecycle.write_text( - json.dumps(mutated_lifecycle), encoding="utf-8" - ) - os.replace( - staged_lifecycle, changed_output / "lifecycle-result.json" - ) - changed_worker.join(2) - self.assertFalse(changed_worker.is_alive()) - self.assertEqual(changed_recovered, []) - self.assertEqual(len(changed_errors), 1) - self.assertIsInstance(changed_errors[0], ScoringError) - self.assertIn("lifecycle publication changed", str(changed_errors[0])) - self.assertTrue(changed_alias.is_symlink()) - self.assertTrue((changed_control / "control.sock").exists()) - self.assertFalse(changed_successor_started.is_set()) - - missing_root = self.root / "receipt-only" / "blind" / "blind-missing" - for name in ("input", "session", "output"): - (missing_root / name).mkdir(parents=True, exist_ok=True) - missing_output = missing_root / "output" - missing_control = missing_output / "missing-control" - missing_control.mkdir() - missing_alias = Path(tempfile.gettempdir()) / ( - "iop-score-missing-" - + hashlib.sha256(str(missing_output).encode("utf-8")).hexdigest()[:16] - ) - os.symlink(missing_output, missing_alias, target_is_directory=True) - self.addCleanup( - lambda: missing_alias.unlink() - if missing_alias.exists() or missing_alias.is_symlink() - else None - ) - missing_locator = SupervisorLocator( - os.getpid(), - "missing-start-identity", - str(missing_alias / "missing-control" / "control.sock"), - "missing-challenge", - str(missing_alias / "missing-control"), - "2026-08-11T00:00:00+00:00", - ) - missing_receipt = dict(receipt) - missing_receipt["challenge_digest"] = hashlib.sha256( - missing_locator.challenge.encode("utf-8") - ).hexdigest() - (missing_control / "cleanup-receipt.json").write_text( - json.dumps(missing_receipt), encoding="utf-8" - ) - with mock.patch.object( - scoring_module, "_POST_CLEANUP_TIMEOUT_SECONDS", 0.05 - ), mock.patch.object( - scoring_module, "_POST_CLEANUP_QUIET_SECONDS", 0.01 - ): - with self.assertRaisesRegex( - ScoringError, "lifecycle publication is incomplete" - ): - scoring_module._recover_runner( - missing_root, - missing_locator, - invocation_digest, - control_target=missing_control, - ) - self.assertTrue(missing_alias.is_symlink()) - self.assertFalse((missing_output / "lifecycle-result.json").exists()) - - def test_interrupted_evaluator_is_stopped_before_retry(self): - attempt = self._attempt() - - class InterruptingAdapter(FakeScoringAdapter): - def __init__(self): - super().__init__() - self.first = True - self.worker: threading.Thread | None = None - self.worker_result = None - self.worker_error: BaseException | None = None - self.successor_started = False - self.locator: SupervisorLocator | None = None - self.callback_error: BaseException | None = None - self.control_alias: Path | None = None - - def invoke(self, cell, blind, task_payload, timeout, on_started): - if not self.first: - self.successor_started = True - if self.worker is not None and self.worker.is_alive(): - raise AssertionError( - "successor started before survivor cleanup" - ) - return super().invoke( - cell, blind, task_payload, timeout, on_started - ) - self.first = False - self.invocations.append((blind, task_payload)) - ready = threading.Event() - locator_box: list[SupervisorLocator] = [] - alias = Path(tempfile.gettempdir()) / ( - "iop-score-test-" - + hashlib.sha256(blind.output_dir.encode("utf-8")).hexdigest()[:16] - ) - os.symlink(blind.output_dir, alias, target_is_directory=True) - self.control_alias = alias - spec = InvocationSpec( - argv=( - sys.executable, - "-u", - "-c", - "import sys,time; sys.stdin.buffer.read(); " - "print('START', flush=True); time.sleep(30)", - ), - cwd=blind.root, - env=env_pairs( - {"PATH": os.environ.get("PATH", "/usr/bin:/bin")} - ), - submission_mode=SUBMISSION_STDIN_ONCE, - completion_mode=COMPLETION_EXIT_AFTER_IDLE, - timeout=timeout, - evidence_dir=blind.output_dir, - task_payload=b"evaluate", - control_dir=str(alias / "live-control"), - ) - - def run() -> None: - try: - def commit(locator): - try: - on_started(locator, spec_digest(spec)) - locator_box.append(locator) - except BaseException as exc: - self.callback_error = exc - finally: - ready.set() - if self.callback_error is not None: - raise self.callback_error - - self.worker_result = run_invocation( - spec, - parse_event=lambda _stream, _line: None, - on_started=commit, - ) - except BaseException as exc: - self.worker_error = exc - - self.worker = threading.Thread(target=run) - self.worker.start() - if not ready.wait(15): - raise KeyboardInterrupt( - "evaluator locator was not published: " - + repr(self.worker_error) - ) - if self.callback_error is not None: - raise KeyboardInterrupt(repr(self.callback_error)) - deadline = time.monotonic() + 5 - while not recover_invocation( - locator_box[0], stop=False - ).caller_launched: - if time.monotonic() >= deadline: - raise AssertionError("evaluator did not launch") - time.sleep(0.01) - self.locator = locator_box[0] - raise KeyboardInterrupt("simulated scoring controller loss") - - adapter = InterruptingAdapter() - - def cleanup() -> None: - if adapter.worker is not None and adapter.worker.is_alive(): - if adapter.locator is not None: - try: - recover_invocation(adapter.locator, stop=True) - except Exception: - pass - adapter.worker.join(5) - if adapter.control_alias is not None and adapter.control_alias.is_symlink(): - adapter.control_alias.unlink() - - self.addCleanup(cleanup) - with self.assertRaises(KeyboardInterrupt): - score_run(self.store, self.run, self.manifest, adapter=adapter) - self.assertIsNone(adapter.callback_error, repr(adapter.callback_error)) - self.assertIsNone(adapter.worker_error, repr(adapter.worker_error)) - self.assertIsNone(adapter.worker_result, repr(adapter.worker_result)) - first_root = Path(attempt.root) / "scoring" / "score-000001" - self.assertTrue((first_root / "runner.json").is_file()) - self.assertFalse((first_root / "result.json").exists()) - prior = { - path: path.read_bytes() for path in first_root.rglob("*") if path.is_file() - } - - summary = score_run( - self.store, - self.run, - self.manifest, - adapter=adapter, - retry_scoring_failed=True, - ) - self.assertEqual((summary.scored, summary.scoring_failed), (1, 0)) - self.assertTrue(adapter.successor_started) - self.assertIsNotNone(adapter.worker) - adapter.worker.join(5) # type: ignore[union-attr] - self.assertFalse(adapter.worker.is_alive()) # type: ignore[union-attr] - self.assertIsNone(adapter.worker_error) - self.assertIsNotNone(adapter.worker_result) - self.assertTrue(adapter.worker_result.harness.cleanup_complete) - self.assertFalse(adapter.worker_result.process_group_alive) - receipt = json.loads( - ( - Path(adapter.invocations[0][0].output_dir) - / "live-control" - / "cleanup-receipt.json" - ).read_text() - ) - self.assertEqual(receipt["reason"], "recovered_stop") - self.assertTrue(receipt["cleanup_complete"]) - self.assertFalse(receipt["process_group_alive"]) - for path, data in prior.items(): - self.assertEqual(path.read_bytes(), data) - self.assertTrue( - (Path(attempt.root) / "scoring" / "score-000002").is_dir() - ) - - def test_mutated_input_and_runtime_secret_fail_before_scored(self): - attempt = self._attempt() - mutated = FakeScoringAdapter(modes=["mutate", "success"]) - first = score_run( - self.store, self.run, self.manifest, adapter=mutated - ) - self.assertEqual((first.scored, first.scoring_failed), (0, 1)) - first_root = Path(attempt.root) / "scoring" / "score-000001" - first_bytes = { - path: path.read_bytes() for path in first_root.rglob("*") if path.is_file() - } - result = json.loads((first_root / "result.json").read_text()) - self.assertEqual(result["reason"], "input_mutated") - self.assertNotIn("worksheet", result) - - retry = score_run( - self.store, - self.run, - self.manifest, - adapter=mutated, - retry_scoring_failed=True, - ) - self.assertEqual((retry.scored, retry.scoring_failed), (1, 0)) - self.assertEqual( - first_bytes, - { - path: path.read_bytes() - for path in first_root.rglob("*") - if path.is_file() - }, - ) - - secret = "runtime-evaluator-secret-literal" - self.run = self.store.create(self.manifest, self.manifest_path.read_bytes()) - secret_attempt = self._attempt() - leaking = FakeScoringAdapter( - modes=["secret"], sensitive_value=secret - ) - leaked = score_run( - self.store, self.run, self.manifest, adapter=leaking - ) - self.assertEqual((leaked.scored, leaked.scoring_failed), (0, 1)) - durable = b"".join( - path.read_bytes() - for path in Path(self.run.root).rglob("*") - if path.is_file() - ) - self.assertNotIn(secret.encode("utf-8"), durable) - secret_result = json.loads( - ( - Path(secret_attempt.root) - / "scoring" - / "score-000001" - / "result.json" - ).read_text() - ) - self.assertEqual(secret_result["reason"], "runtime_secret_leak") - self.assertNotIn("worksheet", secret_result) - - def test_tampered_score_fails_closed_without_rewrite(self): - attempt = self._attempt() - score_run(self.store, self.run, self.manifest, adapter=FakeScoringAdapter()) - result = Path(attempt.root) / "scoring" / "score-000001" / "result.json" - record = json.loads(result.read_text()) - record["worksheet"]["total"] = 0 - result.write_text( - json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n" - ) - before = result.read_bytes() - with self.assertRaises(ScoringError): - score_run(self.store, self.run, self.manifest, adapter=FakeScoringAdapter()) - self.assertEqual(result.read_bytes(), before) - - def test_execution_preset_evaluator_uses_manifest_stage_binding(self): - raw = json.loads(self.manifest_path.read_text()) - raw["evaluator"]["iop"] = { - "request_model": "judge-model", - "requested_effort": "xhigh", - "route_kind": "execution_preset", - "route_id": "judge-preset", - "expected_bindings": [ - {"stage": stage, "model": "judge-model"} - for stage in ("selector", "plan", "work", "review") - ], - } - raw["output_root"] = "agent-test/runs/preset-bench" - path = self.root / "preset.json" - path.write_text(json.dumps(raw)) - self.manifest_path = path - self.manifest = load_manifest(path, repo_root=self.root) - self.run = self.store.create(self.manifest, path.read_bytes()) - self._attempt() - adapter = FakeScoringAdapter() - summary = score_run(self.store, self.run, self.manifest, adapter=adapter) - self.assertEqual(summary.scored, 1) - self.assertEqual(adapter.preflights, 1) - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/agent_benchmark/skill_contract_test.py b/scripts/agent_benchmark/skill_contract_test.py deleted file mode 100644 index 8a35cfec..00000000 --- a/scripts/agent_benchmark/skill_contract_test.py +++ /dev/null @@ -1,1062 +0,0 @@ -""" -Credential-free contract tests binding the benchmark skill to the CLI surface. - -Covers: template/frontmatter invariants, project rule routing, documented -command forms against --help and required options, supported/unsupported capability matrix, -absence of the internal workspace API from user routing, provider/dispatcher/secret/prepare prohibitions, -durable state vs isolated cache boundaries, and mutation resistance. - -Invokes only help and tracked text; never runs a stateful benchmark command. -""" - -from __future__ import annotations - -import json -import re -import subprocess -import sys -import unittest -from pathlib import Path - -# Ensure repo root is importable -_REPO_ROOT = Path(__file__).resolve().parent.parent.parent -if str(_REPO_ROOT) not in sys.path: - sys.path.insert(0, str(_REPO_ROOT)) - -_SKILL_DIR = _REPO_ROOT / "agent-ops" / "skills" / "project" / "iop-agent-comparison-benchmark" -_SKILL_FILE = _SKILL_DIR / "SKILL.md" -_DEPLOY_SKILL_FILE = ( - _REPO_ROOT / "agent-ops" / "skills" / "project" / "dev-runtime-deploy" / "SKILL.md" -) -_RULES_FILE = _REPO_ROOT / "agent-ops" / "rules" / "project" / "rules.md" -_CLI_SCRIPT = _REPO_ROOT / "scripts" / "agent_comparison_benchmark.py" - -# Commands documented by the CLI --help -_CLI_HELP_COMMANDS = {"validate", "preflight", "run", "resume", "status", "score", "report"} - -# Cached exact option sets per subcommand, derived from each subcommand --help. -_CLI_OPTION_CACHE: dict[str, set[str]] = {} - - -class BenchmarkSkillContractTest(unittest.TestCase): - """Contract tests for the iop-agent-comparison-benchmark skill.""" - - # ------------------------------------------------------------------ - # Helper methods for section parsing & semantic assertions - # ------------------------------------------------------------------ - - def _get_section(self, skill_text: str, section_name: str) -> str: - in_section = False - lines = [] - target = f"## {section_name.strip()}" - for line in skill_text.splitlines(): - if line.startswith("## "): - if line.strip().lower() == target.lower(): - in_section = True - continue - elif in_section: - break - if in_section: - lines.append(line) - return "\n".join(lines) - - def _get_procedure_step(self, skill_text: str, command: str) -> str: - """Extract the numbered `Delegate ` Procedure step body.""" - procedure = self._get_section(skill_text, "Procedure") - pattern = rf"\d+\.\s+\*\*Delegate {command}.*?(?=\n\d+\.|\Z)" - match = re.search(pattern, procedure, re.DOTALL) - self.assertTrue( - match, - f"Procedure step delegating '{command}' must be present", - ) - return match.group(0) - - def _get_cli_help_commands(self) -> set[str]: - result = subprocess.run( - [sys.executable, str(_CLI_SCRIPT), "--help"], - capture_output=True, - text=True, - cwd=str(_REPO_ROOT), - ) - self.assertEqual( - result.returncode, - 0, - f"CLI --help exited {result.returncode}: {result.stderr}", - ) - commands: set[str] = set() - in_subparsers = False - for line in result.stdout.splitlines(): - stripped = line.strip() - if "positional arguments:" in line: - in_subparsers = True - continue - if in_subparsers and stripped and not stripped.startswith("-"): - if stripped == "options:": - continue - if stripped.startswith("{") and stripped.endswith("}"): - for cmd in stripped[1:-1].split(","): - commands.add(cmd) - else: - commands.add(stripped.split()[0]) - return commands - - def _assert_no_public_prepare(self, skill_text: str) -> None: - """Parse all sections; fail if public prepare is exposed as an operation or trigger.""" - allowed_prepare_fragments = ( - "no public `prepare` operation was exposed or referenced.", - "do not expose a public `prepare` operation.", - ) - for line in skill_text.splitlines(): - stripped = line.strip() - if "prepare" in stripped.lower(): - lower = stripped.lower() - if not any(fragment in lower for fragment in allowed_prepare_fragments): - self.fail( - "Section or step exposes or mentions prepare operation " - f"outside the approved prohibition forms: '{stripped}'" - ) - - def _assert_provider_prohibition(self, skill_text: str) -> None: - """Assert provider APIs/services are prohibited and never invoked in any section.""" - prohibitions = self._get_section(skill_text, "Prohibitions") - self.assertRegex( - prohibitions, - r"(?i)do not invoke.*provider", - "Prohibitions must explicitly forbid provider invocations", - ) - allowed_provider_fragments = ( - "no caller or provider was invoked outside the deterministic cli.", - "do not invoke a caller or provider outside the deterministic benchmark cli.", - "stop without fallback, fabricated evidence, ad-hoc provider calls, subagents, or orchestration dispatchers.", - ) - for line in skill_text.splitlines(): - stripped = line.strip() - if "provider" in stripped.lower(): - lower = stripped.lower() - if not any(fragment in lower for fragment in allowed_provider_fragments): - self.fail( - "Line contains an unapproved provider operation or " - f"prohibition form: '{stripped}'" - ) - - def _get_cli_subcommand_options(self, cmd: str) -> set[str]: - """Derive the exact long-option set for a subcommand from its live --help output.""" - if cmd in _CLI_OPTION_CACHE: - return set(_CLI_OPTION_CACHE[cmd]) - result = subprocess.run( - [sys.executable, str(_CLI_SCRIPT), cmd, "--help"], - capture_output=True, - text=True, - cwd=str(_REPO_ROOT), - ) - self.assertEqual( - result.returncode, - 0, - f"CLI {cmd} --help exited {result.returncode}: {result.stderr}", - ) - options = set(re.findall(r"--[a-z][a-z0-9-]*", result.stdout)) - options.discard("--help") - _CLI_OPTION_CACHE[cmd] = set(options) - return options - - def _documented_command_options(self, cmd_line: str) -> set[str]: - """Extract the documented long-option set, excluding --help and value placeholders.""" - options: set[str] = set() - for match in re.finditer(r"--[a-z][a-z0-9-]*", cmd_line): - token = match.group(0) - if token != "--help": - options.add(token) - return options - - def _assert_command_options(self, skill_text: str) -> None: - """Parse each documented CLI invocation and require exact option parity with subcommand --help.""" - procedure = self._get_section(skill_text, "Procedure") - for cmd in ("validate", "preflight", "run", "resume", "status", "score", "report"): - pattern = rf"python3 scripts/agent_comparison_benchmark\.py {cmd}\b[^\n]*" - matches = re.findall(pattern, procedure) - self.assertTrue(matches, f"Documented command line for '{cmd}' missing from Procedure") - cli_options = self._get_cli_subcommand_options(cmd) - for index, cmd_line in enumerate(matches, start=1): - documented = self._documented_command_options(cmd_line) - self.assertEqual( - documented, - cli_options, - f"Documented options {sorted(documented)} for '{cmd}' invocation " - f"#{index} must exactly match CLI --help options " - f"{sorted(cli_options)}: '{cmd_line}'", - ) - - def _assert_boundary_wording(self, skill_text: str) -> None: - """Assert consistent durable-state, isolated cache, read-only testbed, and output boundaries.""" - self.assertIn("Durable run/attempt state", skill_text) - self.assertIn("agent-test/runs///", skill_text) - self.assertRegex( - skill_text, - r"fresh and isolated for every cell, repetition, and attempt", - "Skill text must specify per-cell/per-repetition/per-attempt freshness and isolation", - ) - self.assertIn("../iop-s2", skill_text) - self.assertIn("read-only", skill_text) - forbidden_phrases = [ - "do not cache or persist state between invocations", - "do not read or write files outside the benchmark workspace", - "The benchmark workspace root is fixed at `../iop-s2`.", - "Output is contained within the benchmark workspace.", - "fresh, isolated per run, and never shared across runs", - ] - for phrase in forbidden_phrases: - self.assertNotIn(phrase, skill_text, f"Forbidden contradictory boundary phrase found: {phrase}") - - prohibitions = self._get_section(skill_text, "Prohibitions") - self.assertRegex( - prohibitions, - r"(?i)do not share session or cache state within a run", - "Prohibitions must explicitly forbid session or cache state sharing within a run", - ) - - for line in skill_text.splitlines(): - stripped = line.strip() - lower = stripped.lower() - if ( - "session" in lower - and "cache" in lower - and any(token in lower for token in ("share", "shared", "sharing")) - ): - allowed_boundary_fragments = ( - "caller sessions, output workspaces, and caches are fresh and isolated for every cell, repetition, and attempt; session or cache state is never shared within a run or across runs.", - "do not share session or cache state within a run across cells, repetitions, or attempts, or across run invocations.", - ) - if not any(fragment in lower for fragment in allowed_boundary_fragments): - self.fail( - "Line contains an unapproved affirmative or contradictory " - f"session/cache sharing statement: '{stripped}'" - ) - elif "cache" in lower and "sharing" in lower: - self.fail(f"Line contains an unapproved cache-sharing statement: '{stripped}'") - - def _assert_error_ordering(self, skill_text: str) -> None: - """Assert invalid state is handled before execution preflight blockers.""" - procedure = self._get_section(skill_text, "Procedure") - for cmd in ("run", "resume", "status", "score", "report"): - pattern = rf"\d+\.\s+\*\*Delegate {cmd}.*?(?=\n\d+\.|\Z)" - match = re.search(pattern, procedure, re.DOTALL) - self.assertTrue(match, f"Procedure step for '{cmd}' missing") - step_text = match.group(0) - if cmd in ("run", "resume"): - pos_invalid = step_text.find("benchmark state is unavailable") - pos_blocked = step_text.find("preflight blocked") - self.assertTrue( - pos_invalid != -1 - and pos_blocked != -1 - and pos_invalid < pos_blocked, - f"In step '{cmd}', invalid state must precede preflight blockers", - ) - if cmd == "report": - self.assertIn( - "benchmark report is unavailable", - step_text, - f"In step '{cmd}', invalid state must be reported", - ) - - def _assert_capabilities(self, skill_text: str) -> None: - """Assert report is a supported operation alongside run/resume.""" - procedure = self._get_section(skill_text, "Procedure") - self.assertNotIn("capability-unavailable: caller-adapter", procedure) - self.assertIn("append a fresh all-cell preflight before attempt allocation", procedure) - self.assertIn("invoke each eligible cell exactly once", procedure) - # report must be delegated, not gated - self.assertIn("report", procedure) - - def _assert_preflight_contract(self, skill_text: str) -> None: - """Require exact all-cell append semantics and fail-closed blockers.""" - procedure = self._get_section(skill_text, "Procedure") - validation = self._get_section(skill_text, "Validation") - prohibitions = self._get_section(skill_text, "Prohibitions") - self.assertIn( - "python3 scripts/agent_comparison_benchmark.py preflight --manifest ", - procedure, - ) - self.assertIn( - "records one fresh live observation for every immutable matrix cell", - procedure, - ) - self.assertIn("including direct and execution-preset routes", procedure) - self.assertIn("in canonical matrix order", procedure) - self.assertIn("registration_required", procedure) - self.assertIn("implementation_gap", procedure) - self.assertIn("Never bypass the blocker", procedure) - self.assertIn("substitute a route/model/effort", procedure) - self.assertIn( - "Preflight evidence is append-only, covers every immutable matrix cell in canonical order", - validation, - ) - self.assertIn("created no scored attempt", validation) - self.assertIn("Do not bypass a preflight blocker", prohibitions) - self.assertIn("Do not claim execution-preset fixture validation as live readiness", prohibitions) - for obsolete in ( - "records only direct-cell observations", - "Generic preset cells are local contract validation only", - "Preflight evidence is append-only, direct-only", - "Direct preflight never allocates a scored attempt", - "fresh direct preflight", - ): - self.assertNotIn(obsolete, skill_text) - - def _assert_scoring_contract(self, skill_text: str) -> None: - procedure = self._get_section(skill_text, "Procedure") - validation = self._get_section(skill_text, "Validation") - prohibitions = self._get_section(skill_text, "Prohibitions") - self.assertIn( - "python3 scripts/agent_comparison_benchmark.py score --manifest --run-id [--retry-scoring-failed]", - procedure, - ) - self.assertIn("immutable `unscored`", procedure) - self.assertIn("without invoking the evaluator or assigning zero", procedure) - self.assertIn("manifest-bound fresh Codex evaluator session", procedure) - self.assertIn("exact immutable manifest-selected rubric", procedure) - self.assertIn("closed supported catalog", procedure) - self.assertIn("`landing-quality-v1`", procedure) - self.assertIn("`one-shot-agent-comparison-v1`", procedure) - self.assertIn("no substitute rubric or reinterpretation is permitted", procedure) - self.assertIn("allocates a new score id and preserves every prior byte", procedure) - self.assertIn("failed product, harness, process, or required artifact gate", procedure) - self.assertIn("`scoring_failed` used no fallback", validation) - self.assertIn("Do not retry scoring implicitly", prohibitions) - self.assertIn("convert `unscored`/`scoring_failed` to zero", prohibitions) - - def _assert_no_secret_operational_language(self, skill_text: str) -> None: - operational_text = "\n".join( - self._get_section(skill_text, section) - for section in ("Inputs", "Preflight", "Procedure") - ) - self.assertNotRegex( - operational_text, - r"(?i)\b(secret|credential|api_key|token)\b", - "Operational sections must not mention secrets or credentials", - ) - - def _assert_execution_resolution_contract(self, skill_text: str) -> None: - """Bind run/resume terminal resolution, exit-69 scope, explicit retry, and independent process axes.""" - validation = self._get_section(skill_text, "Validation") - output_format = self._get_section(skill_text, "Output format") - stop_conditions = self._get_section(skill_text, "Stop conditions") - safety = self._get_section(skill_text, "Safety rules") - - # each stateful command step must independently retain terminal resolution - expected_resolution = ( - "Exit 0 when every manifest slot has complete terminal evidence (`unresolved=0`); " - "independent failure counts remain in stdout and are classified by `score`." - ) - for command in ("run", "resume"): - step = self._get_procedure_step(skill_text, command) - self.assertIn( - expected_resolution, - step, - f"Procedure step for '{command}' must bind exit 0 to unresolved=0 resolution " - f"with independent failure counts", - ) - self.assertRegex( - step, - r"Exit 69 only for preflight blockers or incomplete evidence", - f"Procedure step for '{command}' must limit exit 69 to preflight blockers " - f"or incomplete evidence", - ) - self.assertIn( - "Never performs an implicit retry of a failed gate.", - step, - f"Procedure step for '{command}' must forbid implicit failed-gate retry", - ) - # failed execution retry requires explicit --retry-failed only - self.assertIn( - "Retry is explicit only (`--retry-failed`)", - validation, - "Validation must require explicit --retry-failed retry", - ) - self.assertIn( - "--retry-failed`", - stop_conditions, - "Stop conditions must reference explicit --retry-failed resume gating", - ) - self.assertIn( - "complete terminal evidence may continue to the fresh nine-cell preflight", - stop_conditions, - "Release qualification must continue after evidence-complete product failure without retry", - ) - self.assertIn( - "it does not require every gate to pass", - safety, - "Safety rules must separate resolution (unresolved=0) from all-gates success", - ) - # valid resolved terminal process failure axes remain visible as - for placeholder in ( - "process_exited=", - "process_signalled=", - "process_timed_out=", - "process_cancelled=", - "process_not_started=", - ): - self.assertIn( - placeholder, - output_format, - f"Terminal process axis must remain visible as {placeholder}", - ) - # completeness invariants stay zero (no slot running, all artifacts run) - self.assertIn("running=0", output_format) - self.assertIn("artifact_not_run=0", output_format) - - def _assert_full_skill_contract(self, skill_text: str) -> None: - """Validate complete contract on skill text.""" - self.assertIn("## Purpose", skill_text) - self.assertIn("## When to use", skill_text) - self.assertIn("## Preflight", skill_text) - self.assertIn("## Procedure", skill_text) - self.assertIn("## Validation", skill_text) - self.assertIn("## Safety rules", skill_text) - self.assertIn("## Stop conditions", skill_text) - self.assertIn("## Prohibitions", skill_text) - self._assert_command_options(skill_text) - self._assert_no_public_prepare(skill_text) - self._assert_provider_prohibition(skill_text) - self._assert_boundary_wording(skill_text) - self._assert_error_ordering(skill_text) - self._assert_capabilities(skill_text) - self._assert_preflight_contract(skill_text) - self._assert_scoring_contract(skill_text) - self._assert_execution_resolution_contract(skill_text) - self._assert_no_secret_operational_language(skill_text) - self.assertIn("product_succeeded=", skill_text) - self.assertIn("harness_passed=", skill_text) - self.assertIn("process_exited=", skill_text) - self.assertIn("artifact_passed=", skill_text) - self.assertIn("five-cell direct manifest", skill_text) - for qualification_rule in ( - "fresh `ready=5`", - "exactly five fresh attempts", - "terminal controller/product/harness/process/web-validation evidence for every slot", - "no exhausted browser/CDP infrastructure block", - "Product failure, upstream HTTP rejection, generated-missing after caller failure, and timeout remain measured outcomes", - "do not trigger an implicit retry", - "Edge pre-ingress incompatibility or an exhausted browser/CDP infrastructure block stops qualification", - ): - self.assertIn(qualification_rule, skill_text) - self.assertNotIn("requires all four gates for all five cells", skill_text) - - # ------------------------------------------------------------------ - # Template / frontmatter invariants - # ------------------------------------------------------------------ - - def test_skill_file_exists(self) -> None: - self.assertTrue(_SKILL_FILE.is_file(), f"{_SKILL_FILE} must exist") - - def test_frontmatter_name(self) -> None: - content = _SKILL_FILE.read_text(encoding="utf-8") - self.assertIn("name: iop-agent-comparison-benchmark", content) - - def test_frontmatter_keys(self) -> None: - content = _SKILL_FILE.read_text(encoding="utf-8") - frontmatter = content.split("---", 2)[1] - keys = [ - line.partition(":")[0] - for line in frontmatter.splitlines() - if line.strip() - ] - self.assertEqual(keys, ["name", "description"]) - - def test_frontmatter_description_present(self) -> None: - content = _SKILL_FILE.read_text(encoding="utf-8") - self.assertRegex(content, r"description: .+", re.MULTILINE) - - def test_dev_runtime_deploy_contract_is_authoritative_and_runnable(self) -> None: - content = _DEPLOY_SKILL_FILE.read_text(encoding="utf-8") - frontmatter = content.split("---", 2)[1] - keys = [ - line.partition(":")[0] - for line in frontmatter.splitlines() - if line.strip() - ] - self.assertEqual(keys, ["name", "description"]) - self.assertIn("git fetch --prune origin dev main --tags", content) - self.assertIn("git remote prune origin", content) - self.assertIn( - "git ls-remote --heads origin 'refs/heads/release/*'", - content, - ) - self.assertIn("stale local remote-tracking ref", content) - self.assertIn( - "go list ./apps/control-plane/... ./apps/edge/... ./apps/node/... ./packages/go/... ./scripts/...", - content, - ) - self.assertNotIn("./cmd/...", content) - - def test_required_sections_present(self) -> None: - content = _SKILL_FILE.read_text(encoding="utf-8") - for section in ( - "## Purpose", - "## When to use", - "## Preflight", - "## Procedure", - "## Validation", - "## Prohibitions", - ): - self.assertIn(section, content, f"Missing section: {section}") - - # ------------------------------------------------------------------ - # Routing - # ------------------------------------------------------------------ - - def test_project_rules_routes_benchmark(self) -> None: - rules_text = _RULES_FILE.read_text(encoding="utf-8") - self.assertIn( - "agent-ops/skills/project/iop-agent-comparison-benchmark/SKILL.md", - rules_text, - "Project rules must route to the benchmark skill", - ) - - def test_project_rules_routes_trigger_keywords(self) -> None: - rules_text = _RULES_FILE.read_text(encoding="utf-8") - for keyword in ("validate", "run", "resume", "status", "score", "report"): - self.assertIn( - keyword, - rules_text, - f"Project rules must mention trigger keyword: {keyword}", - ) - - # ------------------------------------------------------------------ - # CLI parity & documented command options matching --help - # ------------------------------------------------------------------ - - def test_skill_commands_match_cli_help(self) -> None: - """Documented procedure commands must be a subset of CLI --help commands.""" - skill_text = _SKILL_FILE.read_text(encoding="utf-8") - procedure_section = self._get_section(skill_text, "Procedure") - - documented_commands: set[str] = set() - for cmd in _CLI_HELP_COMMANDS: - if f"`{cmd}`" in procedure_section or f"`{cmd}`" in skill_text: - documented_commands.add(cmd) - - cli_commands = self._get_cli_help_commands() - self.assertTrue( - documented_commands.issubset(cli_commands), - f"Documented commands {documented_commands} must be subset of CLI commands {cli_commands}", - ) - - def test_documented_command_options_match_cli_help(self) -> None: - """Documented command forms in Procedure must include all required CLI options.""" - skill_text = _SKILL_FILE.read_text(encoding="utf-8") - self._assert_command_options(skill_text) - - def test_skill_manifest_required_for_all_commands(self) -> None: - """Skill must specify manifest required for every manifest command.""" - skill_text = _SKILL_FILE.read_text(encoding="utf-8") - inputs_section = self._get_section(skill_text, "Inputs") - preflight_section = self._get_section(skill_text, "Preflight") - self.assertIn("required for validate, preflight, run, resume, status, score, report", inputs_section) - self.assertIn("For validate/preflight/run/resume/status/score/report: confirm a manifest path is provided", preflight_section) - - def test_cli_help_exits_zero(self) -> None: - result = subprocess.run( - [sys.executable, str(_CLI_SCRIPT), "--help"], - capture_output=True, - text=True, - cwd=str(_REPO_ROOT), - ) - self.assertEqual(result.returncode, 0) - - def test_cli_error_ordering_and_invalid_state(self) -> None: - """CLI must exit 69 with 'error: benchmark state is unavailable' for invalid manifest/state before capability gates.""" - schema_fixture = str(_REPO_ROOT / "scripts" / "fixtures" / "agent-comparison-benchmark-manifest.schema.json") - for cmd in ("run", "resume", "status"): - args = [sys.executable, str(_CLI_SCRIPT), cmd, "--manifest", schema_fixture] - if cmd != "run": - args.extend(["--run-id", "dummy-run-id"]) - res = subprocess.run(args, capture_output=True, text=True, cwd=str(_REPO_ROOT)) - self.assertEqual(res.returncode, 69, f"{cmd} with invalid state should exit 69") - self.assertIn("error: benchmark state is unavailable", res.stderr) - skill_text = _SKILL_FILE.read_text(encoding="utf-8") - self._assert_error_ordering(skill_text) - - # ------------------------------------------------------------------ - # Capability gates - # ------------------------------------------------------------------ - - def test_run_resume_are_available_in_skill(self) -> None: - """Run/resume must document execution rather than a capability gate.""" - skill_text = _SKILL_FILE.read_text(encoding="utf-8") - self.assertNotIn("capability-unavailable: caller-adapter", skill_text) - self.assertIn("append a fresh all-cell preflight before attempt allocation", skill_text) - - def test_cli_help_exits_zero(self) -> None: - """Skill must not expose a public prepare operation across all steps and sections.""" - skill_text = _SKILL_FILE.read_text(encoding="utf-8") - self._assert_no_public_prepare(skill_text) - - # ------------------------------------------------------------------ - # Provider prohibition - # ------------------------------------------------------------------ - - def test_provider_prohibition(self) -> None: - """Skill must explicitly prohibit provider API invocations.""" - skill_text = _SKILL_FILE.read_text(encoding="utf-8") - self._assert_provider_prohibition(skill_text) - - # ------------------------------------------------------------------ - # No dispatcher / secret / fallback language - # ------------------------------------------------------------------ - - def test_no_dispatcher_reference(self) -> None: - """Skill must not reference dispatch.py or orchestration dispatchers in procedure.""" - skill_text = _SKILL_FILE.read_text(encoding="utf-8") - procedure_text = self._get_section(skill_text, "Procedure") - self.assertNotIn( - "dispatch.py", - procedure_text, - "Procedure must not reference dispatch.py", - ) - self.assertNotIn( - "orchestration dispatcher", - procedure_text.lower(), - "Procedure must not use orchestration dispatcher language", - ) - - def test_no_secret_language(self) -> None: - """Skill must not reference credential discovery or secrets in operational sections.""" - skill_text = _SKILL_FILE.read_text(encoding="utf-8") - self._assert_no_secret_operational_language(skill_text) - - def test_no_fallback_language(self) -> None: - """Skill must not suggest fallback behavior.""" - skill_text = _SKILL_FILE.read_text(encoding="utf-8") - procedure_text = self._get_section(skill_text, "Procedure") - self.assertNotIn( - "fallback", - procedure_text.lower(), - "Procedure must not suggest fallback behavior", - ) - - # ------------------------------------------------------------------ - # No internal workspace API exposure - # ------------------------------------------------------------------ - - def test_no_internal_api_in_user_routing(self) -> None: - """The internal workspace API must not be a user command.""" - skill_text = _SKILL_FILE.read_text(encoding="utf-8") - internal_apis = ["RunStore", "load_manifest", "ManifestDigestError"] - when_to_use = self._get_section(skill_text, "When to use") - procedure = self._get_section(skill_text, "Procedure") - for api in internal_apis: - for line in when_to_use.splitlines() + procedure.splitlines(): - stripped = line.strip() - if api in stripped and (stripped.startswith("- ") or stripped.startswith("1.") or stripped.startswith("2.")): - self.fail(f"Internal API {api} exposed in user-facing section: {stripped}") - - # ------------------------------------------------------------------ - # Safety rules & boundaries - # ------------------------------------------------------------------ - - def test_safety_rules_present(self) -> None: - """Skill must document safety rules.""" - skill_text = _SKILL_FILE.read_text(encoding="utf-8") - self.assertIn("## Safety rules", skill_text) - - def test_fixed_testbed_provenance(self) -> None: - """Skill must reference the read-only ../iop-s2 testbed.""" - skill_text = _SKILL_FILE.read_text(encoding="utf-8") - self.assertIn("../iop-s2", skill_text) - self.assertIn("read-only", skill_text) - - def test_testbed_output_state_boundary_wording(self) -> None: - """Skill must document read-only ../iop-s2 provenance, durable state, and contained agent-test/runs/ output.""" - skill_text = _SKILL_FILE.read_text(encoding="utf-8") - self._assert_boundary_wording(skill_text) - - # ------------------------------------------------------------------ - # CLI help verification - # ------------------------------------------------------------------ - - def test_cli_help_documents_only_supported_commands(self) -> None: - """CLI --help should document exactly the seven public state commands.""" - cli_commands = self._get_cli_help_commands() - self.assertEqual( - cli_commands, - _CLI_HELP_COMMANDS, - f"CLI commands must be exactly {_CLI_HELP_COMMANDS}, got {cli_commands}", - ) - - # ------------------------------------------------------------------ - # CLI delegation verification - # ------------------------------------------------------------------ - - def test_cli_validate_help(self) -> None: - """validate subcommand must exist in CLI.""" - result = subprocess.run( - [sys.executable, str(_CLI_SCRIPT), "validate", "--help"], - capture_output=True, - text=True, - cwd=str(_REPO_ROOT), - ) - self.assertEqual(result.returncode, 0) - self.assertIn("--manifest", result.stdout) - - def test_cli_preflight_help(self) -> None: - """preflight exists and accepts only the manifest input.""" - result = subprocess.run( - [sys.executable, str(_CLI_SCRIPT), "preflight", "--help"], - capture_output=True, - text=True, - cwd=str(_REPO_ROOT), - ) - self.assertEqual(result.returncode, 0) - self.assertIn("--manifest", result.stdout) - self.assertNotIn("--run-id", result.stdout) - - def test_cli_run_help(self) -> None: - """run subcommand must exist in CLI and require --manifest.""" - result = subprocess.run( - [sys.executable, str(_CLI_SCRIPT), "run", "--help"], - capture_output=True, - text=True, - cwd=str(_REPO_ROOT), - ) - self.assertEqual(result.returncode, 0) - self.assertIn("--manifest", result.stdout) - - def test_cli_status_help(self) -> None: - """status subcommand must exist in CLI and require --manifest and --run-id.""" - result = subprocess.run( - [sys.executable, str(_CLI_SCRIPT), "status", "--help"], - capture_output=True, - text=True, - cwd=str(_REPO_ROOT), - ) - self.assertEqual(result.returncode, 0) - self.assertIn("--manifest", result.stdout) - self.assertIn("--run-id", result.stdout) - - def test_cli_resume_help(self) -> None: - """resume subcommand must exist in CLI and require --manifest and --run-id.""" - result = subprocess.run( - [sys.executable, str(_CLI_SCRIPT), "resume", "--help"], - capture_output=True, - text=True, - cwd=str(_REPO_ROOT), - ) - self.assertEqual(result.returncode, 0) - self.assertIn("--manifest", result.stdout) - self.assertIn("--run-id", result.stdout) - - def test_cli_score_help(self) -> None: - """score exposes only manifest, run id, and explicit scoring retry.""" - result = subprocess.run( - [sys.executable, str(_CLI_SCRIPT), "score", "--help"], - capture_output=True, - text=True, - cwd=str(_REPO_ROOT), - ) - self.assertEqual(result.returncode, 0) - self.assertEqual( - set(re.findall(r"--[a-z][a-z0-9-]*", result.stdout)) - {"--help"}, - {"--manifest", "--run-id", "--retry-scoring-failed"}, - ) - - def test_cli_run_is_not_documented_as_capability_unavailable(self) -> None: - """Contract tests must not execute a stateful run just to prove availability.""" - skill_text = _SKILL_FILE.read_text(encoding="utf-8") - run_step = re.search( - r"\d+\.\s+\*\*Delegate run.*?(?=\n\d+\.|\Z)", - self._get_section(skill_text, "Procedure"), - re.DOTALL, - ) - self.assertIsNotNone(run_step) - self.assertNotIn( - "capability unavailable", run_step.group(0) # type: ignore[union-attr] - ) - - def test_score_contract_is_append_only_and_no_zero(self) -> None: - self._assert_scoring_contract(_SKILL_FILE.read_text(encoding="utf-8")) - - # ------------------------------------------------------------------ - # Independent mutation regression coverage - # - # Each unsafe variant is verified by its own test method so every - # mutated suite is independently non-zero while the original suite is - # zero (verified by test_base_skill_text_satisfies_full_contract). - # ------------------------------------------------------------------ - - def _skill_base_text(self) -> str: - return _SKILL_FILE.read_text(encoding="utf-8") - - def test_base_skill_text_satisfies_full_contract(self) -> None: - """Original skill text must satisfy the complete semantic contract (exit-zero baseline).""" - self._assert_full_skill_contract(self._skill_base_text()) - - def test_mutation_unknown_option_on_run_command(self) -> None: - """An unknown option on the documented run command must fail exact option parity.""" - base = self._skill_base_text() - mutated = base.replace( - "python3 scripts/agent_comparison_benchmark.py run --manifest ", - "python3 scripts/agent_comparison_benchmark.py run --manifest --bogus-option", - ) - self.assertNotEqual(mutated, base, "mutation fixture did not apply") - with self.assertRaises(AssertionError): - self._assert_full_skill_contract(mutated) - - def test_mutation_unknown_option_on_later_run_command(self) -> None: - """An unknown option on a later documented invocation must also fail parity.""" - base = self._skill_base_text() - command = " - Run: `python3 scripts/agent_comparison_benchmark.py run --manifest `" - mutated = base.replace( - command, - command - + "\n - Run again: `python3 scripts/agent_comparison_benchmark.py run --manifest --bogus-option`", - ) - self.assertNotEqual(mutated, base, "mutation fixture did not apply") - with self.assertRaises(AssertionError): - self._assert_full_skill_contract(mutated) - - def test_mutation_missing_required_manifest_on_run_command(self) -> None: - """Removing the required --manifest option from run must fail exact option parity.""" - base = self._skill_base_text() - mutated = base.replace( - "python3 scripts/agent_comparison_benchmark.py run --manifest ", - "python3 scripts/agent_comparison_benchmark.py run", - ) - self.assertNotEqual(mutated, base, "mutation fixture did not apply") - with self.assertRaises(AssertionError): - self._assert_full_skill_contract(mutated) - - def test_mutation_public_prepare_trigger(self) -> None: - """A public prepare benchmark trigger in When-to-use must fail the prepare prohibition.""" - base = self._skill_base_text() - mutated = base.replace( - "- User requests benchmark manifest validation: `validate`, `validate manifest`, `manifest 검증`", - "- User requests benchmark manifest validation: `validate`, `validate manifest`, `manifest 검증`\n- User requests benchmark preparation: `prepare benchmark`", - ) - self.assertNotEqual(mutated, base, "mutation fixture did not apply") - with self.assertRaises(AssertionError): - self._assert_full_skill_contract(mutated) - - def test_mutation_affirmative_provider_invocation(self) -> None: - """An affirmative provider invocation must fail the provider prohibition.""" - base = self._skill_base_text() - mutated = base.replace( - "- Do not invoke a caller or provider outside the deterministic benchmark CLI.", - "- Invoke provider APIs when needed for execution.", - ) - self.assertNotEqual(mutated, base, "mutation fixture did not apply") - with self.assertRaises(AssertionError): - self._assert_full_skill_contract(mutated) - - def test_mutation_additive_provider_contradiction(self) -> None: - """An additive provider instruction must not bypass the full validator.""" - base = self._skill_base_text() - mutated = base.replace( - "- Do not invoke a caller or provider outside the deterministic benchmark CLI.", - "- Do not invoke a caller or provider outside the deterministic benchmark CLI.\n" - "- Do not skip provider API invocation when a benchmark is requested.", - ) - self.assertNotEqual(mutated, base, "mutation fixture did not apply") - with self.assertRaises(AssertionError): - self._assert_full_skill_contract(mutated) - - def test_mutation_additive_prepare_contradiction(self) -> None: - """An additive prepare instruction must not bypass the prepare prohibition.""" - base = self._skill_base_text() - mutated = base.replace( - "- [ ] No public `prepare` operation was exposed or referenced.", - "- [ ] No public `prepare` operation was exposed or referenced.\n" - "- Do not delay public prepare when the caller requests it.", - ) - self.assertNotEqual(mutated, base, "mutation fixture did not apply") - with self.assertRaises(AssertionError): - self._assert_full_skill_contract(mutated) - - def test_mutation_within_run_session_or_cache_sharing(self) -> None: - """Permitting session/cache sharing within a run must fail the boundary wording.""" - base = self._skill_base_text() - mutated = base.replace( - "- Do not share session or cache state within a run across cells, repetitions, or attempts, or across run invocations.", - "- Caller sessions and caches are shared within a run across cells.", - ) - self.assertNotEqual(mutated, base, "mutation fixture did not apply") - with self.assertRaises(AssertionError): - self._assert_full_skill_contract(mutated) - - def test_mutation_additive_cache_sharing_contradiction(self) -> None: - """An additive cache-sharing instruction must fail the boundary contract.""" - base = self._skill_base_text() - mutated = base.replace( - "- Do not share session or cache state within a run across cells, repetitions, or attempts, or across run invocations.", - "- Do not share session or cache state within a run across cells, repetitions, or attempts, or across run invocations.\n" - "- Do not prevent sharing cache within a run across cells when convenient.", - ) - self.assertNotEqual(mutated, base, "mutation fixture did not apply") - with self.assertRaises(AssertionError): - self._assert_full_skill_contract(mutated) - - def test_mutation_weakened_per_run_wording(self) -> None: - """Re-introducing the weaker per-run wording must fail the per-attempt boundary check.""" - base = self._skill_base_text() - mutated = base.replace( - "- Caller sessions, output workspaces, and caches are fresh and isolated for every cell, repetition, and attempt; session or cache state is never shared within a run or across runs.", - "- Caller sessions and caches are fresh, isolated per run, and never shared across runs.", - ) - self.assertNotEqual(mutated, base, "mutation fixture did not apply") - with self.assertRaises(AssertionError): - self._assert_full_skill_contract(mutated) - - def test_mutation_removes_run_execution_contract(self) -> None: - """Removing the ready execution branch must fail the capability check.""" - base = self._skill_base_text() - mutated = base.replace( - "invoke each eligible cell exactly once", - "skip each eligible cell", - ) - self.assertNotEqual(mutated, base, "mutation fixture did not apply") - with self.assertRaises(AssertionError): - self._assert_full_skill_contract(mutated) - - def test_mutation_missing_public_preflight_delegation(self) -> None: - base = self._skill_base_text() - mutated = base.replace( - "python3 scripts/agent_comparison_benchmark.py preflight --manifest ", - "python3 scripts/agent_comparison_benchmark.py validate --manifest ", - ) - self.assertNotEqual(mutated, base, "mutation fixture did not apply") - with self.assertRaises(AssertionError): - self._assert_full_skill_contract(mutated) - - def test_mutation_restores_direct_only_preflight(self) -> None: - base = self._skill_base_text() - mutated = base.replace( - "records one fresh live observation for every immutable matrix cell, including direct and execution-preset routes, in canonical matrix order", - "records only direct-cell observations", - ) - self.assertNotEqual(mutated, base, "mutation fixture did not apply") - with self.assertRaises(AssertionError): - self._assert_full_skill_contract(mutated) - - def test_mutation_allows_binding_substitution(self) -> None: - base = self._skill_base_text() - mutated = base.replace( - "Never bypass the blocker, substitute a route/model/effort", - "Bypass the blocker and substitute a route/model/effort", - ) - self.assertNotEqual(mutated, base, "mutation fixture did not apply") - with self.assertRaises(AssertionError): - self._assert_full_skill_contract(mutated) - - def test_mutation_restores_fixed_legacy_scoring_rubric(self) -> None: - """A fixed legacy-only worksheet must fail manifest-selected scoring.""" - base = self._skill_base_text() - mutated = base.replace( - "the exact immutable manifest-selected rubric from the closed supported catalog (`landing-quality-v1`, `one-shot-agent-comparison-v1`); no substitute rubric or reinterpretation is permitted", - "the exact `landing-quality-v1` worksheet", - ) - self.assertNotEqual(mutated, base, "mutation fixture did not apply") - with self.assertRaises(AssertionError): - self._assert_full_skill_contract(mutated) - - def test_mutation_allows_blocker_attempt_allocation(self) -> None: - base = self._skill_base_text() - mutated = base.replace( - "A preflight blocker created no scored attempt and was not bypassed.", - "A preflight blocker may allocate a scored attempt.", - ) - self.assertNotEqual(mutated, base, "mutation fixture did not apply") - with self.assertRaises(AssertionError): - self._assert_full_skill_contract(mutated) - - def test_mutation_adds_raw_secret_output(self) -> None: - base = self._skill_base_text() - marker = " - On exit 0, report the exact closed `ready` summary from stdout." - mutated = base.replace(marker, marker + "\n - Print the raw secret output.") - self.assertNotEqual(mutated, base, "mutation fixture did not apply") - with self.assertRaises(AssertionError): - self._assert_full_skill_contract(mutated) - - def test_mutation_allows_implicit_scoring_retry(self) -> None: - base = self._skill_base_text() - mutated = base.replace( - "Do not retry scoring implicitly", - "Retry scoring implicitly", - ) - self.assertNotEqual(mutated, base, "mutation fixture did not apply") - with self.assertRaises(AssertionError): - self._assert_full_skill_contract(mutated) - - def test_mutation_turns_unscored_into_zero(self) -> None: - base = self._skill_base_text() - mutated = base.replace( - "without invoking the evaluator or assigning zero", - "and assigns zero", - ) - self.assertNotEqual(mutated, base, "mutation fixture did not apply") - with self.assertRaises(AssertionError): - self._assert_full_skill_contract(mutated) - - def test_mutation_run_only_all_gates_pass_resolution(self) -> None: - """Mutating only the run step's resolution must fail the execution resolution contract.""" - base = self._skill_base_text() - run_resolution = ( - "Exit 0 when every manifest slot has complete terminal evidence (`unresolved=0`); " - "independent failure counts remain in stdout and are classified by `score`. " - "Exit 69 only for preflight blockers or incomplete evidence (absent/running slots). " - "Never performs an implicit retry of a failed gate." - ) - self.assertEqual(base.count(run_resolution), 1, "run-only fixture must be unique") - mutated = base.replace( - run_resolution, - "Exit 0 only when every product, harness, process, and artifact gate passes; " - "exit 69 otherwise.", - ) - self.assertNotEqual(mutated, base, "mutation fixture did not apply") - with self.assertRaises(AssertionError): - self._assert_full_skill_contract(mutated) - - def test_mutation_resume_only_all_gates_pass_resolution(self) -> None: - """Mutating only the resume step's resolution must fail the execution resolution contract.""" - base = self._skill_base_text() - resume_resolution = ( - "Exit 0 when every manifest slot has complete terminal evidence (`unresolved=0`); " - "independent failure counts remain in stdout and are classified by `score`. " - "Exit 69 only for preflight blockers or incomplete evidence. " - "Never performs an implicit retry of a failed gate." - ) - self.assertEqual(base.count(resume_resolution), 1, "resume-only fixture must be unique") - mutated = base.replace( - resume_resolution, - "Exit 0 only when every product, harness, process, and artifact gate passes; " - "exit 69 otherwise.", - ) - self.assertNotEqual(mutated, base, "mutation fixture did not apply") - with self.assertRaises(AssertionError): - self._assert_full_skill_contract(mutated) - - def test_mutation_allows_implicit_execution_retry(self) -> None: - """Implicit failed-execution retry must fail the explicit --retry-failed contract.""" - base = self._skill_base_text() - mutated = base.replace( - "Retry is explicit only (`--retry-failed`); a failed terminal gate is never reinterpreted as success.", - "Retry failed terminal gates implicitly when they are observed.", - ) - self.assertNotEqual(mutated, base, "mutation fixture did not apply") - with self.assertRaises(AssertionError): - self._assert_full_skill_contract(mutated) - - def test_mutation_hardcodes_terminal_process_failure_count(self) -> None: - """Hardcoding a terminal process failure axis to zero must fail the independent process-axis contract.""" - base = self._skill_base_text() - mutated = base.replace( - "process_exited= process_signalled= process_timed_out= " - "process_cancelled= process_not_started=", - "process_exited= process_signalled=0 process_timed_out= " - "process_cancelled= process_not_started=", - ) - self.assertNotEqual(mutated, base, "mutation fixture did not apply") - with self.assertRaises(AssertionError): - self._assert_full_skill_contract(mutated) - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/agent_benchmark/web_validation.py b/scripts/agent_benchmark/web_validation.py deleted file mode 100644 index 9c5c132e..00000000 --- a/scripts/agent_benchmark/web_validation.py +++ /dev/null @@ -1,1105 +0,0 @@ -"""Fail-closed S12 web evidence and immutable artifact validation. - -The record written by this module is deliberately self-contained: it binds the -fixture and generated workspace files, browser observations, both manifest -viewports, screenshot bytes, ordered gates, and the attempt measurement. The -loader revalidates the referenced bytes instead of trusting a summary flag. -""" -from __future__ import annotations - -import hashlib -import json -import math -import os -import re -import stat -from dataclasses import dataclass -from pathlib import Path, PurePosixPath -from types import SimpleNamespace -from typing import Any - -from scripts.agent_benchmark.browser_cdp import ( - BrowserError, - BrowserRenderer, - RenderObservation, -) -from scripts.agent_benchmark.lifecycle import publish_bytes_no_replace -from scripts.agent_benchmark.manifest import VIEWPORT_ID_RE -from scripts.agent_benchmark.measurement import ( - AttemptMeasurement, - MEASUREMENT_FILENAME, -) - -WEB_VALIDATION_FILENAME = "web-validation.json" -WEB_VALIDATION_VERSION = 2 -WEB_STATUSES = ("passed", "failed", "blocked", "not_run") -WEB_GATES = ( - "generated_files", - "static_safety", - "images", - "network", - "console", - "responsive", - "accessibility", -) -GENERATED_FILES = ("index.html", "script.js", "styles.css") -MAX_WORKSPACE_FILES = 256 -MAX_WORKSPACE_BYTES = 32 * 1024 * 1024 -MAX_EVIDENCE_BYTES = 64 * 1024 * 1024 -DIGEST_RE = re.compile(r"^sha256:[0-9a-f]{64}$") -TOKEN_RE = re.compile(r"^[a-z0-9][a-z0-9_]{0,95}$") -UNRECOVERABLE_BROWSER_REASONS = { - "browser_process_cleanup_failed", - "output_unavailable", - "screenshot_cleanup_failed", - "screenshot_collision", -} - - -class WebValidationError(Exception): - """The S12 record or one of its referenced artifacts is untrusted.""" - - -@dataclass(frozen=True) -class WebValidation: - status: str - record: dict[str, Any] - - -def _digest(data: bytes) -> str: - return "sha256:" + hashlib.sha256(data).hexdigest() - - -def _bytes(value: dict[str, Any]) -> bytes: - return ( - json.dumps( - value, - sort_keys=True, - separators=(",", ":"), - ensure_ascii=True, - ).encode("ascii") - + b"\n" - ) - - -def _regular(path: Path) -> bytes: - flags = os.O_RDONLY | os.O_CLOEXEC | os.O_NONBLOCK - if hasattr(os, "O_NOFOLLOW"): - flags |= os.O_NOFOLLOW - try: - fd = os.open(path, flags) - except OSError as exc: - raise WebValidationError("web validation is unavailable") from exc - try: - info = os.fstat(fd) - if not stat.S_ISREG(info.st_mode) or info.st_size > MAX_EVIDENCE_BYTES: - raise WebValidationError("web validation must be a bounded regular file") - data = bytearray() - while len(data) < info.st_size: - chunk = os.read(fd, min(1024 * 1024, info.st_size - len(data))) - if not chunk: - raise WebValidationError("web validation changed while reading") - data.extend(chunk) - if os.read(fd, 1): - raise WebValidationError("web validation changed while reading") - return bytes(data) - except OSError as exc: - raise WebValidationError("web validation is unavailable") from exc - finally: - os.close(fd) - - -def _canonical_path(value: Any) -> str: - if not isinstance(value, str) or not value or "\\" in value or ":" in value: - raise WebValidationError("web validation path is invalid") - path = PurePosixPath(value) - if path.is_absolute() or str(path) != value or any( - part in ("", ".", "..") for part in path.parts - ): - raise WebValidationError("web validation path is invalid") - return value - - -def _safe_read(root: Path, relative: str) -> bytes: - """Read a bounded regular file without following any path component.""" - relative = _canonical_path(relative) - directory_flags = os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC - file_flags = os.O_RDONLY | os.O_CLOEXEC | os.O_NONBLOCK - if hasattr(os, "O_NOFOLLOW"): - directory_flags |= os.O_NOFOLLOW - file_flags |= os.O_NOFOLLOW - descriptors: list[int] = [] - try: - current = os.open(root, directory_flags) - descriptors.append(current) - parts = PurePosixPath(relative).parts - for component in parts[:-1]: - current = os.open(component, directory_flags, dir_fd=current) - descriptors.append(current) - fd = os.open(parts[-1], file_flags, dir_fd=current) - descriptors.append(fd) - info = os.fstat(fd) - if not stat.S_ISREG(info.st_mode) or info.st_size > MAX_WORKSPACE_BYTES: - raise OSError("not a bounded regular file") - data = bytearray() - while len(data) < info.st_size: - chunk = os.read(fd, min(1024 * 1024, info.st_size - len(data))) - if not chunk: - raise OSError("short read") - data.extend(chunk) - if os.read(fd, 1): - raise OSError("file grew during read") - return bytes(data) - finally: - for descriptor in reversed(descriptors): - try: - os.close(descriptor) - except OSError: - pass - - -def _observed_file(root: Path, relative: str) -> tuple[str, str, int]: - try: - data = _safe_read(root, relative) - except FileNotFoundError: - return "missing", "", 0 - except (OSError, ValueError, WebValidationError): - return ( - ("non_regular", "", 0) - if os.path.lexists(root / relative) - else ("missing", "", 0) - ) - return "regular", _digest(data), len(data) - - -def _file_fact( - root: Path, - relative: str, - kind: str, - *, - expected: bytes | None = None, -) -> dict[str, Any]: - state, digest, size = _observed_file(root, relative) - expected_digest = "" if expected is None else _digest(expected) - if state == "regular" and expected is not None and digest != expected_digest: - state = "mismatch" - return { - "path": relative, - "kind": kind, - "state": state, - "digest": digest, - "size": size, - "expected_digest": expected_digest, - } - - -def _workspace_entries(root: Path) -> tuple[str, ...]: - entries: list[str] = [] - for current, directories, files in os.walk(root, followlinks=False): - current_path = Path(current) - for name in sorted([*directories, *files]): - path = current_path / name - relative = path.relative_to(root).as_posix() - entries.append(relative) - if len(entries) > MAX_WORKSPACE_FILES: - raise WebValidationError("workspace evidence exceeds file budget") - return tuple(sorted(entries)) - - -def _expected_directories(paths: set[str]) -> set[str]: - directories: set[str] = set() - for raw in paths: - parent = PurePosixPath(raw).parent - while str(parent) != ".": - directories.add(str(parent)) - parent = parent.parent - return directories - - -def _workspace_snapshot(workspace: Path, manifest) -> dict[str, Any]: - inputs = [ - _file_fact( - workspace, - asset.workspace_path, - "fixture", - expected=asset.content, - ) - for asset in sorted(manifest.fixture.assets, key=lambda item: item.workspace_path) - ] - generated = [ - _file_fact(workspace, name, "generated") for name in GENERATED_FILES - ] - expected_paths = {item["path"] for item in [*inputs, *generated]} - allowed_entries = expected_paths | _expected_directories(expected_paths) - extras = [ - path for path in _workspace_entries(workspace) if path not in allowed_entries - ] - return {"inputs": inputs, "generated": generated, "extra_paths": extras} - - -def _gate( - ident: str, - passed: bool, - reason: str, - source: str, - evidence: list[str], -) -> dict[str, Any]: - return { - "id": ident, - "passed": passed, - "reason": "" if passed else reason, - "source": source, - "evidence": evidence, - } - - -def _generated_gate(snapshot: dict[str, Any]) -> dict[str, Any]: - facts = [*snapshot["inputs"], *snapshot["generated"]] - passed = all(item["state"] == "regular" for item in facts) and not snapshot[ - "extra_paths" - ] - if snapshot["extra_paths"]: - reason = "unexpected_content" - else: - failed = next((item for item in facts if item["state"] != "regular"), None) - reason = "" if failed is None else f"{failed['kind']}_{failed['state']}" - evidence = [ - f"{item['kind']}:{item['path']}:{item['state']}" for item in facts - ] + [f"extra:{item}" for item in snapshot["extra_paths"]] - return _gate("generated_files", passed, reason, "workspace", evidence) - - -def _static_gate(workspace: Path, generated_gate: dict[str, Any], manifest) -> dict[str, Any]: - if not generated_gate["passed"]: - return _gate( - "static_safety", - False, - "workspace_not_closed", - "source_scan", - ["generated_files=false"], - ) - try: - text = b"\n".join(_safe_read(workspace, name) for name in GENERATED_FILES).decode( - "utf-8", "strict" - ) - except (OSError, UnicodeDecodeError, WebValidationError): - return _gate( - "static_safety", False, "source_unavailable", "source_scan", [] - ) - reason = "" - if re.search(r"(?:@import|\bimport\s*(?:\(|[\"']))", text, re.I): - reason = "external_or_module_reference" - elif re.search(r"\b(?:react|vue|angular|bootstrap|tailwind)\b", text, re.I): - reason = "framework_reference" - else: - declared = {asset.workspace_path for asset in manifest.fixture.assets} - for reference in re.findall(r"(?:src|href)\s*=\s*[\"']([^\"']+)", text, re.I): - path = reference.split("?", 1)[0].split("#", 1)[0] - if not path or path in GENERATED_FILES or reference.startswith("#"): - continue - if re.match(r"^(?:https?:|//|data:)", reference, re.I) or path not in declared: - reason = "undeclared_reference" - break - return _gate( - "static_safety", - not reason, - reason, - "source_scan", - [f"generated={','.join(GENERATED_FILES)}"], - ) - - -def _viewport_record(view) -> dict[str, Any]: - return { - "id": view.id, - "width": view.width, - "height": view.height, - "screenshot": { - "file": view.screenshot, - "digest": view.screenshot_digest, - "size": view.screenshot_size, - }, - "images": [dict(item) for item in view.image_facts], - "layout": dict(view.layout), - "accessibility": dict(view.accessibility), - } - - -def _runtime_gates(manifest, render: RenderObservation) -> dict[str, dict[str, Any]]: - viewports = tuple(render.viewports) - expected_assets = { - asset.workspace_path - for asset in manifest.fixture.assets - if asset.workspace_path.startswith("assets/") - } - image_failures: list[str] = [] - responsive_failures: list[str] = [] - accessibility_failures: list[str] = [] - for view in viewports: - images = { - item.get("src"): item - for item in view.image_facts - if isinstance(item, dict) and isinstance(item.get("src"), str) - } - for path in sorted(expected_assets): - item = images.get(path) - if not item or not ( - item.get("complete") is True - and isinstance(item.get("natural_width"), int) - and item["natural_width"] > 0 - and isinstance(item.get("natural_height"), int) - and item["natural_height"] > 0 - and item.get("visible") is True - and isinstance(item.get("alt"), str) - and bool(item["alt"].strip()) - ): - image_failures.append(f"{view.id}:{path}") - layout = view.layout - if ( - layout.get("scroll_width") != layout.get("client_width") - or layout.get("clipped") != 0 - or layout.get("overlaps") != 0 - ): - responsive_failures.append(view.id) - accessibility = view.accessibility - controls = accessibility.get("controls") - ax = accessibility.get("ax") - if not ( - accessibility.get("h1_count") == 1 - and accessibility.get("heading_progression") is True - and isinstance(accessibility.get("main_count"), int) - and accessibility["main_count"] >= 1 - and isinstance(accessibility.get("landmarks"), int) - and accessibility["landmarks"] >= 1 - and isinstance(controls, list) - and controls - and all( - item.get("name") is True - and item.get("focused") is True - and item.get("focus_visible") is True - and isinstance(item.get("tab_index"), int) - and item["tab_index"] >= 0 - and isinstance(item.get("contrast"), (int, float)) - and not isinstance(item.get("contrast"), bool) - and math.isfinite(item["contrast"]) - and item["contrast"] >= 4.5 - for item in controls - if isinstance(item, dict) - ) - and isinstance(ax, dict) - and isinstance(ax.get("non_ignored"), int) - and ax["non_ignored"] > 0 - and isinstance(ax.get("named"), int) - and ax["named"] > 0 - ): - accessibility_failures.append(view.id) - expected_viewports = [item.id for item in manifest.viewports] - observed_viewports = [item.id for item in viewports] - if observed_viewports != expected_viewports: - responsive_failures.append("viewport_set") - - request_failures = [ - item - for item in render.requests - if not item.get("allowed") or int(item.get("status", 0)) >= 400 - ] - console_failures = [ - item - for item in render.console - if item.get("kind") == "exception" - or str(item.get("level", "")).lower() - in {"error", "warning", "warn", "assert"} - ] - return { - "images": _gate( - "images", - not image_failures and bool(expected_assets), - "image_evidence_failed" if image_failures else "image_fixture_missing", - "browser_dom", - image_failures or [f"asset:{path}" for path in sorted(expected_assets)], - ), - "network": _gate( - "network", - not request_failures, - "request_failed", - "browser_fetch", - [f"requests={len(render.requests)}"], - ), - "console": _gate( - "console", - not console_failures, - "console_error", - "browser_console", - [f"events={len(render.console)}"], - ), - "responsive": _gate( - "responsive", - not responsive_failures, - "layout_failed", - "browser_layout", - responsive_failures or [f"viewport:{item}" for item in observed_viewports], - ), - "accessibility": _gate( - "accessibility", - not accessibility_failures, - "accessibility_failed", - "browser_dom_ax", - accessibility_failures - or [f"viewport:{item}" for item in observed_viewports], - ), - } - - -def _not_observed_gates(reason: str, source: str) -> dict[str, dict[str, Any]]: - return { - ident: _gate(ident, False, reason, source, [f"reason={reason}"]) - for ident in WEB_GATES - } - - -def _reason_token(value: Any, fallback: str) -> str: - text = str(value or "").strip().lower().replace("-", "_").replace(" ", "_") - return text if TOKEN_RE.fullmatch(text) else fallback - - -def build_web_validation( - manifest, - attempt, - measurement: AttemptMeasurement, - render: RenderObservation | None, - *, - blocked: str = "", -) -> WebValidation: - workspace = Path( - attempt.workspace_dir if hasattr(attempt, "workspace_dir") else attempt - ) - attempt_root = Path( - attempt.attempt_root - if hasattr(attempt, "attempt_root") - else workspace.parent - ) - try: - workspace_mode = os.lstat(workspace).st_mode - attempt_mode = os.lstat(attempt_root).st_mode - except OSError as exc: - raise WebValidationError("web validation workspace is unavailable") from exc - if not stat.S_ISDIR(workspace_mode) or not stat.S_ISDIR(attempt_mode): - raise WebValidationError("web validation workspace is invalid") - snapshot = _workspace_snapshot(workspace, manifest) - generated_gate = _generated_gate(snapshot) - static_gate = _static_gate(workspace, generated_gate, manifest) - browser = {"status": "not_observed", "product": "", "origin": ""} - requests: list[dict[str, Any]] = [] - console: list[dict[str, Any]] = [] - viewports: list[dict[str, Any]] = [] - reason = "" - if blocked: - status = "blocked" - reason = _reason_token(blocked, "browser_failure") - gates = _not_observed_gates(reason, "browser") - gates["generated_files"] = generated_gate - gates["static_safety"] = static_gate - elif render is None: - status = "failed" - reason = generated_gate["reason"] or "render_not_run" - gates = _not_observed_gates("render_not_run", "pipeline") - gates["generated_files"] = generated_gate - gates["static_safety"] = static_gate - else: - browser = { - "status": "observed", - "product": render.browser, - "origin": render.origin, - } - requests = [dict(item) for item in render.requests] - console = [dict(item) for item in render.console] - viewports = [_viewport_record(item) for item in render.viewports] - gates = _runtime_gates(manifest, render) - gates["generated_files"] = generated_gate - gates["static_safety"] = static_gate - passed = all(gates[ident]["passed"] for ident in WEB_GATES) - status = "passed" if passed else "failed" - if not passed: - reason = next(gates[ident]["reason"] for ident in WEB_GATES if not gates[ident]["passed"]) - - screenshots = [ - {"id": item["id"], **item["screenshot"]} for item in viewports - ] - measurement_bytes = _regular(attempt_root / MEASUREMENT_FILENAME) - record = { - "record": "web-validation", - "web_validation_version": WEB_VALIDATION_VERSION, - "status": status, - "reason": reason, - "attempt": { - "run_id": measurement.run_id, - "cell_id": measurement.cell_id, - "repetition": measurement.repetition, - "attempt": measurement.attempt, - }, - "manifest_digest": manifest.digest, - "fixture_checksum": manifest.fixture.checksum, - "measurement_digest": _digest(measurement_bytes), - "workspace": snapshot, - "browser": browser, - "requests": requests, - "console": console, - "viewports": viewports, - "screenshots": screenshots, - "gates": [gates[ident] for ident in WEB_GATES], - } - web = WebValidation(status, record) - _validate_record(record, attempt_root, manifest=manifest) - return web - - -def validate_web_attempt( - manifest, - attempt_root: str | Path, - prepared, - measurement: AttemptMeasurement, - result=None, - *, - browser_binary: str = "chromium", -) -> WebValidation: - workspace = Path(prepared.workspace_dir) - render = None - blocked = "" - generated_ready = all( - _observed_file(workspace, name)[0] == "regular" for name in GENERATED_FILES - ) - if generated_ready: - try: - render = BrowserRenderer(browser_binary).render( - workspace_root=workspace, - output_root=attempt_root, - viewports=manifest.viewports, - timeout_seconds=manifest.timeout.run_seconds, - ) - except BrowserError as exc: - blocked = _reason_token(exc, "browser_failure") - if blocked in UNRECOVERABLE_BROWSER_REASONS: - raise WebValidationError( - "browser evidence cleanup or collision is invalid" - ) from exc - except (FileNotFoundError, OSError): - blocked = "browser_start_failed" - return build_web_validation( - manifest, prepared, measurement, render, blocked=blocked - ) - - -def _require_fields(value: Any, fields: set[str], message: str) -> dict[str, Any]: - if not isinstance(value, dict) or set(value) != fields: - raise WebValidationError(message) - return value - - -def _is_int(value: Any, *, minimum: int = 0) -> bool: - return not isinstance(value, bool) and isinstance(value, int) and value >= minimum - - -def _is_number(value: Any, *, minimum: float | None = None) -> bool: - if isinstance(value, bool) or not isinstance(value, (int, float)): - return False - if not math.isfinite(value): - return False - return minimum is None or value >= minimum - - -def _validate_fact(item: Any, expected_kind: str) -> dict[str, Any]: - fact = _require_fields( - item, - {"path", "kind", "state", "digest", "size", "expected_digest"}, - "web validation workspace fact is invalid", - ) - _canonical_path(fact["path"]) - if fact["kind"] != expected_kind or fact["state"] not in { - "regular", - "missing", - "non_regular", - "mismatch", - }: - raise WebValidationError("web validation workspace fact is invalid") - if not _is_int(fact["size"]): - raise WebValidationError("web validation workspace fact is invalid") - if fact["state"] in {"regular", "mismatch"}: - if not isinstance(fact["digest"], str) or not DIGEST_RE.fullmatch(fact["digest"]): - raise WebValidationError("web validation workspace fact is invalid") - elif fact["digest"] != "" or fact["size"] != 0: - raise WebValidationError("web validation workspace fact is invalid") - if expected_kind == "fixture": - if not isinstance(fact["expected_digest"], str) or not DIGEST_RE.fullmatch( - fact["expected_digest"] - ): - raise WebValidationError("web validation workspace fact is invalid") - if fact["state"] == "regular" and fact["digest"] != fact["expected_digest"]: - raise WebValidationError("web validation workspace fact is inconsistent") - if fact["state"] == "mismatch" and fact["digest"] == fact["expected_digest"]: - raise WebValidationError("web validation workspace fact is inconsistent") - elif fact["expected_digest"] != "": - raise WebValidationError("web validation workspace fact is invalid") - return fact - - -def _validate_rect(value: Any) -> None: - rect = _require_fields( - value, - {"x", "y", "width", "height", "right", "bottom"}, - "web validation image rectangle is invalid", - ) - if not all(_is_number(item) for item in rect.values()) or not all( - _is_number(rect[key], minimum=0) for key in ("width", "height") - ): - raise WebValidationError("web validation image rectangle is invalid") - if not math.isclose(rect["right"], rect["x"] + rect["width"], abs_tol=0.01) or not math.isclose( - rect["bottom"], rect["y"] + rect["height"], abs_tol=0.01 - ): - raise WebValidationError("web validation image rectangle is inconsistent") - - -def _validate_viewport(value: Any) -> dict[str, Any]: - view = _require_fields( - value, - {"id", "width", "height", "screenshot", "images", "layout", "accessibility"}, - "web validation viewport is invalid", - ) - if not isinstance(view["id"], str) or not VIEWPORT_ID_RE.fullmatch(view["id"]): - raise WebValidationError("web validation viewport is invalid") - if not _is_int(view["width"], minimum=1) or not _is_int(view["height"], minimum=1): - raise WebValidationError("web validation viewport is invalid") - screenshot = _require_fields( - view["screenshot"], - {"file", "digest", "size"}, - "web validation screenshot is invalid", - ) - _canonical_path(screenshot["file"]) - if ( - "/" in screenshot["file"] - or screenshot["file"] != f"screenshot-{view['id']}.png" - or not DIGEST_RE.fullmatch(str(screenshot["digest"])) - or not _is_int(screenshot["size"], minimum=1) - ): - raise WebValidationError("web validation screenshot is invalid") - if not isinstance(view["images"], list): - raise WebValidationError("web validation image evidence is invalid") - for raw in view["images"]: - image = _require_fields( - raw, - { - "src", - "alt", - "complete", - "natural_width", - "natural_height", - "visible", - "rect", - }, - "web validation image evidence is invalid", - ) - if ( - not isinstance(image["src"], str) - or not isinstance(image["alt"], str) - or not isinstance(image["complete"], bool) - or not _is_int(image["natural_width"]) - or not _is_int(image["natural_height"]) - or not isinstance(image["visible"], bool) - ): - raise WebValidationError("web validation image evidence is invalid") - _validate_rect(image["rect"]) - layout = _require_fields( - view["layout"], - {"scroll_width", "client_width", "clipped", "overlaps"}, - "web validation layout evidence is invalid", - ) - if not all(_is_int(item) for item in layout.values()): - raise WebValidationError("web validation layout evidence is invalid") - accessibility = _require_fields( - view["accessibility"], - { - "h1_count", - "headings", - "heading_progression", - "main_count", - "landmarks", - "controls", - "ax", - }, - "web validation accessibility evidence is invalid", - ) - if ( - not all( - _is_int(accessibility[key]) - for key in ("h1_count", "main_count", "landmarks") - ) - or not isinstance(accessibility["heading_progression"], bool) - or not isinstance(accessibility["headings"], list) - or not all(_is_int(item, minimum=1) and item <= 6 for item in accessibility["headings"]) - or not isinstance(accessibility["controls"], list) - ): - raise WebValidationError("web validation accessibility evidence is invalid") - expected_progression = all( - accessibility["headings"][index] - <= accessibility["headings"][index - 1] + 1 - for index in range(1, len(accessibility["headings"])) - ) - if ( - accessibility["heading_progression"] != expected_progression - or accessibility["h1_count"] - != sum(1 for item in accessibility["headings"] if item == 1) - ): - raise WebValidationError( - "web validation accessibility evidence is inconsistent" - ) - for raw in accessibility["controls"]: - control = _require_fields( - raw, - {"name", "tab_index", "focused", "focus_visible", "contrast"}, - "web validation control evidence is invalid", - ) - if ( - not isinstance(control["name"], bool) - or isinstance(control["tab_index"], bool) - or not isinstance(control["tab_index"], int) - or not isinstance(control["focused"], bool) - or not isinstance(control["focus_visible"], bool) - or not _is_number(control["contrast"], minimum=0) - ): - raise WebValidationError("web validation control evidence is invalid") - ax = _require_fields( - accessibility["ax"], - {"nodes", "non_ignored", "named"}, - "web validation accessibility tree is invalid", - ) - if not all(_is_int(item) for item in ax.values()): - raise WebValidationError("web validation accessibility tree is invalid") - if not 0 <= ax["named"] <= ax["non_ignored"] <= ax["nodes"]: - raise WebValidationError("web validation accessibility tree is inconsistent") - if ( - view["layout"]["client_width"] > view["width"] - or view["layout"]["scroll_width"] < view["layout"]["client_width"] - ): - raise WebValidationError("web validation layout evidence is inconsistent") - return view - - -def _validate_record(record: Any, attempt_root: Path, *, manifest=None) -> None: - fields = { - "record", - "web_validation_version", - "status", - "reason", - "attempt", - "manifest_digest", - "fixture_checksum", - "measurement_digest", - "workspace", - "browser", - "requests", - "console", - "viewports", - "screenshots", - "gates", - } - record = _require_fields(record, fields, "web validation schema is invalid") - if ( - record["record"] != "web-validation" - or record["web_validation_version"] != WEB_VALIDATION_VERSION - or record["status"] not in WEB_STATUSES - or not isinstance(record["reason"], str) - or (record["reason"] and not TOKEN_RE.fullmatch(record["reason"])) - ): - raise WebValidationError("web validation schema is invalid") - identity = _require_fields( - record["attempt"], - {"run_id", "cell_id", "repetition", "attempt"}, - "web validation identity is invalid", - ) - if ( - not all(isinstance(identity[key], str) and identity[key] for key in ("run_id", "cell_id")) - or not _is_int(identity["repetition"], minimum=1) - or not _is_int(identity["attempt"], minimum=1) - ): - raise WebValidationError("web validation identity is invalid") - for field in ("manifest_digest", "fixture_checksum", "measurement_digest"): - if not isinstance(record[field], str) or not DIGEST_RE.fullmatch(record[field]): - raise WebValidationError("web validation digest is invalid") - - workspace = _require_fields( - record["workspace"], - {"inputs", "generated", "extra_paths"}, - "web validation workspace evidence is invalid", - ) - if not all(isinstance(workspace[field], list) for field in workspace): - raise WebValidationError("web validation workspace evidence is invalid") - inputs = [_validate_fact(item, "fixture") for item in workspace["inputs"]] - generated = [_validate_fact(item, "generated") for item in workspace["generated"]] - if ( - [item["path"] for item in inputs] != sorted(item["path"] for item in inputs) - or [item["path"] for item in generated] != list(GENERATED_FILES) - or len({item["path"] for item in [*inputs, *generated]}) != len(inputs) + len(generated) - or workspace["extra_paths"] != sorted(workspace["extra_paths"]) - ): - raise WebValidationError("web validation workspace evidence is invalid") - for path in workspace["extra_paths"]: - _canonical_path(path) - - workspace_root = attempt_root / "workspace" - for fact in [*inputs, *generated]: - state, digest, size = _observed_file(workspace_root, fact["path"]) - expected_state = "regular" if fact["state"] == "mismatch" else fact["state"] - if state != expected_state or digest != fact["digest"] or size != fact["size"]: - raise WebValidationError("web validation workspace artifact changed") - expected_paths = {item["path"] for item in [*inputs, *generated]} - allowed_entries = expected_paths | _expected_directories(expected_paths) - extras = [ - path for path in _workspace_entries(workspace_root) if path not in allowed_entries - ] - if extras != workspace["extra_paths"]: - raise WebValidationError("web validation workspace artifact changed") - - browser = _require_fields( - record["browser"], - {"status", "product", "origin"}, - "web validation browser evidence is invalid", - ) - if browser["status"] not in {"observed", "not_observed"} or not all( - isinstance(browser[key], str) for key in ("product", "origin") - ): - raise WebValidationError("web validation browser evidence is invalid") - if browser["status"] == "observed": - if not browser["product"] or not re.fullmatch(r"http://127\.0\.0\.1:[0-9]+", browser["origin"]): - raise WebValidationError("web validation browser evidence is invalid") - elif browser["product"] or browser["origin"]: - raise WebValidationError("web validation browser evidence is invalid") - - if not isinstance(record["requests"], list): - raise WebValidationError("web validation request evidence is invalid") - for raw in record["requests"]: - if not isinstance(raw, dict) or raw.get("kind") not in {"local", "external"}: - raise WebValidationError("web validation request evidence is invalid") - if raw["kind"] == "local": - request = _require_fields( - raw, - {"kind", "path", "allowed", "status"}, - "web validation request evidence is invalid", - ) - if not isinstance(request["path"], str) or not request["path"].startswith("/"): - raise WebValidationError("web validation request evidence is invalid") - else: - request = _require_fields( - raw, - {"kind", "url_digest", "allowed", "status"}, - "web validation request evidence is invalid", - ) - if not isinstance(request["url_digest"], str) or not DIGEST_RE.fullmatch(request["url_digest"]): - raise WebValidationError("web validation request evidence is invalid") - if not isinstance(request["allowed"], bool) or not _is_int(request["status"]): - raise WebValidationError("web validation request evidence is invalid") - if request["kind"] == "external" and ( - request["allowed"] or request["status"] != 0 - ): - raise WebValidationError("web validation request evidence is inconsistent") - if request["kind"] == "local" and request["allowed"] != ( - request["status"] < 400 - ): - raise WebValidationError("web validation request evidence is inconsistent") - if not isinstance(record["console"], list): - raise WebValidationError("web validation console evidence is invalid") - for raw in record["console"]: - item = _require_fields( - raw, - {"kind", "level"}, - "web validation console evidence is invalid", - ) - if item["kind"] not in {"console", "exception", "log"} or not isinstance(item["level"], str): - raise WebValidationError("web validation console evidence is invalid") - - if not isinstance(record["viewports"], list): - raise WebValidationError("web validation viewport evidence is invalid") - viewports = [_validate_viewport(item) for item in record["viewports"]] - if len({item["id"] for item in viewports}) != len(viewports): - raise WebValidationError("web validation viewport evidence is invalid") - if not isinstance(record["screenshots"], list): - raise WebValidationError("web validation screenshot evidence is invalid") - expected_screenshots = [ - {"id": item["id"], **item["screenshot"]} for item in viewports - ] - if record["screenshots"] != expected_screenshots: - raise WebValidationError("web validation screenshot evidence is inconsistent") - for screenshot in record["screenshots"]: - data = _regular(attempt_root / screenshot["file"]) - if ( - not data.startswith(b"\x89PNG\r\n\x1a\n") - or len(data) != screenshot["size"] - or _digest(data) != screenshot["digest"] - ): - raise WebValidationError("web validation screenshot artifact is invalid") - referenced_screenshots = {item["file"] for item in record["screenshots"]} - present_screenshots = { - item.name - for item in attempt_root.iterdir() - if item.name.startswith("screenshot-") and item.name.endswith(".png") - } - if present_screenshots != referenced_screenshots: - raise WebValidationError("web validation screenshot set is invalid") - - if not isinstance(record["gates"], list) or [ - item.get("id") if isinstance(item, dict) else None for item in record["gates"] - ] != list(WEB_GATES): - raise WebValidationError("web validation gates are invalid") - for raw in record["gates"]: - gate = _require_fields( - raw, - {"id", "passed", "reason", "source", "evidence"}, - "web validation gates are invalid", - ) - if ( - not isinstance(gate["passed"], bool) - or not isinstance(gate["reason"], str) - or not isinstance(gate["source"], str) - or not gate["source"] - or not isinstance(gate["evidence"], list) - or not all(isinstance(item, str) and item for item in gate["evidence"]) - or (gate["passed"] and gate["reason"]) - or (not gate["passed"] and not TOKEN_RE.fullmatch(gate["reason"])) - ): - raise WebValidationError("web validation gates are invalid") - - gate_manifest = manifest - if gate_manifest is None: - gate_manifest = SimpleNamespace( - fixture=SimpleNamespace( - assets=tuple( - SimpleNamespace(workspace_path=item["path"]) - for item in inputs - ) - ), - viewports=tuple( - SimpleNamespace( - id=item["id"], width=item["width"], height=item["height"] - ) - for item in viewports - ), - ) - generated_gate = _generated_gate(workspace) - static_gate = _static_gate(workspace_root, generated_gate, gate_manifest) - if browser["status"] == "observed": - projected_render = SimpleNamespace( - requests=tuple(record["requests"]), - console=tuple(record["console"]), - viewports=tuple( - SimpleNamespace( - id=item["id"], - width=item["width"], - height=item["height"], - image_facts=tuple(item["images"]), - layout=item["layout"], - accessibility=item["accessibility"], - ) - for item in viewports - ), - ) - expected_gates = _runtime_gates(gate_manifest, projected_render) - expected_gates["generated_files"] = generated_gate - expected_gates["static_safety"] = static_gate - elif record["status"] == "blocked": - expected_gates = _not_observed_gates(record["reason"], "browser") - expected_gates["generated_files"] = generated_gate - expected_gates["static_safety"] = static_gate - elif record["status"] == "not_run": - expected_gates = _not_observed_gates(record["reason"], "lifecycle") - else: - expected_gates = _not_observed_gates("render_not_run", "pipeline") - expected_gates["generated_files"] = generated_gate - expected_gates["static_safety"] = static_gate - if record["gates"] != [expected_gates[ident] for ident in WEB_GATES]: - raise WebValidationError("web validation gate evidence is inconsistent") - - all_passed = all(item["passed"] for item in record["gates"]) - observed = browser["status"] == "observed" - if record["status"] == "passed": - if not all_passed or not observed or not viewports or record["reason"]: - raise WebValidationError("web validation status is inconsistent") - elif record["status"] == "failed": - expected_reason = next( - item["reason"] for item in record["gates"] if not item["passed"] - ) - if all_passed or record["reason"] != expected_reason: - raise WebValidationError("web validation status is inconsistent") - elif record["status"] == "blocked": - if observed or viewports or record["screenshots"] or not record["reason"]: - raise WebValidationError("web validation status is inconsistent") - elif record["status"] == "not_run": - if observed or viewports or record["screenshots"] or not record["reason"].startswith("lifecycle_"): - raise WebValidationError("web validation status is inconsistent") - - measurement = _regular(attempt_root / MEASUREMENT_FILENAME) - if _digest(measurement) != record["measurement_digest"]: - raise WebValidationError("web validation measurement artifact changed") - if manifest is not None: - _validate_manifest_binding(record, manifest) - - -def _validate_manifest_binding(record: dict[str, Any], manifest) -> None: - if ( - record["manifest_digest"] != manifest.digest - or record["fixture_checksum"] != manifest.fixture.checksum - ): - raise WebValidationError("web validation manifest binding is invalid") - inputs = record["workspace"]["inputs"] - expected_inputs = [ - (asset.workspace_path, _digest(asset.content)) - for asset in sorted(manifest.fixture.assets, key=lambda item: item.workspace_path) - ] - if [(item["path"], item["expected_digest"]) for item in inputs] != expected_inputs: - raise WebValidationError("web validation fixture binding is invalid") - if record["browser"]["status"] == "observed": - expected_viewports = [ - (item.id, item.width, item.height) for item in manifest.viewports - ] - observed_viewports = [ - (item["id"], item["width"], item["height"]) - for item in record["viewports"] - ] - if observed_viewports != expected_viewports: - raise WebValidationError("web validation viewport binding is invalid") - if record["status"] == "passed" and any( - item["state"] != "regular" for item in record["workspace"]["inputs"] - ): - raise WebValidationError("web validation fixture status is inconsistent") - - -def validate_web_validation_manifest(record: WebValidation, manifest) -> None: - """Rebind an already loaded record to the immutable run manifest.""" - _validate_manifest_binding(record.record, manifest) - - -def publish_web_validation(attempt_root: str | Path, record: WebValidation) -> Path: - root = Path(attempt_root) - if not isinstance(record, WebValidation) or record.status != record.record.get("status"): - raise WebValidationError("web validation object is invalid") - _validate_record(record.record, root) - path = root / WEB_VALIDATION_FILENAME - try: - publish_bytes_no_replace(path, _bytes(record.record)) - except Exception as exc: - raise WebValidationError( - "web validation publication refused an existing target" - ) from exc - return path - - -def load_web_validation( - attempt_root: str | Path, *, manifest=None -) -> WebValidation: - root = Path(attempt_root) - raw = _regular(root / WEB_VALIDATION_FILENAME) - try: - record = json.loads(raw.decode("ascii")) - except (UnicodeDecodeError, json.JSONDecodeError) as exc: - raise WebValidationError("web validation is invalid") from exc - if not isinstance(record, dict) or _bytes(record) != raw: - raise WebValidationError("web validation is not canonical") - _validate_record(record, root, manifest=manifest) - return WebValidation(record["status"], record) diff --git a/scripts/agent_benchmark/web_validation_test.py b/scripts/agent_benchmark/web_validation_test.py deleted file mode 100644 index 40ba1231..00000000 --- a/scripts/agent_benchmark/web_validation_test.py +++ /dev/null @@ -1,580 +0,0 @@ -from __future__ import annotations - -import copy -import hashlib -import json -import tempfile -import unittest -from pathlib import Path -from types import SimpleNamespace -from unittest import mock - -from scripts.agent_benchmark import web_validation as web_validation_module -from scripts.agent_benchmark.browser_cdp import ( - BrowserError, - RenderObservation, - ViewportObservation, -) -from scripts.agent_benchmark.web_validation import ( - WEB_GATES, - WebValidationError, - build_web_validation, - load_web_validation, - publish_web_validation, - validate_web_attempt, -) - - -def _digest(data: bytes) -> str: - return "sha256:" + hashlib.sha256(data).hexdigest() - - -class WebValidationTest(unittest.TestCase): - def setUp(self) -> None: - self.temporary = tempfile.TemporaryDirectory() - self.addCleanup(self.temporary.cleanup) - self.attempt = Path(self.temporary.name) / "attempt" - self.attempt.mkdir() - self.workspace = self.attempt / "workspace" - self.workspace.mkdir() - (self.workspace / "assets").mkdir() - (self.workspace / "brief").mkdir() - self.asset_content = { - "assets/a.svg": b"", - "assets/b.svg": b"", - "brief/reference.txt": b"reference\n", - } - for relative, content in self.asset_content.items(): - (self.workspace / relative).write_bytes(content) - (self.workspace / "index.html").write_text( - "

Ready

A" - "Bgo" - "
", - encoding="utf-8", - ) - (self.workspace / "styles.css").write_text( - "body{color:#111;background:#fff}img{width:20px}" - "a:focus{outline:2px solid #05f}", - encoding="utf-8", - ) - (self.workspace / "script.js").write_text( - "document.body.dataset.ready='1';", encoding="utf-8" - ) - (self.attempt / "attempt-measurement.json").write_bytes(b"measurement\n") - - def _manifest(self): - assets = tuple( - SimpleNamespace(workspace_path=path, content=content) - for path, content in sorted(self.asset_content.items()) - ) - return SimpleNamespace( - digest="sha256:" + "a" * 64, - fixture=SimpleNamespace( - checksum="sha256:" + "b" * 64, - assets=assets, - ), - viewports=( - SimpleNamespace(id="desktop", width=800, height=600), - SimpleNamespace(id="mobile", width=375, height=600), - ), - timeout=SimpleNamespace(run_seconds=20), - ) - - @staticmethod - def _measurement(reason: str = "success"): - return SimpleNamespace( - run_id="run", - cell_id="cell", - repetition=1, - attempt=1, - ) - - def _view(self, ident: str, width: int, *, suffix: str = "") -> ViewportObservation: - screenshot = f"screenshot-{ident}.png" - png = b"\x89PNG\r\n\x1a\n" + ident.encode("ascii") + suffix.encode("ascii") - (self.attempt / screenshot).write_bytes(png) - images = tuple( - { - "src": path, - "alt": path, - "complete": True, - "natural_width": 20, - "natural_height": 20, - "visible": True, - "rect": { - "x": 0, - "y": 0, - "width": 20, - "height": 20, - "right": 20, - "bottom": 20, - }, - } - for path in ("assets/a.svg", "assets/b.svg") - ) - accessibility = { - "h1_count": 1, - "headings": [1], - "heading_progression": True, - "main_count": 1, - "landmarks": 1, - "controls": [ - { - "name": True, - "tab_index": 0, - "focused": True, - "focus_visible": True, - "contrast": 7.0, - } - ], - "ax": {"nodes": 4, "non_ignored": 3, "named": 2}, - } - return ViewportObservation( - ident, - width, - 600, - screenshot, - _digest(png), - len(png), - images, - { - "scroll_width": width, - "client_width": width, - "clipped": 0, - "overlaps": 0, - }, - accessibility, - ) - - def _render(self) -> RenderObservation: - return RenderObservation( - "Chromium/Test", - "http://127.0.0.1:12345", - ( - {"kind": "local", "path": "/index.html", "allowed": True, "status": 200}, - {"kind": "local", "path": "/assets/a.svg", "allowed": True, "status": 200}, - {"kind": "local", "path": "/assets/b.svg", "allowed": True, "status": 200}, - ), - (), - (self._view("desktop", 800), self._view("mobile", 375)), - ) - - def _build(self, render=None, reason: str = "success"): - return build_web_validation( - self._manifest(), - self.workspace, - self._measurement(reason), - self._render() if render is None else render, - ) - - def _publish_valid(self): - record = self._build() - publish_web_validation(self.attempt, record) - return record - - def _rewrite_record(self, mutate) -> None: - path = self.attempt / "web-validation.json" - record = json.loads(path.read_text(encoding="ascii")) - mutate(record) - path.write_text( - json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n", - encoding="ascii", - ) - - def _reproject_runtime_gates(self, record) -> None: - manifest = self._manifest() - render = SimpleNamespace( - requests=tuple(record["requests"]), - console=tuple(record["console"]), - viewports=tuple( - SimpleNamespace( - id=item["id"], - width=item["width"], - height=item["height"], - image_facts=tuple(item["images"]), - layout=item["layout"], - accessibility=item["accessibility"], - ) - for item in record["viewports"] - ), - ) - gates = web_validation_module._runtime_gates(manifest, render) - generated = web_validation_module._generated_gate(record["workspace"]) - gates["generated_files"] = generated - gates["static_safety"] = web_validation_module._static_gate( - self.workspace, generated, manifest - ) - record["screenshots"] = [ - {"id": item["id"], **item["screenshot"]} - for item in record["viewports"] - ] - record["gates"] = [gates[ident] for ident in WEB_GATES] - passed = all(item["passed"] for item in record["gates"]) - record["status"] = "passed" if passed else "failed" - record["reason"] = "" if passed else next( - item["reason"] for item in record["gates"] if not item["passed"] - ) - - def test_valid_record_binds_complete_evidence_and_is_immutable(self): - record = self._publish_valid() - loaded = load_web_validation(self.attempt, manifest=self._manifest()) - self.assertEqual(loaded.status, "passed") - self.assertEqual([item["id"] for item in loaded.record["gates"]], list(WEB_GATES)) - self.assertEqual(len(loaded.record["workspace"]["inputs"]), 3) - self.assertEqual(len(loaded.record["viewports"]), 2) - self.assertEqual(len(loaded.record["screenshots"]), 2) - self.assertEqual(record.record, loaded.record) - before = (self.attempt / "web-validation.json").read_bytes() - with self.assertRaises(WebValidationError): - publish_web_validation(self.attempt, record) - self.assertEqual((self.attempt / "web-validation.json").read_bytes(), before) - - def test_static_workspace_fault_matrix_is_recorded_failed(self): - cases = { - "fixture_mutation": lambda: (self.workspace / "assets/a.svg").write_bytes(b"changed"), - "fixture_symlink": self._replace_fixture_with_symlink, - "extra_nested": lambda: (self.workspace / "assets/extra.svg").write_text("extra"), - "external_reference": lambda: (self.workspace / "index.html").write_text( - "

x

x
" - ), - } - for name, mutate in cases.items(): - with self.subTest(name=name): - self.tearDown() - self.setUp() - mutate() - record = self._build() - self.assertEqual(record.status, "failed") - self.assertFalse(all(item["passed"] for item in record.record["gates"])) - - def _replace_fixture_with_symlink(self) -> None: - target = self.workspace / "assets/a.svg" - target.unlink() - target.symlink_to(self.workspace / "assets/b.svg") - - def test_runtime_gate_one_fault_matrix(self): - cases = {} - render = self._render() - image_views = list(render.viewports) - image_facts = [dict(item) for item in image_views[0].image_facts] - image_facts[0]["complete"] = False - image_views[0] = SimpleNamespace(**{ - **image_views[0].__dict__, "image_facts": tuple(image_facts) - }) - cases["images"] = SimpleNamespace(**{**render.__dict__, "viewports": tuple(image_views)}) - - render = self._render() - cases["network"] = SimpleNamespace(**{ - **render.__dict__, - "requests": (*render.requests, {"kind": "external", "url_digest": _digest(b"x"), "allowed": False, "status": 0}), - }) - render = self._render() - cases["console"] = SimpleNamespace(**{ - **render.__dict__, "console": ({"kind": "exception", "level": "error"},) - }) - render = self._render() - views = list(render.viewports) - views[0] = SimpleNamespace(**{ - **views[0].__dict__, - "layout": {"scroll_width": 801, "client_width": 800, "clipped": 1, "overlaps": 0}, - }) - cases["responsive"] = SimpleNamespace(**{**render.__dict__, "viewports": tuple(views)}) - render = self._render() - views = list(render.viewports) - accessibility = copy.deepcopy(views[0].accessibility) - accessibility["controls"][0]["focus_visible"] = False - views[0] = SimpleNamespace(**{**views[0].__dict__, "accessibility": accessibility}) - cases["accessibility"] = SimpleNamespace(**{**render.__dict__, "viewports": tuple(views)}) - - for gate, faulty in cases.items(): - with self.subTest(gate=gate): - record = self._build(faulty) - self.assertEqual(record.status, "failed") - gates = {item["id"]: item for item in record.record["gates"]} - self.assertFalse(gates[gate]["passed"]) - - def test_lifecycle_non_success_still_validates_workspace(self): - for reason in ("nonzero_exit", "timed_out", "cancelled", "controller_lost"): - with self.subTest(reason=reason): - record = build_web_validation( - self._manifest(), self.workspace, self._measurement(reason), None - ) - self.assertEqual(record.status, "failed") - self.assertEqual(record.record["reason"], "render_not_run") - gates = {item["id"]: item for item in record.record["gates"]} - self.assertTrue(gates["generated_files"]["passed"]) - self.assertTrue(gates["static_safety"]["passed"]) - self.assertFalse(gates["images"]["passed"]) - self.assertFalse(gates["responsive"]["passed"]) - - def test_browser_discovery_or_start_failure_is_blocked(self): - prepared = SimpleNamespace( - workspace_dir=str(self.workspace), attempt_root=str(self.attempt) - ) - for error in (FileNotFoundError("missing"), OSError("start")): - with self.subTest(error=type(error).__name__), mock.patch( - "scripts.agent_benchmark.web_validation.BrowserRenderer.render", - side_effect=error, - ): - record = validate_web_attempt( - self._manifest(), - self.attempt, - prepared, - self._measurement(), - ) - self.assertEqual(record.status, "blocked") - self.assertFalse(record.record["screenshots"]) - - with mock.patch( - "scripts.agent_benchmark.web_validation.BrowserRenderer.render", - side_effect=BrowserError("screenshot_collision"), - ), self.assertRaises(WebValidationError): - validate_web_attempt( - self._manifest(), - self.attempt, - prepared, - self._measurement(), - ) - - def test_missing_generated_files_are_failed_not_not_run(self): - (self.workspace / "index.html").unlink() - record = build_web_validation( - self._manifest(), self.workspace, self._measurement(), None - ) - self.assertEqual(record.status, "failed") - self.assertFalse(record.record["gates"][0]["passed"]) - - def test_status_gate_contradiction_and_unknown_nested_field_are_rejected(self): - self._publish_valid() - self._rewrite_record( - lambda record: record["gates"][0].update( - {"passed": False, "reason": "generated_missing"} - ) - ) - with self.assertRaises(WebValidationError): - load_web_validation(self.attempt) - - (self.attempt / "web-validation.json").unlink() - self._publish_valid() - self._rewrite_record(lambda record: record["browser"].update({"unknown": 1})) - with self.assertRaises(WebValidationError): - load_web_validation(self.attempt) - - (self.attempt / "web-validation.json").unlink() - self._publish_valid() - self._rewrite_record( - lambda record: record["viewports"][0]["images"][0].update( - {"complete": False} - ) - ) - with self.assertRaises(WebValidationError): - load_web_validation(self.attempt) - - (self.attempt / "web-validation.json").unlink() - self._publish_valid() - self._rewrite_record( - lambda record: record["gates"][0]["evidence"].append("invented") - ) - with self.assertRaises(WebValidationError): - load_web_validation(self.attempt) - - def test_screenshot_corruption_missing_nonregular_and_symlink_are_rejected(self): - variants = ("corrupt", "missing", "directory", "symlink", "extra") - for variant in variants: - with self.subTest(variant=variant): - self.tearDown() - self.setUp() - record = self._publish_valid() - target = self.attempt / record.record["screenshots"][0]["file"] - if variant == "corrupt": - target.write_bytes(b"not-png") - elif variant == "extra": - (self.attempt / "screenshot-extra.png").write_bytes( - b"\x89PNG\r\n\x1a\nextra" - ) - else: - target.unlink() - if variant == "directory": - target.mkdir() - elif variant == "symlink": - target.symlink_to(self.attempt / record.record["screenshots"][1]["file"]) - with self.assertRaises(WebValidationError): - load_web_validation(self.attempt) - - def test_fixture_generated_extra_and_measurement_changes_are_rejected(self): - variants = ("fixture", "generated", "extra", "measurement") - for variant in variants: - with self.subTest(variant=variant): - self.tearDown() - self.setUp() - self._publish_valid() - if variant == "fixture": - (self.workspace / "assets/a.svg").write_bytes(b"changed") - elif variant == "generated": - (self.workspace / "script.js").write_text("changed") - elif variant == "extra": - (self.workspace / "extra.txt").write_text("extra") - else: - (self.attempt / "attempt-measurement.json").write_bytes(b"changed") - with self.assertRaises(WebValidationError): - load_web_validation(self.attempt) - - def test_manifest_fixture_and_viewport_binding_are_strict(self): - self._publish_valid() - manifest = self._manifest() - manifest.fixture.checksum = "sha256:" + "c" * 64 - with self.assertRaises(WebValidationError): - load_web_validation(self.attempt, manifest=manifest) - - manifest = self._manifest() - manifest.viewports = tuple(reversed(manifest.viewports)) - with self.assertRaises(WebValidationError): - load_web_validation(self.attempt, manifest=manifest) - - def test_observed_records_require_exact_manifest_viewports(self): - def failed_render(): - render = self._render() - return SimpleNamespace( - **{ - **render.__dict__, - "console": ({"kind": "exception", "level": "error"},), - } - ) - - valid_failed = self._build(failed_render()) - self.assertEqual(valid_failed.status, "failed") - publish_web_validation(self.attempt, valid_failed) - loaded = load_web_validation(self.attempt, manifest=self._manifest()) - self.assertEqual(loaded.status, "failed") - self.assertEqual(len(loaded.record["viewports"]), 2) - - def missing(record): - removed = record["viewports"].pop() - (self.attempt / removed["screenshot"]["file"]).unlink() - - def empty(record): - for item in record["viewports"]: - (self.attempt / item["screenshot"]["file"]).unlink() - record["viewports"] = [] - - def duplicate(record): - removed = record["viewports"][1] - (self.attempt / removed["screenshot"]["file"]).unlink() - record["viewports"] = [ - copy.deepcopy(record["viewports"][0]), - copy.deepcopy(record["viewports"][0]), - ] - - def reordered(record): - record["viewports"].reverse() - - def foreign(record): - item = record["viewports"][1] - old = self.attempt / item["screenshot"]["file"] - item["id"] = "foreign" - item["screenshot"]["file"] = "screenshot-foreign.png" - old.rename(self.attempt / item["screenshot"]["file"]) - - for name, mutate in ( - ("missing", missing), - ("empty", empty), - ("duplicate", duplicate), - ("reordered", reordered), - ("foreign", foreign), - ): - with self.subTest(case=name): - self.tearDown() - self.setUp() - record = self._build(failed_render()) - publish_web_validation(self.attempt, record) - path = self.attempt / "web-validation.json" - raw = json.loads(path.read_text(encoding="ascii")) - mutate(raw) - self._reproject_runtime_gates(raw) - path.write_text( - json.dumps(raw, sort_keys=True, separators=(",", ":")) + "\n", - encoding="ascii", - ) - before = path.read_bytes() - with self.assertRaises(WebValidationError): - load_web_validation(self.attempt, manifest=self._manifest()) - self.assertEqual(path.read_bytes(), before) - - def test_manifest_viewport_id_grammar_round_trips(self): - manifest = self._manifest() - manifest.viewports = ( - SimpleNamespace(id="mobile.small+wide", width=375, height=600), - ) - render = RenderObservation( - "Chromium/Test", - "http://127.0.0.1:12345", - ( - { - "kind": "local", - "path": "/index.html", - "allowed": True, - "status": 200, - }, - { - "kind": "local", - "path": "/assets/a.svg", - "allowed": True, - "status": 200, - }, - { - "kind": "local", - "path": "/assets/b.svg", - "allowed": True, - "status": 200, - }, - ), - (), - (self._view("mobile.small+wide", 375),), - ) - record = build_web_validation( - manifest, - self.workspace, - self._measurement(), - render, - ) - self.assertEqual(record.status, "passed") - publish_web_validation(self.attempt, record) - loaded = load_web_validation(self.attempt, manifest=manifest) - self.assertEqual( - [item["id"] for item in loaded.record["viewports"]], - ["mobile.small+wide"], - ) - - - def test_negative_tab_index_is_failed_evidence_not_schema_error(self): - """Signed negative tab_index must serialize as failed evidence, not raise.""" - render = self._render() - views = list(render.viewports) - # Modify both viewports to have tab_index=-1 - for i, view in enumerate(views): - accessibility = copy.deepcopy(view.accessibility) - accessibility["controls"][0]["tab_index"] = -1 - views[i] = SimpleNamespace(**{ - **view.__dict__, "accessibility": accessibility - }) - render = SimpleNamespace(**{ - **render.__dict__, "viewports": tuple(views) - }) - record = self._build(render) - self.assertEqual(record.status, "failed") - gates = {item["id"]: item for item in record.record["gates"]} - self.assertFalse(gates["accessibility"]["passed"]) - self.assertIn("accessibility_failed", record.record["reason"]) - for vp in record.record["viewports"]: - for ctrl in vp["accessibility"]["controls"]: - self.assertEqual(ctrl["tab_index"], -1) - publish_web_validation(self.attempt, record) - loaded = load_web_validation(self.attempt, manifest=self._manifest()) - self.assertEqual(loaded.status, "failed") - for vp in loaded.record["viewports"]: - for ctrl in vp["accessibility"]["controls"]: - self.assertEqual(ctrl["tab_index"], -1) - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/agent_benchmark/workspace.py b/scripts/agent_benchmark/workspace.py deleted file mode 100644 index 32075d2a..00000000 --- a/scripts/agent_benchmark/workspace.py +++ /dev/null @@ -1,680 +0,0 @@ -""" -workspace.py - Fixture-seeded workspace and fresh caller-session materialization. - -Provides deterministic, standard-library-only workspace preparation beneath an -already allocated empty attempt root. -""" - -from __future__ import annotations - -import datetime -import hashlib -import json -import os -import posixpath -import re -import secrets -import shutil -import subprocess -from dataclasses import dataclass -from pathlib import Path -from typing import Any, Optional - -from scripts.agent_benchmark.manifest import ( - AssetMapping, - Manifest, - digest_workspace_inputs, -) - -RUN_ID_RE = re.compile(r"^run-\d{8}T\d{6}Z-[0-9a-f]{12}$") -CELL_ID_RE = re.compile(r"^[a-z0-9][a-z0-9_-]{0,63}$") - - -class WorkspaceError(Exception): - """Base exception for workspace preparation failures.""" - - -class WorkspaceValidationError(WorkspaceError): - """Raised when identity or manifest configuration is invalid for workspace preparation.""" - - -class WorkspacePathError(WorkspaceError): - """Raised when an attempt root or asset path violates containment, symlink, or collision rules.""" - - -class WorkspaceChecksumError(WorkspaceError): - """Raised when initial fixture or materialized workspace checksum fails verification.""" - - -class TestbedError(WorkspaceError): - """Raised when the testbed provenance check fails or testbed is dirty/mutated.""" - - -@dataclass(frozen=True) -class AttemptIdentity: - run_id: str - cell_id: str - repetition: int - attempt: int - - -class _PreparationOwnership: - """Tracks filesystem paths created and owned by a workspace preparation transaction.""" - - def __init__(self, attempt_root: Path) -> None: - self.attempt_root = attempt_root.resolve() - self.created_paths: list[Path] = [] - - def register(self, path: Path) -> None: - resolved = path.resolve() - if resolved not in self.created_paths: - self.created_paths.append(resolved) - - def unregister(self, path: Path) -> None: - resolved = path.resolve() - if resolved in self.created_paths: - self.created_paths.remove(resolved) - - def rollback(self) -> None: - """Remove all registered paths created by this invocation in reverse order.""" - for path in reversed(self.created_paths): - if not path.exists() and not path.is_symlink(): - continue - try: - if path.is_dir() and not path.is_symlink(): - shutil.rmtree(path, ignore_errors=True) - else: - path.unlink(missing_ok=True) - except OSError: - pass - - -def _destinations_conflict(path1: str, path2: str) -> bool: - """Return True if path1 and path2 conflict as identical or ancestor/descendant paths.""" - if path1 == path2: - return True - return path2.startswith(path1 + "/") or path1.startswith(path2 + "/") - - - -@dataclass(frozen=True) -class TestbedProvenance: - path: str - branch: str - head: str - status_digest: str - clean: bool - - -@dataclass(frozen=True) -class PreparedWorkspace: - identity: AttemptIdentity - attempt_root: str - workspace_dir: str - session_dir: str - session_id: str - session_is_fresh: bool - workspace_checksum: str - setup_cache_policy: str - testbed_provenance: TestbedProvenance - prepared_at: str - - -def _find_repo_root(start_path: Path) -> Path: - """Find repository root by searching upward for Makefile or .git.""" - candidate = start_path.resolve() - for _ in range(20): - if (candidate / "Makefile").is_file() or (candidate / ".git").exists(): - return candidate - parent = candidate.parent - if parent == candidate: - break - candidate = parent - return Path.cwd().resolve() - - -def validate_attempt_identity( - identity: AttemptIdentity, manifest: Manifest | None = None -) -> None: - """Validate format and bounds of an AttemptIdentity. - - Raises: - WorkspaceValidationError: If any field fails format or boundary checks. - """ - if not isinstance(identity, AttemptIdentity): - raise WorkspaceValidationError("identity must be an AttemptIdentity instance") - - if not isinstance(identity.run_id, str) or not RUN_ID_RE.match(identity.run_id): - raise WorkspaceValidationError(f"invalid run_id format '{identity.run_id}'") - - if not isinstance(identity.cell_id, str) or not CELL_ID_RE.match(identity.cell_id): - raise WorkspaceValidationError(f"invalid cell_id format '{identity.cell_id}'") - - if ( - isinstance(identity.repetition, bool) - or not isinstance(identity.repetition, int) - or identity.repetition < 1 - ): - raise WorkspaceValidationError("repetition must be a positive integer >= 1") - - if ( - isinstance(identity.attempt, bool) - or not isinstance(identity.attempt, int) - or identity.attempt < 1 - ): - raise WorkspaceValidationError("attempt must be a positive integer >= 1") - - if manifest is not None: - valid_cell_ids = {cell.id for cell in manifest.matrix} - if identity.cell_id not in valid_cell_ids: - raise WorkspaceValidationError( - f"cell_id '{identity.cell_id}' not found in manifest matrix" - ) - if identity.repetition > manifest.repetitions: - raise WorkspaceValidationError( - f"repetition {identity.repetition} exceeds manifest repetitions ({manifest.repetitions})" - ) - - -def inspect_testbed_provenance(testbed_path: str | Path) -> TestbedProvenance: - """Capture Git provenance of the runtime testbed directory. - - Requires a clean Git working copy. Never modifies the repository. - - Raises: - TestbedError: If directory does not exist, is not a git repo, or is dirty. - """ - path = Path(testbed_path).resolve() - if not path.exists() or not path.is_dir(): - raise TestbedError( - f"testbed directory '{path}' does not exist or is not a directory" - ) - - try: - proc_status = subprocess.run( - ["git", "status", "--porcelain=v1", "--untracked-files=all"], - cwd=path, - capture_output=True, - text=True, - check=False, - ) - except Exception as exc: - raise TestbedError(f"failed to run git status in testbed '{path}': {exc}") - - if proc_status.returncode != 0: - raise TestbedError(f"git status returned non-zero exit code in testbed '{path}'") - - status_output = proc_status.stdout.strip() - if status_output: - raise TestbedError(f"testbed repository '{path}' is dirty:\n{status_output}") - - try: - proc_branch = subprocess.run( - ["git", "branch", "--show-current"], - cwd=path, - capture_output=True, - text=True, - check=False, - ) - branch = proc_branch.stdout.strip() - if not branch: - proc_head_ref = subprocess.run( - ["git", "rev-parse", "--abbrev-ref", "HEAD"], - cwd=path, - capture_output=True, - text=True, - check=False, - ) - branch = proc_head_ref.stdout.strip() - except Exception as exc: - raise TestbedError(f"failed to inspect git branch in testbed '{path}': {exc}") - - try: - proc_head = subprocess.run( - ["git", "rev-parse", "HEAD"], - cwd=path, - capture_output=True, - text=True, - check=False, - ) - except Exception as exc: - raise TestbedError(f"failed to inspect git HEAD in testbed '{path}': {exc}") - - if proc_head.returncode != 0: - raise TestbedError(f"git rev-parse HEAD failed in testbed '{path}'") - - head = proc_head.stdout.strip() - status_raw = f"branch:{branch}\nhead:{head}\nstatus:{status_output}".encode("utf-8") - status_digest = "sha256:" + hashlib.sha256(status_raw).hexdigest() - - return TestbedProvenance( - path=str(path), - branch=branch, - head=head, - status_digest=status_digest, - clean=True, - ) - - -def prepare_workspace( - manifest: Manifest, - attempt_root: str | Path, - identity: AttemptIdentity, - repo_root: str | Path | None = None, -) -> PreparedWorkspace: - """Materialize clean workspace and session under an allocated empty attempt root. - - Args: - manifest: Validated benchmark manifest object. - attempt_root: Path to an exclusively allocated empty attempt root directory. - identity: Frozen AttemptIdentity specifying run/cell/repetition/attempt. - repo_root: Optional repository root path for resolving testbed/assets. - - Returns: - PreparedWorkspace dataclass containing materialized metadata. - - Raises: - WorkspaceValidationError: If identity or manifest schema/policy checks fail. - WorkspacePathError: If path containment, symlink, or root canonicality fails. - WorkspaceChecksumError: If fixture or workspace content digest verification fails. - TestbedError: If testbed provenance inspection fails or testbed is dirty/modified. - """ - attempt_root_input = Path(attempt_root) - - if repo_root is None: - repo_root_path = _find_repo_root(attempt_root_input) - else: - repo_root_path = Path(repo_root).resolve() - - # Phase 1: Validate complete plan before any visible publication. - validated_inputs = _validate_complete_plan( - manifest, attempt_root_input, identity, repo_root_path - ) - - resolved_attempt_root = validated_inputs["resolved_attempt_root"] - owned = _PreparationOwnership(resolved_attempt_root) - - # Phase 2: Stage preparation beneath the validated empty attempt root in a private owned directory. - # Phase 3: Verify staged content (checksum, postflight). - # Phase 4: Publish metadata and prepared.json. - # All three phases are bounded; any exception triggers rollback of function-owned artifacts. - try: - staging = _stage_preparation(validated_inputs, owned) - _verify_staged_preparation(staging, manifest) - prepared = _publish_preparation(staging, owned) - except Exception: - _rollback_owned_preparation(owned) - raise - - return prepared - - -def _validate_complete_plan( - manifest: Manifest, - attempt_root_input: Path, - identity: AttemptIdentity, - repo_root_path: Path, -) -> dict[str, Any]: - """Validate the entire preparation plan without creating any files. - - Returns a dict of resolved paths and validated state for staging. - """ - # 1. Validate identity against manifest - validate_attempt_identity(identity, manifest) - - if manifest.session_policy != "fresh": - raise WorkspaceValidationError( - f"unsupported session_policy '{manifest.session_policy}', expected 'fresh'" - ) - if manifest.setup_cache_policy != "isolated": - raise WorkspaceValidationError( - f"unsupported setup_cache_policy '{manifest.setup_cache_policy}', expected 'isolated'" - ) - - # 2. Path & Symlink validation on attempt_root - raw_path = attempt_root_input - if raw_path.is_symlink(): - raise WorkspacePathError("attempt_root cannot be a symlink") - - # Walk up parent segments to ensure no symlinks in path - curr = raw_path - while curr != curr.parent: - if curr.is_symlink(): - raise WorkspacePathError(f"path segment '{curr}' in attempt_root is a symlink") - if curr == repo_root_path: - break - curr = curr.parent - - resolved_attempt_root = raw_path.resolve() - - # Verify expected canonical relative path - rep_segment = f"repetition-{identity.repetition:04d}" - att_segment = f"attempt-{identity.attempt:06d}" - expected_rel = ( - Path(manifest.output_root) - / identity.run_id - / "cells" - / identity.cell_id - / rep_segment - / att_segment - ) - expected_abs = (repo_root_path / expected_rel).resolve() - - if resolved_attempt_root != expected_abs: - raise WorkspacePathError( - f"attempt_root '{resolved_attempt_root}' does not match expected canonical path '{expected_abs}'" - ) - - if not resolved_attempt_root.exists() or not resolved_attempt_root.is_dir(): - raise WorkspacePathError( - f"attempt_root '{resolved_attempt_root}' does not exist or is not a directory" - ) - - # Verify attempt_root is empty - if list(resolved_attempt_root.iterdir()): - raise WorkspacePathError(f"attempt_root '{resolved_attempt_root}' is not empty") - - # 3. Testbed preflight provenance check (must not be beneath attempt_root) - testbed_path = (repo_root_path / manifest.testbed).resolve() - try: - testbed_path.relative_to(resolved_attempt_root) - raise WorkspacePathError("testbed cannot be located beneath attempt_root") - except ValueError: - pass - - testbed_before = inspect_testbed_provenance(testbed_path) - - # 4. Check fixture checksum before copying - computed_fixture_checksum = digest_workspace_inputs(manifest.fixture.assets) - if computed_fixture_checksum != manifest.fixture.checksum: - raise WorkspaceChecksumError( - f"declared fixture checksum '{manifest.fixture.checksum}' does not match computed fixture asset digest '{computed_fixture_checksum}'" - ) - - # 5. Validate all asset sources exist, aren't symlinks, are within repo root - # and check for destination collisions before any creation. - workspace_dir = resolved_attempt_root / "workspace" - if workspace_dir.exists(): - raise WorkspacePathError("workspace child directory already exists") - - asset_validations: list[dict[str, Any]] = [] - workspace_destinations: set[str] = set() - for asset in manifest.fixture.assets: - raw_src_path = repo_root_path / asset.source - if raw_src_path.is_symlink(): - raise WorkspacePathError(f"asset source '{asset.source}' cannot be a symlink") - src_path = raw_src_path.resolve() - if src_path.is_symlink(): - raise WorkspacePathError(f"asset source '{asset.source}' cannot be a symlink") - if not src_path.exists() or not src_path.is_file(): - raise WorkspacePathError( - f"asset source '{asset.source}' does not exist or is not a regular file" - ) - try: - src_path.relative_to(repo_root_path) - except ValueError: - raise WorkspacePathError(f"asset source '{asset.source}' escapes repository root") - - wp = asset.workspace_path - if "\\" in wp or ":" in wp: - raise WorkspacePathError(f"asset workspace_path '{wp}' contains invalid characters") - wp_norm = posixpath.normpath(wp) - if wp != wp_norm or wp.startswith("/") or wp.startswith(".."): - raise WorkspacePathError( - f"asset workspace_path '{wp}' is not in canonical posix relative form" - ) - - target_file = workspace_dir / wp_norm - try: - target_file.relative_to(workspace_dir) - except ValueError: - raise WorkspacePathError(f"asset workspace_path '{wp}' escapes workspace directory") - - # Check for destination collisions before copying - if target_file.exists(): - raise WorkspacePathError( - f"asset destination '{wp}' already exists in workspace" - ) - - # Check parent path for symlinks - parent_check = target_file.parent - while parent_check != workspace_dir: - if parent_check.is_symlink(): - raise WorkspacePathError( - f"symlink detected in asset target path '{parent_check}'" - ) - parent_check = parent_check.parent - - for existing in workspace_destinations: - if _destinations_conflict(existing, wp_norm): - raise WorkspacePathError( - f"asset workspace_path '{wp_norm}' conflicts with '{existing}'" - ) - workspace_destinations.add(wp_norm) - - asset_validations.append({ - "source": asset.source, - "src_path": src_path, - "workspace_path": wp_norm, - "target_path": target_file, - "content": asset.content, - }) - - # 6. Verify prompt is not copied unless declared as an asset - prompt_declared = any( - a.source == manifest.fixture.prompt or a.workspace_path == manifest.fixture.prompt - for a in manifest.fixture.assets - ) - if not prompt_declared: - prompt_in_ws = workspace_dir / manifest.fixture.prompt - if prompt_in_ws.exists(): - raise WorkspacePathError( - "prompt file materialized in workspace without being declared as an asset" - ) - - return { - "resolved_attempt_root": resolved_attempt_root, - "workspace_dir": workspace_dir, - "testbed_path": testbed_path, - "testbed_before": testbed_before, - "asset_validations": asset_validations, - "prompt_declared": prompt_declared, - "identity": identity, - } - - -def _stage_preparation( - validated_inputs: dict[str, Any], owned: _PreparationOwnership -) -> dict[str, Any]: - """Create workspace/ and session/ directories and copy assets in private staging. - - This is the staging phase. Any failure here triggers rollback of owned entries. - """ - resolved_attempt_root = validated_inputs["resolved_attempt_root"] - - staging_dir = resolved_attempt_root / f".staging-{secrets.token_hex(6)}" - staging_dir.mkdir(parents=False, exist_ok=False) - owned.register(staging_dir) - - staging_workspace_dir = staging_dir / "workspace" - staging_workspace_dir.mkdir(parents=False, exist_ok=False) - - materialized_assets: list[AssetMapping] = [] - for av in validated_inputs["asset_validations"]: - src_path = av["src_path"] - target_file = staging_workspace_dir / av["workspace_path"] - content = src_path.read_bytes() - if av["content"] and av["content"] != content: - raise WorkspaceChecksumError( - f"asset content for '{av['source']}' does not match resolved file content" - ) - target_file.parent.mkdir(parents=True, exist_ok=True) - target_file.write_bytes(content) - materialized_assets.append( - AssetMapping(source=av["source"], workspace_path=av["workspace_path"], content=content) - ) - - staging_session_dir = staging_dir / "session" - staging_session_dir.mkdir(parents=False, exist_ok=False) - - return { - "resolved_attempt_root": resolved_attempt_root, - "staging_dir": staging_dir, - "workspace_dir": staging_workspace_dir, - "session_dir": staging_session_dir, - "materialized_assets": materialized_assets, - "validated_inputs": validated_inputs, - } - - -def _verify_staged_preparation( - staging: dict[str, Any], manifest: Manifest -) -> None: - """Verify staged workspace checksum and testbed postflight.""" - workspace_dir = staging["workspace_dir"] - testbed_path = staging["validated_inputs"]["testbed_path"] - testbed_before = staging["validated_inputs"]["testbed_before"] - - # Recompute workspace checksum after materialization - actual_assets: list[AssetMapping] = [] - for root, _, files in os.walk(workspace_dir): - for f in files: - fp = Path(root) / f - if fp.is_symlink(): - raise WorkspacePathError(f"symlink created in workspace directory '{fp}'") - rel_wp = fp.relative_to(workspace_dir).as_posix() - actual_assets.append( - AssetMapping(source="", workspace_path=rel_wp, content=fp.read_bytes()) - ) - - computed_workspace_checksum = digest_workspace_inputs(actual_assets) - if computed_workspace_checksum != manifest.fixture.checksum: - raise WorkspaceChecksumError( - f"materialized workspace checksum '{computed_workspace_checksum}' does not match declared fixture checksum '{manifest.fixture.checksum}'" - ) - - # Postflight testbed check - testbed_after = inspect_testbed_provenance(testbed_path) - if testbed_before != testbed_after: - raise TestbedError("testbed repository state was modified during workspace preparation") - - # Store testbed_after and workspace_checksum in staging for publish phase - staging["testbed_after"] = testbed_after - staging["workspace_checksum"] = computed_workspace_checksum - - -def _publish_owned_directory( - staging_dir: Path, - final_dir: Path, - owned: _PreparationOwnership, - collision_message: str, -) -> None: - """Exclusively create final_dir, register ownership, and move staged contents into it.""" - try: - final_dir.mkdir(exist_ok=False) - except (FileExistsError, OSError): - raise WorkspacePathError(collision_message) - - owned.register(final_dir) - - for item in staging_dir.iterdir(): - item.rename(final_dir / item.name) - - -def _publish_preparation( - staging: dict[str, Any], owned: _PreparationOwnership -) -> PreparedWorkspace: - """Generate session ID, build metadata, publish final entries, and write prepared.json.""" - validated_inputs = staging["validated_inputs"] - identity = validated_inputs["identity"] - - token = secrets.token_hex(6) - session_id = ( - f"session-{identity.run_id}-{identity.cell_id}-" - f"rep{identity.repetition:04d}-att{identity.attempt:06d}-{token}" - ) - session_is_fresh = True - - staging_dir = staging["staging_dir"] - staging_workspace_dir = staging["workspace_dir"] - staging_session_dir = staging["session_dir"] - resolved_attempt_root = staging["resolved_attempt_root"] - testbed_after = staging["testbed_after"] - workspace_checksum = staging["workspace_checksum"] - - final_workspace_dir = resolved_attempt_root / "workspace" - _publish_owned_directory( - staging_workspace_dir, - final_workspace_dir, - owned, - "workspace child directory already exists in attempt_root", - ) - - final_session_dir = resolved_attempt_root / "session" - _publish_owned_directory( - staging_session_dir, - final_session_dir, - owned, - "session child directory already exists in attempt_root", - ) - - if staging_dir.exists(): - shutil.rmtree(staging_dir, ignore_errors=True) - owned.unregister(staging_dir) - - prepared_at = datetime.datetime.now(datetime.timezone.utc).isoformat() - prepared = PreparedWorkspace( - identity=identity, - attempt_root=str(resolved_attempt_root), - workspace_dir=str(final_workspace_dir), - session_dir=str(final_session_dir), - session_id=session_id, - session_is_fresh=session_is_fresh, - workspace_checksum=workspace_checksum, - setup_cache_policy="isolated", - testbed_provenance=testbed_after, - prepared_at=prepared_at, - ) - - prepared_data = { - "identity": { - "run_id": identity.run_id, - "cell_id": identity.cell_id, - "repetition": identity.repetition, - "attempt": identity.attempt, - }, - "attempt_root": prepared.attempt_root, - "workspace_dir": prepared.workspace_dir, - "session_dir": prepared.session_dir, - "session_id": prepared.session_id, - "session_is_fresh": prepared.session_is_fresh, - "workspace_checksum": prepared.workspace_checksum, - "setup_cache_policy": prepared.setup_cache_policy, - "testbed_provenance": { - "path": prepared.testbed_provenance.path, - "branch": prepared.testbed_provenance.branch, - "head": prepared.testbed_provenance.head, - "status_digest": prepared.testbed_provenance.status_digest, - "clean": prepared.testbed_provenance.clean, - }, - "prepared_at": prepared.prepared_at, - } - - prepared_json_path = resolved_attempt_root / "prepared.json" - if prepared_json_path.exists() or prepared_json_path.is_symlink(): - raise WorkspacePathError("prepared.json already exists in attempt_root") - - try: - with prepared_json_path.open("x", encoding="utf-8") as f: - json.dump(prepared_data, f, indent=2) - except FileExistsError: - raise WorkspacePathError("prepared.json already exists in attempt_root") - - owned.register(prepared_json_path) - - return prepared - - -def _rollback_owned_preparation(owned: _PreparationOwnership) -> None: - """Remove paths explicitly registered as created by this preparation transaction.""" - owned.rollback() diff --git a/scripts/agent_benchmark/workspace_test.py b/scripts/agent_benchmark/workspace_test.py deleted file mode 100644 index 0f766315..00000000 --- a/scripts/agent_benchmark/workspace_test.py +++ /dev/null @@ -1,758 +0,0 @@ -""" -workspace_test.py - Comprehensive tests for workspace materialization and isolation. - -Covers exact run/cell/repetition/attempt grammar, path containment, symlink/collision rejection, -asset mapping, prompt exclusion, checksum verification, session freshness, testbed provenance, -and cross-attempt isolation. -""" - -from __future__ import annotations - -import json -import os -import shutil -import subprocess -import tempfile -import unittest -from pathlib import Path - -from scripts.agent_benchmark.manifest import ( - AssetMapping, - Fixture, - IopCell, - Manifest, - MatrixCell, - Timeout, - Viewport, - digest_manifest_and_resolved_inputs, - digest_workspace_inputs, - load_manifest, -) -from scripts.agent_benchmark.workspace import ( - AttemptIdentity, - PreparedWorkspace, - TestbedError, - TestbedProvenance, - WorkspaceChecksumError, - WorkspaceError, - WorkspacePathError, - WorkspaceValidationError, - inspect_testbed_provenance, - prepare_workspace, - validate_attempt_identity, -) - - -class BaseWorkspaceTest(unittest.TestCase): - """Base test class providing temporary Git repositories and testbed fixtures.""" - - def setUp(self) -> None: - self.tmp_dir_obj = tempfile.TemporaryDirectory() - self.tmp_dir = Path(self.tmp_dir_obj.name).resolve() - - self.repo_root = self.tmp_dir / "repo" - self.repo_root.mkdir() - - # Initialize git repo in repo_root - subprocess.run(["git", "init"], cwd=self.repo_root, capture_output=True, check=True) - subprocess.run( - ["git", "config", "user.name", "Test"], cwd=self.repo_root, capture_output=True, check=True - ) - subprocess.run( - ["git", "config", "user.email", "test@example.com"], - cwd=self.repo_root, - capture_output=True, - check=True, - ) - - # Initialize git repo in testbed (iop-s2) - self.testbed_dir = self.tmp_dir / "iop-s2" - self.testbed_dir.mkdir() - (self.testbed_dir / "README.md").write_text("testbed content", encoding="utf-8") - subprocess.run(["git", "init"], cwd=self.testbed_dir, capture_output=True, check=True) - subprocess.run( - ["git", "config", "user.name", "Test"], cwd=self.testbed_dir, capture_output=True, check=True - ) - subprocess.run( - ["git", "config", "user.email", "test@example.com"], - cwd=self.testbed_dir, - capture_output=True, - check=True, - ) - subprocess.run(["git", "add", "."], cwd=self.testbed_dir, capture_output=True, check=True) - subprocess.run( - ["git", "commit", "-m", "initial testbed commit"], - cwd=self.testbed_dir, - capture_output=True, - check=True, - ) - - # Create fixture files under repo_root - self.fixture_dir = self.repo_root / "scripts" / "fixtures" / "bench" - self.fixture_dir.mkdir(parents=True) - - self.prompt_rel = "scripts/fixtures/bench/prompt.md" - self.prompt_file = self.repo_root / self.prompt_rel - self.prompt_content = b"# Test Prompt\nDo task.\n" - self.prompt_file.write_bytes(self.prompt_content) - - self.ref_rel = "scripts/fixtures/bench/ref.txt" - self.ref_file = self.repo_root / self.ref_rel - self.ref_content = b"Reference data content\n" - self.ref_file.write_bytes(self.ref_content) - - subprocess.run(["git", "add", "."], cwd=self.repo_root, capture_output=True, check=True) - subprocess.run( - ["git", "commit", "-m", "add fixture files"], - cwd=self.repo_root, - capture_output=True, - check=True, - ) - - # Assets list (initially only ref.txt is an asset, prompt.md is separate prompt file) - self.assets = [ - AssetMapping(source=self.ref_rel, workspace_path="data/ref.txt", content=self.ref_content) - ] - self.fixture_checksum = digest_workspace_inputs(self.assets) - - self.run_id = "run-20260809T161730Z-0123456789ab" - self.output_root_rel = "agent-test/runs/bench-01" - - self.manifest_raw = { - "pipeline_version": "2", - "environment": "dev", - "testbed": "../iop-s2", - "repetitions": 2, - "session_policy": "fresh", - "setup_cache_policy": "isolated", - "timeout": { - "run_seconds": 300, - "idle_seconds": 30, - "quiet_seconds": 10, - "cleanup_grace_seconds": 5, - }, - "viewports": [{"id": "desktop_1080", "width": 1920, "height": 1080}], - "rubric_version": "landing-quality-v1", - "evaluator": {"caller": "codex", "iop": {"request_model": "judge", "requested_effort": "high", "route_kind": "direct", "route_id": "judge", "expected_bindings": [{"stage": "request", "model": "judge", "effort": "high"}]}}, - "output_root": self.output_root_rel, - "fixture": { - "version": "v1.0", - "prompt": self.prompt_rel, - "assets": [{"source": self.ref_rel, "workspace_path": "data/ref.txt"}], - "checksum": self.fixture_checksum, - }, - "matrix": [ - { - "id": "cell-1", - "caller": "claude", - "iop": { - "request_model": "claude-sonnet-4-20250514", - "requested_effort": "high", - "route_kind": "direct", - "route_id": "claude-direct", - "expected_bindings": [ - {"stage": "request", "model": "claude-sonnet-4-20250514", "effort": "high"} - ], - }, - }, - { - "id": "cell-2", - "caller": "agy", - "iop": { - "request_model": "gemini-2.0-flash", - "requested_effort": "high", - "route_kind": "direct", - "route_id": "agy-direct", - "expected_bindings": [ - {"stage": "request", "model": "gemini-2.0-flash", "effort": "high"} - ], - }, - }, - ], - } - - self.manifest_file = self.repo_root / "manifest.json" - self.manifest_file.write_text(json.dumps(self.manifest_raw, indent=2), encoding="utf-8") - self.manifest = load_manifest(self.manifest_file, repo_root=self.repo_root) - - def tearDown(self) -> None: - self.tmp_dir_obj.cleanup() - - def make_attempt_root( - self, cell_id: str = "cell-1", repetition: int = 1, attempt: int = 1 - ) -> tuple[AttemptIdentity, Path]: - identity = AttemptIdentity( - run_id=self.run_id, cell_id=cell_id, repetition=repetition, attempt=attempt - ) - rep_segment = f"repetition-{repetition:04d}" - att_segment = f"attempt-{attempt:06d}" - attempt_root = ( - self.repo_root - / self.output_root_rel - / self.run_id - / "cells" - / cell_id - / rep_segment - / att_segment - ) - attempt_root.mkdir(parents=True, exist_ok=True) - return identity, attempt_root - - -class TestAttemptIdentityValidation(BaseWorkspaceTest): - """Tests for AttemptIdentity format and boundary validation.""" - - def test_valid_identity(self) -> None: - identity = AttemptIdentity( - run_id=self.run_id, cell_id="cell-1", repetition=1, attempt=1 - ) - validate_attempt_identity(identity, self.manifest) - - def test_invalid_run_id_rejected(self) -> None: - bad_run_ids = [ - "invalid_run_id", - "run-20260809161730Z-0123456789ab", # missing T - "run-20260809T161730Z-0123456789aG", # uppercase G - "run-20260809T161730Z-short", - ] - for run_id in bad_run_ids: - identity = AttemptIdentity(run_id=run_id, cell_id="cell-1", repetition=1, attempt=1) - with self.assertRaises(WorkspaceValidationError): - validate_attempt_identity(identity, self.manifest) - - def test_invalid_cell_id_rejected(self) -> None: - # Invalid format or missing from manifest - identity_bad_fmt = AttemptIdentity( - run_id=self.run_id, cell_id="-invalid", repetition=1, attempt=1 - ) - with self.assertRaises(WorkspaceValidationError): - validate_attempt_identity(identity_bad_fmt, self.manifest) - - identity_missing = AttemptIdentity( - run_id=self.run_id, cell_id="cell-nonexistent", repetition=1, attempt=1 - ) - with self.assertRaises(WorkspaceValidationError): - validate_attempt_identity(identity_missing, self.manifest) - - def test_invalid_repetition_rejected(self) -> None: - # Non-positive or exceeding manifest repetitions (2) - for rep in [0, -1, 3, True]: - identity = AttemptIdentity( - run_id=self.run_id, cell_id="cell-1", repetition=rep, attempt=1 - ) - with self.assertRaises(WorkspaceValidationError): - validate_attempt_identity(identity, self.manifest) - - def test_invalid_attempt_rejected(self) -> None: - for att in [0, -1, True]: - identity = AttemptIdentity( - run_id=self.run_id, cell_id="cell-1", repetition=1, attempt=att - ) - with self.assertRaises(WorkspaceValidationError): - validate_attempt_identity(identity, self.manifest) - - -class TestAttemptRootPathRules(BaseWorkspaceTest): - """Tests path rules for attempt_root.""" - - def test_attempt_root_does_not_exist(self) -> None: - identity = AttemptIdentity( - run_id=self.run_id, cell_id="cell-1", repetition=1, attempt=1 - ) - non_existent = ( - self.repo_root - / self.output_root_rel - / self.run_id - / "cells" - / "cell-1" - / "repetition-0001" - / "attempt-000001" - ) - with self.assertRaises(WorkspacePathError): - prepare_workspace( - self.manifest, non_existent, identity, repo_root=self.repo_root - ) - - def test_attempt_root_is_file(self) -> None: - identity, attempt_root = self.make_attempt_root() - attempt_root.rmdir() - attempt_root.write_text("file instead of dir", encoding="utf-8") - with self.assertRaises(WorkspacePathError): - prepare_workspace( - self.manifest, attempt_root, identity, repo_root=self.repo_root - ) - - def test_attempt_root_is_symlink(self) -> None: - identity, attempt_root = self.make_attempt_root() - attempt_root.rmdir() - real_target = self.tmp_dir / "real_target" - real_target.mkdir() - attempt_root.symlink_to(real_target) - with self.assertRaises(WorkspacePathError): - prepare_workspace( - self.manifest, attempt_root, identity, repo_root=self.repo_root - ) - - def test_attempt_root_parent_is_symlink(self) -> None: - identity = AttemptIdentity( - run_id=self.run_id, cell_id="cell-1", repetition=1, attempt=1 - ) - parent_dir = ( - self.repo_root - / self.output_root_rel - / self.run_id - / "cells" - / "cell-1" - / "repetition-0001" - ) - parent_dir.mkdir(parents=True, exist_ok=True) - real_parent = self.tmp_dir / "real_parent" - real_parent.mkdir() - attempt_root_real = real_parent / "attempt-000001" - attempt_root_real.mkdir() - - symlink_att = parent_dir / "attempt-000001" - symlink_att.symlink_to(attempt_root_real) - - with self.assertRaises(WorkspacePathError): - prepare_workspace( - self.manifest, symlink_att, identity, repo_root=self.repo_root - ) - - def test_attempt_root_not_empty(self) -> None: - identity, attempt_root = self.make_attempt_root() - (attempt_root / "existing.txt").write_text("existing", encoding="utf-8") - with self.assertRaises(WorkspacePathError): - prepare_workspace( - self.manifest, attempt_root, identity, repo_root=self.repo_root - ) - - def test_attempt_root_canonical_path_mismatch(self) -> None: - identity = AttemptIdentity( - run_id=self.run_id, cell_id="cell-1", repetition=1, attempt=1 - ) - wrong_path = ( - self.repo_root - / self.output_root_rel - / self.run_id - / "cells" - / "cell-1" - / "repetition-0002" # mismatched repetition - / "attempt-000001" - ) - wrong_path.mkdir(parents=True, exist_ok=True) - with self.assertRaises(WorkspacePathError): - prepare_workspace(self.manifest, wrong_path, identity, repo_root=self.repo_root) - - def test_exclusive_child_collision(self) -> None: - identity, attempt_root = self.make_attempt_root() - (attempt_root / "workspace").mkdir() - with self.assertRaises(WorkspacePathError): - prepare_workspace( - self.manifest, attempt_root, identity, repo_root=self.repo_root - ) - - -class TestWorkspaceMaterialization(BaseWorkspaceTest): - """Tests for workspace asset copying, prompt exclusion, checksums, and metadata.""" - - def test_successful_workspace_preparation(self) -> None: - identity, attempt_root = self.make_attempt_root() - prepared = prepare_workspace( - self.manifest, attempt_root, identity, repo_root=self.repo_root - ) - - self.assertEqual(prepared.identity, identity) - self.assertEqual(prepared.workspace_checksum, self.manifest.fixture.checksum) - self.assertTrue(prepared.session_is_fresh) - self.assertEqual(prepared.setup_cache_policy, "isolated") - - # Verify directories exist - ws_path = Path(prepared.workspace_dir) - session_path = Path(prepared.session_dir) - self.assertTrue(ws_path.is_dir()) - self.assertTrue(session_path.is_dir()) - - # Verify session directory is empty - self.assertEqual(list(session_path.iterdir()), []) - - # Verify asset materialization - asset_file = ws_path / "data" / "ref.txt" - self.assertTrue(asset_file.is_file()) - self.assertEqual(asset_file.read_bytes(), self.ref_content) - - # Verify prepared.json content - prep_json = Path(prepared.attempt_root) / "prepared.json" - self.assertTrue(prep_json.is_file()) - data = json.loads(prep_json.read_text(encoding="utf-8")) - self.assertEqual(data["session_id"], prepared.session_id) - self.assertEqual(data["workspace_checksum"], prepared.workspace_checksum) - - def test_prompt_exclusion_when_not_declared(self) -> None: - # Prompt file exists in fixture.prompt, but is not listed in fixture.assets - identity, attempt_root = self.make_attempt_root() - prepared = prepare_workspace( - self.manifest, attempt_root, identity, repo_root=self.repo_root - ) - ws_path = Path(prepared.workspace_dir) - - # Prompt should not exist in workspace - prompt_ws = ws_path / self.prompt_rel - self.assertFalse(prompt_ws.exists()) - - def test_prompt_included_when_declared_as_asset(self) -> None: - # Update manifest to declare prompt.md as an asset too - new_assets = [ - {"source": self.ref_rel, "workspace_path": "data/ref.txt"}, - {"source": self.prompt_rel, "workspace_path": "prompt.md"}, - ] - asset_objs = [ - AssetMapping(source=self.ref_rel, workspace_path="data/ref.txt", content=self.ref_content), - AssetMapping(source=self.prompt_rel, workspace_path="prompt.md", content=self.prompt_content), - ] - checksum = digest_workspace_inputs(asset_objs) - self.manifest_raw["fixture"]["assets"] = new_assets - self.manifest_raw["fixture"]["checksum"] = checksum - - self.manifest_file.write_text(json.dumps(self.manifest_raw, indent=2), encoding="utf-8") - manifest = load_manifest(self.manifest_file, repo_root=self.repo_root) - - identity, attempt_root = self.make_attempt_root() - prepared = prepare_workspace(manifest, attempt_root, identity, repo_root=self.repo_root) - ws_path = Path(prepared.workspace_dir) - - prompt_ws = ws_path / "prompt.md" - self.assertTrue(prompt_ws.is_file()) - self.assertEqual(prompt_ws.read_bytes(), self.prompt_content) - - def test_fixture_checksum_mismatch_rejected(self) -> None: - # Corrupt declared checksum in manifest - self.manifest_raw["fixture"]["checksum"] = "sha256:" + "0" * 64 - self.manifest_file.write_text(json.dumps(self.manifest_raw, indent=2), encoding="utf-8") - - # Manually create manifest with corrupt checksum to bypass load_manifest validation - dummy_manifest = Manifest( - pipeline_version=self.manifest.pipeline_version, - environment=self.manifest.environment, - testbed=self.manifest.testbed, - repetitions=self.manifest.repetitions, - session_policy=self.manifest.session_policy, - setup_cache_policy=self.manifest.setup_cache_policy, - timeout=self.manifest.timeout, - viewports=self.manifest.viewports, - rubric_version=self.manifest.rubric_version, - evaluator=self.manifest.evaluator, - output_root=self.manifest.output_root, - fixture=Fixture( - version=self.manifest.fixture.version, - prompt=self.manifest.fixture.prompt, - assets=self.manifest.fixture.assets, - checksum="sha256:" + "0" * 64, - prompt_content=self.manifest.fixture.prompt_content, - ), - matrix=self.manifest.matrix, - digest=self.manifest.digest, - ) - - identity, attempt_root = self.make_attempt_root() - with self.assertRaises(WorkspaceChecksumError): - prepare_workspace(dummy_manifest, attempt_root, identity, repo_root=self.repo_root) - - def test_symlink_asset_source_rejected(self) -> None: - symlink_source = self.repo_root / "scripts" / "fixtures" / "bench" / "symlink.txt" - symlink_source.symlink_to(self.ref_file) - - symlink_rel = "scripts/fixtures/bench/symlink.txt" - self.manifest_raw["fixture"]["assets"] = [ - {"source": symlink_rel, "workspace_path": "data/symlink.txt"} - ] - asset_objs = [ - AssetMapping(source=symlink_rel, workspace_path="data/symlink.txt", content=self.ref_content) - ] - self.manifest_raw["fixture"]["checksum"] = digest_workspace_inputs(asset_objs) - self.manifest_file.write_text(json.dumps(self.manifest_raw, indent=2), encoding="utf-8") - - # Create manifest directly to bypass load_manifest symlink check - dummy_manifest = Manifest( - pipeline_version=self.manifest.pipeline_version, - environment=self.manifest.environment, - testbed=self.manifest.testbed, - repetitions=self.manifest.repetitions, - session_policy=self.manifest.session_policy, - setup_cache_policy=self.manifest.setup_cache_policy, - timeout=self.manifest.timeout, - viewports=self.manifest.viewports, - rubric_version=self.manifest.rubric_version, - evaluator=self.manifest.evaluator, - output_root=self.manifest.output_root, - fixture=Fixture( - version=self.manifest.fixture.version, - prompt=self.manifest.fixture.prompt, - assets=(AssetMapping(source=symlink_rel, workspace_path="data/symlink.txt", content=self.ref_content),), - checksum=self.manifest_raw["fixture"]["checksum"], - prompt_content=self.manifest.fixture.prompt_content, - ), - matrix=self.manifest.matrix, - digest=self.manifest.digest, - ) - - identity, attempt_root = self.make_attempt_root() - with self.assertRaises(WorkspacePathError): - prepare_workspace(dummy_manifest, attempt_root, identity, repo_root=self.repo_root) - - def test_escaping_workspace_path_rejected(self) -> None: - bad_assets = [ - {"source": self.ref_rel, "workspace_path": "../data/ref.txt"}, - ] - self.manifest_raw["fixture"]["assets"] = bad_assets - self.manifest_file.write_text(json.dumps(self.manifest_raw, indent=2), encoding="utf-8") - - with self.assertRaises(Exception): - load_manifest(self.manifest_file, repo_root=self.repo_root) - - def test_source_drift_failure_leaves_attempt_root_empty_and_retryable(self) -> None: - """R1: Mutate a fixture source after manifest load, prove rollback and retry.""" - identity, attempt_root = self.make_attempt_root() - - # Capture original ref.txt content - original_ref_content = self.ref_file.read_bytes() - - # Mutate the source file after manifest load (simulates source drift) - self.ref_file.write_bytes(b"corrupted reference data\n") - - with self.assertRaises(WorkspaceChecksumError): - prepare_workspace( - self.manifest, attempt_root, identity, repo_root=self.repo_root - ) - - # Assert the attempt root is empty (rollback succeeded) - self.assertEqual(list(attempt_root.iterdir()), []) - - # Restore the original source - self.ref_file.write_bytes(original_ref_content) - - # Prove the same attempt root can succeed on retry - # (need to recreate it since rollback removed it) - identity2, attempt_root2 = self.make_attempt_root() - prepared = prepare_workspace( - self.manifest, attempt_root2, identity2, repo_root=self.repo_root - ) - self.assertEqual(prepared.identity, identity2) - self.assertTrue(Path(prepared.workspace_dir).is_dir()) - self.assertTrue(Path(prepared.session_dir).is_dir()) - - def test_postflight_failure_leaves_attempt_root_empty(self) -> None: - """R1: Deterministic mocked postflight failure proves rollback of all owned entries.""" - import unittest.mock - - identity, attempt_root = self.make_attempt_root() - - # Mock inspect_testbed_provenance to succeed on first call (preflight) but fail on second (postflight) - original_inspect = inspect_testbed_provenance - call_count = {"n": 0} - - def mock_inspect(path): - call_count["n"] += 1 - if call_count["n"] == 1: - return original_inspect(path) # preflight succeeds - # postflight raises TestbedError - from scripts.agent_benchmark.workspace import TestbedError as TE - raise TE("mocked postflight: testbed modified") - - with unittest.mock.patch( - "scripts.agent_benchmark.workspace.inspect_testbed_provenance", - side_effect=mock_inspect, - ): - with self.assertRaises(TestbedError): - prepare_workspace( - self.manifest, attempt_root, identity, repo_root=self.repo_root - ) - - # Assert the attempt root is empty (rollback succeeded) - self.assertEqual(list(attempt_root.iterdir()), []) - - # Assert prepared.json does not exist - prepared_json = attempt_root / "prepared.json" - self.assertFalse(prepared_json.exists()) - - def test_ancestor_destination_collision_rejected_before_mutation(self) -> None: - """R1: Asset destinations with ancestor/file conflict are rejected before mutation.""" - new_assets = [ - {"source": self.ref_rel, "workspace_path": "data"}, - {"source": self.prompt_rel, "workspace_path": "data/prompt.md"}, - ] - asset_objs = [ - AssetMapping(source=self.ref_rel, workspace_path="data", content=self.ref_content), - AssetMapping(source=self.prompt_rel, workspace_path="data/prompt.md", content=self.prompt_content), - ] - checksum = digest_workspace_inputs(asset_objs) - self.manifest_raw["fixture"]["assets"] = new_assets - self.manifest_raw["fixture"]["checksum"] = checksum - self.manifest_file.write_text(json.dumps(self.manifest_raw, indent=2), encoding="utf-8") - manifest = load_manifest(self.manifest_file, repo_root=self.repo_root) - - identity, attempt_root = self.make_attempt_root() - - with self.assertRaises(WorkspacePathError): - prepare_workspace(manifest, attempt_root, identity, repo_root=self.repo_root) - - # Assert attempt_root remains completely empty (no staging, workspace, or session created) - self.assertEqual(list(attempt_root.iterdir()), []) - - def test_concurrent_collision_preserves_unrelated_entries(self) -> None: - """R1: Concurrent collision content not created by this preparation is preserved on rollback.""" - import unittest.mock - import scripts.agent_benchmark.workspace - - identity, attempt_root = self.make_attempt_root() - - def mock_verify_with_collision(staging, manifest): - # Simulate a caller/external collision creating workspace/caller-owned.txt after validation - caller_ws = attempt_root / "workspace" - caller_ws.mkdir(exist_ok=True) - sentinel = caller_ws / "caller-owned.txt" - sentinel.write_text("caller owned content", encoding="utf-8") - raise TestbedError("mocked failure during preparation") - - with unittest.mock.patch( - "scripts.agent_benchmark.workspace._verify_staged_preparation", - side_effect=mock_verify_with_collision, - ): - with self.assertRaises(TestbedError): - prepare_workspace(self.manifest, attempt_root, identity, repo_root=self.repo_root) - - # Assert caller-owned collision file was preserved on rollback - sentinel = attempt_root / "workspace" / "caller-owned.txt" - self.assertTrue(sentinel.is_file()) - self.assertEqual(sentinel.read_text(encoding="utf-8"), "caller owned content") - - def test_empty_publication_collision_preserves_unrelated_directory(self) -> None: - """R1: Empty concurrent collision directory created before final publication is preserved on rollback.""" - import unittest.mock - import scripts.agent_benchmark.workspace - - identity, attempt_root = self.make_attempt_root() - - empty_inode: int | None = None - orig_publish = scripts.agent_benchmark.workspace._publish_owned_directory - - def mock_publish_with_empty_collision(staging_dir, final_dir, owned, collision_message): - nonlocal empty_inode - if final_dir.name == "workspace": - final_dir.mkdir(exist_ok=False) - empty_inode = final_dir.stat().st_ino - return orig_publish(staging_dir, final_dir, owned, collision_message) - - with unittest.mock.patch( - "scripts.agent_benchmark.workspace._publish_owned_directory", - side_effect=mock_publish_with_empty_collision, - ): - with self.assertRaises(WorkspacePathError): - prepare_workspace(self.manifest, attempt_root, identity, repo_root=self.repo_root) - - caller_ws = attempt_root / "workspace" - self.assertTrue(caller_ws.is_dir()) - self.assertIsNotNone(empty_inode) - self.assertEqual(caller_ws.stat().st_ino, empty_inode) - self.assertEqual(list(caller_ws.iterdir()), []) - self.assertFalse((attempt_root / "session").exists()) - self.assertFalse((attempt_root / "prepared.json").exists()) - self.assertEqual(list(attempt_root.iterdir()), [caller_ws]) - - -class TestTestbedProvenanceAndNonMutation(BaseWorkspaceTest): - """Tests for runtime testbed provenance checking and non-mutation.""" - - def test_clean_testbed_provenance(self) -> None: - prov = inspect_testbed_provenance(self.testbed_dir) - self.assertTrue(prov.clean) - self.assertTrue(prov.status_digest.startswith("sha256:")) - self.assertTrue(len(prov.head) > 0) - - def test_dirty_testbed_rejected(self) -> None: - # Create an uncommitted file in testbed_dir - (self.testbed_dir / "dirty.txt").write_text("dirty", encoding="utf-8") - - with self.assertRaises(TestbedError): - inspect_testbed_provenance(self.testbed_dir) - - identity, attempt_root = self.make_attempt_root() - with self.assertRaises(TestbedError): - prepare_workspace(self.manifest, attempt_root, identity, repo_root=self.repo_root) - - def test_testbed_unaffected_by_preparation(self) -> None: - prov_before = inspect_testbed_provenance(self.testbed_dir) - - identity, attempt_root = self.make_attempt_root() - prepared = prepare_workspace( - self.manifest, attempt_root, identity, repo_root=self.repo_root - ) - - prov_after = inspect_testbed_provenance(self.testbed_dir) - self.assertEqual(prov_before, prov_after) - - # Assert no file from testbed_dir was copied into workspace_dir - ws_files = list(Path(prepared.workspace_dir).rglob("*")) - ws_rel_paths = {f.name for f in ws_files} - self.assertNotIn("README.md", ws_rel_paths) # testbed README.md is not in workspace - - -class TestCrossAttemptIsolation(BaseWorkspaceTest): - """API-2: Prove cross-attempt isolation and source integrity.""" - - def test_cross_attempt_isolation_and_source_integrity(self) -> None: - # Prepare 4 attempt roots (2 cells x 2 repetitions) - testbed_before = inspect_testbed_provenance(self.testbed_dir) - - attempts_config = [ - ("cell-1", 1, 1), - ("cell-1", 2, 1), - ("cell-2", 1, 1), - ("cell-2", 2, 1), - ] - - prepared_list: list[PreparedWorkspace] = [] - for cell_id, rep, att in attempts_config: - identity, attempt_root = self.make_attempt_root( - cell_id=cell_id, repetition=rep, attempt=att - ) - prep = prepare_workspace( - self.manifest, attempt_root, identity, repo_root=self.repo_root - ) - prepared_list.append(prep) - - # 1. Assert four distinct workspace/session identities - session_ids = {p.session_id for p in prepared_list} - self.assertEqual(len(session_ids), 4) - - attempt_roots = {p.attempt_root for p in prepared_list} - self.assertEqual(len(attempt_roots), 4) - - # 2. Assert identical initial workspace digests matching manifest.fixture.checksum - workspace_checksums = {p.workspace_checksum for p in prepared_list} - self.assertEqual(workspace_checksums, {self.manifest.fixture.checksum}) - - # 3. Mutate one session (session #1) and prove peer sessions remain empty - session1_path = Path(prepared_list[0].session_dir) - sentinel = session1_path / "history-sentinel" - sentinel.write_text("owned", encoding="utf-8") - self.assertTrue(sentinel.is_file()) - - for peer in prepared_list[1:]: - self.assertEqual(list(Path(peer.session_dir).iterdir()), []) - - # 4. Capture testbed branch/HEAD/status digest before and after and assert equality - testbed_after = inspect_testbed_provenance(self.testbed_dir) - self.assertEqual(testbed_before, testbed_after) - - # 5. Assert no path under testbed appears beneath any workspace - testbed_files = {f.name for f in self.testbed_dir.rglob("*") if f.is_file()} - for prep in prepared_list: - ws_files = {f.name for f in Path(prep.workspace_dir).rglob("*") if f.is_file()} - # Intersection of testbed files and workspace files should be empty - # (testbed contains README.md; workspace contains data/ref.txt) - self.assertEqual(testbed_files.intersection(ws_files), set()) - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/agent_comparison_benchmark.py b/scripts/agent_comparison_benchmark.py deleted file mode 100644 index 0d8eb4d4..00000000 --- a/scripts/agent_comparison_benchmark.py +++ /dev/null @@ -1,319 +0,0 @@ -#!/usr/bin/env python3 -""" -Public CLI for the agent comparison benchmark manifest. - -Usage: - python3 scripts/agent_comparison_benchmark.py validate --manifest PATH - python3 scripts/agent_comparison_benchmark.py preflight --manifest PATH - python3 scripts/agent_comparison_benchmark.py run --manifest PATH - python3 scripts/agent_comparison_benchmark.py resume --manifest PATH --run-id RUN_ID - python3 scripts/agent_comparison_benchmark.py status --manifest PATH --run-id RUN_ID - python3 scripts/agent_comparison_benchmark.py score --manifest PATH --run-id RUN_ID - -Exits: - 0 - manifest is valid or every matrix preflight cell is ready - 64 - usage error (missing args, bad flags) - 69 - validation/state failed or preflight is blocked -""" - -from __future__ import annotations - -import argparse -import os -import sys -from pathlib import Path -from collections.abc import Mapping - -# Ensure the repo root is on sys.path for imports. -_REPO_ROOT = Path(__file__).resolve().parent.parent -if str(_REPO_ROOT) not in sys.path: - sys.path.insert(0, str(_REPO_ROOT)) - -from scripts.agent_benchmark.manifest import CALLER_ENUM, ManifestError, load_manifest -from scripts.agent_benchmark.attempts import ( - AttemptStateError, - CapabilityUnavailable, - ExecutionAdapter, - RunStore, - preflight_manifest, - run_slots, -) -from scripts.agent_benchmark.live_iop import ( - build_live_adapter_registry, - build_live_scoring_adapter, -) -from scripts.agent_benchmark.scoring import ScoringError, score_run -from scripts.agent_benchmark.reporting import ReportError, publish_report -from scripts.agent_benchmark.workspace import prepare_workspace - -EXIT_VALID = 0 -EXIT_USAGE = 64 -EXIT_INVALID = 69 - - -def build_adapter_registry( - environment: Mapping[str, str] | None = None, -) -> dict[str, ExecutionAdapter]: - """Build the exact three-caller registry from explicit live inputs only.""" - registry = build_live_adapter_registry( - os.environ if environment is None else environment - ) - if tuple(registry) != CALLER_ENUM: - raise RuntimeError("caller adapter registry is invalid") - return registry - - -class _SanitizedArgumentParser(argparse.ArgumentParser): - def error(self, message: str) -> None: - print("error: invalid usage", file=sys.stderr) - sys.exit(EXIT_USAGE) - - -def _build_parser() -> argparse.ArgumentParser: - parser = _SanitizedArgumentParser( - prog="agent_comparison_benchmark", - description="Agent comparison benchmark manifest tools.", - ) - sub = parser.add_subparsers(dest="command", required=True) - p_validate = sub.add_parser( - "validate", - help="Validate a benchmark manifest JSON file.", - ) - p_validate.add_argument( - "--manifest", - required=True, - help="Path to the manifest JSON file.", - ) - for command in ("preflight", "run", "resume", "status", "score", "report"): - entry = sub.add_parser(command, help=f"Safely {command} benchmark state.") - entry.add_argument("--manifest", required=True, help="Path to the manifest JSON file.") - if command in {"resume", "status", "score", "report"}: - entry.add_argument("--run-id", required=True, help="Harness-generated run id.") - if command == "resume": - entry.add_argument("--retry-failed", action="store_true") - if command == "score": - entry.add_argument("--retry-scoring-failed", action="store_true") - return parser - - -def _cmd_validate(args: argparse.Namespace) -> int: - manifest_path = Path(args.manifest) - if not manifest_path.is_file(): - print("error: manifest is unavailable", file=sys.stderr) - return EXIT_INVALID - - try: - load_manifest(manifest_path) - except ManifestError as exc: - print(f"error: {exc}", file=sys.stderr) - return EXIT_INVALID - except Exception: - print("error: manifest validation failed", file=sys.stderr) - return EXIT_INVALID - - print("ok: manifest is valid") - return EXIT_VALID - - -def _cmd_state(args: argparse.Namespace) -> int: - try: - manifest_path = Path(args.manifest) - manifest = load_manifest(manifest_path, repo_root=_REPO_ROOT) - raw = manifest_path.read_bytes() - store = RunStore(_REPO_ROOT) - if args.command == "run": - run = store.create(manifest, raw) - else: - run = store.open(manifest, args.run_id, raw) - if args.command == "status": - status = store.status(run, manifest) - attempts = status["attempts"] - outcomes = status["outcomes"] - attempt_summary = " ".join( - f"{state}={attempts[state]}" - for state in ( - "completed", "timed_out", "cancelled", "interrupted", "running" - ) - ) - axes = " ".join( - f"{axis}_{name}={outcomes[axis][name]}" - for axis in ("product", "harness", "process", "artifact") - for name in outcomes[axis] - ) - print( - f"ok: status run_id={run.run_id} " - f"unresolved={outcomes['unresolved']} {attempt_summary} {axes}" - ) - return EXIT_VALID - - completed = run_slots( - store, - run, - manifest, - adapters=build_adapter_registry(), - prepare=lambda bound_manifest, attempt: prepare_workspace( - bound_manifest, - attempt.root, - attempt.identity, - repo_root=_REPO_ROOT, - ), - retry_failed=bool(getattr(args, "retry_failed", False)), - ) - status = store.status(run, manifest) - preflight = status["preflight"] - preflight_summary = ( - f"run_id={run.run_id} status={preflight['latest_status']} " - f"ready={preflight['ready']} " - f"registration_required={preflight['registration_required']} " - f"implementation_gap={preflight['implementation_gap']}" - ) - if preflight["latest_status"] != "ready": - print("error: preflight blocked " + preflight_summary, file=sys.stderr) - return EXIT_INVALID - - attempts = status["attempts"] - attempt_summary = " ".join( - f"{state}={attempts[state]}" - for state in ( - "completed", "timed_out", "cancelled", "interrupted", "running" - ) - ) - outcomes = status["outcomes"] - unresolved = outcomes["unresolved"] - axes = " ".join( - f"{prefix}_{name}={counts[name]}" - for prefix, counts in ( - ("product", outcomes["product"]), - ("harness", outcomes["harness"]), - ("process", outcomes["process"]), - ("artifact", outcomes["artifact"]), - ) - for name in counts - ) - summary = ( - f"run_id={run.run_id} executed={len(completed)} " - f"unresolved={unresolved} {attempt_summary} {axes}" - ) - if unresolved: - print("error: benchmark execution failed " + summary, file=sys.stderr) - return EXIT_INVALID - print(f"ok: {args.command} " + summary) - return EXIT_VALID - except CapabilityUnavailable: - print("error: capability unavailable", file=sys.stderr) - except Exception: - print("error: benchmark state is unavailable", file=sys.stderr) - return EXIT_INVALID - - -def _cmd_preflight(args: argparse.Namespace) -> int: - try: - manifest_path = Path(args.manifest) - manifest = load_manifest(manifest_path, repo_root=_REPO_ROOT) - raw = manifest_path.read_bytes() - run, record = preflight_manifest( - RunStore(_REPO_ROOT), - manifest, - raw, - adapters=build_adapter_registry(), - ) - counts = {status: 0 for status in ("ready", "registration_required", "implementation_gap")} - for result in record["results"]: - counts[result["status"]] += 1 - summary = ( - f"run_id={run.run_id} status={record['status']} " - f"ready={counts['ready']} " - f"registration_required={counts['registration_required']} " - f"implementation_gap={counts['implementation_gap']}" - ) - if record["status"] == "ready": - print("ok: preflight " + summary) - return EXIT_VALID - print("error: preflight blocked " + summary, file=sys.stderr) - except CapabilityUnavailable: - print("error: capability unavailable", file=sys.stderr) - except Exception: - print("error: benchmark preflight is unavailable", file=sys.stderr) - return EXIT_INVALID - - -def _cmd_score(args: argparse.Namespace) -> int: - run_id = str(args.run_id) - try: - manifest_path = Path(args.manifest) - manifest = load_manifest(manifest_path, repo_root=_REPO_ROOT) - raw = manifest_path.read_bytes() - store = RunStore(_REPO_ROOT) - run = store.open(manifest, run_id, raw) - summary = score_run( - store, - run, - manifest, - adapter=build_live_scoring_adapter(os.environ), - retry_scoring_failed=bool(args.retry_scoring_failed), - ) - counts = ( - f"run_id={summary.run_id} scored={summary.scored} " - f"unscored={summary.unscored} " - f"scoring_failed={summary.scoring_failed} blocked={summary.blocked}" - ) - if summary.scoring_failed or summary.blocked: - print("error: benchmark scoring failed " + counts, file=sys.stderr) - return EXIT_INVALID - print("ok: score " + counts) - return EXIT_VALID - except (ManifestError, ScoringError, AttemptStateError, OSError): - print( - f"error: benchmark scoring is unavailable run_id={run_id}", - file=sys.stderr, - ) - except Exception: - print( - f"error: benchmark scoring is unavailable run_id={run_id}", - file=sys.stderr, - ) - return EXIT_INVALID - - -def _cmd_report(args: argparse.Namespace) -> int: - run_id = str(args.run_id) - try: - manifest_path = Path(args.manifest) - manifest = load_manifest(manifest_path, repo_root=_REPO_ROOT) - raw = manifest_path.read_bytes() - store = RunStore(_REPO_ROOT) - run = store.open(manifest, run_id, raw) - path = publish_report(store, run, manifest) - rel = str(path.relative_to(_REPO_ROOT)) - print(f"ok: report run_id={run_id} path={rel}") - return EXIT_VALID - except (ManifestError, ReportError, AttemptStateError, OSError): - print("error: benchmark report is unavailable", file=sys.stderr) - except Exception: - print("error: benchmark report is unavailable", file=sys.stderr) - return EXIT_INVALID - - -def main(argv: list[str] | None = None) -> int: - parser = _build_parser() - try: - args = parser.parse_args(argv) - except SystemExit as exc: - return exc.code if isinstance(exc.code, int) else EXIT_USAGE - - if args.command == "validate": - return _cmd_validate(args) - if args.command == "preflight": - return _cmd_preflight(args) - if args.command == "score": - return _cmd_score(args) - if args.command == "report": - return _cmd_report(args) - if args.command in {"run", "resume", "status"}: - return _cmd_state(args) - - return EXIT_USAGE - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/fixtures/agent-comparison-benchmark-direct-preflight.example.json b/scripts/fixtures/agent-comparison-benchmark-direct-preflight.example.json deleted file mode 100644 index b76a8888..00000000 --- a/scripts/fixtures/agent-comparison-benchmark-direct-preflight.example.json +++ /dev/null @@ -1,109 +0,0 @@ -{ - "pipeline_version": "2", - "environment": "dev", - "testbed": "../iop-s2", - "repetitions": 1, - "session_policy": "fresh", - "setup_cache_policy": "isolated", - "timeout": { - "run_seconds": 300, - "idle_seconds": 30, - "quiet_seconds": 10, - "cleanup_grace_seconds": 5 - }, - "viewports": [ - {"id": "desktop_1080", "width": 1920, "height": 1080}, - {"id": "mobile_375", "width": 375, "height": 812} - ], - "rubric_version": "landing-quality-v1", - "evaluator": { - "caller": "codex", - "iop": { - "request_model": "gpt-5.6-luna", - "requested_effort": "xhigh", - "route_kind": "direct", - "route_id": "gpt-5.6-luna", - "expected_bindings": [ - {"stage": "request", "model": "gpt-5.6-luna", "effort": "xhigh"} - ] - } - }, - "output_root": "agent-test/runs/bench-01-direct-preflight", - "fixture": { - "version": "product-card-v2", - "prompt": "scripts/fixtures/agent-comparison-benchmark/prompt.md", - "assets": [ - {"source": "scripts/fixtures/agent-comparison-benchmark/reference.txt", "workspace_path": "brief/reference.txt"}, - {"source": "scripts/fixtures/agent-comparison-benchmark/aurora-grid.svg", "workspace_path": "assets/aurora-grid.svg"}, - {"source": "scripts/fixtures/agent-comparison-benchmark/orbit-rings.svg", "workspace_path": "assets/orbit-rings.svg"} - ], - "checksum": "sha256:fb16198fd4c3576f880f047ed7de54dddc160b0f70c2ba55b435cf078c61828e" - }, - "matrix": [ - { - "id": "claude-sonnet-direct", - "caller": "claude", - "iop": { - "request_model": "claude-sonnet-5", - "requested_effort": "max", - "route_kind": "direct", - "route_id": "claude-sonnet-5", - "expected_bindings": [ - {"stage": "request", "model": "claude-sonnet-5", "effort": "max"} - ] - } - }, - { - "id": "claude-gemini-direct", - "caller": "claude", - "iop": { - "request_model": "gemini-3.6-flash", - "requested_effort": "high", - "route_kind": "direct", - "route_id": "gemini-3.6-flash", - "expected_bindings": [ - {"stage": "request", "model": "gemini-3.6-flash", "effort": "high"} - ] - } - }, - { - "id": "claude-gpt-direct", - "caller": "claude", - "iop": { - "request_model": "gpt-5.6-luna", - "requested_effort": "xhigh", - "route_kind": "direct", - "route_id": "gpt-5.6-luna", - "expected_bindings": [ - {"stage": "request", "model": "gpt-5.6-luna", "effort": "xhigh"} - ] - } - }, - { - "id": "agy-gemini-direct", - "caller": "agy", - "iop": { - "request_model": "gemini-3.6-flash", - "requested_effort": "high", - "route_kind": "direct", - "route_id": "gemini-3.6-flash", - "expected_bindings": [ - {"stage": "request", "model": "gemini-3.6-flash", "effort": "high"} - ] - } - }, - { - "id": "codex-gpt-direct", - "caller": "codex", - "iop": { - "request_model": "gpt-5.6-luna", - "requested_effort": "xhigh", - "route_kind": "direct", - "route_id": "gpt-5.6-luna", - "expected_bindings": [ - {"stage": "request", "model": "gpt-5.6-luna", "effort": "xhigh"} - ] - } - } - ] -} diff --git a/scripts/fixtures/agent-comparison-benchmark-iop-one-shot.json b/scripts/fixtures/agent-comparison-benchmark-iop-one-shot.json deleted file mode 100644 index 8da3c124..00000000 --- a/scripts/fixtures/agent-comparison-benchmark-iop-one-shot.json +++ /dev/null @@ -1,178 +0,0 @@ -{ - "pipeline_version": "2", - "environment": "dev", - "testbed": "../iop-s2", - "execution_order_seed": "bench-02-c01-c09-v1", - "repetitions": 1, - "session_policy": "fresh", - "setup_cache_policy": "isolated", - "timeout": { - "run_seconds": 300, - "idle_seconds": 30, - "quiet_seconds": 10, - "cleanup_grace_seconds": 5 - }, - "viewports": [ - {"id": "desktop_1080", "width": 1920, "height": 1080}, - {"id": "mobile_375", "width": 375, "height": 812} - ], - "rubric_version": "one-shot-agent-comparison-v1", - "evaluator": { - "caller": "codex", - "iop": { - "request_model": "gpt-5.6-luna", - "requested_effort": "xhigh", - "route_kind": "direct", - "route_id": "gpt-5.6-luna", - "expected_bindings": [ - {"stage": "request", "model": "gpt-5.6-luna", "effort": "xhigh"} - ] - } - }, - "output_root": "agent-test/runs/bench-02", - "fixture": { - "version": "product-card-v2", - "prompt": "scripts/fixtures/agent-comparison-benchmark/prompt.md", - "assets": [ - {"source": "scripts/fixtures/agent-comparison-benchmark/reference.txt", "workspace_path": "brief/reference.txt"}, - {"source": "scripts/fixtures/agent-comparison-benchmark/aurora-grid.svg", "workspace_path": "assets/aurora-grid.svg"}, - {"source": "scripts/fixtures/agent-comparison-benchmark/orbit-rings.svg", "workspace_path": "assets/orbit-rings.svg"} - ], - "checksum": "sha256:fb16198fd4c3576f880f047ed7de54dddc160b0f70c2ba55b435cf078c61828e" - }, - "matrix": [ - { - "id": "c01-claude-sonnet-direct", - "caller": "claude", - "iop": { - "request_model": "claude-sonnet-5", - "requested_effort": "max", - "route_kind": "direct", - "route_id": "claude-sonnet-5", - "expected_bindings": [ - {"stage": "request", "model": "claude-sonnet-5", "effort": "max"} - ] - } - }, - { - "id": "c02-claude-gemini-direct", - "caller": "claude", - "iop": { - "request_model": "gemini-3.6-flash", - "requested_effort": "high", - "route_kind": "direct", - "route_id": "gemini-3.6-flash", - "expected_bindings": [ - {"stage": "request", "model": "gemini-3.6-flash", "effort": "high"} - ] - } - }, - { - "id": "c03-agy-gemini-direct", - "caller": "agy", - "iop": { - "request_model": "gemini-3.6-flash", - "requested_effort": "high", - "route_kind": "direct", - "route_id": "gemini-3.6-flash", - "expected_bindings": [ - {"stage": "request", "model": "gemini-3.6-flash", "effort": "high"} - ] - } - }, - { - "id": "c04-claude-gpt-direct", - "caller": "claude", - "iop": { - "request_model": "gpt-5.6-luna", - "requested_effort": "xhigh", - "route_kind": "direct", - "route_id": "gpt-5.6-luna", - "expected_bindings": [ - {"stage": "request", "model": "gpt-5.6-luna", "effort": "xhigh"} - ] - } - }, - { - "id": "c05-codex-gpt-direct", - "caller": "codex", - "iop": { - "request_model": "gpt-5.6-luna", - "requested_effort": "xhigh", - "route_kind": "direct", - "route_id": "gpt-5.6-luna", - "expected_bindings": [ - {"stage": "request", "model": "gpt-5.6-luna", "effort": "xhigh"} - ] - } - }, - { - "id": "c06-claude-gemini-hybrid", - "caller": "claude", - "iop": { - "request_model": "gemini-hybrid", - "requested_effort": "high", - "route_kind": "execution_preset", - "route_id": "gemini-hybrid", - "expected_bindings": [ - {"stage": "selector", "model": "gemini-3.6-flash", "effort": "high"}, - {"stage": "plan", "model": "gemini-3.6-flash", "effort": "high"}, - {"stage": "work", "model": "ornith-fast"}, - {"stage": "review", "model": "gemini-3.6-flash", "effort": "high"}, - {"stage": "repair", "model": "gemini-3.6-flash", "effort": "high"} - ] - } - }, - { - "id": "c07-agy-gemini-hybrid", - "caller": "agy", - "iop": { - "request_model": "gemini-hybrid", - "requested_effort": "high", - "route_kind": "execution_preset", - "route_id": "gemini-hybrid", - "expected_bindings": [ - {"stage": "selector", "model": "gemini-3.6-flash", "effort": "high"}, - {"stage": "plan", "model": "gemini-3.6-flash", "effort": "high"}, - {"stage": "work", "model": "ornith-fast"}, - {"stage": "review", "model": "gemini-3.6-flash", "effort": "high"}, - {"stage": "repair", "model": "gemini-3.6-flash", "effort": "high"} - ] - } - }, - { - "id": "c08-claude-gpt-hybrid", - "caller": "claude", - "iop": { - "request_model": "gpt-hybrid", - "requested_effort": "xhigh", - "route_kind": "execution_preset", - "route_id": "gpt-hybrid", - "expected_bindings": [ - {"stage": "selector", "model": "gpt-5.6-terra", "effort": "high"}, - {"stage": "plan", "model": "gpt-5.6-terra", "effort": "high"}, - {"stage": "work", "model": "ornith-fast"}, - {"stage": "review", "model": "gpt-5.6-terra", "effort": "high"}, - {"stage": "repair", "model": "gpt-5.6-terra", "effort": "high"} - ] - } - }, - { - "id": "c09-codex-gpt-hybrid", - "caller": "codex", - "iop": { - "request_model": "gpt-hybrid", - "requested_effort": "xhigh", - "route_kind": "execution_preset", - "route_id": "gpt-hybrid", - "expected_bindings": [ - {"stage": "selector", "model": "gpt-5.6-terra", "effort": "high"}, - {"stage": "plan", "model": "gpt-5.6-terra", "effort": "high"}, - {"stage": "work", "model": "ornith-fast"}, - {"stage": "review", "model": "gpt-5.6-terra", "effort": "high"}, - {"stage": "repair", "model": "gpt-5.6-terra", "effort": "high"} - ] - } - } - ] -} diff --git a/scripts/fixtures/agent-comparison-benchmark-manifest.example.json b/scripts/fixtures/agent-comparison-benchmark-manifest.example.json deleted file mode 100644 index 76111e14..00000000 --- a/scripts/fixtures/agent-comparison-benchmark-manifest.example.json +++ /dev/null @@ -1,92 +0,0 @@ -{ - "pipeline_version": "2", - "environment": "dev", - "testbed": "../iop-s2", - "repetitions": 1, - "session_policy": "fresh", - "setup_cache_policy": "isolated", - "timeout": { - "run_seconds": 300, - "idle_seconds": 30, - "quiet_seconds": 10, - "cleanup_grace_seconds": 5 - }, - "viewports": [ - {"id": "desktop_1080", "width": 1920, "height": 1080}, - {"id": "mobile_375", "width": 375, "height": 812} - ], - "rubric_version": "landing-quality-v1", - "evaluator": { - "caller": "codex", - "iop": { - "request_model": "gpt-5.6-luna", - "requested_effort": "xhigh", - "route_kind": "direct", - "route_id": "gpt-5.6-luna", - "expected_bindings": [ - {"stage": "request", "model": "gpt-5.6-luna", "effort": "xhigh"} - ] - } - }, - "output_root": "agent-test/runs/bench-01", - "fixture": { - "version": "product-card-v2", - "prompt": "scripts/fixtures/agent-comparison-benchmark/prompt.md", - "assets": [ - {"source": "scripts/fixtures/agent-comparison-benchmark/reference.txt", "workspace_path": "brief/reference.txt"}, - {"source": "scripts/fixtures/agent-comparison-benchmark/aurora-grid.svg", "workspace_path": "assets/aurora-grid.svg"}, - {"source": "scripts/fixtures/agent-comparison-benchmark/orbit-rings.svg", "workspace_path": "assets/orbit-rings.svg"} - ], - "checksum": "sha256:fb16198fd4c3576f880f047ed7de54dddc160b0f70c2ba55b435cf078c61828e" - }, - "matrix": [ - { - "id": "claude-generic-preset", - "caller": "claude", - "iop": { - "request_model": "claude-sonnet-5", - "requested_effort": "high", - "route_kind": "execution_preset", - "route_id": "claude-generic", - "expected_bindings": [ - {"stage": "selector", "model": "claude-sonnet-5"}, - {"stage": "plan", "model": "claude-sonnet-5"}, - {"stage": "work", "model": "claude-sonnet-5"}, - {"stage": "review", "model": "claude-sonnet-5"} - ] - } - }, - { - "id": "agy-generic-preset", - "caller": "agy", - "iop": { - "request_model": "gemini-3.6-flash", - "requested_effort": "high", - "route_kind": "execution_preset", - "route_id": "agy-generic", - "expected_bindings": [ - {"stage": "selector", "model": "gemini-3.6-flash"}, - {"stage": "plan", "model": "gemini-3.6-flash"}, - {"stage": "work", "model": "gemini-3.6-flash"}, - {"stage": "review", "model": "gemini-3.6-flash"} - ] - } - }, - { - "id": "codex-generic-preset", - "caller": "codex", - "iop": { - "request_model": "gpt-5.6-luna", - "requested_effort": "xhigh", - "route_kind": "execution_preset", - "route_id": "codex-generic", - "expected_bindings": [ - {"stage": "selector", "model": "gpt-5.6-luna"}, - {"stage": "plan", "model": "gpt-5.6-luna"}, - {"stage": "work", "model": "gpt-5.6-luna"}, - {"stage": "review", "model": "gpt-5.6-luna"} - ] - } - } - ] -} diff --git a/scripts/fixtures/agent-comparison-benchmark-manifest.schema.json b/scripts/fixtures/agent-comparison-benchmark-manifest.schema.json deleted file mode 100644 index 1cfa2740..00000000 --- a/scripts/fixtures/agent-comparison-benchmark-manifest.schema.json +++ /dev/null @@ -1,240 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://iop.local/schemas/agent-comparison-benchmark-manifest.schema.json", - "title": "Agent comparison benchmark manifest", - "description": "Closed Draft 2020-12 schema for the benchmark manifest. Every object rejects unknown members. The loader accepts only the subset used by this packet.", - "type": "object", - "additionalProperties": false, - "required": [ - "pipeline_version", - "environment", - "testbed", - "fixture", - "matrix", - "session_policy", - "setup_cache_policy", - "timeout", - "viewports", - "rubric_version", - "evaluator", - "output_root" - ], - "properties": { - "pipeline_version": { "const": "2" }, - "environment": { "const": "dev" }, - "testbed": { "const": "../iop-s2" }, - "repetitions": { - "type": "integer", - "minimum": 1, - "default": 1 - }, - "execution_order_seed": { - "type": "string", - "pattern": "^[a-z0-9][a-z0-9_-]{0,63}$" - }, - "session_policy": { "const": "fresh" }, - "setup_cache_policy": { "const": "isolated" }, - "timeout": { "$ref": "#/$defs/timeout" }, - "viewports": { "$ref": "#/$defs/viewports" }, - "rubric_version": { - "enum": ["landing-quality-v1", "one-shot-agent-comparison-v1"] - }, - "evaluator": { "$ref": "#/$defs/evaluator" }, - "output_root": { "$ref": "#/$defs/output_root" }, - "fixture": { "$ref": "#/$defs/fixture" }, - "matrix": { "$ref": "#/$defs/matrix" } - }, - "$defs": { - "bounded_token": { - "type": "string", - "pattern": "^[a-z0-9][a-z0-9_.+-]{0,31}$" - }, - "timeout": { - "type": "object", - "additionalProperties": false, - "required": ["run_seconds", "idle_seconds", "quiet_seconds", "cleanup_grace_seconds"], - "properties": { - "run_seconds": { "type": "integer", "minimum": 1, "maximum": 86400 }, - "idle_seconds": { "type": "integer", "minimum": 1, "maximum": 600 }, - "quiet_seconds": { "type": "integer", "minimum": 1, "maximum": 60 }, - "cleanup_grace_seconds": { "type": "integer", "minimum": 1, "maximum": 60 } - } - }, - "viewport_record": { - "type": "object", - "additionalProperties": false, - "required": ["id", "width", "height"], - "properties": { - "id": { "$ref": "#/$defs/bounded_token" }, - "width": { "type": "integer", "minimum": 1, "maximum": 8192 }, - "height": { "type": "integer", "minimum": 1, "maximum": 8192 } - } - }, - "viewports": { - "type": "array", - "minItems": 1, - "items": { "$ref": "#/$defs/viewport_record" }, - "uniqueItems": true - }, - "output_root": { - "type": "string", - "pattern": "^agent-test/runs/[^/]+$" - }, - "asset_mapping": { - "type": "object", - "additionalProperties": false, - "required": ["source", "workspace_path"], - "properties": { - "source": { - "type": "string", - "pattern": "^[^/][^:]*$" - }, - "workspace_path": { - "type": "string", - "pattern": "^[^/][^:]*$" - } - } - }, - "fixture": { - "type": "object", - "additionalProperties": false, - "required": ["version", "prompt", "assets", "checksum"], - "properties": { - "version": { "$ref": "#/$defs/bounded_token" }, - "prompt": { - "type": "string", - "pattern": "^[^/][^:]*$" - }, - "assets": { - "type": "array", - "minItems": 1, - "items": { "$ref": "#/$defs/asset_mapping" } - }, - "checksum": { - "type": "string", - "pattern": "^sha256:[0-9a-f]{64}$" - } - } - }, - "binding": { - "type": "object", - "additionalProperties": false, - "required": ["stage", "model"], - "properties": { - "stage": { - "type": "string", - "enum": ["request", "selector", "plan", "work", "review", "repair"] - }, - "model": { "$ref": "#/$defs/bounded_token" }, - "effort": { "$ref": "#/$defs/bounded_token" } - } - }, - "expected_binding": { - "type": "object", - "additionalProperties": false, - "required": ["stage", "model"], - "properties": { - "stage": { - "type": "string", - "enum": ["request", "selector", "plan", "work", "review", "repair"] - }, - "model": { "$ref": "#/$defs/bounded_token" }, - "effort": { "$ref": "#/$defs/bounded_token" } - } - }, - "iop_cell": { - "type": "object", - "additionalProperties": false, - "required": ["request_model", "requested_effort", "route_kind", "route_id", "expected_bindings"], - "properties": { - "request_model": { "$ref": "#/$defs/bounded_token" }, - "requested_effort": { "$ref": "#/$defs/bounded_token" }, - "route_kind": { "type": "string", "enum": ["direct", "execution_preset"] }, - "route_id": { "$ref": "#/$defs/bounded_token" }, - "expected_bindings": { - "type": "array", - "minItems": 1, - "items": { "$ref": "#/$defs/expected_binding" }, - "uniqueItems": true - } - }, - "allOf": [ - { - "if": { - "properties": { "route_kind": { "const": "direct" } } - }, - "then": { - "properties": { - "expected_bindings": { - "type": "array", - "minItems": 1, - "maxItems": 1, - "items": { - "type": "object", - "properties": { - "stage": { "const": "request" } - } - } - } - } - } - }, - { - "if": { - "properties": { "route_kind": { "const": "execution_preset" } } - }, - "then": { - "properties": { - "expected_bindings": { - "type": "array", - "minItems": 4, - "maxItems": 5, - "items": { - "type": "object", - "properties": { - "stage": { "enum": ["selector", "plan", "work", "review", "repair"] } - } - }, - "allOf": [ - { "contains": { "properties": { "stage": { "const": "selector" } }, "required": ["stage"] }, "minContains": 1, "maxContains": 1 }, - { "contains": { "properties": { "stage": { "const": "plan" } }, "required": ["stage"] }, "minContains": 1, "maxContains": 1 }, - { "contains": { "properties": { "stage": { "const": "work" } }, "required": ["stage"] }, "minContains": 1, "maxContains": 1 }, - { "contains": { "properties": { "stage": { "const": "review" } }, "required": ["stage"] }, "minContains": 1, "maxContains": 1 }, - { "contains": { "properties": { "stage": { "const": "repair" } }, "required": ["stage"] }, "minContains": 0, "maxContains": 1 } - ] - } - } - } - } - ] - }, - "evaluator": { - "type": "object", - "additionalProperties": false, - "required": ["caller", "iop"], - "properties": { - "caller": { "const": "codex" }, - "iop": { "$ref": "#/$defs/iop_cell" } - } - }, - "cell": { - "type": "object", - "additionalProperties": false, - "required": ["id", "caller", "iop"], - "properties": { - "id": { - "type": "string", - "pattern": "^[a-z0-9][a-z0-9_-]{0,63}$" - }, - "caller": { "type": "string", "enum": ["claude", "agy", "codex"] }, - "iop": { "$ref": "#/$defs/iop_cell" } - } - }, - "matrix": { - "type": "array", - "minItems": 1, - "items": { "$ref": "#/$defs/cell" }, - "uniqueItems": true - } - } -} diff --git a/scripts/fixtures/agent-comparison-benchmark-recovery-qualification.json b/scripts/fixtures/agent-comparison-benchmark-recovery-qualification.json deleted file mode 100644 index 574a35d7..00000000 --- a/scripts/fixtures/agent-comparison-benchmark-recovery-qualification.json +++ /dev/null @@ -1,135 +0,0 @@ -{ - "pipeline_version": "2", - "environment": "dev", - "testbed": "../iop-s2", - "execution_order_seed": "bench-02-recovery-qualification-v1", - "repetitions": 1, - "session_policy": "fresh", - "setup_cache_policy": "isolated", - "timeout": { - "run_seconds": 300, - "idle_seconds": 30, - "quiet_seconds": 10, - "cleanup_grace_seconds": 5 - }, - "viewports": [ - {"id": "desktop_1080", "width": 1920, "height": 1080}, - {"id": "mobile_375", "width": 375, "height": 812} - ], - "rubric_version": "one-shot-agent-comparison-v1", - "evaluator": { - "caller": "codex", - "iop": { - "request_model": "gpt-5.6-luna", - "requested_effort": "xhigh", - "route_kind": "direct", - "route_id": "gpt-5.6-luna", - "expected_bindings": [ - {"stage": "request", "model": "gpt-5.6-luna", "effort": "xhigh"} - ] - } - }, - "output_root": "agent-test/runs/bench-02-recovery", - "fixture": { - "version": "product-card-v2", - "prompt": "scripts/fixtures/agent-comparison-benchmark/prompt.md", - "assets": [ - {"source": "scripts/fixtures/agent-comparison-benchmark/reference.txt", "workspace_path": "brief/reference.txt"}, - {"source": "scripts/fixtures/agent-comparison-benchmark/aurora-grid.svg", "workspace_path": "assets/aurora-grid.svg"}, - {"source": "scripts/fixtures/agent-comparison-benchmark/orbit-rings.svg", "workspace_path": "assets/orbit-rings.svg"} - ], - "checksum": "sha256:fb16198fd4c3576f880f047ed7de54dddc160b0f70c2ba55b435cf078c61828e" - }, - "matrix": [ - { - "id": "c01-claude-sonnet-direct", - "caller": "claude", - "iop": { - "request_model": "claude-sonnet-5", - "requested_effort": "max", - "route_kind": "direct", - "route_id": "claude-sonnet-5", - "expected_bindings": [ - {"stage": "request", "model": "claude-sonnet-5", "effort": "max"} - ] - } - }, - { - "id": "c03-agy-gemini-direct", - "caller": "agy", - "iop": { - "request_model": "gemini-3.6-flash", - "requested_effort": "high", - "route_kind": "direct", - "route_id": "gemini-3.6-flash", - "expected_bindings": [ - {"stage": "request", "model": "gemini-3.6-flash", "effort": "high"} - ] - } - }, - { - "id": "c04-claude-gpt-direct", - "caller": "claude", - "iop": { - "request_model": "gpt-5.6-luna", - "requested_effort": "xhigh", - "route_kind": "direct", - "route_id": "gpt-5.6-luna", - "expected_bindings": [ - {"stage": "request", "model": "gpt-5.6-luna", "effort": "xhigh"} - ] - } - }, - { - "id": "c06-claude-gemini-hybrid", - "caller": "claude", - "iop": { - "request_model": "gemini-hybrid", - "requested_effort": "high", - "route_kind": "execution_preset", - "route_id": "gemini-hybrid", - "expected_bindings": [ - {"stage": "selector", "model": "gemini-3.6-flash", "effort": "high"}, - {"stage": "plan", "model": "gemini-3.6-flash", "effort": "high"}, - {"stage": "work", "model": "ornith-fast"}, - {"stage": "review", "model": "gemini-3.6-flash", "effort": "high"}, - {"stage": "repair", "model": "gemini-3.6-flash", "effort": "high"} - ] - } - }, - { - "id": "c07-agy-gemini-hybrid", - "caller": "agy", - "iop": { - "request_model": "gemini-hybrid", - "requested_effort": "high", - "route_kind": "execution_preset", - "route_id": "gemini-hybrid", - "expected_bindings": [ - {"stage": "selector", "model": "gemini-3.6-flash", "effort": "high"}, - {"stage": "plan", "model": "gemini-3.6-flash", "effort": "high"}, - {"stage": "work", "model": "ornith-fast"}, - {"stage": "review", "model": "gemini-3.6-flash", "effort": "high"}, - {"stage": "repair", "model": "gemini-3.6-flash", "effort": "high"} - ] - } - }, - { - "id": "c08-claude-gpt-hybrid", - "caller": "claude", - "iop": { - "request_model": "gpt-hybrid", - "requested_effort": "xhigh", - "route_kind": "execution_preset", - "route_id": "gpt-hybrid", - "expected_bindings": [ - {"stage": "selector", "model": "gpt-5.6-terra", "effort": "high"}, - {"stage": "plan", "model": "gpt-5.6-terra", "effort": "high"}, - {"stage": "work", "model": "ornith-fast"}, - {"stage": "review", "model": "gpt-5.6-terra", "effort": "high"}, - {"stage": "repair", "model": "gpt-5.6-terra", "effort": "high"} - ] - } - } - ] -} diff --git a/scripts/fixtures/agent-comparison-benchmark-report.expected.md b/scripts/fixtures/agent-comparison-benchmark-report.expected.md deleted file mode 100644 index 01b3b1c6..00000000 --- a/scripts/fixtures/agent-comparison-benchmark-report.expected.md +++ /dev/null @@ -1,110 +0,0 @@ -# Agent comparison benchmark report - -## Run identity - -| field | value | -|---|---| -| run_id | run-20260811T010203Z-123456abcdef | -| manifest_digest | sha256:39715007db41882abcfe5a3fd5f8e3cc4bcadce0ef31c17df4f3347825184359 | -| pipeline_version | 2 | - -## Immutable conditions - -| field | value | -|---|---| -| environment | dev | -| fixture | landing-v1 (sha256:4b5c9dcfe799d21f86a4462a2ada35275d72d5fe919b2adb069eeb3f4ac72fcb) | -| rubric | landing-quality-v1 | -| session_policy | fresh | -| setup_cache_policy | isolated | -| evaluator | codex/judge-model/xhigh | - -## Execution preflight - -| sequence | status | results | -|---:|---|---:| -| — | unavailable | 0 | - -## Attempt outcomes - -| cell | repetition | attempt | controller | product | harness | process | artifact | scoring | total | rank | -|---|---:|---:|---|---|---|---|---|---|---:|---:| -| cell-sentinel | 1 | 1 | completed | succeeded | passed | exited | passed | scored | 99 | 1 | -| cell-sentinel | 1 | 2 | completed | succeeded | passed | exited | passed | scored | 99 | 1 | -| cell-sentinel | 1 | 3 | completed | failed | passed | exited | failed | unscored | — | — | -| cell-sentinel | 1 | 4 | completed | succeeded | passed | exited | passed | scoring_failed | — | — | -| cell-sentinel | 1 | 5 | completed | succeeded | passed | exited | passed | blocked | — | — | - -## Quality score breakdown - -| cell/repetition/attempt | category | score | max | -|---|---|---:|---:| -| cell-sentinel/r1/a1 | task_fidelity | 24 | 25 | -| cell-sentinel/r1/a1 | visual_hierarchy | 25 | 25 | -| cell-sentinel/r1/a1 | responsive_composition | 20 | 20 | -| cell-sentinel/r1/a1 | typography_readability | 15 | 15 | -| cell-sentinel/r1/a1 | polish_consistency | 15 | 15 | -| cell-sentinel/r1/a2 | task_fidelity | 24 | 25 | -| cell-sentinel/r1/a2 | visual_hierarchy | 25 | 25 | -| cell-sentinel/r1/a2 | responsive_composition | 20 | 20 | -| cell-sentinel/r1/a2 | typography_readability | 15 | 15 | -| cell-sentinel/r1/a2 | polish_consistency | 15 | 15 | - -## Timing and token evidence - -| cell/repetition/attempt | time observations | token observations | -|---|---|---| -| cell-sentinel/r1/a1 | total_duration=1 ns; clock=harness_monotonic; source=harness; submitted_at,first_output_at=unavailable; reason=not_observed; source=harness; first_write_observed_at,first_write_mtime=unavailable; reason=not_observed; source=workspace_poll | cache_write_tokens,cached_input_tokens,input_tokens,model_calls,model_duration,output_tokens,queue_duration,reasoning_tokens,tool_calls,tool_duration,total_duration,total_tokens=unavailable; reason=not_reported; source=harness | -| cell-sentinel/r1/a2 | total_duration=1 ns; clock=harness_monotonic; source=harness; submitted_at,first_output_at=unavailable; reason=not_observed; source=harness; first_write_observed_at,first_write_mtime=unavailable; reason=not_observed; source=workspace_poll | cache_write_tokens,cached_input_tokens,input_tokens,model_calls,model_duration,output_tokens,queue_duration,reasoning_tokens,tool_calls,tool_duration,total_duration,total_tokens=unavailable; reason=not_reported; source=harness | -| cell-sentinel/r1/a3 | total_duration=1 ns; clock=harness_monotonic; source=harness; submitted_at,first_output_at=unavailable; reason=not_observed; source=harness; first_write_observed_at,first_write_mtime=unavailable; reason=not_observed; source=workspace_poll | cache_write_tokens,cached_input_tokens,input_tokens,model_calls,model_duration,output_tokens,queue_duration,reasoning_tokens,tool_calls,tool_duration,total_duration,total_tokens=unavailable; reason=not_reported; source=harness | -| cell-sentinel/r1/a4 | total_duration=1 ns; clock=harness_monotonic; source=harness; submitted_at,first_output_at=unavailable; reason=not_observed; source=harness; first_write_observed_at,first_write_mtime=unavailable; reason=not_observed; source=workspace_poll | cache_write_tokens,cached_input_tokens,input_tokens,model_calls,model_duration,output_tokens,queue_duration,reasoning_tokens,tool_calls,tool_duration,total_duration,total_tokens=unavailable; reason=not_reported; source=harness | -| cell-sentinel/r1/a5 | total_duration=1 ns; clock=harness_monotonic; source=harness; submitted_at,first_output_at=unavailable; reason=not_observed; source=harness; first_write_observed_at,first_write_mtime=unavailable; reason=not_observed; source=workspace_poll | cache_write_tokens,cached_input_tokens,input_tokens,model_calls,model_duration,output_tokens,queue_duration,reasoning_tokens,tool_calls,tool_duration,total_duration,total_tokens=unavailable; reason=not_reported; source=harness | - -## Web validation and scoring provenance - -| cell/repetition/attempt | web gates | screenshots | score_id | evaluator | scoring condition | -|---|---|---|---|---|---| -| cell-sentinel/r1/a1 | generated_files=pass, static_safety=pass, images=pass, network=pass, console=pass, responsive=pass, accessibility=pass | screenshot-desktop.png, screenshot-mobile.png | score-000001 | codex/judge-route/judge-model/xhigh | recorded | -| cell-sentinel/r1/a2 | generated_files=pass, static_safety=pass, images=pass, network=pass, console=pass, responsive=pass, accessibility=pass | screenshot-desktop.png, screenshot-mobile.png | score-000001 | codex/judge-route/judge-model/xhigh | recorded | -| cell-sentinel/r1/a3 | generated_files=pass, static_safety=pass, images=fail, network=fail, console=fail, responsive=fail, accessibility=fail | unavailable | — | unavailable | product_failed, process_nonzero_exit, gate_images, gate_network | -| cell-sentinel/r1/a4 | generated_files=pass, static_safety=pass, images=pass, network=pass, console=pass, responsive=pass, accessibility=pass | screenshot-desktop.png, screenshot-mobile.png | score-000001 | codex/judge-route/judge-model/xhigh | invalid_worksheet | -| cell-sentinel/r1/a5 | generated_files=pass, static_safety=pass, images=pass, network=pass, console=pass, responsive=pass, accessibility=pass | screenshot-desktop.png, screenshot-mobile.png | — | unavailable | evaluator_preflight_blocked | - -## Limitations - -- Values marked `unavailable` retain the producing source and reason; they are not inferred as zero. -- Automatic web gates establish eligibility only and contribute no quality points. -- Equal scored totals share a competition rank; unscored and scoring-failed attempts do not receive a rank. - -## Raw evidence index - -| contained pointer | -|---| -| [raw]() | -| [raw]() | -| [raw]() | -| [raw]() | -| [raw]() | -| [raw]() | -| [raw]() | -| [raw]() | -| [raw]() | -| [raw]() | -| [raw]() | -| [raw]() | -| [raw]() | -| [raw]() | -| [raw]() | -| [raw]() | -| [raw]() | -| [raw]() | -| [raw]() | -| [raw]() | -| [raw]() | -| [raw]() | -| [raw]() | -| [raw]() | -| [raw]() | -| [raw]() | -| [raw]() | -| [raw]() | diff --git a/scripts/fixtures/agent-comparison-benchmark-supported-direct.example.json b/scripts/fixtures/agent-comparison-benchmark-supported-direct.example.json deleted file mode 100644 index a2e138c6..00000000 --- a/scripts/fixtures/agent-comparison-benchmark-supported-direct.example.json +++ /dev/null @@ -1,96 +0,0 @@ -{ - "pipeline_version": "2", - "environment": "dev", - "testbed": "../iop-s2", - "repetitions": 1, - "session_policy": "fresh", - "setup_cache_policy": "isolated", - "timeout": { - "run_seconds": 300, - "idle_seconds": 30, - "quiet_seconds": 10, - "cleanup_grace_seconds": 5 - }, - "viewports": [ - {"id": "desktop_1080", "width": 1920, "height": 1080}, - {"id": "mobile_375", "width": 375, "height": 812} - ], - "rubric_version": "landing-quality-v1", - "evaluator": { - "caller": "codex", - "iop": { - "request_model": "gpt-5.6-luna", - "requested_effort": "xhigh", - "route_kind": "direct", - "route_id": "gpt-5.6-luna", - "expected_bindings": [ - {"stage": "request", "model": "gpt-5.6-luna", "effort": "xhigh"} - ] - } - }, - "output_root": "agent-test/runs/bench-01-supported-direct", - "fixture": { - "version": "product-card-v2", - "prompt": "scripts/fixtures/agent-comparison-benchmark/prompt.md", - "assets": [ - {"source": "scripts/fixtures/agent-comparison-benchmark/reference.txt", "workspace_path": "brief/reference.txt"}, - {"source": "scripts/fixtures/agent-comparison-benchmark/aurora-grid.svg", "workspace_path": "assets/aurora-grid.svg"}, - {"source": "scripts/fixtures/agent-comparison-benchmark/orbit-rings.svg", "workspace_path": "assets/orbit-rings.svg"} - ], - "checksum": "sha256:fb16198fd4c3576f880f047ed7de54dddc160b0f70c2ba55b435cf078c61828e" - }, - "matrix": [ - { - "id": "claude-sonnet-direct", - "caller": "claude", - "iop": { - "request_model": "claude-sonnet-5", - "requested_effort": "max", - "route_kind": "direct", - "route_id": "claude-sonnet-5", - "expected_bindings": [ - {"stage": "request", "model": "claude-sonnet-5", "effort": "max"} - ] - } - }, - { - "id": "claude-gemini-direct", - "caller": "claude", - "iop": { - "request_model": "gemini-3.6-flash", - "requested_effort": "high", - "route_kind": "direct", - "route_id": "gemini-3.6-flash", - "expected_bindings": [ - {"stage": "request", "model": "gemini-3.6-flash", "effort": "high"} - ] - } - }, - { - "id": "claude-gpt-direct", - "caller": "claude", - "iop": { - "request_model": "gpt-5.6-luna", - "requested_effort": "xhigh", - "route_kind": "direct", - "route_id": "gpt-5.6-luna", - "expected_bindings": [ - {"stage": "request", "model": "gpt-5.6-luna", "effort": "xhigh"} - ] - } - }, - { - "id": "codex-gpt-direct", - "caller": "codex", - "iop": { - "request_model": "gpt-5.6-luna", - "requested_effort": "xhigh", - "route_kind": "direct", - "route_id": "gpt-5.6-luna", - "expected_bindings": [ - {"stage": "request", "model": "gpt-5.6-luna", "effort": "xhigh"} - ] - } - } - ] -} diff --git a/scripts/fixtures/agent-comparison-benchmark/agy-iop-stream.jsonl b/scripts/fixtures/agent-comparison-benchmark/agy-iop-stream.jsonl deleted file mode 100644 index 477c4b25..00000000 --- a/scripts/fixtures/agent-comparison-benchmark/agy-iop-stream.jsonl +++ /dev/null @@ -1,3 +0,0 @@ -{"conversation_id":"fixture-conversation","event":"init","init":{"cwd":"/workspace","model":"Gemini 3.6 Flash","permission_mode":"sandbox","tools":[]}} -{"event":"step_update","step_update":{"conversation_id":"fixture-conversation","duration_seconds":0.012,"state":"DONE","step_index":0,"step_type":"agent_response","text_delta":"fixture content must be redacted","usage":{"cache_read_tokens":1,"input_tokens":5,"output_tokens":4,"thinking_tokens":2,"total_tokens":12}}} -{"event":"result","result":{"conversation_id":"fixture-conversation","duration_seconds":0.012,"num_turns":1,"response":"fixture content must be redacted","status":"SUCCESS","usage":{"cache_read_tokens":1,"input_tokens":5,"output_tokens":4,"thinking_tokens":2,"total_tokens":12}}} diff --git a/scripts/fixtures/agent-comparison-benchmark/aurora-grid.svg b/scripts/fixtures/agent-comparison-benchmark/aurora-grid.svg deleted file mode 100644 index 77ed2941..00000000 --- a/scripts/fixtures/agent-comparison-benchmark/aurora-grid.svg +++ /dev/null @@ -1,42 +0,0 @@ - - Aurora service grid - Abstract luminous nodes connected across a dark blue operational grid. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/scripts/fixtures/agent-comparison-benchmark/claude-iop-stream.jsonl b/scripts/fixtures/agent-comparison-benchmark/claude-iop-stream.jsonl deleted file mode 100644 index 755d3d29..00000000 --- a/scripts/fixtures/agent-comparison-benchmark/claude-iop-stream.jsonl +++ /dev/null @@ -1,3 +0,0 @@ -{"type":"system","subtype":"init","session_id":"claude-session-fixture","model":"claude-sonnet","cwd":"[redacted]"} -{"type":"assistant","session_id":"claude-session-fixture","message":{"model":"claude-sonnet","stop_reason":"end_turn","content":[{"type":"text","text":"[redacted]"}]}} -{"type":"result","subtype":"success","session_id":"claude-session-fixture","duration_ms":1234,"duration_api_ms":1000,"usage":{"input_tokens":11,"output_tokens":22,"cache_read_input_tokens":5},"result":"[redacted]"} diff --git a/scripts/fixtures/agent-comparison-benchmark/codex-iop-stream.jsonl b/scripts/fixtures/agent-comparison-benchmark/codex-iop-stream.jsonl deleted file mode 100644 index ef0de3a4..00000000 --- a/scripts/fixtures/agent-comparison-benchmark/codex-iop-stream.jsonl +++ /dev/null @@ -1,4 +0,0 @@ -{"type":"thread.started","thread_id":"public-fixture"} -{"type":"item.completed","item":{"id":"call-1","type":"command_execution","duration_ms":7.25}} -{"type":"turn.completed","status":"completed","usage":{"input_tokens":31,"cached_input_tokens":8,"cache_write_input_tokens":6,"output_tokens":12,"reasoning_output_tokens":4},"item":{"content":"public fixture content"}} -{"type":"adapter.idle","adapter":"codex_iop","nonce":"fixture-nonce-0001","child_exit":0} diff --git a/scripts/fixtures/agent-comparison-benchmark/orbit-rings.svg b/scripts/fixtures/agent-comparison-benchmark/orbit-rings.svg deleted file mode 100644 index d51e8bea..00000000 --- a/scripts/fixtures/agent-comparison-benchmark/orbit-rings.svg +++ /dev/null @@ -1,37 +0,0 @@ - - Layered orbit rings - Abstract rings and markers arranged around a bright central operating point. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/scripts/fixtures/agent-comparison-benchmark/prompt.md b/scripts/fixtures/agent-comparison-benchmark/prompt.md deleted file mode 100644 index 851eba58..00000000 --- a/scripts/fixtures/agent-comparison-benchmark/prompt.md +++ /dev/null @@ -1,14 +0,0 @@ -Build a polished, responsive product card for the fictional product described in `brief/reference.txt`. - -Requirements: - -- Create exactly these implementation files at the workspace root: `index.html`, `styles.css`, and `script.js`. -- Use both provided local images, `assets/aurora-grid.svg` and `assets/orbit-rings.svg`, as visible `` content with meaningful `alt` text. -- Compose one compact card with a small product header, one hero area, and one primary CTA using the supplied copy. Do not add feature, workflow, testimonial, pricing, or footer sections. -- Provide one tiny keyboard-accessible toggle in `script.js` for the supplied status detail; the detail must remain readable when JavaScript is unavailable. -- Support the supplied desktop and mobile viewports without horizontal overflow, clipped primary content, or overlapping controls. -- Use semantic HTML, visible focus states, sufficient text/background contrast, a logical heading order, and labels for interactive controls. -- Do not use external network assets, frameworks, package managers, build tools, inline data URLs, or generated replacements for the provided images. -- Do not add credentials, private endpoints, analytics, trackers, or production data. - -Finish only after all three required files exist and the page can be served as static files from the workspace root. diff --git a/scripts/fixtures/agent-comparison-benchmark/reference.txt b/scripts/fixtures/agent-comparison-benchmark/reference.txt deleted file mode 100644 index 79aaac87..00000000 --- a/scripts/fixtures/agent-comparison-benchmark/reference.txt +++ /dev/null @@ -1,7 +0,0 @@ -PRODUCT: Lumen Atlas -EYEBROW: Operational clarity, at a glance -HEADLINE: Turn scattered signals into a shared operating picture. -SUMMARY: Lumen Atlas brings service health, ownership, and live operational context into one focused workspace so teams can decide with confidence. -PRIMARY CTA: Explore the workspace -STATUS TOGGLE: Show live context -STATUS DETAIL: Ownership, service health, and recent changes stay connected in one calm view.