From 8f00606c0339e1e9ace5f7c8d019582ba6843513 Mon Sep 17 00:00:00 2001 From: toki Date: Wed, 12 Aug 2026 21:01:51 +0900 Subject: [PATCH] =?UTF-8?q?fix(benchmark):=20=EA=B2=B0=EA=B3=BC=20?= =?UTF-8?q?=EA=B2=BD=EA=B3=84=EB=A5=BC=20=EB=8F=85=EB=A6=BD=20=EC=B6=95?= =?UTF-8?q?=EC=9C=BC=EB=A1=9C=20=EB=B6=84=EB=A6=AC=ED=95=9C=EB=8B=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 제품 결과와 harness·process·artifact 실패가 하나의 성공 값으로 덮이지 않도록 durable evidence와 모든 소비자 계약을 함께 마이그레이션한다. --- .../iop-agent-comparison-benchmark/SKILL.md | 13 +- .../testing/agent-comparison-benchmark.md | 11 +- .../CODE_REVIEW-cloud-G10.md | 162 ++++++-- .../08+07_comparison_rerun/PLAN-cloud-G10.md | 376 ++++++++++++++---- .../code_review_cloud_G10_3.log | 57 +++ .../plan_cloud_G10_3.log | 113 ++++++ docs/agent-comparison-benchmark-dev-guide.md | 37 +- scripts/agent_benchmark/agy_iop.py | 50 ++- scripts/agent_benchmark/agy_iop_test.py | 20 +- scripts/agent_benchmark/attempts.py | 348 +++++++++++++--- scripts/agent_benchmark/attempts_test.py | 164 ++++++-- scripts/agent_benchmark/claude_iop.py | 33 +- scripts/agent_benchmark/claude_iop_test.py | 58 +-- scripts/agent_benchmark/codex_iop.py | 21 +- scripts/agent_benchmark/codex_iop_test.py | 22 +- .../connectivity_integration_test.py | 267 +++++++------ scripts/agent_benchmark/lifecycle.py | 293 +++++++++++--- scripts/agent_benchmark/lifecycle_test.py | 218 +++++++--- scripts/agent_benchmark/live_iop.py | 21 +- scripts/agent_benchmark/measurement.py | 65 ++- scripts/agent_benchmark/measurement_test.py | 24 +- scripts/agent_benchmark/reporting.py | 13 +- scripts/agent_benchmark/scoring.py | 159 +++++++- scripts/agent_benchmark/scoring_test.py | 158 +++++++- .../agent_benchmark/skill_contract_test.py | 6 + scripts/agent_benchmark/web_validation.py | 14 +- .../agent_benchmark/web_validation_test.py | 13 +- scripts/agent_comparison_benchmark.py | 42 +- ...nt-comparison-benchmark-report.expected.md | 16 +- 29 files changed, 2169 insertions(+), 625 deletions(-) create mode 100644 agent-task/m-iop-one-shot-agent-model-comparison/08+07_comparison_rerun/code_review_cloud_G10_3.log create mode 100644 agent-task/m-iop-one-shot-agent-model-comparison/08+07_comparison_rerun/plan_cloud_G10_3.log diff --git a/agent-ops/skills/project/iop-agent-comparison-benchmark/SKILL.md b/agent-ops/skills/project/iop-agent-comparison-benchmark/SKILL.md index 2073cbc7..cfaea914 100644 --- a/agent-ops/skills/project/iop-agent-comparison-benchmark/SKILL.md +++ b/agent-ops/skills/project/iop-agent-comparison-benchmark/SKILL.md @@ -65,25 +65,25 @@ Route agent comparison benchmark requests to the deterministic CLI while enforci - The CLI creates one run and uses its single writer to append a fresh all-cell preflight before attempt allocation. - On `registration_required` or `implementation_gap`, it prints `error: preflight blocked ...` to stderr with exit 69, allocates no attempt, and preserves the run id for a later resume. - On `ready`, it binds the exact caller, cell, fresh workspace, session, and attempt identity, then must invoke each eligible cell exactly once with the fixture task. - - Exit 0 only when every retained attempt is successful; otherwise report the exact closed execution failure summary from stderr with exit 69. + - Exit 0 only when every latest slot has `product=succeeded`, `harness=passed`, `process=exited` with exit code 0 and no signal, and `artifact=passed`; otherwise report the exact independent-axis summary from stderr with exit 69. 6. **Delegate resume to the CLI** - Run: `python3 scripts/agent_comparison_benchmark.py resume --manifest --run-id [--retry-failed]` - On missing or invalid manifest or state, the CLI prints `error: benchmark state is unavailable` to stderr with exit 69 (or `error: invalid usage` with exit 64) before changing the run. - The CLI opens the exact immutable run and uses its single writer to append a fresh all-cell preflight before attempt allocation. - On `registration_required` or `implementation_gap`, it prints `error: preflight blocked ...` to stderr with exit 69 and allocates no attempt. - - On `ready`, it reconciles interrupted state, skips successful slots, preserves prior attempt bytes, and allocates a new attempt only for eligible work. `--retry-failed` admits a new attempt for failed, timed-out, or cancelled slots. + - On `ready`, it reconciles interrupted state, skips only slots whose latest product/harness/process/artifact gates all pass, preserves prior attempt bytes, and allocates a new attempt only for eligible work. `--retry-failed` admits a new attempt for any latest terminal attempt whose independent gates do not all pass. - It must invoke each eligible cell exactly once with a new workspace and session identity. 7. **Delegate status to the CLI** - Run: `python3 scripts/agent_comparison_benchmark.py status --manifest --run-id ` - On missing or invalid manifest or state, the CLI prints `error: benchmark state is unavailable` to stderr with exit 69 (or `error: invalid usage` with exit 64). - - On success, the CLI prints `ok: ` to stdout with exit 0. + - On success, the CLI prints the controller counts, product/harness/process/artifact counts, and `unresolved=` to stdout with exit 0. - Report the exact CLI output. 8. **Delegate score to the CLI** - Run: `python3 scripts/agent_comparison_benchmark.py score --manifest --run-id [--retry-scoring-failed]` - - The CLI classifies lifecycle or required web-gate failures as immutable `unscored`, without invoking the evaluator or assigning zero. + - The CLI classifies each failed product, harness, process, or required artifact gate with its own immutable `unscored` reason, without invoking the evaluator or assigning zero. - Eligible attempts receive an opaque blind workspace, one manifest-bound fresh Codex evaluator session, and the exact immutable manifest-selected rubric from the closed supported catalog (`landing-quality-v1`, `one-shot-agent-comparison-v1`); no substitute rubric or reinterpretation is permitted. - A prior `scored` result is terminal. A prior `scoring_failed` result is retried only with `--retry-scoring-failed`, which allocates a new score id and preserves every prior byte. - On exit 0, report the exact closed `scored`, `unscored`, `scoring_failed`, and `blocked` counts from stdout. @@ -100,6 +100,7 @@ Route agent comparison benchmark requests to the deterministic CLI while enforci - [ ] Preflight evidence is append-only, covers every immutable matrix cell in canonical order, and uses only closed status/count fields. - [ ] A preflight blocker created no scored attempt and was not bypassed. - [ ] An ineligible execution attempt became `unscored` without an evaluator invocation or a zero score. +- [ ] Product, harness, process, and artifact outcomes remain separately visible in run/resume/status output and report rows. - [ ] Each eligible score id used one opaque blind workspace and one fresh evaluator session; retry preserved prior bytes and used a new id. - [ ] `scoring_failed` used no fallback, synthetic worksheet, or implicit retry. - [ ] No caller or provider was invoked outside the deterministic CLI. @@ -150,7 +151,7 @@ For run/resume ready completion: ``` command: exit_code: 0 -stdout: ok: run_id= completed= unresolved=0 success= failed= timed_out= cancelled= interrupted= running=0 +stdout: ok: run_id= executed= unresolved=0 completed= timed_out= cancelled= interrupted= running=0 product_succeeded= product_failed=0 product_unknown=0 harness_passed= harness_failed=0 process_exited= process_signalled=0 process_timed_out=0 process_cancelled=0 process_not_started=0 artifact_passed= artifact_failed=0 artifact_blocked=0 artifact_not_run=0 stderr: (none) ``` @@ -172,6 +173,8 @@ stderr: - Preflight never allocates a scored attempt. Every immutable matrix cell requires its own fresh live observation. - Run/resume append a fresh all-cell preflight under the run writer before any attempt allocation; a blocker allocates no attempt. - Ready execution binds one exact cell and immutable attempt identity to one fresh workspace/session and one task submission. +- Product, harness, process, and artifact are independent gates. Controller state `completed` only means the invocation controller reached a terminal state. +- Release qualification runs the five-cell direct manifest as one unscored canary and requires all four gates for all five cells before a fresh nine-cell preflight; it does not allocate hybrid or scored execution. - Scoring copies only anonymous generated files, two local images, and screenshots into an opaque run-owned blind tree; the identity mapping remains outside that tree. - Scoring records `unscored`, `scored`, and `scoring_failed` append-only, and a retry always allocates a fresh score id/session. - The internal workspace API (`RunStore`, `Manifest`, etc.) is not a user command. Do not expose it. diff --git a/agent-spec/testing/agent-comparison-benchmark.md b/agent-spec/testing/agent-comparison-benchmark.md index 14429c62..b5d35c44 100644 --- a/agent-spec/testing/agent-comparison-benchmark.md +++ b/agent-spec/testing/agent-comparison-benchmark.md @@ -68,11 +68,11 @@ Claude Code, agy, Codex가 IOP를 경유해 수행하는 동일 과업을 설정 |------|------| | manifest 검증 | caller, IOP direct/preset route, model, effort, fixture, 반복 횟수, timeout, evaluator와 `agent-test/runs/` 경로를 검증하고 canonical digest를 만든다. | | 연결 preflight | Claude Code, agy, Codex의 binary/config와 IOP endpoint·auth·model·effort·stream binding을 확인하고 `ready`, `registration_required`, `implementation_gap`으로 분류한다. | -| 격리 실행과 재개 | 각 cell/repetition을 동일 checksum의 clean workspace와 fresh caller session에서 실행하며, 종료·idle·timeout·cancel·cleanup을 bounded하게 처리한다. 실패한 attempt는 덮어쓰지 않고 명시적 재개 시 새 attempt로 남긴다. | +| 격리 실행과 재개 | 각 cell/repetition을 동일 checksum의 clean workspace와 fresh caller session에서 실행하며, 제품 결과·harness 정합성·process 종료를 독립 결과로 보존한다. top-level state는 `running`, `completed`, `timed_out`, `cancelled`, `interrupted`의 controller 상태만 나타내며, 실패한 attempt는 덮어쓰지 않는다. | | 측정과 evidence | 제출, 첫 출력, 첫 파일 쓰기, model/tool/queue, finish/idle 시간을 관측 source와 함께 정규화한다. token은 보고 주체와 미제공 상태를 보존하며 임의 추정값을 authoritative 값과 섞지 않는다. | -| 웹 자동 검증 | 필수 HTML/CSS/JS와 로컬 이미지, 외부 asset 금지, desktop/mobile render, console/asset 오류, 반응형·접근성 gate와 screenshot을 확인한다. | +| 웹 자동 검증 | product/harness 성공 여부와 무관하게 모든 terminal workspace에서 필수 HTML/CSS/JS와 로컬 이미지, 외부 asset 금지, desktop/mobile render, console/asset 오류, 반응형·접근성 gate와 screenshot을 확인한다. | | 익명 품질 채점 | 필수 자동 gate를 통과한 결과만 identity를 가린 뒤 manifest에 고정된 fresh evaluator로 100점 rubric을 평가한다. 부적격 결과는 `unscored`, 평가 실패는 `scoring_failed`로 남기며 retry는 새 scoring attempt id를 사용한다. | -| 상태와 보고 | `validate`, `preflight`, `run`, `resume`, `status`, `score`, `report` CLI를 제공하고, 성공·실패·blocked·unscored·scoring_failed·동점을 raw evidence 포인터와 함께 deterministic Markdown으로 만든다. | +| 상태와 보고 | `validate`, `preflight`, `run`, `resume`, `status`, `score`, `report` CLI를 제공하고, controller/product/harness/process/artifact/scoring 축과 동점을 raw evidence 포인터와 함께 deterministic Markdown으로 만든다. | ## 범위 @@ -105,7 +105,9 @@ flowchart LR - manifest의 matrix cell은 stable id, caller, IOP route kind, requested/effective model과 effort를 가진다. unsupported alias나 effort는 다른 값으로 대체하지 않고 fail-closed한다. - run state는 `agent-test/runs///` 아래에 격리되며 manifest digest가 다른 상태를 재개하지 않는다. - preflight는 scored attempt가 아니며, 실행 중 실패·timeout·cancel과 scoring 실패는 기존 attempt를 수정하지 않고 보존한다. -- Claude result가 마지막 active assistant snapshot을 직접 완성하면 adapter가 `finish`와 `idle`을 함께 투영하고, assistant가 이미 finish를 냈으면 result는 `idle`만 투영한다. synthetic API error와 agy ERROR result는 parser malformed로 바꾸지 않고 process terminal에 실패 판정을 맡긴다. +- caller parser는 raw terminal 문자열 대신 `CallerEvent(finish|idle)`, `CallerTerminal(succeeded|failed)`와 typed metric만 반환한다. Claude result가 마지막 active assistant snapshot을 직접 완성하면 adapter가 typed finish와 idle을 함께 투영하고, assistant가 이미 finish를 냈으면 result는 idle만 투영한다. synthetic API error와 agy ERROR result는 `product=failed`, `harness=passed`가 될 수 있으며 parser malformed는 `product=unknown`, `harness=failed`로 구분한다. +- durable lifecycle/measurement/attempt evidence는 `product`, `harness`, `process` 객체를 그대로 저장한다. `run` exit 0과 scoring eligibility는 product succeeded, harness passed, process exited/exit 0/no signal, artifact passed를 모두 요구한다. +- 배포 qualification은 동일 clean source에서 5-cell direct manifest를 unscored canary로 한 번 실행해 네 gate 5/5를 확인한 뒤 fresh C01-C09 preflight `ready=9`까지만 수행한다. hybrid 또는 scored C01-C09 실행은 후속 승인 전에는 할당하지 않는다. - lifecycle supervisor는 exit watcher와 출력 reader를 join한 뒤 하나의 child return code를 동결해 lifecycle result와 cleanup receipt가 동일한 exit/signal을 갖게 한다. 불일치 evidence는 resume에서 fail-closed한다. - raw credential과 private endpoint는 tracked manifest, event, log, screenshot과 report에 기록하지 않는다. - report는 run state의 canonical evidence에서 생성되며 성공하지 않은 결과를 0점으로 변환하거나 동점에 임의 순위를 부여하지 않는다. @@ -127,5 +129,6 @@ flowchart LR ## 변경 기록 +- 2026-08-12: caller terminal을 closed typed observation으로 바꾸고 product/harness/process 결과, failure-inclusive artifact gate, 독립 CLI/report/scoring gate와 direct-first qualification을 구현했다. - 2026-08-12: official Claude result-direct/API-error 및 agy ERROR terminal을 lifecycle 계약에 맞게 분리하고, timeout cleanup result/receipt가 같은 child exit snapshot을 사용하도록 동기화했다. - 2026-08-12: `[bench-01]` 종료 감사에서 확인한 421개 benchmark test, manifest/CLI 계약과 구현 evidence를 기준으로 생성했다. diff --git a/agent-task/m-iop-one-shot-agent-model-comparison/08+07_comparison_rerun/CODE_REVIEW-cloud-G10.md b/agent-task/m-iop-one-shot-agent-model-comparison/08+07_comparison_rerun/CODE_REVIEW-cloud-G10.md index f3426709..2de59bc9 100644 --- a/agent-task/m-iop-one-shot-agent-model-comparison/08+07_comparison_rerun/CODE_REVIEW-cloud-G10.md +++ b/agent-task/m-iop-one-shot-agent-model-comparison/08+07_comparison_rerun/CODE_REVIEW-cloud-G10.md @@ -1,57 +1,147 @@ - + -# Code Review Reference - REVIEW_REVIEW_TEST +# Code Review Reference - REVIEW_REFACTOR + +> **[IMPLEMENTING AGENT — READ FIRST] Filling in this file is the mandatory final step of implementation.** +> The task is NOT complete until every implementation-owned section below is filled in. +> Complete the `Implementation Checklist`; the final checklist item is mandatory before saving. +> Fill implementation-owned sections, then stop with active files in place and report ready for review. +> Execute the plan's selected root cause, scope, files, and dependency decisions as written. Do not choose another owner, narrow/expand the write boundary, or replace a fix with another verification attempt. +> If implementation is blocked, record the exact blocker, attempted commands/output, and resume condition only in implementation-owned evidence fields. +> Do not ask the user directly, present choices, call user-input tools, create control-plane stop files, or classify the next state. +> Finalization (`Code Review Result`, log rename, `complete.log`, archive moves, `Review-Only Checklist`) is review-agent-only. ## Overview date=2026-08-12 -task=m-iop-one-shot-agent-model-comparison/08+07_comparison_rerun, plan=3, tag=REVIEW_REVIEW_TEST +task=m-iop-one-shot-agent-model-comparison/08+07_comparison_rerun, plan=4, tag=REVIEW_REFACTOR ## Archive Evidence Snapshot -- Prior plan: `agent-task/m-iop-one-shot-agent-model-comparison/08+07_comparison_rerun/plan_cloud_G10_2.log` -- Prior review: `agent-task/m-iop-one-shot-agent-model-comparison/08+07_comparison_rerun/code_review_cloud_G10_2.log` -- Verdict: `FAIL`; Required R1 Claude result-direct/API-error terminal projection, Required R2 agy top-level official JSONL, Required R3 timeout result/receipt exit-code race. -- Retained run evidence: C02 `out_of_order_event`; C03/C06/C08 `malformed_event`; C01 `timed_out` with lifecycle exit `143` and receipt exit `null`; C05/C09 successful. Public resume appended a ready=9 preflight but correctly rejected the inconsistent terminal and did not retry a failed cell. +- Previous active packet is preserved at `agent-task/m-iop-one-shot-agent-model-comparison/08+07_comparison_rerun/plan_cloud_G10_3.log` and `agent-task/m-iop-one-shot-agent-model-comparison/08+07_comparison_rerun/code_review_cloud_G10_3.log`; it had implemented R1-R6 but had not completed full local verification, deployment, public smoke, or a fresh scored run. +- R1-R6 remain selected, committed compatibility fixes. In particular, they cover Claude result-direct/API-error shapes, agy structured output and ERROR terminal handling, supervisor exit/receipt coherence, deterministic Plan rendering, the pinned Claude beta, and provider-independent Plan arrays. +- Retained run evidence already showed that a single lifecycle result conflates product and harness causes: valid caller errors became missing/malformed lifecycle evidence, parser/order defects became attempt failures, and web validation was skipped solely because lifecycle was non-success. +- The interrupted test left only `/tmp/iop-s0-interrupted-test-tmp7hc4t2o6`; no benchmark, unittest, deploy, or target Codex process remains active. + +## For the Review Agent + +> **[REVIEW AGENT ONLY]** Compare every item with source and rerun applicable commands fresh. Verify the implementation preserved R1-R6 and did not allocate a hybrid or scored nine-cell run. Finalization, verdict, log renames, `complete.log`, archive moves, and any next-state classification are review-agent only. ## Implementation Item Completion | Item | Status | |---|---| -| REVIEW_REVIEW_TEST-1 Claude terminal projection | [x] | -| REVIEW_REVIEW_TEST-2 agy request/error compatibility | [x] | -| REVIEW_REVIEW_TEST-3 lifecycle exit coherence | [x] | -| REVIEW_REVIEW_TEST-4 deterministic Plan rendering | [x] | -| REVIEW_REVIEW_TEST-5 pinned Claude beta compatibility | [x] | -| REVIEW_REVIEW_TEST-6 provider-independent Plan arrays | [x] | -| REVIEW_REVIEW_TEST-7 complete local verification | [ ] | -| REVIEW_REVIEW_TEST-8 clean release deployment | [ ] | -| REVIEW_REVIEW_TEST-9 public smokes/preflight | [ ] | -| REVIEW_REVIEW_TEST-10 scored run/report | [ ] | +| REVIEW_REFACTOR-1 typed invocation outcomes | [ ] | +| REVIEW_REFACTOR-2 durable projections, artifact gate, and reporting | [ ] | +| REVIEW_REFACTOR-3 deterministic and clean-build qualification | [ ] | +| REVIEW_REFACTOR-4 direct-first live convergence gate | [ ] | -## Implementation Evidence +## Implementation Checklist -- `python3 -m unittest scripts.agent_benchmark.claude_iop_test scripts.agent_benchmark.agy_iop_test scripts.agent_benchmark.lifecycle_test` passed 55 tests after R1-R3 implementation. -- `python3 -m unittest discover -s scripts/agent_benchmark -p '*_test.py'` passed 437 tests. -- `go test -count=1 ./apps/edge/internal/openai` passed with official agy structured-output request coverage. -- Sequential `go test -count=1` passed every package returned by `go list ./apps/control-plane/... ./apps/edge/... ./apps/node/... ./packages/go/...`, excluding only the declared `agenttask` boundary. -- Sanitized replay of retained C02/C06/C08 streams now projects C02 as `finish,idle` and both synthetic API-error cells as no success terminal without parser failure. -- Post-deploy official agy direct and `gemini-hybrid` smokes reached `result.status=SUCCESS`; Claude direct reached `result/subtype=success`. The first Claude hybrid smoke did not repeat the immediate API error, but raw-free Edge metrics exposed three Plan validation failures and the call exceeded the 180-second non-scored boundary, creating R4 before scored execution. -- R4 replaces free-form PlanMD generation with a strict stage-owned `goal`/`steps`/`verification` JSON schema, exact closed-object decoding (including duplicate-key rejection), bounded field validation, and deterministic rendering of the frozen operator template. Focused Plan/template/executor regressions and the complete `apps/edge/internal/openai` package pass. -- The current feature tree passes all 437 Python benchmark tests, every relevant Go package under Control Plane/Edge/Node/shared runtime (excluding only the declared `agenttask` boundary), `git diff --check`, and manifest validation. Clean release-tree repetition remains pending commit/merge. -- After deploying release `1f748bd9bd7fcc843074847a57ed523a28db75fd`, official Claude Code `2.1.228` direct smoke exposed R5 before any scored run: its nominal `result/subtype=success` carried `is_error=true` and a sanitized unsupported-beta 400 for `advisor-tool-2026-03-01`. The scored-run allowance remains unused. -- R5 adds the exact pinned-caller beta to the closed admission inventory and consumes it at the Chat bridge without forwarding it or creating capability authority. Direct Claude Code mapping, marked single-request admission, and unknown-beta rejection regressions pass. -- After the R5 release, official Claude GPT hybrid reached the configured `gpt-5.6-terra` Plan provider twice but returned a caller-visible server error. Raw-free Edge evidence classified both Plan terminals as malformed, proving routing and credential selection succeeded while the remaining R4 Markdown-in-JSON string shape failed provider-independent decoding. -- R6 changes only the private Plan response contract: `steps` and `verification` are bounded arrays of non-empty one-line strings, while Edge deterministically adds Markdown bullets and newlines. It does not change routes, credentials, retries, templates, or any caller-visible schema. +- [ ] [REVIEW_REFACTOR-1] Replace caller/parser string terminals and the overloaded lifecycle result with closed typed caller, product, harness, and process outcomes. +- [ ] [REVIEW_REFACTOR-2] Migrate attempt, measurement, web validation, scoring, CLI, report, project skill, guide, and living spec to preserve the three outcome axes and validate every terminal workspace. +- [ ] [REVIEW_REFACTOR-3] Run the complete deterministic benchmark suite, manifest validation, diff checks, and clean release source/build verification. +- [ ] [REVIEW_REFACTOR-4] Deploy the same clean source ref and pass the five-cell direct canary plus fresh C01-C09 ready=9 without allocating a hybrid or scored run. +- [ ] Fill implementation-owned sections in CODE_REVIEW-*-G??.md with actual implementation notes and verification output. ## Review-Only Checklist -- [ ] Verify every R1-R6 regression exercises the exact production lifecycle path and remains fail-closed for unknown data. -- [ ] Verify no credential value, ambient caller config, direct provider call, manifest change, hidden retry, or provider reselection was introduced. -- [ ] Verify the prior run is immutable, exactly one plan-3 scored run exists, and all nine attempts plus web gates succeed. -- [ ] Verify clean build/deployment identities, 4/4 node and 8/8 provider health, ready=9, and report pointer consistency. -- [ ] Append final verdict and routing signals, then complete the mandated archive/next-state action. +> **[REVIEW AGENT ONLY]** Implementing agents must not modify or check this section. + +- [ ] Verify caller product error, parser/order error, process terminal, cleanup, and artifact result cannot overwrite one another. +- [ ] Verify no compatibility `success`/`terminal_reason` alias remains available to new consumers. +- [ ] Verify failure workspaces receive uniform automatic gates and screenshots when renderable. +- [ ] Verify scoring eligibility requires product, harness/process, and artifact gates independently. +- [ ] Verify R1-R6, manifest, fixture, rubric, routes, credentials, and retry policy were not weakened. +- [ ] Verify local tests are fresh, clean deployment source/build identities match, direct canary is 5/5, and ready=9 is fresh. +- [ ] Verify no hybrid or new nine-cell scored run was allocated and no old failed run was retried/rewritten. +- [ ] Append one verdict and verified `review_rework_count` / `evidence_integrity_failure` signals. +- [ ] Archive the active review to `code_review_cloud_G10_4.log` and plan to `plan_cloud_G10_4.log` only through the code-review skill. +- [ ] If PASS, write `complete.log`, preserve milestone-task metadata, and move the task directory to its dated archive path; otherwise write the required next filesystem state. + +## Deviations from Plan + +_Record any deviations and rationale here._ + +## Key Design Decisions + +_Record the exact closed vocabularies, precedence rules, schema version decisions, and compatibility decisions here._ + +## Reviewer Checkpoints + +- Product success comes only from one caller-declared typed success terminal plus its required finish/idle evidence. +- A valid caller-declared product error may coexist with harness `passed`; malformed/contradictory output yields product `unknown` and harness `failed`. +- Timeout, cancellation, nonzero/signal exit, not-started, and cleanup failure remain independently queryable. +- Artifact validation runs for every terminal workspace; renderer/workspace unavailability is explicit. +- CLI/report rows and scoring reasons expose the independent axes without a misleading aggregate success. +- Live verification stops after the direct canary and ready=9; hybrid/scored execution requires a later authorized state. + +## Verification Results + +### REVIEW_REFACTOR-1 focused caller/lifecycle suite + +```bash +python3 -m unittest \ + scripts.agent_benchmark.lifecycle_test \ + scripts.agent_benchmark.claude_iop_test \ + scripts.agent_benchmark.agy_iop_test \ + scripts.agent_benchmark.codex_iop_test \ + scripts.agent_benchmark.connectivity_integration_test +``` + +_Paste actual stdout/stderr and exit code._ + +### REVIEW_REFACTOR-2 projection/validation/scoring/report suite + +```bash +python3 -m unittest \ + scripts.agent_benchmark.attempts_test \ + scripts.agent_benchmark.measurement_test \ + scripts.agent_benchmark.web_validation_test \ + scripts.agent_benchmark.scoring_test \ + scripts.agent_benchmark.reporting_test \ + scripts.agent_benchmark.skill_contract_test +``` + +_Paste actual stdout/stderr and exit code._ + +### REVIEW_REFACTOR-3 complete deterministic verification + +```bash +python3 -m unittest discover -s scripts/agent_benchmark -p '*_test.py' +python3 scripts/agent_comparison_benchmark.py validate \ + --manifest scripts/fixtures/agent-comparison-benchmark-iop-one-shot.json +git diff --check +git status --short --branch +``` + +_Paste actual stdout/stderr, exit codes, commit/push identity, clean remote source state, build identities, and deployment health._ + +### REVIEW_REFACTOR-4 direct-first live convergence + +```bash +python3 scripts/agent_comparison_benchmark.py preflight \ + --manifest scripts/fixtures/agent-comparison-benchmark-direct-preflight.example.json +python3 scripts/agent_comparison_benchmark.py run \ + --manifest scripts/fixtures/agent-comparison-benchmark-direct-preflight.example.json +python3 scripts/agent_comparison_benchmark.py preflight \ + --manifest scripts/fixtures/agent-comparison-benchmark-iop-one-shot.json +``` + +_Paste sanitized exact CLI output, direct run id/status with five independent gates, ready=9 evidence, and proof that no hybrid or scored nine-cell run was allocated._ + +--- + +> **[IMPLEMENTING AGENT — BEFORE SAVING]** Fill every implementation-owned section, then leave active files in place. ## Section Ownership -Implementation status/evidence is implementation-owned. Review checklist/verdict is review-only. +| Section | Owner | Note | +|---|---|---| +| Header, Overview, Archive Evidence, Review Agent Instructions | Fixed | Do not modify | +| Implementation Item Completion and Implementation Checklist | Implementing agent checks status only | Text/order fixed | +| Review-Only Checklist | Review agent | Implementer must not modify | +| Deviations, Key Design Decisions | Implementing agent | Replace placeholders with actual content | +| Reviewer Checkpoints | Fixed | Plan-derived acceptance | +| Verification Results | Implementing agent, then reviewer | Record actual output; command changes require a deviation entry | +| Code Review Result | Review agent appends | Not present in stub | diff --git a/agent-task/m-iop-one-shot-agent-model-comparison/08+07_comparison_rerun/PLAN-cloud-G10.md b/agent-task/m-iop-one-shot-agent-model-comparison/08+07_comparison_rerun/PLAN-cloud-G10.md index d344dfdb..ef9ac2b7 100644 --- a/agent-task/m-iop-one-shot-agent-model-comparison/08+07_comparison_rerun/PLAN-cloud-G10.md +++ b/agent-task/m-iop-one-shot-agent-model-comparison/08+07_comparison_rerun/PLAN-cloud-G10.md @@ -1,113 +1,343 @@ - + -# Official caller terminal compatibility and evidence coherence +# Benchmark outcome boundaries and direct-first convergence ## For the Implementing Agent -Implement R1-R6 exactly as selected below. Do not alter benchmark inputs, route aliases, credentials, scoring, or retained run evidence. Use only the public benchmark CLI or its documented official-caller smoke path for live caller/provider execution. Run deterministic local qualification first, then clean-build and deploy the release, run non-scored direct/hybrid smokes, require ready=9, and execute exactly one fresh scored run. Fill the implementation-owned sections of `CODE_REVIEW-cloud-G10.md` with sanitized actual evidence and leave both active files in place for review. +Implement the selected result-boundary migration exactly as written. Preserve commit `58fdb322` and all earlier R1-R6 compatibility fixes; do not deploy or start another nine-cell scored run until the new gates pass. Run every verification command, fill the implementation-owned sections of `CODE_REVIEW-cloud-G10.md` with actual output, keep both active files in place, and report ready for review. If blocked, record only the exact blocker, attempted command/output, and resume condition in the review evidence; do not ask the user, call user-input tools, create control-plane stop files, classify the next state, archive logs, or write `complete.log`. ## Background -Plan 2 fixed the first live compatibility set and deployed clean release `04f7c39372ae526a51c5583926aefaf4bdc76394`. Public preflight was ready=9, but the single scored run `run-20260812T074548Z-d15500c16009` exposed three narrower boundary defects: Claude result terminal sequencing, official agy planner request compatibility plus error-terminal classification, and a supervisor exit-watcher/receipt race. The run is immutable and must not be retried or rewritten. +The stopped session completed and pushed R6 as `58fdb322`, then was interrupted while running the full Python benchmark suite. The current design still projects caller-declared product failure, parser/order failure, process failure, cleanup failure, timeout, and cancellation through one `terminal_reason` and one `success` boolean; attempt state, web validation, scoring, CLI summaries, and the report then reuse that mixed value. This packet replaces that overloaded boundary with three independent results—product execution, harness integrity, and artifact validation—and requires a five-cell direct canary before any hybrid or scored matrix execution. ## Archive Evidence Snapshot -- Prior plan: `agent-task/m-iop-one-shot-agent-model-comparison/08+07_comparison_rerun/plan_cloud_G10_2.log` -- Prior review: `agent-task/m-iop-one-shot-agent-model-comparison/08+07_comparison_rerun/code_review_cloud_G10_2.log` -- Verdict: `FAIL`; Required R1 Claude result-direct/API-error terminal projection, Required R2 agy top-level official JSONL, Required R3 timeout result/receipt exit-code race. -- Retained run evidence: C02 `out_of_order_event`; C03/C06/C08 `malformed_event`; C01 `timed_out` with lifecycle exit `143` and receipt exit `null`; C05/C09 successful. Public resume appended a ready=9 preflight but correctly rejected the inconsistent terminal and did not retry a failed cell. +- Previous active packet is preserved at `agent-task/m-iop-one-shot-agent-model-comparison/08+07_comparison_rerun/plan_cloud_G10_3.log` and `agent-task/m-iop-one-shot-agent-model-comparison/08+07_comparison_rerun/code_review_cloud_G10_3.log`; it had implemented R1-R6 but had not completed full local verification, deployment, public smoke, or a fresh scored run. +- R1-R6 remain selected, committed compatibility fixes. In particular, they cover Claude result-direct/API-error shapes, agy structured output and ERROR terminal handling, supervisor exit/receipt coherence, deterministic Plan rendering, the pinned Claude beta, and provider-independent Plan arrays. +- Retained run evidence already showed that a single lifecycle result conflates product and harness causes: valid caller errors became missing/malformed lifecycle evidence, parser/order defects became attempt failures, and web validation was skipped solely because lifecycle was non-success. +- The interrupted test left only `/tmp/iop-s0-interrupted-test-tmp7hc4t2o6`; no benchmark, unittest, deploy, or target Codex process remains active. ## Finding Resolution Map -| ID | Reviewer Evidence | Root Cause | Selected Fix | Mode | Acceptance | +| ID | Evidence | Root Cause | Selected Fix | Mode | Acceptance | |---|---|---|---|---|---| -| R1 | C02 emitted `idle` without `finish`; C06/C08 valid API-error terminals returned an empty tuple | Claude result handling does not distinguish result-direct completion and violates the no-observation parser contract | emit `finish,idle` for direct completion, `idle` after prior finish, and `None` for a valid API-error terminal; add lifecycle-backed tests | direct-fix | Claude adapter tests and retained-shape replay pass without out-of-order/malformed classification | -| R2 | C03's structurally redacted evidence initially showed only top-level keys, but the retained raw stream and caller log prove the payload is nested and the model request first failed with Edge `400 INVALID_ARGUMENT: request body is invalid` | Edge's pinned Gemini request decoder lacks the official agy planner's structured-output request fields; the adapter then maps the caller's nested `result.status=ERROR` to `malformed_event` instead of preserving the upstream/process failure | extend the Gemini bridge's exact structured-output request contract and tests; treat a valid agy ERROR result as non-success terminal evidence without parser failure | direct-fix | Gemini bridge structured-output regression and agy error lifecycle regression pass; non-scored official agy direct smoke succeeds | -| R3 | C01 result exit `143` differs from authenticated receipt exit `null` | supervisor snapshots returncode before joining the concurrent exit watcher | join watcher/IO, refresh the authoritative child returncode, then write the one receipt; add deterministic race/recovery regression | direct-fix | timeout/recovery test proves result and receipt exit/signal equality and public resume can reconcile | -| R4 | Post-deploy raw-free metrics recorded three `plan/validation` failures among six marked single-request calls; Claude hybrid did not complete before the non-scored 180-second boundary | Plan stage asks the provider for free-form PlanMD and rejects harmless format drift after the model call | request a stage-owned strict JSON object (`goal`, `steps`, `verification`), validate the bounded fields, and render the configured PlanMD template deterministically inside Edge | direct-fix | repeated plan-stage fixtures cannot create malformed PlanMD; official Claude hybrid smoke reaches a success terminal within the scored timeout budget | -| R5 | After the R4 release deployment, pinned Claude Code `2.1.228` returned `result/subtype=success` with `is_error=true` and `API Error: 400 unsupported anthropic-beta "advisor-tool-2026-03-01"` before either direct or hybrid model execution | the exact official caller now emits a compatibility beta absent from Edge's closed admission set | admit and consume `advisor-tool-2026-03-01` without forwarding it or granting any route/tool/workspace authority; retain rejection for every unknown beta | direct-fix | direct and marked-preset header regressions pass; official Claude direct and Gemini hybrid smokes terminate with `is_error=false` | -| R6 | After the R5 release deployment, official Claude `gpt-hybrid` reached the configured GPT Plan provider twice but returned `is_error=true`; raw-free Edge terminal evidence classified both Plan responses as `malformed` | R4 still makes the model serialize Markdown bullet prefixes and newline layout inside two JSON string fields, leaving a provider-format dependency after structured output succeeds | make `steps` and `verification` bounded arrays of non-empty one-line strings in the closed response schema, then let Edge own bullet prefixes and newline rendering | direct-fix | array-schema/template regressions pass; official Claude GPT hybrid terminates with `is_error=false` without a retry or route change | +| R7 | Claude API error and agy `status=ERROR` are structurally valid product failures, yet adapters currently emit no success terminal and lifecycle falls through to `nonzero_exit`/`missing_idle`; parser defects share the same reason field | parser output is an open union of strings, metrics, and tuples, and `InvocationResult.success` is computed from the harness terminal reason | replace string terminals with closed typed caller observations and freeze separate `ProductOutcome`, `HarnessOutcome`, and `ProcessOutcome` values in one invocation result | direct-fix | success, upstream product error, malformed stream, nonzero exit, timeout, cancellation, cleanup failure, duplicate and out-of-order cases preserve the correct independent axes | +| R8 | `attempt.json`, measurement, web validation, scoring eligibility, CLI summaries, skill prose, and Markdown report all treat lifecycle/attempt success as the one outcome | consumers copy the overloaded lifecycle reason instead of projecting independent gates | migrate durable schemas and consumers atomically; artifact validation runs against every terminal workspace and scoring requires product success + harness pass + artifact pass | direct-fix | deterministic fixtures expose separate product/harness/process/artifact columns and never label a parser-clean product error as harness failure or suppress artifact evidence because product failed | +| R9 | previous loops deployed after deterministic tests but discovered new direct-path caller variants only in scored execution | deployment qualification had no immutable direct-only all-caller gate and jumped from unit tests/ad-hoc smokes to the nine-cell scored run | after local and clean-build verification, deploy once, run the existing five-cell direct manifest as an unscored canary, require all five product/harness/artifact gates, then refresh the nine-cell preflight; hybrid execution and scored C01-C09 remain review-gated | direct-fix | no hybrid or scored run is allocated until direct canary 5/5, fresh ready=9, and source/build identity evidence all pass | -## Modified Files Summary +## Analysis +### Files Read + +- `scripts/agent_benchmark/lifecycle.py` +- `scripts/agent_benchmark/lifecycle_test.py` - `scripts/agent_benchmark/claude_iop.py` - `scripts/agent_benchmark/claude_iop_test.py` - `scripts/agent_benchmark/agy_iop.py` - `scripts/agent_benchmark/agy_iop_test.py` -- `scripts/agent_benchmark/lifecycle.py` -- `scripts/agent_benchmark/lifecycle_test.py` and/or the existing attempts recovery test module containing the closest deterministic oracle -- `apps/edge/internal/openai/gemini_types.go` -- `apps/edge/internal/openai/gemini_handler.go` -- `apps/edge/internal/openai/gemini_handler_test.go` -- `apps/edge/internal/openai/single_request_plan_stage.go` -- `apps/edge/internal/openai/single_request_plan_stage_test.go` -- `apps/edge/internal/openai/single_request_provider_stage.go` -- `apps/edge/internal/openai/single_request_executor_test.go` -- `packages/go/singlerequesttemplate/template.go` -- `packages/go/singlerequesttemplate/template_test.go` -- `apps/edge/internal/openai/anthropic_types.go` -- `apps/edge/internal/openai/anthropic_bridge_test.go` -- `apps/edge/internal/openai/single_request_handler_test.go` -- this active review plus existing run/preflight/report pointer files after successful fresh execution +- `scripts/agent_benchmark/codex_iop.py` +- `scripts/agent_benchmark/codex_iop_test.py` +- `scripts/agent_benchmark/live_iop.py` +- `scripts/agent_benchmark/connectivity_integration_test.py` +- `scripts/agent_benchmark/attempts.py` +- `scripts/agent_benchmark/attempts_test.py` +- `scripts/agent_benchmark/measurement.py` +- `scripts/agent_benchmark/measurement_test.py` +- `scripts/agent_benchmark/web_validation.py` +- `scripts/agent_benchmark/web_validation_test.py` +- `scripts/agent_benchmark/scoring.py` +- `scripts/agent_benchmark/scoring_test.py` +- `scripts/agent_benchmark/reporting.py` +- `scripts/agent_benchmark/reporting_test.py` +- `scripts/agent_comparison_benchmark.py` +- `scripts/agent_benchmark/skill_contract_test.py` +- `scripts/fixtures/agent-comparison-benchmark-report.expected.md` +- `agent-ops/skills/project/iop-agent-comparison-benchmark/SKILL.md` +- `docs/agent-comparison-benchmark-dev-guide.md` +- `agent-spec/testing/agent-comparison-benchmark.md` +- `agent-roadmap/sdd/knowledge-tool-optimization-extension/iop-one-shot-agent-model-comparison/SDD.md` +- `agent-test/local/rules.md` +- `agent-test/dev/rules.md` +- `agent-test/local/edge-smoke.md` +- `agent-test/local/testing-smoke.md` +- `agent-ops/rules/project/domain/testing/rules.md` +- `agent-ops/rules/project/domain/edge/rules.md` +- `agent-ops/skills/project/dev-runtime-deploy/SKILL.md` +- `agent-ops/skills/project/e2e-smoke/SKILL.md` +- `agent-task/m-iop-one-shot-agent-model-comparison/08+07_comparison_rerun/plan_cloud_G10_3.log` +- `agent-task/m-iop-one-shot-agent-model-comparison/08+07_comparison_rerun/code_review_cloud_G10_3.log` -## Implementation Checklist +### SDD Criteria -- [x] [REVIEW_REVIEW_TEST-1] Implement R1 Claude terminal projection and lifecycle-backed regressions. -- [x] [REVIEW_REVIEW_TEST-2] Implement R2 official agy structured-output request compatibility and error-terminal classification regressions. -- [x] [REVIEW_REVIEW_TEST-3] Implement R3 authoritative supervisor exit snapshot and timeout/recovery regression. -- [x] [REVIEW_REVIEW_TEST-4] Implement R4 schema-bound Plan fields and deterministic configured-template rendering. -- [x] [REVIEW_REVIEW_TEST-5] Implement R5 pinned Claude beta compatibility without forwarding or widening internal authority. -- [x] [REVIEW_REVIEW_TEST-6] Implement R6 provider-independent array fields and Edge-owned Plan bullet rendering. -- [ ] [REVIEW_REVIEW_TEST-7] Run the complete Python benchmark suite and relevant Go/runtime regressions from a clean tree. -- [ ] [REVIEW_REVIEW_TEST-8] Commit/push, merge the clean release branch, rebuild/deploy Edge and every Node, and verify 4/4 nodes plus 8/8 healthy providers. -- [ ] [REVIEW_REVIEW_TEST-9] Run public non-scored direct and hybrid smoke coverage and a fresh ready=9 preflight. -- [ ] [REVIEW_REVIEW_TEST-10] Execute exactly one fresh scored run, require success=9 and all web gates, update existing id pointers, and generate the human-readable Markdown report. -- [ ] Fill `CODE_REVIEW-cloud-G10.md` with exact sanitized commands, release identities, deployment health, run id, terminal summary, and report path. +- SDD: `agent-roadmap/sdd/knowledge-tool-optimization-extension/iop-one-shot-agent-model-comparison/SDD.md`; status `승인됨`, lock `해제`, user review `없음`. +- First-line milestone tasks: `claude-standalone`, `gemini-standalone`, `gpt-standalone`, `gemini-hybrid`, `gpt-hybrid`. +- Target scenarios: S04-S08 require each caller/direct/hybrid product result plus terminal/timing/usage evidence; S09 explicitly requires automatic validation for both success and failure workspaces; S10 requires scoring separate from automatic gates; S11-S12 require source-labelled measurements and a complete failure-inclusive report; S13-S14 retain official agy and managed credential constraints. +- Evidence Map rows S04-S08 and the common completion rule drive the product/harness separation; S09 drives failure-workspace artifact validation; S10-S12 drive separate eligibility/report projections. Therefore the checklist migrates the result schema and every consumer before any new live execution. -## Verification +### Verification Context -1. `python3 -m unittest discover -s scripts/agent_benchmark -p '*_test.py'` passes. -2. Targeted Claude/agy/lifecycle/attempt recovery regressions pass on the benchmark runner's supported Python. -3. Clean release validation/build tests pass and all deployed binaries report the same clean VCS revision. -4. Control Plane reports four connected nodes and eight available healthy providers. -5. Public non-scored direct/hybrid smokes succeed, then public preflight reports ready=9. -6. Exactly one plan-3 scored run reports success=9 with all required web validations, and the public report command writes Markdown. +- No neutral verification handoff was supplied. Repository-native evidence came from the source/tests above, the approved SDD, local/dev test rules, the retained plan/review pair, and direct inspection of the stopped session/process tree. +- Preconditions: HEAD and `origin/feature/iop-one-shot-agent-model-comparison` are both `58fdb322`; tracked worktree was clean before this plan write; no benchmark/test/deploy process remained; archived predecessor `07+06_caller_write_contracts/complete.log` satisfies index 07. +- Constraints: preserve immutable failed runs, fixture/manifest/rubric/routes/credentials/retry policy, R1-R6, and public caller paths; never print secrets; use only the benchmark CLI for live canaries/scored work. +- Gap and confidence: deterministic fixtures cover every selected semantic axis, but live caller/provider compatibility remains external and must be proven after a same-ref clean deployment. Confidence is high for the structural root cause and moderate for field convergence until the direct canary passes; hybrid compatibility stays an explicit later gate. -## Constraints and Exclusions +#### External Verification Preflight -- Never print, persist, or pass provider source keys to callers. Read the managed benchmark principal only from `token/.iop-bench` into a process-local variable. -- Do not mutate `agent-test/runs/bench-02/run-20260812T074548Z-d15500c16009` or use `--retry-failed` on it. -- Do not change manifest/task/rubric/checksums, route models/efforts/stages, retry policy, or failure classification to manufacture success. -- Do not add a retry, provider reselection, alternate route, or direct provider invocation. -- Parser compatibility must remain exact to the pinned official caller version and fail closed on unknown structures. +- Runner/workdir: `ssh toki@toki-labs.com`, `/Users/toki/agent-work/iop-dev`, as declared by `agent-test/dev/rules.md`; native dev-runtime, not the compose profile. +- Source state: fetch `origin/dev`, `origin/main`, and tags; require a clean checkout at the selected release ref and record branch, HEAD, dirty state, ancestry, and source sync before build. +- Artifacts/config: `build/dev-runtime/bin/edge`, mac/Linux ARM64/Windows AMD64 Node binaries, and `build/dev-runtime/edge.yaml`; require `go version -m`, SHA-256, config check, refresh help/dry-run, and identical source identity. +- Commands/runtime: record `python3`, `git`, `go`, `claude --version`, `agy --version`, `codex --version`, caller help surfaces, managed Edge/CP/Node processes and ports `18082`, `18083`, `18084`, `19093`, `19101`; confirm 4 connected Nodes and all expected provider snapshots. +- Hosts/OS: remote runner macOS/ARM64, GX10 Linux/ARM64, OneXPlayer and RTX5090 Windows/AMD64 per inventory. A mismatch requires the documented clean sync/rebuild/redeploy/restart step; do not reuse a stale binary. -## Analysis +### Test Coverage Gaps -### Outcome and Acceptance +- Typed caller observations: current tests cover caller-specific success variants and malformed input but do not assert a common closed product terminal type across all three callers. +- Independent outcomes: current lifecycle tests assert only `success` and `terminal_reason`; add Cartesian boundary tests for product error vs parser failure vs process/cleanup failure. +- Durable migration: current attempt/measurement validation hard-codes the old result field set and `success == terminal_reason == success`; add canonical round-trip and tamper tests for all three axes. +- Failure-workspace validation: current web test explicitly expects `not_run` for every non-success lifecycle; replace it with generated/static/browser evidence tests independent of product outcome. +- CLI/report contract: current summaries and golden report expose `execution` and `terminal`; add independent columns/counts and skill contract assertions. +- Live convergence: no deterministic test substitutes for exact current official caller/provider behavior; require the direct canary in this packet and keep hybrid execution as a later reviewed gate before scored execution. -The task is complete only when deterministic regressions prove the six boundary corrections, the clean release is deployed across the managed dev runtime, public readiness is 9/9, and one fresh immutable scored run is 9/9 with a generated Markdown report. +### Symbol References -### Scope and Ownership - -The caller adapters own their exact JSONL interpretation and redaction. The Gemini ingress owns the exact official agy request-to-Chat conversion. The lifecycle supervisor owns the authoritative process exit projection shared by its result and receipt. Deployment and benchmark commands only consume these contracts; they do not reinterpret them. +- Replace parser string terminals `"finish"`, `"idle"`, and `"malformed"` at `ClaudeStreamParser`, `AgyEventParser`, `CodexJSONLParser`, lifecycle test parser lambdas, and connectivity fixtures with closed observation values. +- Replace `InvocationResult.success`, `InvocationResult.terminal_reason`, and `InvocationResult.finish_then_idle_then_quiet` call sites found in `agy_iop.py`, `live_iop.py`, `attempts.py`, `measurement.py`, `web_validation.py`, `scoring.py`, `reporting.py`, their tests, and CLI projections. Do not retain compatibility properties that let new consumers silently collapse the axes again. +- `AttemptMeasurement.terminal_reason` and report `execution/terminal` projections are schema migrations; update all constructor, serialization, loader, tamper, golden fixture, and skill-documentation references in the modified-file boundary. +- Top-level attempt state changes from an overloaded success result to controller lifecycle only: `running | completed | timed_out | cancelled | interrupted`. Retry and unresolved decisions consume the nested product/harness/process/artifact gates, not the state name. ### Split Judgment -Keep one plan because the scored-run acceptance depends on one indivisible evidence invariant: every caller terminal must project through the shared lifecycle into a coherent immutable attempt before the same release can be accepted. Splitting would require multiple scored reruns or accept an invalid intermediate evidence state. +Keep one atomic plan. The stable persisted attempt invariant is: one invocation publishes one coherent product outcome, harness outcome, process outcome, measurement, artifact outcome, eligibility decision, CLI status, and report row. Splitting the producer types from durable validation/consumer migration would either require compatibility aliases that preserve the defect or create an invalid intermediate schema; live rollout is included as integration evidence only after the atomic deterministic migration passes. -### Routing +### Scope Rationale -- finalizer=`finalize-task-policy.sh`, mode=`pair` -- build=`grade-boundary/cloud/G10`; review=`official-review/cloud/G10` -- risks=`temporal_state,concurrent_consistency,boundary_contract,structured_interpretation,variant_product` -- `review_rework_count=3`; `evidence_integrity_failure=true` +- No Edge/Node product code, outer API contract, route binding, credential plane, provider retry, manifest, fixture, rubric, or SDD decision changes are allowed; R1-R6 already own those compatibility fixes. +- Historical run trees and `preflight_id.log`/`run_id.log` remain immutable. A new scored run is explicitly excluded from implementation ownership; after canaries and ready=9, the reviewer verifies the gate and decides the next authorized state. +- Common Agent-Ops rules/skills remain untouched. Only the project benchmark skill is updated because its public CLI outcome contract changes. + +### Final Routing + +- `evaluation_mode=first-pass`; all build/review closures are true, with no capability gap. +- finalizer=`finalize-task-policy.sh pair`; build=`grade-boundary/cloud/G10`, catalog=`worker/cloud/G10`, filename=`PLAN-cloud-G10.md`; review=`official-review/cloud/G10`, catalog=`review/cloud/G10`, filename=`CODE_REVIEW-cloud-G10.md`. +- Grade scores: build `2/2/2/2/2`, review `2/2/2/2/2` for scope/state/blast/evidence/verification. +- `large_indivisible_context=true`; positive loop risks=`temporal_state,concurrent_consistency,boundary_contract,structured_interpretation,variant_product` (5). +- Recovery signals: `review_rework_count=3`, `evidence_integrity_failure=true`; both risk and recovery boundaries match, while the route basis remains grade-boundary. + +## Implementation Checklist + +- [ ] [REVIEW_REFACTOR-1] Replace caller/parser string terminals and the overloaded lifecycle result with closed typed caller, product, harness, and process outcomes. +- [ ] [REVIEW_REFACTOR-2] Migrate attempt, measurement, web validation, scoring, CLI, report, project skill, guide, and living spec to preserve the three outcome axes and validate every terminal workspace. +- [ ] [REVIEW_REFACTOR-3] Run the complete deterministic benchmark suite, manifest validation, diff checks, and clean release source/build verification. +- [ ] [REVIEW_REFACTOR-4] Deploy the same clean source ref and pass the five-cell direct canary plus fresh C01-C09 ready=9 without allocating a hybrid or scored run. +- [ ] Fill implementation-owned sections in CODE_REVIEW-*-G??.md with actual implementation notes and verification output. + +### [REVIEW_REFACTOR-1] Typed invocation outcomes + +**Problem:** `scripts/agent_benchmark/lifecycle.py:262-334,1552-1654,1718-1839` admits `str | ParsedMetric | tuple` parser values, treats unknown values as harness reasons, and computes `success` from the same reason used for process/cleanup. `claude_iop.py:280-361`, `agy_iop.py:439-496`, and `codex_iop.py:314-336` encode valid product errors by omitting terminal evidence or raising parser errors. + +**Solution:** Introduce frozen closed observations in `lifecycle.py`: `CallerEvent(kind=finish|idle)`, `CallerTerminal(status=succeeded|failed, reason=)`, plus existing `ParsedMetric`. Parser return type becomes `CallerObservation | tuple[CallerObservation, ...] | None`; raw strings are rejected. Freeze three results in `InvocationResult`: + +```python +# before: scripts/agent_benchmark/lifecycle.py:313-334 +class InvocationResult: + success: bool + terminal_reason: str + exit_code: Optional[int] + signal: Optional[int] + finish_then_idle_then_quiet: bool + +# after +class ProductOutcome: + status: str # succeeded | failed | unknown + reason: str # caller_success | caller_error | unavailable + +class HarnessOutcome: + status: str # passed | failed + reason: str # success | parser_error | malformed_event | ... + ordered_terminal: bool + cleanup_complete: bool + +class ProcessOutcome: + status: str # exited | signalled | timed_out | cancelled | not_started + exit_code: Optional[int] + signal: Optional[int] + +class InvocationResult: + product: ProductOutcome + harness: HarnessOutcome + process: ProcessOutcome + ... +``` + +The exact reason vocabularies must be constants validated at construction and serialization. A valid caller failure emits `CallerTerminal(failed, caller_error)` and may still have a clean parser/order/cleanup; malformed or contradictory caller output sets `product=unknown`, `harness=failed`; timeout/cancel/launch/cleanup remain process/harness results and cannot fabricate a product failure. Product success requires exactly one succeeded caller terminal plus the caller-specific finish/idle sequence; process exit alone never creates product success. Update each official caller adapter to emit the same types and keep metrics independently. + +**Modified Files and Checklist:** + +- [ ] `scripts/agent_benchmark/lifecycle.py`: add closed types/vocabularies, typed parser application, independent result construction, and canonical evidence records. +- [ ] `scripts/agent_benchmark/claude_iop.py`: emit typed finish/idle and success/API-error terminal observations. +- [ ] `scripts/agent_benchmark/agy_iop.py`: emit typed success/ERROR terminals; reserve parser failure for malformed structure. +- [ ] `scripts/agent_benchmark/codex_iop.py`: emit typed turn success and bridge idle; explicitly type unsuccessful terminal turns. +- [ ] `scripts/agent_benchmark/live_iop.py`: bind metrics and connectivity using typed outcomes without collapsing them. +- [ ] `scripts/agent_benchmark/lifecycle_test.py`, `claude_iop_test.py`, `agy_iop_test.py`, `codex_iop_test.py`, `connectivity_integration_test.py`: replace constructors/expectations and add independent-axis regressions. + +**Test Strategy:** Add named tests `test_product_error_can_have_clean_harness_and_process`, `test_parser_failure_leaves_product_unknown`, `test_timeout_cancel_and_cleanup_do_not_fabricate_product`, and caller-specific valid-error/success tests. Use local fake executables and current JSONL fixtures; no network. + +**Verification:** + +```bash +python3 -m unittest \ + scripts.agent_benchmark.lifecycle_test \ + scripts.agent_benchmark.claude_iop_test \ + scripts.agent_benchmark.agy_iop_test \ + scripts.agent_benchmark.codex_iop_test \ + scripts.agent_benchmark.connectivity_integration_test +``` + +Expected: exit 0; success and product-error fixtures have harness `passed`, malformed fixtures have product `unknown` and harness `failed`, and timeout/cancel/process failure retain their own process status. + +### [REVIEW_REFACTOR-2] Durable projections, artifact gate, and reporting + +**Problem:** `attempts.py:1192-1214,1368-1464,1608-1870`, `measurement.py:478-541,675-716`, `web_validation.py:491-579`, `scoring.py:943-964`, `agent_comparison_benchmark.py:145-176`, and `reporting.py:349-494` reuse lifecycle success as execution, validation eligibility, scoring eligibility, and report truth. This violates S09 for failed workspaces and makes the report unable to distinguish product defects from harness defects. + +**Solution:** Bump the internal lifecycle/journal, attempt-result, measurement, web-validation, and report fixture schema versions where defined; new writers emit only the new version and never rewrite historical evidence. A versioned read-only legacy decoder may project an old proven success as product succeeded/harness passed, but every legacy non-success must keep product `unknown` and preserve only the harness/process fact actually evidenced—never infer a product failure. Persist exact nested `product`, `harness`, and `process` objects in lifecycle result/terminal, measurement, and attempt record. Make top-level attempt state controller-only: `running | completed | timed_out | cancelled | interrupted`; retry/unresolved logic reads the independent nested gates. Run generated/static/browser validation for every terminal workspace, including product/harness failure, and mark only genuinely unavailable renderer/workspace evidence as `not_run`/`blocked`. Scoring eligibility requires product `succeeded`, harness `passed`, process acceptable, and web `passed`, with a separate reason for each failed gate. + +Replace the CLI and report projection with explicit fields: + +```text +product=succeeded|failed|unknown +harness=passed|failed +process=exited|signalled|timed_out|cancelled|not_started +artifact=passed|failed|blocked|not_run +scoring=scored|unscored|scoring_failed|blocked +``` + +`run` exit 0 still requires the latest attempt of every slot to have product succeeded, harness passed, acceptable process cleanup, and artifact passed. `status` and Markdown must display each axis; no aggregate named `success` may hide which axis failed. Update the project skill, dev guide, golden report, and living spec to document the same exact contract. + +**Modified Files and Checklist:** + +- [ ] `scripts/agent_benchmark/attempts.py`, `attempts_test.py`: migrate canonical attempt/result validation, terminal transitions, status projection, tamper and recovery tests. +- [ ] `scripts/agent_benchmark/measurement.py`, `measurement_test.py`: persist all three outcomes and their sources. +- [ ] `scripts/agent_benchmark/web_validation.py`, `web_validation_test.py`: validate every terminal workspace independent of product/harness status. +- [ ] `scripts/agent_benchmark/scoring.py`, `scoring_test.py`: use explicit gate reasons and invoke evaluator only when all eligibility axes pass. +- [ ] `scripts/agent_benchmark/reporting.py`, `reporting_test.py`, `scripts/fixtures/agent-comparison-benchmark-report.expected.md`: render independent outcome columns and failure-inclusive artifact evidence. +- [ ] `scripts/agent_comparison_benchmark.py`, `scripts/agent_benchmark/skill_contract_test.py`, `agent-ops/skills/project/iop-agent-comparison-benchmark/SKILL.md`: align public summaries and exact output contract. +- [ ] `docs/agent-comparison-benchmark-dev-guide.md`, `agent-spec/testing/agent-comparison-benchmark.md`: document the new source of truth and direct-first gate. +- [ ] `agent-task/m-iop-one-shot-agent-model-comparison/08+07_comparison_rerun/CODE_REVIEW-cloud-G10.md`: record actual migration and verification evidence. + +**Test Strategy:** Add canonical round-trip/tamper tests for each nested result, web validation of a failed-product workspace that still produces screenshots/gates, scoring exclusions for each independent axis, CLI status count assertions, and a golden report row for product failure with harness pass plus parser failure with product unknown. + +**Verification:** + +```bash +python3 -m unittest \ + scripts.agent_benchmark.attempts_test \ + scripts.agent_benchmark.measurement_test \ + scripts.agent_benchmark.web_validation_test \ + scripts.agent_benchmark.scoring_test \ + scripts.agent_benchmark.reporting_test \ + scripts.agent_benchmark.skill_contract_test +``` + +Expected: exit 0; durable tampering is rejected, failed-product workspaces receive artifact evidence, evaluator invocation occurs only for all-pass attempts, and the golden report shows the separate axes. + +### [REVIEW_REFACTOR-3] Deterministic and clean-build qualification + +**Problem:** The stopped session was interrupted during the complete suite, so commit `58fdb322` and this migration have no complete fresh deterministic result. A live deployment before this gate would repeat the prior loop. + +**Solution:** Run the entire Python suite from the clean feature tree, validate the fixed manifest, run diff checks, commit/push the migration, then follow `dev-runtime-deploy` through clean remote sync, sequential relevant Go tests, rebuild of all four binaries, config checks, and source/build identity capture. Do not start a caller canary until every binary is proven to contain the same clean ref. + +**Modified Files and Checklist:** + +- [ ] `agent-task/m-iop-one-shot-agent-model-comparison/08+07_comparison_rerun/CODE_REVIEW-cloud-G10.md`: record full local output and clean release/build identities without secrets. + +**Test Strategy:** No new test file; this item executes the complete existing suite fresh (`unittest` has no result cache) and validates the actual manifest plus release build boundary. + +**Verification:** + +```bash +python3 -m unittest discover -s scripts/agent_benchmark -p '*_test.py' +python3 scripts/agent_comparison_benchmark.py validate \ + --manifest scripts/fixtures/agent-comparison-benchmark-iop-one-shot.json +git diff --check +git status --short --branch +``` + +Expected: all tests and validation exit 0, `git diff --check` is silent, and only the intentional plan/review plus implementation changes exist before commit; after commit/push, the remote release clean-build procedure proves matching source and binary identities. + +### [REVIEW_REFACTOR-4] Direct-first live convergence gate + +**Problem:** The prior packet planned ad-hoc direct/hybrid smoke after deployment but did not define a single deterministic gate whose failure prevents scored execution. Consequently new exact caller variants were discovered inside the costly nine-cell run. + +**Solution:** After the same-ref deployment and 4/4 Node plus provider health checks, use only the public benchmark CLI with the existing `scripts/fixtures/agent-comparison-benchmark-direct-preflight.example.json` five-cell direct manifest. Run `preflight`, then exactly one unscored canary `run`; require five latest attempts with product succeeded, harness passed, acceptable process, artifact passed, no running/interrupted state, and no secret leakage. Only after the direct canary passes may the one-shot manifest receive a fresh `preflight` with ready=9. Stop there: do not call any hybrid canary or the scored nine-cell `run` in this implementation packet; those start only from a later reviewed/authorized packet. + +**Modified Files and Checklist:** + +- [ ] `agent-task/m-iop-one-shot-agent-model-comparison/08+07_comparison_rerun/CODE_REVIEW-cloud-G10.md`: record sanitized deployment, five direct cell outcomes, hybrid caller/route outcomes, and ready=9 preflight id/output. + +**Test Strategy:** External integration evidence is mandatory because current official CLI/provider versions are the compatibility boundary. The direct canary is unscored, uses fresh workspaces/sessions, and creates append-only run evidence; no retry or route substitution is allowed. Hybrid/scored execution is deliberately deferred rather than represented by an undefined ad-hoc command. + +**Verification:** + +```bash +python3 scripts/agent_comparison_benchmark.py preflight \ + --manifest scripts/fixtures/agent-comparison-benchmark-direct-preflight.example.json +python3 scripts/agent_comparison_benchmark.py run \ + --manifest scripts/fixtures/agent-comparison-benchmark-direct-preflight.example.json +python3 scripts/agent_comparison_benchmark.py preflight \ + --manifest scripts/fixtures/agent-comparison-benchmark-iop-one-shot.json +``` + +Expected: direct preflight is ready=5, the direct canary returns product/harness/artifact 5/5 with no unresolved attempt, and final one-shot preflight is ready=9. No hybrid or nine-cell scored run id is allocated. + +## Modified Files Summary + +| File | Items | +|---|---| +| `scripts/agent_benchmark/lifecycle.py` | REVIEW_REFACTOR-1 | +| `scripts/agent_benchmark/lifecycle_test.py` | REVIEW_REFACTOR-1 | +| `scripts/agent_benchmark/claude_iop.py` | REVIEW_REFACTOR-1 | +| `scripts/agent_benchmark/claude_iop_test.py` | REVIEW_REFACTOR-1 | +| `scripts/agent_benchmark/agy_iop.py` | REVIEW_REFACTOR-1 | +| `scripts/agent_benchmark/agy_iop_test.py` | REVIEW_REFACTOR-1 | +| `scripts/agent_benchmark/codex_iop.py` | REVIEW_REFACTOR-1 | +| `scripts/agent_benchmark/codex_iop_test.py` | REVIEW_REFACTOR-1 | +| `scripts/agent_benchmark/live_iop.py` | REVIEW_REFACTOR-1 | +| `scripts/agent_benchmark/connectivity_integration_test.py` | REVIEW_REFACTOR-1 | +| `scripts/agent_benchmark/attempts.py` | REVIEW_REFACTOR-2 | +| `scripts/agent_benchmark/attempts_test.py` | REVIEW_REFACTOR-2 | +| `scripts/agent_benchmark/measurement.py` | REVIEW_REFACTOR-2 | +| `scripts/agent_benchmark/measurement_test.py` | REVIEW_REFACTOR-2 | +| `scripts/agent_benchmark/web_validation.py` | REVIEW_REFACTOR-2 | +| `scripts/agent_benchmark/web_validation_test.py` | REVIEW_REFACTOR-2 | +| `scripts/agent_benchmark/scoring.py` | REVIEW_REFACTOR-2 | +| `scripts/agent_benchmark/scoring_test.py` | REVIEW_REFACTOR-2 | +| `scripts/agent_benchmark/reporting.py` | REVIEW_REFACTOR-2 | +| `scripts/agent_benchmark/reporting_test.py` | REVIEW_REFACTOR-2 | +| `scripts/fixtures/agent-comparison-benchmark-report.expected.md` | REVIEW_REFACTOR-2 | +| `scripts/agent_comparison_benchmark.py` | REVIEW_REFACTOR-2 | +| `scripts/agent_benchmark/skill_contract_test.py` | REVIEW_REFACTOR-2 | +| `agent-ops/skills/project/iop-agent-comparison-benchmark/SKILL.md` | REVIEW_REFACTOR-2 | +| `docs/agent-comparison-benchmark-dev-guide.md` | REVIEW_REFACTOR-2 | +| `agent-spec/testing/agent-comparison-benchmark.md` | REVIEW_REFACTOR-2 | +| `agent-task/m-iop-one-shot-agent-model-comparison/08+07_comparison_rerun/CODE_REVIEW-cloud-G10.md` | REVIEW_REFACTOR-2, REVIEW_REFACTOR-3, REVIEW_REFACTOR-4 | ## Dependencies and Execution Order -R1-R6 are independent code fixes but must all pass local regressions before one clean release is built. Deployment must finish before non-scored smokes; smokes and ready=9 must pass before the only fresh scored run. +- Predecessor `07+06_caller_write_contracts` is satisfied by `agent-task/archive/2026/08/m-iop-one-shot-agent-model-comparison/07+06_caller_write_contracts/complete.log`. +- Execute items 1 and 2 as one schema migration, then item 3. Item 4 starts only after item 3 and same-ref deployment pass. The direct canary precedes ready=9. Hybrid and scored execution are outside this packet. -## Final Routing +## Final Verification -- finalizer=`finalize-task-policy.sh pair` -- build=`grade-boundary/cloud/G10`; review=`official-review/cloud/G10` -- catalog routes=`worker/cloud/G10`, `review/cloud/G10` +```bash +python3 -m unittest discover -s scripts/agent_benchmark -p '*_test.py' +python3 scripts/agent_comparison_benchmark.py validate \ + --manifest scripts/fixtures/agent-comparison-benchmark-iop-one-shot.json +git diff --check +git status --short --branch +``` + +Expected local outcome: all tests/validation pass, diff check is silent, and repository state contains only intentional changes. Cached test output is not accepted; rerun fresh. + +After clean same-ref build/deploy, record 4/4 connected Nodes, expected healthy providers, direct preflight ready=5, direct product/harness/artifact 5/5, and final ready=9. Verify no hybrid or scored C01-C09 `run` was allocated by this packet. + +After completing all code changes, fill implementation-owned sections in `CODE_REVIEW-*-G??.md`. diff --git a/agent-task/m-iop-one-shot-agent-model-comparison/08+07_comparison_rerun/code_review_cloud_G10_3.log b/agent-task/m-iop-one-shot-agent-model-comparison/08+07_comparison_rerun/code_review_cloud_G10_3.log new file mode 100644 index 00000000..f3426709 --- /dev/null +++ b/agent-task/m-iop-one-shot-agent-model-comparison/08+07_comparison_rerun/code_review_cloud_G10_3.log @@ -0,0 +1,57 @@ + + +# Code Review Reference - REVIEW_REVIEW_TEST + +## Overview + +date=2026-08-12 +task=m-iop-one-shot-agent-model-comparison/08+07_comparison_rerun, plan=3, tag=REVIEW_REVIEW_TEST + +## Archive Evidence Snapshot + +- Prior plan: `agent-task/m-iop-one-shot-agent-model-comparison/08+07_comparison_rerun/plan_cloud_G10_2.log` +- Prior review: `agent-task/m-iop-one-shot-agent-model-comparison/08+07_comparison_rerun/code_review_cloud_G10_2.log` +- Verdict: `FAIL`; Required R1 Claude result-direct/API-error terminal projection, Required R2 agy top-level official JSONL, Required R3 timeout result/receipt exit-code race. +- Retained run evidence: C02 `out_of_order_event`; C03/C06/C08 `malformed_event`; C01 `timed_out` with lifecycle exit `143` and receipt exit `null`; C05/C09 successful. Public resume appended a ready=9 preflight but correctly rejected the inconsistent terminal and did not retry a failed cell. + +## Implementation Item Completion + +| Item | Status | +|---|---| +| REVIEW_REVIEW_TEST-1 Claude terminal projection | [x] | +| REVIEW_REVIEW_TEST-2 agy request/error compatibility | [x] | +| REVIEW_REVIEW_TEST-3 lifecycle exit coherence | [x] | +| REVIEW_REVIEW_TEST-4 deterministic Plan rendering | [x] | +| REVIEW_REVIEW_TEST-5 pinned Claude beta compatibility | [x] | +| REVIEW_REVIEW_TEST-6 provider-independent Plan arrays | [x] | +| REVIEW_REVIEW_TEST-7 complete local verification | [ ] | +| REVIEW_REVIEW_TEST-8 clean release deployment | [ ] | +| REVIEW_REVIEW_TEST-9 public smokes/preflight | [ ] | +| REVIEW_REVIEW_TEST-10 scored run/report | [ ] | + +## Implementation Evidence + +- `python3 -m unittest scripts.agent_benchmark.claude_iop_test scripts.agent_benchmark.agy_iop_test scripts.agent_benchmark.lifecycle_test` passed 55 tests after R1-R3 implementation. +- `python3 -m unittest discover -s scripts/agent_benchmark -p '*_test.py'` passed 437 tests. +- `go test -count=1 ./apps/edge/internal/openai` passed with official agy structured-output request coverage. +- Sequential `go test -count=1` passed every package returned by `go list ./apps/control-plane/... ./apps/edge/... ./apps/node/... ./packages/go/...`, excluding only the declared `agenttask` boundary. +- Sanitized replay of retained C02/C06/C08 streams now projects C02 as `finish,idle` and both synthetic API-error cells as no success terminal without parser failure. +- Post-deploy official agy direct and `gemini-hybrid` smokes reached `result.status=SUCCESS`; Claude direct reached `result/subtype=success`. The first Claude hybrid smoke did not repeat the immediate API error, but raw-free Edge metrics exposed three Plan validation failures and the call exceeded the 180-second non-scored boundary, creating R4 before scored execution. +- R4 replaces free-form PlanMD generation with a strict stage-owned `goal`/`steps`/`verification` JSON schema, exact closed-object decoding (including duplicate-key rejection), bounded field validation, and deterministic rendering of the frozen operator template. Focused Plan/template/executor regressions and the complete `apps/edge/internal/openai` package pass. +- The current feature tree passes all 437 Python benchmark tests, every relevant Go package under Control Plane/Edge/Node/shared runtime (excluding only the declared `agenttask` boundary), `git diff --check`, and manifest validation. Clean release-tree repetition remains pending commit/merge. +- After deploying release `1f748bd9bd7fcc843074847a57ed523a28db75fd`, official Claude Code `2.1.228` direct smoke exposed R5 before any scored run: its nominal `result/subtype=success` carried `is_error=true` and a sanitized unsupported-beta 400 for `advisor-tool-2026-03-01`. The scored-run allowance remains unused. +- R5 adds the exact pinned-caller beta to the closed admission inventory and consumes it at the Chat bridge without forwarding it or creating capability authority. Direct Claude Code mapping, marked single-request admission, and unknown-beta rejection regressions pass. +- After the R5 release, official Claude GPT hybrid reached the configured `gpt-5.6-terra` Plan provider twice but returned a caller-visible server error. Raw-free Edge evidence classified both Plan terminals as malformed, proving routing and credential selection succeeded while the remaining R4 Markdown-in-JSON string shape failed provider-independent decoding. +- R6 changes only the private Plan response contract: `steps` and `verification` are bounded arrays of non-empty one-line strings, while Edge deterministically adds Markdown bullets and newlines. It does not change routes, credentials, retries, templates, or any caller-visible schema. + +## Review-Only Checklist + +- [ ] Verify every R1-R6 regression exercises the exact production lifecycle path and remains fail-closed for unknown data. +- [ ] Verify no credential value, ambient caller config, direct provider call, manifest change, hidden retry, or provider reselection was introduced. +- [ ] Verify the prior run is immutable, exactly one plan-3 scored run exists, and all nine attempts plus web gates succeed. +- [ ] Verify clean build/deployment identities, 4/4 node and 8/8 provider health, ready=9, and report pointer consistency. +- [ ] Append final verdict and routing signals, then complete the mandated archive/next-state action. + +## Section Ownership + +Implementation status/evidence is implementation-owned. Review checklist/verdict is review-only. diff --git a/agent-task/m-iop-one-shot-agent-model-comparison/08+07_comparison_rerun/plan_cloud_G10_3.log b/agent-task/m-iop-one-shot-agent-model-comparison/08+07_comparison_rerun/plan_cloud_G10_3.log new file mode 100644 index 00000000..d344dfdb --- /dev/null +++ b/agent-task/m-iop-one-shot-agent-model-comparison/08+07_comparison_rerun/plan_cloud_G10_3.log @@ -0,0 +1,113 @@ + + +# Official caller terminal compatibility and evidence coherence + +## For the Implementing Agent + +Implement R1-R6 exactly as selected below. Do not alter benchmark inputs, route aliases, credentials, scoring, or retained run evidence. Use only the public benchmark CLI or its documented official-caller smoke path for live caller/provider execution. Run deterministic local qualification first, then clean-build and deploy the release, run non-scored direct/hybrid smokes, require ready=9, and execute exactly one fresh scored run. Fill the implementation-owned sections of `CODE_REVIEW-cloud-G10.md` with sanitized actual evidence and leave both active files in place for review. + +## Background + +Plan 2 fixed the first live compatibility set and deployed clean release `04f7c39372ae526a51c5583926aefaf4bdc76394`. Public preflight was ready=9, but the single scored run `run-20260812T074548Z-d15500c16009` exposed three narrower boundary defects: Claude result terminal sequencing, official agy planner request compatibility plus error-terminal classification, and a supervisor exit-watcher/receipt race. The run is immutable and must not be retried or rewritten. + +## Archive Evidence Snapshot + +- Prior plan: `agent-task/m-iop-one-shot-agent-model-comparison/08+07_comparison_rerun/plan_cloud_G10_2.log` +- Prior review: `agent-task/m-iop-one-shot-agent-model-comparison/08+07_comparison_rerun/code_review_cloud_G10_2.log` +- Verdict: `FAIL`; Required R1 Claude result-direct/API-error terminal projection, Required R2 agy top-level official JSONL, Required R3 timeout result/receipt exit-code race. +- Retained run evidence: C02 `out_of_order_event`; C03/C06/C08 `malformed_event`; C01 `timed_out` with lifecycle exit `143` and receipt exit `null`; C05/C09 successful. Public resume appended a ready=9 preflight but correctly rejected the inconsistent terminal and did not retry a failed cell. + +## Finding Resolution Map + +| ID | Reviewer Evidence | Root Cause | Selected Fix | Mode | Acceptance | +|---|---|---|---|---|---| +| R1 | C02 emitted `idle` without `finish`; C06/C08 valid API-error terminals returned an empty tuple | Claude result handling does not distinguish result-direct completion and violates the no-observation parser contract | emit `finish,idle` for direct completion, `idle` after prior finish, and `None` for a valid API-error terminal; add lifecycle-backed tests | direct-fix | Claude adapter tests and retained-shape replay pass without out-of-order/malformed classification | +| R2 | C03's structurally redacted evidence initially showed only top-level keys, but the retained raw stream and caller log prove the payload is nested and the model request first failed with Edge `400 INVALID_ARGUMENT: request body is invalid` | Edge's pinned Gemini request decoder lacks the official agy planner's structured-output request fields; the adapter then maps the caller's nested `result.status=ERROR` to `malformed_event` instead of preserving the upstream/process failure | extend the Gemini bridge's exact structured-output request contract and tests; treat a valid agy ERROR result as non-success terminal evidence without parser failure | direct-fix | Gemini bridge structured-output regression and agy error lifecycle regression pass; non-scored official agy direct smoke succeeds | +| R3 | C01 result exit `143` differs from authenticated receipt exit `null` | supervisor snapshots returncode before joining the concurrent exit watcher | join watcher/IO, refresh the authoritative child returncode, then write the one receipt; add deterministic race/recovery regression | direct-fix | timeout/recovery test proves result and receipt exit/signal equality and public resume can reconcile | +| R4 | Post-deploy raw-free metrics recorded three `plan/validation` failures among six marked single-request calls; Claude hybrid did not complete before the non-scored 180-second boundary | Plan stage asks the provider for free-form PlanMD and rejects harmless format drift after the model call | request a stage-owned strict JSON object (`goal`, `steps`, `verification`), validate the bounded fields, and render the configured PlanMD template deterministically inside Edge | direct-fix | repeated plan-stage fixtures cannot create malformed PlanMD; official Claude hybrid smoke reaches a success terminal within the scored timeout budget | +| R5 | After the R4 release deployment, pinned Claude Code `2.1.228` returned `result/subtype=success` with `is_error=true` and `API Error: 400 unsupported anthropic-beta "advisor-tool-2026-03-01"` before either direct or hybrid model execution | the exact official caller now emits a compatibility beta absent from Edge's closed admission set | admit and consume `advisor-tool-2026-03-01` without forwarding it or granting any route/tool/workspace authority; retain rejection for every unknown beta | direct-fix | direct and marked-preset header regressions pass; official Claude direct and Gemini hybrid smokes terminate with `is_error=false` | +| R6 | After the R5 release deployment, official Claude `gpt-hybrid` reached the configured GPT Plan provider twice but returned `is_error=true`; raw-free Edge terminal evidence classified both Plan responses as `malformed` | R4 still makes the model serialize Markdown bullet prefixes and newline layout inside two JSON string fields, leaving a provider-format dependency after structured output succeeds | make `steps` and `verification` bounded arrays of non-empty one-line strings in the closed response schema, then let Edge own bullet prefixes and newline rendering | direct-fix | array-schema/template regressions pass; official Claude GPT hybrid terminates with `is_error=false` without a retry or route change | + +## Modified Files Summary + +- `scripts/agent_benchmark/claude_iop.py` +- `scripts/agent_benchmark/claude_iop_test.py` +- `scripts/agent_benchmark/agy_iop.py` +- `scripts/agent_benchmark/agy_iop_test.py` +- `scripts/agent_benchmark/lifecycle.py` +- `scripts/agent_benchmark/lifecycle_test.py` and/or the existing attempts recovery test module containing the closest deterministic oracle +- `apps/edge/internal/openai/gemini_types.go` +- `apps/edge/internal/openai/gemini_handler.go` +- `apps/edge/internal/openai/gemini_handler_test.go` +- `apps/edge/internal/openai/single_request_plan_stage.go` +- `apps/edge/internal/openai/single_request_plan_stage_test.go` +- `apps/edge/internal/openai/single_request_provider_stage.go` +- `apps/edge/internal/openai/single_request_executor_test.go` +- `packages/go/singlerequesttemplate/template.go` +- `packages/go/singlerequesttemplate/template_test.go` +- `apps/edge/internal/openai/anthropic_types.go` +- `apps/edge/internal/openai/anthropic_bridge_test.go` +- `apps/edge/internal/openai/single_request_handler_test.go` +- this active review plus existing run/preflight/report pointer files after successful fresh execution + +## Implementation Checklist + +- [x] [REVIEW_REVIEW_TEST-1] Implement R1 Claude terminal projection and lifecycle-backed regressions. +- [x] [REVIEW_REVIEW_TEST-2] Implement R2 official agy structured-output request compatibility and error-terminal classification regressions. +- [x] [REVIEW_REVIEW_TEST-3] Implement R3 authoritative supervisor exit snapshot and timeout/recovery regression. +- [x] [REVIEW_REVIEW_TEST-4] Implement R4 schema-bound Plan fields and deterministic configured-template rendering. +- [x] [REVIEW_REVIEW_TEST-5] Implement R5 pinned Claude beta compatibility without forwarding or widening internal authority. +- [x] [REVIEW_REVIEW_TEST-6] Implement R6 provider-independent array fields and Edge-owned Plan bullet rendering. +- [ ] [REVIEW_REVIEW_TEST-7] Run the complete Python benchmark suite and relevant Go/runtime regressions from a clean tree. +- [ ] [REVIEW_REVIEW_TEST-8] Commit/push, merge the clean release branch, rebuild/deploy Edge and every Node, and verify 4/4 nodes plus 8/8 healthy providers. +- [ ] [REVIEW_REVIEW_TEST-9] Run public non-scored direct and hybrid smoke coverage and a fresh ready=9 preflight. +- [ ] [REVIEW_REVIEW_TEST-10] Execute exactly one fresh scored run, require success=9 and all web gates, update existing id pointers, and generate the human-readable Markdown report. +- [ ] Fill `CODE_REVIEW-cloud-G10.md` with exact sanitized commands, release identities, deployment health, run id, terminal summary, and report path. + +## Verification + +1. `python3 -m unittest discover -s scripts/agent_benchmark -p '*_test.py'` passes. +2. Targeted Claude/agy/lifecycle/attempt recovery regressions pass on the benchmark runner's supported Python. +3. Clean release validation/build tests pass and all deployed binaries report the same clean VCS revision. +4. Control Plane reports four connected nodes and eight available healthy providers. +5. Public non-scored direct/hybrid smokes succeed, then public preflight reports ready=9. +6. Exactly one plan-3 scored run reports success=9 with all required web validations, and the public report command writes Markdown. + +## Constraints and Exclusions + +- Never print, persist, or pass provider source keys to callers. Read the managed benchmark principal only from `token/.iop-bench` into a process-local variable. +- Do not mutate `agent-test/runs/bench-02/run-20260812T074548Z-d15500c16009` or use `--retry-failed` on it. +- Do not change manifest/task/rubric/checksums, route models/efforts/stages, retry policy, or failure classification to manufacture success. +- Do not add a retry, provider reselection, alternate route, or direct provider invocation. +- Parser compatibility must remain exact to the pinned official caller version and fail closed on unknown structures. + +## Analysis + +### Outcome and Acceptance + +The task is complete only when deterministic regressions prove the six boundary corrections, the clean release is deployed across the managed dev runtime, public readiness is 9/9, and one fresh immutable scored run is 9/9 with a generated Markdown report. + +### Scope and Ownership + +The caller adapters own their exact JSONL interpretation and redaction. The Gemini ingress owns the exact official agy request-to-Chat conversion. The lifecycle supervisor owns the authoritative process exit projection shared by its result and receipt. Deployment and benchmark commands only consume these contracts; they do not reinterpret them. + +### Split Judgment + +Keep one plan because the scored-run acceptance depends on one indivisible evidence invariant: every caller terminal must project through the shared lifecycle into a coherent immutable attempt before the same release can be accepted. Splitting would require multiple scored reruns or accept an invalid intermediate evidence state. + +### Routing + +- finalizer=`finalize-task-policy.sh`, mode=`pair` +- build=`grade-boundary/cloud/G10`; review=`official-review/cloud/G10` +- risks=`temporal_state,concurrent_consistency,boundary_contract,structured_interpretation,variant_product` +- `review_rework_count=3`; `evidence_integrity_failure=true` + +## Dependencies and Execution Order + +R1-R6 are independent code fixes but must all pass local regressions before one clean release is built. Deployment must finish before non-scored smokes; smokes and ready=9 must pass before the only fresh scored run. + +## Final Routing + +- finalizer=`finalize-task-policy.sh pair` +- build=`grade-boundary/cloud/G10`; review=`official-review/cloud/G10` +- catalog routes=`worker/cloud/G10`, `review/cloud/G10` diff --git a/docs/agent-comparison-benchmark-dev-guide.md b/docs/agent-comparison-benchmark-dev-guide.md index dc21359c..45161186 100644 --- a/docs/agent-comparison-benchmark-dev-guide.md +++ b/docs/agent-comparison-benchmark-dev-guide.md @@ -358,7 +358,19 @@ TMPDIR="$credential_smoke_parent" make test-credential-slot-smoke rmdir "$credential_smoke_parent" ``` -### 11.2 Public preflight +### 11.2 Direct-first qualification + +동일 clean source ref를 모든 runtime binary에 배포하고 4/4 Node와 provider health를 확인한 다음, 먼저 기존 5-cell direct manifest를 사용한다. + +```bash +direct_manifest="scripts/fixtures/agent-comparison-benchmark-direct-preflight.example.json" +python3 scripts/agent_comparison_benchmark.py preflight --manifest "$direct_manifest" +python3 scripts/agent_comparison_benchmark.py run --manifest "$direct_manifest" +``` + +direct canary는 unscored이며 각 최신 slot의 `product=succeeded`, `harness=passed`, `process=exited`/exit 0/no signal, `artifact=passed`가 모두 5/5여야 한다. `running`, `interrupted`, `unresolved`는 0이어야 한다. 실패 시 hybrid 또는 9-cell scored run을 할당하지 않는다. + +### 11.3 Public nine-cell preflight 10절의 환경을 같은 shell에 준비한 뒤 실행한다. @@ -384,7 +396,9 @@ preflight는 다음을 함께 확인한다. `registration_required` 또는 `implementation_gap`이면 즉시 중단한다. alias, model, effort, route나 caller를 대체하지 않는다. preflight-only run root는 evidence이므로 삭제하지 않는다. -### 11.3 Scored execution +direct canary 5/5 뒤 fresh one-shot preflight가 `ready=9`인지 확인하고 멈춘다. 이 qualification 단계에서는 hybrid canary나 C01-C09 `run`을 호출하지 않는다. + +### 11.4 Scored execution fresh preflight와 명시적 실행 권한이 있는 현재 plan에서만 다음 명령을 한 번 호출한다. @@ -398,7 +412,7 @@ python3 scripts/agent_comparison_benchmark.py run --manifest "$benchmark_manifes - CLI가 run id를 출력하지 않으면 임의 id나 성공 pointer를 만들지 않는다. - caller나 provider를 CLI 밖에서 별도로 호출해 scored result를 보충하지 않는다. -### 11.4 Status +### 11.5 Status ```bash python3 scripts/agent_comparison_benchmark.py status \ @@ -408,14 +422,15 @@ python3 scripts/agent_comparison_benchmark.py status \ 현재 comparison execution 완료 조건: -- `success + failed + timed_out + cancelled = 9` +- controller terminal 수 `completed + timed_out + cancelled + interrupted = 9` - `running = 0` - `interrupted = 0` - 각 cell/repetition에 retained terminal attempt가 존재 +- 최신 attempt에서 `product_succeeded=9`, `harness_passed=9`, `process_exited=9`, `artifact_passed=9`, `unresolved=0` -실패/timed-out/cancelled는 terminal evidence지만 성공 결과가 아니므로 scoring eligibility와 최종 비교에서 별도로 표시된다. +`completed`는 controller 종료만 뜻하며 product 성공을 뜻하지 않는다. product/harness/process/artifact 실패는 scoring eligibility와 최종 비교에서 각각 별도로 표시된다. -### 11.5 Blind scoring +### 11.6 Blind scoring 유효한 execution run에 대해서만 수행한다. @@ -427,9 +442,9 @@ python3 scripts/agent_comparison_benchmark.py score \ evaluator는 Codex→`gpt-5.6-luna` xhigh direct route다. identity가 제거된 blind workspace만 보며 source cell identity mapping은 blind tree 밖에 유지한다. -automatic gate 실패나 lifecycle failure는 `unscored`이고 0점으로 바꾸지 않는다. `scoring_failed`도 명시적 `--retry-scoring-failed` 권한 없이 재시도하지 않는다. +product, harness, acceptable process 또는 artifact gate 실패는 각각의 reason을 가진 `unscored`이고 0점으로 바꾸지 않는다. `scoring_failed`도 명시적 `--retry-scoring-failed` 권한 없이 재시도하지 않는다. -### 11.6 Report +### 11.7 Report ```bash python3 scripts/agent_comparison_benchmark.py report \ @@ -445,11 +460,11 @@ report는 run root의 immutable evidence를 읽어 idempotent `report.md`를 만 | 범주 | 내용 | |---|---| -| lifecycle | submission, first output, finish, idle, exit, quiet, cleanup와 terminal reason | +| lifecycle | typed caller terminal, submission, first output, finish, idle, quiet와 독립 product/harness/process 결과 | | timeline | submitted, first output, first workspace write observation/mtime, total duration | | usage | input/output/reasoning/cache read/cache write/total tokens, model/tool calls와 duration | | workspace | fresh session identity, fixture checksum, testbed provenance, generated file tree | -| web validation | generated files, static safety, images, network, console, responsive, accessibility | +| web validation | product/harness 결과와 무관하게 모든 terminal workspace에서 생성되는 generated files, static safety, images, network, console, responsive, accessibility | | screenshots | desktop `1920x1080`, mobile `375x812` | | scoring | eligibility, blind allocation, rubric worksheet, score status | @@ -503,7 +518,7 @@ quality rubric: | preflight not ready | blocker를 해결하고 fresh preflight한다. attempt를 할당하지 않는다. | | caller launch 전 interruption | retained evidence를 보존한다. run tree를 직접 수정하지 않는다. | | lifecycle/parser failure | exact retained output으로 source 원인을 수정하고 deterministic regression을 추가한다. | -| terminal failed/timed_out/cancelled | evidence로 보존한다. 암묵 retry하지 않는다. | +| completed이지만 product/harness/process/artifact gate 실패 또는 timed_out/cancelled | 각 축 evidence로 보존한다. 암묵 retry하지 않는다. | | state가 `running`이지만 process가 없음 | manual JSON 수정/삭제/reconcile을 하지 않는다. reviewer evidence로 남기고 승인된 새 plan에서만 다음 상태를 결정한다. | | scoring_failed | 0점 처리하지 않는다. 명시적 retry 권한 없이는 중단한다. | | report unavailable | run evidence를 수정하거나 report를 수작업 생성하지 않는다. | diff --git a/scripts/agent_benchmark/agy_iop.py b/scripts/agent_benchmark/agy_iop.py index 60d8ac4e..dede2900 100644 --- a/scripts/agent_benchmark/agy_iop.py +++ b/scripts/agent_benchmark/agy_iop.py @@ -27,8 +27,14 @@ from scripts.agent_benchmark.connectivity import ( make_result, ) from scripts.agent_benchmark.lifecycle import ( + CALLER_REASON_ERROR, + CALLER_REASON_SUCCESS, + CALLER_STATUS_FAILED, + CALLER_STATUS_SUCCEEDED, COMPLETION_EXIT_AFTER_IDLE, SUBMISSION_ARGV_TASK, + CallerEvent, + CallerTerminal, InvocationResult, InvocationSpec, LifecycleMetricError, @@ -436,50 +442,53 @@ class AgyEventParser: self._result_seen = False self._latest_usage: dict[str, Any] | None = None - def __call__(self, stream: str, raw_line: str) -> str | ParsedMetric | tuple[Any, ...] | None: + def __call__(self, stream: str, raw_line: str) -> Any: return self.parse(stream, raw_line) - def parse(self, stream: str, raw_line: str) -> str | ParsedMetric | tuple[Any, ...] | None: + def parse(self, stream: str, raw_line: str) -> Any: if stream != "stdout": return None try: item = json.loads(raw_line) except (TypeError, json.JSONDecodeError): - return "malformed" + raise AgyAdapterError("malformed agy event") if not isinstance(item, dict): - return "malformed" + raise AgyAdapterError("malformed agy event") event = item.get("event") payload = item.get(event) if isinstance(event, str) else None if not isinstance(payload, dict): - return "malformed" + raise AgyAdapterError("malformed agy event") if event == "init": if self._init_seen or self._result_seen: - return "malformed" + raise AgyAdapterError("malformed agy event") self._init_seen = True return None if event == "step_update": if not self._init_seen or self._result_seen: - return "malformed" + raise AgyAdapterError("malformed agy event") usage = payload.get("usage") if usage is not None: if self._usage_metrics(usage) is None: - return "malformed" + raise AgyAdapterError("malformed agy event") self._latest_usage = usage return None if event != "result" or not self._init_seen or self._result_seen: - return "malformed" + raise AgyAdapterError("malformed agy event") self._result_seen = True if payload.get("status") != "SUCCESS": - # A structurally valid caller error is not a stream mismatch. It - # emits no success terminal and lets the process exit (normally - # non-zero) remain the lifecycle authority. - return None + if payload.get("status") != "ERROR": + raise AgyAdapterError("malformed agy result status") + return ( + CallerTerminal(CALLER_STATUS_FAILED, CALLER_REASON_ERROR), + CallerEvent("finish"), + CallerEvent("idle"), + ) metrics: list[ParsedMetric] = [] usage = payload.get("usage", self._latest_usage) if usage is not None: parsed_usage = self._usage_metrics(usage) if parsed_usage is None: - return "malformed" + raise AgyAdapterError("malformed agy usage") metrics.extend(parsed_usage) try: if "duration_seconds" in payload: @@ -492,8 +501,12 @@ class AgyEventParser: "model_calls", payload["num_turns"], model=self._cell.iop.request_model, )) except LifecycleMetricError: - return "malformed" - return tuple(metrics) + ("finish", "idle") + raise AgyAdapterError("malformed agy metric") + return tuple(metrics) + ( + CallerTerminal(CALLER_STATUS_SUCCEEDED, CALLER_REASON_SUCCESS), + CallerEvent("finish"), + CallerEvent("idle"), + ) def _usage_metrics(self, usage: Any) -> tuple[ParsedMetric, ...] | None: if not isinstance(usage, dict) or not set(usage) <= set(_AGY_USAGE_METRICS): @@ -515,8 +528,9 @@ class AgyEventParser: caller_capability = CallerCapability(AGY_CALLER, capability.route_kinds, capability.efforts) if ( not isinstance(lifecycle, InvocationResult) - or not lifecycle.success - or not lifecycle.finish_then_idle_then_quiet + or lifecycle.product.status != CALLER_STATUS_SUCCEEDED + or lifecycle.harness.status != "passed" + or not lifecycle.harness.ordered_terminal or not self._result_seen ): return make_result(self._cell, caller_capability, requested, closed_gap) diff --git a/scripts/agent_benchmark/agy_iop_test.py b/scripts/agent_benchmark/agy_iop_test.py index 12cbf211..a5920ab8 100644 --- a/scripts/agent_benchmark/agy_iop_test.py +++ b/scripts/agent_benchmark/agy_iop_test.py @@ -216,8 +216,8 @@ class AgyIopTest(unittest.TestCase): parser = AgyEventParser(_cell(), _binding()) fixture = Path("scripts/fixtures/agent-comparison-benchmark/agy-iop-stream.jsonl") result = self._run_lines(fixture.read_text(encoding="utf-8").splitlines(), parser) - self.assertTrue(result.success) - self.assertTrue(result.finish_then_idle_then_quiet) + self.assertTrue(result.product.status == "succeeded") + self.assertTrue(result.harness.ordered_terminal) metrics = {metric.name: metric for metric in result.metrics} self.assertEqual( set(metrics), @@ -242,8 +242,8 @@ class AgyIopTest(unittest.TestCase): ): parser = AgyEventParser(_cell(), _binding()) invocation = self._run_lines(lines, parser) - self.assertFalse(invocation.success) - self.assertEqual(invocation.terminal_reason, reason) + self.assertFalse(invocation.product.status == "succeeded") + self.assertEqual(invocation.harness.reason, "parser_error") parser = AgyEventParser(_cell(), _binding()) error_result = json.dumps({ @@ -252,8 +252,10 @@ class AgyIopTest(unittest.TestCase): invocation = self._run_lines([ json.dumps({"event": "init", "init": {}}), error_result, ], parser, exit_code=1) - self.assertFalse(invocation.success) - self.assertEqual(invocation.terminal_reason, REASON_NONZERO_EXIT) + self.assertFalse(invocation.product.status == "succeeded") + self.assertEqual(invocation.product.status, "failed") + self.assertEqual(invocation.harness.status, "passed") + self.assertEqual(invocation.process.exit_code, 1) def test_latest_step_usage_is_used_only_when_result_omits_usage(self) -> None: parser = AgyEventParser(_cell(), _binding()) @@ -263,7 +265,7 @@ class AgyIopTest(unittest.TestCase): json.dumps({"event": "result", "result": {"status": "SUCCESS", "duration_seconds": 0.2, "num_turns": 1, "response": "private"}}), ] invocation = self._run_lines(lines, parser) - self.assertTrue(invocation.success) + self.assertTrue(invocation.product.status == "succeeded") self.assertEqual({m.name: m.value for m in invocation.metrics}["total_tokens"], 3) def test_malformed_usage_fails_without_partial_metric(self) -> None: @@ -272,8 +274,8 @@ class AgyIopTest(unittest.TestCase): '{"event":"init","init":{}}', '{"event":"result","result":{"status":"SUCCESS","usage":{"input_tokens":"1"}}}', ], parser) - self.assertFalse(invocation.success) - self.assertEqual(invocation.terminal_reason, REASON_MALFORMED_EVENT) + self.assertFalse(invocation.product.status == "succeeded") + self.assertEqual(invocation.harness.reason, "parser_error") self.assertEqual(invocation.metrics, ()) def test_structural_redaction_excludes_response_tools_endpoint_and_secret(self) -> None: diff --git a/scripts/agent_benchmark/attempts.py b/scripts/agent_benchmark/attempts.py index 702535b5..9f4a32ea 100644 --- a/scripts/agent_benchmark/attempts.py +++ b/scripts/agent_benchmark/attempts.py @@ -35,16 +35,25 @@ from scripts.agent_benchmark.connectivity import ( ) from scripts.agent_benchmark.lifecycle import ( + CALLER_REASON_SUCCESS, COMPLETION_MODES, + EVENT_CALLER_TERMINAL, EVENT_FINISH, EVENT_IDLE, EVENT_QUIET, EVENT_SUBMITTED, InvocationResult, + JOURNAL_VERSION, + HARNESS_REASONS, + HARNESS_STATUSES, + PROCESS_STATUSES, + PRODUCT_REASONS, + PRODUCT_STATUSES, LifecycleRecoveryError, RECEIPT_VERSION, REASON_CONTROLLER_LOST, REASON_RECOVERED_STOP, + SOURCE_CALLER_OUTPUT, SOCKET_FILENAME, SUBMISSION_MODES, SupervisorLocator, @@ -82,8 +91,9 @@ PREFLIGHT_RE = re.compile(r"^preflight-([0-9]{6})\.json$") DIGEST_RE = re.compile(r"^sha256:[0-9a-f]{64}$") PREFLIGHT_SCHEMA_VERSION = "1" PREFLIGHT_STATUSES = ("ready", "registration_required", "implementation_gap") -TERMINAL_STATES = frozenset(("success", "failed", "timed_out", "cancelled", "interrupted")) +TERMINAL_STATES = frozenset(("completed", "timed_out", "cancelled", "interrupted")) NONTERMINAL_STATE = "running" +ATTEMPT_RESULT_VERSION = 2 MEASUREMENT_POLICY_REQUIRED_V1 = "required-v1" MEASUREMENT_POLICIES = frozenset((MEASUREMENT_POLICY_REQUIRED_V1,)) MEASUREMENT_POLICY_FILENAME = "attempt-measurement-policy.json" @@ -95,6 +105,7 @@ WEB_VALIDATION_POLICY_FILENAME = "web-validation-policy.json" WEB_VALIDATION_POLICY_RECORD = "web-validation-policy" WEB_VALIDATION_POLICY_VERSION = 1 SUCCESS_EVIDENCE_KINDS = (EVENT_SUBMITTED, EVENT_FINISH, EVENT_IDLE, EVENT_QUIET) +BOUND_EVIDENCE_KINDS = SUCCESS_EVIDENCE_KINDS + (EVENT_CALLER_TERMINAL,) CONTROL_ALIAS_PREFIX = "iop-bench-attempt-" CONTROL_DIRECTORY_NAME = "control" CONTROL_ALIAS_DIGEST_HEX_LENGTH = 24 @@ -108,6 +119,65 @@ RECEIPT_ONLY_TERMINAL_REASONS = frozenset( ) +def _outcome_records(result: InvocationResult) -> dict[str, dict[str, Any]]: + return { + "product": { + "status": result.product.status, + "reason": result.product.reason, + }, + "harness": { + "status": result.harness.status, + "reason": result.harness.reason, + "ordered_terminal": result.harness.ordered_terminal, + "cleanup_complete": result.harness.cleanup_complete, + }, + "process": { + "status": result.process.status, + "exit_code": result.process.exit_code, + "signal": result.process.signal, + }, + } + + +def _unknown_terminal( + reason: str, *, process_status: str = "not_started", + exit_code: int | None = None, signal: int | None = None, +) -> dict[str, dict[str, Any]]: + return { + "product": {"status": "unknown", "reason": "unavailable"}, + "harness": { + "status": "failed", + "reason": reason, + "ordered_terminal": False, + "cleanup_complete": True, + }, + "process": { + "status": process_status, + "exit_code": exit_code, + "signal": signal, + }, + } + + +def _terminal_reason(terminal: Mapping[str, Any]) -> str: + harness = terminal.get("harness") + return str(harness.get("reason") if isinstance(harness, Mapping) else "") + + +def _terminal_passed(terminal: Mapping[str, Any]) -> bool: + process = terminal.get("process") + return ( + isinstance(terminal.get("product"), Mapping) + and terminal["product"].get("status") == "succeeded" + and isinstance(terminal.get("harness"), Mapping) + and terminal["harness"].get("status") == "passed" + and isinstance(process, Mapping) + and process.get("status") == "exited" + and process.get("exit_code") == 0 + and process.get("signal") is None + ) + + class AttemptError(Exception): """Base error whose message is safe to present to a benchmark caller.""" @@ -699,6 +769,7 @@ class RunStore: @staticmethod def _expected_record(run: RunIdentity, identity: AttemptIdentity, state: str) -> dict[str, Any]: return { + "attempt_result_version": ATTEMPT_RESULT_VERSION, "run_id": run.run_id, "manifest_digest": run.manifest_digest, "cell_id": identity.cell_id, @@ -759,7 +830,7 @@ class RunStore: if record["state"] in TERMINAL_STATES: lifecycle = record.get("lifecycle") expected_receipt_reason = ( - lifecycle.get("terminal_reason") + _terminal_reason(lifecycle) if isinstance(lifecycle, dict) else None ) @@ -773,13 +844,13 @@ class RunStore: expected_receipt_reason=expected_receipt_reason, measurement_policy=policy, ) - if "lifecycle" in record and (not isinstance(record["lifecycle"], dict) or set(record["lifecycle"]) != {"terminal_reason"} or not isinstance(record["lifecycle"]["terminal_reason"], str)): - raise AttemptStateError("attempt lifecycle is invalid") + if "lifecycle" in record: + self._validate_outcomes(record["lifecycle"], "attempt lifecycle") if record["state"] in TERMINAL_STATES: pre_registration_interrupted = ( record["state"] == "interrupted" and "locator" not in record - and record.get("lifecycle") == {"terminal_reason": "interrupted"} + and record.get("lifecycle") == _unknown_terminal("interrupted") ) self._validate_web_validation( root, @@ -828,6 +899,33 @@ class RunStore: for attempt in self.attempts(bound_run, slot) ) + def attempt_outcomes(self, attempt: Attempt) -> dict[str, Any]: + """Return the independently validated gates for one terminal attempt.""" + run, root = self._bound_attempt(attempt) + record = self._attempt_record(root, run, attempt.identity) + if record is None or record["state"] not in TERMINAL_STATES: + raise AttemptStateError("attempt outcomes require terminal state") + lifecycle = record.get("lifecycle") + if not isinstance(lifecycle, Mapping): + raise AttemptStateError("attempt lifecycle is unavailable") + try: + artifact = load_web_validation(root, manifest=self.open_manifest_snapshot(run)).status + except WebValidationError: + # The only permitted terminal without a web record is a controller + # interruption before caller registration. It remains unresolved. + artifact = "not_run" + return { + "product": lifecycle["product"]["status"], + "harness": lifecycle["harness"]["status"], + "process": lifecycle["process"]["status"], + "artifact": artifact, + "passed": ( + record["state"] == "completed" + and _terminal_passed(lifecycle) + and artifact == "passed" + ), + } + def allocate(self, run: RunIdentity, slot: Slot) -> Attempt: """Create the exclusive, deliberately empty attempt root.""" existing = self.attempts(run, slot) @@ -885,7 +983,7 @@ class RunStore: raise AttemptStateError("attempt web validation policy is invalid") record["web_validation_policy"] = web_validation_policy if reason is not None: - record["lifecycle"] = {"terminal_reason": reason} + record["lifecycle"] = _unknown_terminal(reason) return record @staticmethod @@ -1022,11 +1120,8 @@ class RunStore: _read_regular_bytes(root / MEASUREMENT_FILENAME, "measurement")).hexdigest() if record["measurement_digest"] != digest or measurement.run_id != run.run_id: raise AttemptStateError("attempt web validation measurement is invalid") - if ( - (measurement.terminal_reason == "success" and web.status == "not_run") - or (measurement.terminal_reason != "success" and web.status != "not_run") - ): - raise AttemptStateError("attempt web validation lifecycle is invalid") + if web.status == "not_run": + raise AttemptStateError("terminal workspace web validation was not run") def open_manifest_snapshot(self, run: RunIdentity) -> Manifest: """Load the exact immutable run manifest used by recovery and evidence.""" @@ -1065,7 +1160,7 @@ class RunStore: if ( policy == WEB_VALIDATION_POLICY_REQUIRED_V1 and record.get("locator") is None - and terminal.get("terminal_reason") == "interrupted" + and _terminal_reason(terminal) == "interrupted" ): path = root / WEB_VALIDATION_FILENAME measurement = root / MEASUREMENT_FILENAME @@ -1189,8 +1284,20 @@ class RunStore: run, root = self._bound_attempt(attempt) record = self._attempt_record(root, run, attempt.identity, absent_ok=True) if record is None: - reason = str((result or {}).get("terminal_reason") or state) - _write_new(root / "attempt.json", _json_bytes(self._initial_record(run, attempt, state, reason=reason))) + terminal_result = result or _unknown_terminal(state) + self._validate_outcomes(terminal_result, "attempt terminal") + if (root / WEB_VALIDATION_FILENAME).exists() or ( + root / WEB_VALIDATION_FILENAME + ).is_symlink(): + self._validate_web_validation( + root, run, attempt.identity, policy=None + ) + initial = self._expected_record(run, attempt.identity, state) + initial["lifecycle"] = { + name: dict(terminal_result[name]) + for name in ("product", "harness", "process") + } + _write_new(root / "attempt.json", _json_bytes(initial)) return Attempt(attempt.identity, attempt.root, state) if record["state"] in TERMINAL_STATES: if record["state"] != state: @@ -1198,7 +1305,8 @@ class RunStore: return Attempt(attempt.identity, attempt.root, state) if record["state"] != NONTERMINAL_STATE: raise AttemptStateError("attempt transition is invalid") - terminal_result = result or {"terminal_reason": state} + terminal_result = result or _unknown_terminal(state) + self._validate_outcomes(terminal_result, "attempt terminal") self._ensure_required_web_validation( root, run, @@ -1208,7 +1316,8 @@ class RunStore: ) record["state"] = state record["lifecycle"] = { - "terminal_reason": str(terminal_result.get("terminal_reason") or state) + name: dict(terminal_result[name]) + for name in ("product", "harness", "process") } _replace(root / "attempt.json", _json_bytes(record)) return Attempt(attempt.identity, attempt.root, state) @@ -1280,7 +1389,7 @@ class RunStore: ) if actual != expected or measurement.spec_digest != expected_digest: raise AttemptStateError("attempt measurement identity is invalid") - if expected_reason is not None and measurement.terminal_reason != expected_reason: + if expected_reason is not None and measurement.harness.reason != expected_reason: raise AttemptStateError("attempt measurement terminal is invalid") if lifecycle is not None: try: @@ -1366,15 +1475,26 @@ class RunStore: @staticmethod def _state_for_reason(reason: str) -> str: - if reason == "success": - return "success" if reason == "timed_out": return "timed_out" if reason == "cancelled": return "cancelled" if reason in RECEIPT_ONLY_TERMINAL_REASONS: return "interrupted" - return "failed" + return "completed" + + @staticmethod + def _state_for_terminal(terminal: Mapping[str, Any]) -> str: + process = terminal.get("process") + status = process.get("status") if isinstance(process, Mapping) else None + if status == "timed_out": + return "timed_out" + if status == "cancelled": + return "cancelled" + reason = _terminal_reason(terminal) + if reason in RECEIPT_ONLY_TERMINAL_REASONS or reason == "interrupted": + return "interrupted" + return "completed" def _read_json_file(self, root: Path, name: str) -> dict[str, Any]: path = root / name @@ -1398,19 +1518,18 @@ class RunStore: def _validate_result_record(self, result: dict[str, Any], locator: SupervisorLocator, expected_digest: str) -> None: fields = { - "record", "success", "terminal_reason", "exit_code", "signal", "submitted", - "finish_then_idle_then_quiet", "cleanup_complete", "process_group_alive", + "record", "product", "harness", "process", "submitted", + "process_group_alive", "submission_mode", "completion_mode", "spec_digest", "locator", "started_at", "ended_at", "duration_ns", "stdout", "stderr", "events", } self._exact_fields(result, fields, "lifecycle result") - required_bools = ("success", "submitted", "finish_then_idle_then_quiet", "cleanup_complete", "process_group_alive") + self._validate_outcomes(result, "lifecycle result") + required_bools = ("submitted", "process_group_alive") if result["record"] != "result" or any(not isinstance(result[key], bool) for key in required_bools): raise AttemptStateError("lifecycle result is invalid") - if result["terminal_reason"] not in TERMINAL_REASONS or result["spec_digest"] != expected_digest: + if result["spec_digest"] != expected_digest: raise AttemptStateError("lifecycle result identity is invalid") - if not self._optional_int(result["exit_code"]) or not self._optional_int(result["signal"]): - raise AttemptStateError("lifecycle result is invalid") if result["submission_mode"] not in SUBMISSION_MODES or result["completion_mode"] not in COMPLETION_MODES: raise AttemptStateError("lifecycle result is invalid") if not all(isinstance(result[key], str) for key in ("started_at", "ended_at")) or not isinstance(result["duration_ns"], int) or isinstance(result["duration_ns"], bool) or result["duration_ns"] < 0: @@ -1428,11 +1547,52 @@ class RunStore: checked = self._exact_fields(event, {"record", "kind", "source", "stream", "monotonic_ns", "source_monotonic_ns", "observed_at", "detail"}, "lifecycle event") if checked["record"] != "event" or not all(isinstance(checked[key], str) for key in ("kind", "source", "stream", "observed_at", "detail")) or not all(isinstance(checked[key], int) and not isinstance(checked[key], bool) and checked[key] >= 0 for key in ("monotonic_ns", "source_monotonic_ns")): raise AttemptStateError("lifecycle event is invalid") - if not result["cleanup_complete"] or result["process_group_alive"] or result["success"] != (result["terminal_reason"] == "success"): + if ( + result["process_group_alive"] + or (not result["harness"]["cleanup_complete"]) + != (result["harness"]["reason"] == "cleanup_failed") + ): raise AttemptStateError("lifecycle terminal outcome is invalid") - if result["success"] and not result["finish_then_idle_then_quiet"]: + if result["harness"]["status"] == "passed" and not result["harness"]["ordered_terminal"]: raise AttemptStateError("lifecycle terminal outcome is invalid") + def _validate_outcomes(self, value: Any, label: str) -> None: + if not isinstance(value, Mapping): + raise AttemptStateError(f"{label} schema is invalid") + product = self._exact_fields( + value.get("product"), {"status", "reason"}, f"{label} product" + ) + harness = self._exact_fields( + value.get("harness"), + {"status", "reason", "ordered_terminal", "cleanup_complete"}, + f"{label} harness", + ) + process = self._exact_fields( + value.get("process"), {"status", "exit_code", "signal"}, + f"{label} process", + ) + if ( + product["status"] not in PRODUCT_STATUSES + or product["reason"] not in PRODUCT_REASONS + or (product["status"] == "succeeded") != (product["reason"] == CALLER_REASON_SUCCESS) + or (product["status"] == "failed") != (product["reason"] == "caller_error") + or harness["status"] not in HARNESS_STATUSES + or harness["reason"] not in HARNESS_REASONS + or not isinstance(harness["ordered_terminal"], bool) + or not isinstance(harness["cleanup_complete"], bool) + or (harness["status"] == "passed") != (harness["reason"] == "success") + or (not harness["cleanup_complete"]) + != (harness["reason"] == "cleanup_failed") + or (harness["status"] == "passed" and not harness["ordered_terminal"]) + or process["status"] not in PROCESS_STATUSES + or not self._optional_int(process["exit_code"]) + or not self._optional_int(process["signal"]) + or (process["status"] == "signalled" and process["signal"] is None) + or (process["status"] in {"exited", "not_started"} and process["signal"] is not None) + or (process["status"] == "not_started" and process["exit_code"] is not None) + ): + raise AttemptStateError(f"{label} is invalid") + @staticmethod def _instant(value: Any, label: str) -> _datetime.datetime: """Parse one produced ISO-8601 instant into a comparable UTC value.""" @@ -1444,11 +1604,11 @@ class RunStore: @staticmethod def _validate_terminal_events(events: list[Any]) -> dict[str, int]: - """Return the unique ordinal of every success-evidence event kind.""" + """Return the unique ordinal of every product/harness evidence event.""" positions: dict[str, int] = {} for index, event in enumerate(events): kind = event["kind"] - if kind not in SUCCESS_EVIDENCE_KINDS: + if kind not in BOUND_EVIDENCE_KINDS: continue if kind in positions: raise AttemptStateError("lifecycle events are invalid") @@ -1458,15 +1618,33 @@ class RunStore: def _validate_terminal_coherence(self, result: dict[str, Any], receipt: dict[str, Any], positions: dict[str, int]) -> None: """Bind result, ordered events and cleanup receipt to one terminal projection.""" submitted, finish, idle, quiet = (positions.get(kind) for kind in SUCCESS_EVIDENCE_KINDS) + caller_terminal = positions.get(EVENT_CALLER_TERMINAL) ordered = finish is not None and idle is not None and quiet is not None and finish < idle < quiet - if ordered != result["finish_then_idle_then_quiet"]: + if ordered != result["harness"]["ordered_terminal"]: raise AttemptStateError("lifecycle ordered evidence is invalid") - if result["exit_code"] != receipt["exit_code"] or result["signal"] != receipt["signal"]: + if result["process"]["exit_code"] != receipt["exit_code"] or result["process"]["signal"] != receipt["signal"]: raise AttemptStateError("lifecycle terminal outcome is invalid") if result["submitted"] and not receipt["caller_launched"]: raise AttemptStateError("lifecycle terminal outcome is invalid") - if result["success"] and (not result["submitted"] or submitted is None or finish is None or submitted > finish): - raise AttemptStateError("lifecycle success evidence is invalid") + if result["product"]["status"] != "unknown": + product = result["product"] + expected_detail = f"status={product['status']} reason={product['reason']}" + terminal_event = ( + None if caller_terminal is None else result["events"][caller_terminal] + ) + if ( + not result["submitted"] + or submitted is None + or finish is None + or idle is None + or caller_terminal is None + or submitted > finish + or submitted > caller_terminal + or caller_terminal > idle + or terminal_event["source"] != SOURCE_CALLER_OUTPUT + or terminal_event["detail"] != expected_detail + ): + raise AttemptStateError("lifecycle product evidence is invalid") started = self._instant(result["started_at"], "lifecycle result") ended = self._instant(result["ended_at"], "lifecycle result") completed = self._instant(receipt["completed_at"], "cleanup receipt") @@ -1476,7 +1654,7 @@ class RunStore: def _validate_receipt_record(self, receipt: dict[str, Any], locator: SupervisorLocator) -> dict[str, Any]: """Validate one cleanup receipt against the registered supervisor identity.""" self._exact_fields(receipt, RECEIPT_FIELDS, "cleanup receipt") - if receipt["receipt_version"] != RECEIPT_VERSION or receipt["supervisor_pid"] != locator.supervisor_pid or receipt["challenge_digest"] != self._public_locator(locator)["challenge_digest"] or receipt["reason"] not in TERMINAL_REASONS or not self._optional_int(receipt["exit_code"]) or not self._optional_int(receipt["signal"]) or not isinstance(receipt["caller_launched"], bool) or receipt["cleanup_complete"] is not True or receipt["process_group_alive"] is not False or not isinstance(receipt["completed_at"], str): + if receipt["receipt_version"] != RECEIPT_VERSION or receipt["supervisor_pid"] != locator.supervisor_pid or receipt["challenge_digest"] != self._public_locator(locator)["challenge_digest"] or receipt["reason"] not in TERMINAL_REASONS or not self._optional_int(receipt["exit_code"]) or not self._optional_int(receipt["signal"]) or not isinstance(receipt["caller_launched"], bool) or not isinstance(receipt["cleanup_complete"], bool) or (not receipt["cleanup_complete"]) != (receipt["reason"] == "cleanup_failed") or receipt["process_group_alive"] is not False or not isinstance(receipt["completed_at"], str): raise AttemptStateError("cleanup receipt is invalid") return receipt @@ -1564,8 +1742,9 @@ class RunStore: raise AttemptStateError("lifecycle journal is invalid") header, terminal = lines[0], lines[-1] self._exact_fields(header, {"record", "journal_version", "spec_digest", "submission_mode", "completion_mode", "started_at"}, "lifecycle journal header") - self._exact_fields(terminal, {"record", "terminal_reason", "success", "cleanup_complete", "process_group_alive", "ended_at"}, "lifecycle journal terminal") - if header["record"] != "header" or header["journal_version"] != 1 or header["spec_digest"] != expected_digest or header["submission_mode"] != result["submission_mode"] or header["completion_mode"] != result["completion_mode"] or header["started_at"] != result["started_at"] or not isinstance(header["started_at"], str) or terminal["record"] != "terminal" or terminal["terminal_reason"] != result["terminal_reason"] or terminal["success"] != result["success"] or terminal["cleanup_complete"] is not True or terminal["process_group_alive"] is not False or terminal["ended_at"] != result["ended_at"] or not isinstance(terminal["ended_at"], str): + self._exact_fields(terminal, {"record", "product", "harness", "process", "process_group_alive", "ended_at"}, "lifecycle journal terminal") + self._validate_outcomes(terminal, "lifecycle journal terminal") + if header["record"] != "header" or header["journal_version"] != JOURNAL_VERSION or header["spec_digest"] != expected_digest or header["submission_mode"] != result["submission_mode"] or header["completion_mode"] != result["completion_mode"] or header["started_at"] != result["started_at"] or not isinstance(header["started_at"], str) or terminal["record"] != "terminal" or terminal["product"] != result["product"] or terminal["harness"] != result["harness"] or terminal["process"] != result["process"] or terminal["process_group_alive"] is not False or terminal["ended_at"] != result["ended_at"] or not isinstance(terminal["ended_at"], str): raise AttemptStateError("lifecycle journal is invalid") for event in lines[1:-1]: self._exact_fields(event, {"record", "kind", "source", "stream", "monotonic_ns", "source_monotonic_ns", "observed_at", "detail"}, "lifecycle journal event") @@ -1599,7 +1778,7 @@ class RunStore: receipt_data = self._validate_receipt_record( self._read_json_file(control, "cleanup-receipt.json"), locator ) - if receipt_data["reason"] != result["terminal_reason"]: + if receipt_data["reason"] != result["harness"]["reason"]: raise AttemptStateError("cleanup receipt is invalid") events = self._validate_terminal_events(result["events"]) self._validate_terminal_coherence(result, receipt_data, events) @@ -1629,7 +1808,7 @@ class RunStore: raise AttemptStateError("lifecycle terminal is unavailable") self._validate_measurement( root, run, attempt.identity, expected_digest, - str(terminal["terminal_reason"]), + _terminal_reason(terminal), record.get("measurement_policy"), terminal, ) self._validate_web_validation( @@ -1638,7 +1817,13 @@ class RunStore: attempt.identity, record.get("web_validation_policy"), ) - fields = ("success", "terminal_reason", "exit_code", "signal", "submitted", "finish_then_idle_then_quiet", "cleanup_complete", "process_group_alive", "spec_digest", "started_at", "ended_at", "duration_ns") + if ( + _outcome_records(invocation)["product"] != terminal["product"] + or _outcome_records(invocation)["harness"] != terminal["harness"] + or _outcome_records(invocation)["process"] != terminal["process"] + ): + raise AttemptStateError("invocation result does not match durable terminal") + fields = ("submitted", "process_group_alive", "spec_digest", "started_at", "ended_at", "duration_ns") if any(getattr(invocation, field) != terminal[field] for field in fields): raise AttemptStateError("invocation result does not match durable terminal") return terminal @@ -1682,7 +1867,7 @@ class RunStore: except MeasurementError as exc: raise AttemptStateError("attempt measurement is invalid") from exc self._validate_measurement( - root, run, attempt.identity, expected_digest, result.terminal_reason, + root, run, attempt.identity, expected_digest, result.harness.reason, record.get("measurement_policy"), ) @@ -1715,7 +1900,7 @@ class RunStore: record = self._attempt_record(root, run, attempt.identity, absent_ok=True) if record is None: terminal = self.publish_terminal( - attempt, "interrupted", result={"terminal_reason": "interrupted"} + attempt, "interrupted", result=_unknown_terminal("interrupted") ) self.release_control_lease(terminal) return terminal @@ -1726,7 +1911,7 @@ class RunStore: raw_locator = record.get("locator") if raw_locator is None: terminal = self.publish_terminal( - attempt, "interrupted", result={"terminal_reason": "interrupted"} + attempt, "interrupted", result=_unknown_terminal("interrupted") ) self.release_control_lease(terminal) return terminal @@ -1740,26 +1925,34 @@ class RunStore: if terminal is not None: self._validate_measurement( root, run, attempt.identity, expected_digest, - str(terminal["terminal_reason"]), + _terminal_reason(terminal), record.get("measurement_policy"), terminal, ) self._ensure_required_web_validation( root, run, attempt.identity, record, terminal ) published = self.publish_terminal( - attempt, self._state_for_reason(terminal["terminal_reason"]), result=terminal + attempt, self._state_for_terminal(terminal), result=terminal ) self.release_control_lease(published) return published closed_receipt = self._closed_cleanup_receipt(root, locator) if closed_receipt is not None: - recovery_terminal = {"terminal_reason": closed_receipt["reason"]} + recovery_terminal = _unknown_terminal( + closed_receipt["reason"], + process_status=( + "signalled" if closed_receipt.get("signal") is not None + else "exited" + ), + exit_code=closed_receipt.get("exit_code"), + signal=closed_receipt.get("signal"), + ) self._ensure_required_web_validation( root, run, attempt.identity, record, recovery_terminal ) published = self.publish_terminal( attempt, - self._state_for_reason(recovery_terminal["terminal_reason"]), + self._state_for_terminal(recovery_terminal), result=recovery_terminal, ) self.release_control_lease(published) @@ -1781,15 +1974,21 @@ class RunStore: raise AttemptStateError("recovery is unverified") from exc if recovered_receipt is None: raise AttemptStateError("recovery is unverified") from exc - recovery_terminal = { - "terminal_reason": recovered_receipt["reason"] - } + recovery_terminal = _unknown_terminal( + recovered_receipt["reason"], + process_status=( + "signalled" if recovered_receipt.get("signal") is not None + else "exited" + ), + exit_code=recovered_receipt.get("exit_code"), + signal=recovered_receipt.get("signal"), + ) self._ensure_required_web_validation( root, run, attempt.identity, record, recovery_terminal ) terminal = self.publish_terminal( attempt, - self._state_for_reason(recovery_terminal["terminal_reason"]), + self._state_for_terminal(recovery_terminal), result=recovery_terminal, ) self.release_control_lease(terminal) @@ -1801,13 +2000,18 @@ class RunStore: self._validate_receipt_record(self._read_json_file(receipt.parent, receipt.name), locator) except AttemptStateError as exc: raise AttemptStateError("recovery cleanup is unverified") from exc - recovery_terminal = {"terminal_reason": outcome.reason} + recovery_terminal = _unknown_terminal( + outcome.reason, + process_status="signalled" if outcome.signal is not None else "exited", + exit_code=outcome.exit_code, + signal=outcome.signal, + ) self._ensure_required_web_validation( root, run, attempt.identity, record, recovery_terminal ) terminal = self.publish_terminal( attempt, - self._state_for_reason(recovery_terminal["terminal_reason"]), + self._state_for_terminal(recovery_terminal), result=recovery_terminal, ) self.release_control_lease(terminal) @@ -1829,7 +2033,7 @@ class RunStore: try: prepare(attempt) except Exception: - _write_new(root / "attempt.json", _json_bytes(self._initial_record(run, attempt, "failed", reason="preparation_failed"))) + _write_new(root / "attempt.json", _json_bytes(self._initial_record(run, attempt, "interrupted", reason="launch_failed"))) raise _write_new( root / "attempt.json", @@ -1854,7 +2058,7 @@ class RunStore: result = invoke(attempt, lambda locator, digest: self.record_locator(attempt, locator, digest)) terminal = self.validate_invocation_terminal(attempt, result) published = self.publish_terminal( - attempt, self._state_for_reason(terminal["terminal_reason"]), result=terminal + attempt, self._state_for_terminal(terminal), result=terminal ) self.release_control_lease(published) return published @@ -1865,9 +2069,31 @@ class RunStore: if bound_run != run: raise AttemptStateError("run identity is invalid") states = {name: 0 for name in sorted(TERMINAL_STATES | {NONTERMINAL_STATE})} + outcomes = { + "product": {name: 0 for name in ("succeeded", "failed", "unknown")}, + "harness": {name: 0 for name in ("passed", "failed")}, + "process": { + name: 0 for name in ( + "exited", "signalled", "timed_out", "cancelled", "not_started" + ) + }, + "artifact": { + name: 0 for name in ("passed", "failed", "blocked", "not_run") + }, + "unresolved": 0, + } for slot in self.slots(manifest): - for attempt in self.attempts(bound_run, slot): + retained = self.attempts(bound_run, slot) + for attempt in retained: states[attempt.state] += 1 + if not retained or retained[-1].state == NONTERMINAL_STATE: + outcomes["unresolved"] += 1 + continue + projection = self.attempt_outcomes(retained[-1]) + for axis in ("product", "harness", "process", "artifact"): + outcomes[axis][projection[axis]] += 1 + if not projection["passed"]: + outcomes["unresolved"] += 1 preflights = self._preflight_records(bound_run, manifest) latest = preflights[-1] if preflights else None projection = { @@ -1885,6 +2111,7 @@ class RunStore: "manifest_digest": bound_run.manifest_digest, "preflight": projection, "attempts": states, + "outcomes": outcomes, } @@ -1986,14 +2213,13 @@ def run_slots( if cell is None: raise AttemptStateError("slot cell identity is invalid") existing = store.attempts(bound_run, slot) - if existing and existing[-1].state == "success": - continue - if existing and existing[-1].state in {"failed", "timed_out", "cancelled"} and not retry_failed: - continue if existing and existing[-1].state not in TERMINAL_STATES: store.reconcile(existing[-1]) existing = store.attempts(bound_run, slot) - if existing[-1].state == "success" or (existing[-1].state in {"failed", "timed_out", "cancelled"} and not retry_failed): + if existing: + if store.attempt_outcomes(existing[-1])["passed"]: + continue + if not retry_failed: continue attempt = store.allocate(bound_run, slot) prepared: PreparedWorkspace | None = None diff --git a/scripts/agent_benchmark/attempts_test.py b/scripts/agent_benchmark/attempts_test.py index eb71f097..d810ee7c 100644 --- a/scripts/agent_benchmark/attempts_test.py +++ b/scripts/agent_benchmark/attempts_test.py @@ -42,10 +42,14 @@ from scripts.agent_benchmark.connectivity import ( make_result, ) from scripts.agent_benchmark.lifecycle import ( + CALLER_REASON_SUCCESS, + CALLER_STATUS_SUCCEEDED, COMPLETION_EXIT_AFTER_IDLE, SUBMISSION_ARGV_TASK, InvocationResult, InvocationSpec, + CallerEvent, + CallerTerminal, LifecycleRecoveryError, REASON_CONTROLLER_LOST, REASON_RECOVERED_STOP, @@ -107,8 +111,14 @@ def _manifest( return load_manifest(path, repo_root=root), raw, path -def _events(_: str, line: str) -> str | None: - return {"FINISH": "finish", "IDLE": "idle"}.get(line.strip()) +def _events(_: str, line: str): + return { + "FINISH": ( + CallerTerminal(CALLER_STATUS_SUCCEEDED, CALLER_REASON_SUCCESS), + CallerEvent("finish"), + ), + "IDLE": CallerEvent("idle"), + }.get(line.strip()) def _preflight_observation(cell, issue_code: str | None = None) -> PreflightObservation: @@ -498,9 +508,9 @@ class AttemptStoreTest(AttemptBase): with self.store.writer(run): first = self.store.allocate(run, Slot("a", 1)) self.assertEqual(list(Path(first.root).iterdir()), []) - terminal = self.store.publish_terminal(first, "failed") + terminal = self.store.publish_terminal(first, "interrupted") with self.assertRaises(AttemptStateError): - self.store.publish_terminal(terminal, "success") + self.store.publish_terminal(terminal, "completed") second = self.store.allocate(run, Slot("a", 1)) self.assertEqual(second.identity.attempt, 2) @@ -527,7 +537,7 @@ class AttemptStoreTest(AttemptBase): run = self.create_run() with self.store.writer(run): attempt = self.store.allocate(run, Slot("a", 1)) - self.store.publish_terminal(attempt, "failed") + self.store.publish_terminal(attempt, "interrupted") record = Path(attempt.root) / "attempt.json" foreign = json.loads(record.read_text(encoding="utf-8")) foreign["run_id"] = "run-20260102T030405Z-ffffffffffff" @@ -649,16 +659,17 @@ class AttemptOrchestrationTest(AttemptBase): return prepare_workspace(manifest, attempt.root, attempt.identity, repo_root=self.root) completed = run_slots(self.store, run, self.manifest, adapters={"claude": self.adapter("success", calls)}, prepare=prepare) - self.assertEqual([item.state for item in completed], ["success"]) + self.assertEqual([item.state for item in completed], ["completed"]) self.assertEqual(calls, ["preflight", "prepare", "invoke"]) attempt_root = Path(completed[0].root) self.assertTrue((attempt_root / "prepared.json").is_file()) state = json.loads((attempt_root / "attempt.json").read_text(encoding="utf-8")) + self.assertEqual(state["attempt_result_version"], 2) alias = Path(state["locator"]["control_dir"]).parent self.assertFalse(os.path.lexists(alias)) self.assertTrue((attempt_root / "control/locator.json").is_file()) self.assertTrue((attempt_root / "control/cleanup-receipt.json").is_file()) - self.assertEqual(self.store.status(run, self.manifest)["attempts"]["success"], 1) + self.assertEqual(self.store.status(run, self.manifest)["attempts"]["completed"], 1) record_path = attempt_root / "attempt.json" original = record_path.read_bytes() corruptions = ( @@ -740,7 +751,7 @@ class AttemptOrchestrationTest(AttemptBase): with self.assertRaisesRegex(RuntimeError, "prepare failure"): run_slots(self.store, run, self.manifest, adapters={"claude": self.adapter("success", calls)}, prepare=fail_prepare) self.assertEqual(calls, ["preflight", "prepare"]) - self.assertEqual(self.store.attempts(run, Slot("a", 1))[-1].state, "failed") + self.assertEqual(self.store.attempts(run, Slot("a", 1))[-1].state, "interrupted") def test_retry_and_skip_preserve_prior_terminal_bytes(self): self._init_testbed() @@ -825,7 +836,7 @@ class AttemptMeasurementTest(AttemptBase): def test_successful_attempt_publishes_one_bound_measurement(self): threads_before = set(threading.enumerate()) run, completed = self._run() - self.assertEqual([item.state for item in completed], ["success"]) + self.assertEqual([item.state for item in completed], ["completed"]) attempt_root = Path(completed[0].root) measurement = load_measurement(attempt_root) @@ -834,7 +845,7 @@ class AttemptMeasurementTest(AttemptBase): (run.run_id, "a", 1, 1), ) self.assertEqual(measurement.caller, "claude") - self.assertEqual(measurement.terminal_reason, "success") + self.assertEqual(measurement.harness.reason, "success") state = json.loads((attempt_root / "attempt.json").read_text(encoding="utf-8")) self.assertEqual(measurement.spec_digest, state["spec_digest"]) self.assertEqual(state["measurement_policy"], "required-v1") @@ -873,7 +884,7 @@ class AttemptMeasurementTest(AttemptBase): _run, completed = self._run(mode) self.assertEqual(completed[0].state, state) measurement = load_measurement(Path(completed[0].root)) - self.assertEqual(measurement.terminal_reason, reason) + self.assertEqual(measurement.harness.reason, reason) self.assertEqual(measurement.observations, ()) for name in ("total_duration", "input_tokens", "model_calls"): self.assertEqual(measurement.usage[name].status, "unavailable") @@ -884,7 +895,7 @@ class AttemptMeasurementTest(AttemptBase): ) def test_failed_attempt_keeps_unavailable_values(self): - self._assert_unavailable_measurement("failed", "failed", "nonzero_exit") + self._assert_unavailable_measurement("failed", "completed", "nonzero_exit") def test_timed_out_attempt_keeps_unavailable_values(self): self._assert_unavailable_measurement("timeout", "timed_out", "timed_out") @@ -911,7 +922,18 @@ class AttemptMeasurementTest(AttemptBase): cases = { "foreign-attempt": {**record, "attempt": {**record["attempt"], "cell_id": "other"}}, "foreign-digest": {**record, "spec_digest": "sha256:" + "0" * 64}, - "rewritten-terminal": {**record, "terminal_reason": "timed_out"}, + "rewritten-terminal": { + **record, + "harness": {**record["harness"], "reason": "timed_out"}, + }, + "rewritten-product": { + **record, + "product": {"status": "failed", "reason": "caller_error"}, + }, + "rewritten-process": { + **record, + "process": {**record["process"], "exit_code": 7}, + }, "invented-total": { **record, "usage": { @@ -933,7 +955,7 @@ class AttemptMeasurementTest(AttemptBase): self.store.status(run, self.manifest) self.assertEqual(before, sidecar.read_bytes()) sidecar.write_bytes(original) - self.assertEqual(self.store.status(run, self.manifest)["attempts"]["success"], 1) + self.assertEqual(self.store.status(run, self.manifest)["attempts"]["completed"], 1) def test_marked_measurement_is_required_and_bound_to_lifecycle_events(self): run, completed = self._run() @@ -1045,7 +1067,7 @@ class AttemptMeasurementTest(AttemptBase): reject_on_status_and_reconcile() finally: AttemptRecoveryTest._restore(marker, saved_marker) - self.assertEqual(self.store.status(run, self.manifest)["attempts"]["success"], 1) + self.assertEqual(self.store.status(run, self.manifest)["attempts"]["completed"], 1) def test_explicitly_unmarked_lower_level_attempt_remains_compatible(self): run = self.create_run() @@ -1060,7 +1082,7 @@ class AttemptMeasurementTest(AttemptBase): self.assertNotIn("measurement_policy", record) self.assertFalse((root / MEASUREMENT_POLICY_FILENAME).exists()) self.assertFalse((root / MEASUREMENT_FILENAME).exists()) - self.assertEqual(self.store.status(run, self.manifest)["attempts"]["success"], 1) + self.assertEqual(self.store.status(run, self.manifest)["attempts"]["completed"], 1) def test_nonregular_measurement_fails_closed_without_blocking(self): run, completed = self._run() @@ -1072,7 +1094,7 @@ class AttemptMeasurementTest(AttemptBase): with self.assertRaises(AttemptStateError): self.store.attempts(run, Slot("a", 1)) AttemptRecoveryTest._restore(sidecar, saved) - self.assertEqual(self.store.attempts(run, Slot("a", 1))[-1].state, "success") + self.assertEqual(self.store.attempts(run, Slot("a", 1))[-1].state, "completed") def test_recovery_commits_only_a_valid_bound_sidecar(self): self._init_testbed() @@ -1121,7 +1143,7 @@ class AttemptMeasurementTest(AttemptBase): sidecar.write_bytes(original) with self.store.writer(run): recovered = self.store.reconcile(attempt) - self.assertEqual(recovered.state, "success") + self.assertEqual(recovered.state, "completed") self.assertEqual(load_measurement(Path(attempt.root)).caller, "claude") @@ -1228,11 +1250,11 @@ class AttemptWebValidationTest(AttemptBase): ) return run, attempt - def test_lifecycle_status_matrix_publishes_not_run_for_non_success(self): + def test_lifecycle_status_matrix_validates_every_terminal_workspace(self): cases = ( - ("success", "success", "failed"), - ("failed", "failed", "not_run"), - ("timeout", "timed_out", "not_run"), + ("success", "completed", "failed"), + ("failed", "completed", "failed"), + ("timeout", "timed_out", "failed"), ) for mode, terminal, web_status in cases: with self.subTest(mode=mode): @@ -1242,9 +1264,7 @@ class AttemptWebValidationTest(AttemptBase): self.assertEqual(completed[0].state, terminal) web = load_web_validation(Path(completed[0].root)) self.assertEqual(web.status, web_status) - if web_status == "not_run": - self.assertTrue(web.record["reason"].startswith("lifecycle_")) - self.assertFalse(any(item["passed"] for item in web.record["gates"])) + self.assertNotEqual(web.status, "not_run") def test_normal_terminal_requires_web_sidecar_before_commit(self): _run, attempt = self._running_required_web(return_result=True) @@ -1259,10 +1279,10 @@ class AttemptWebValidationTest(AttemptBase): before = attempt_record.read_bytes() with self.store.writer(run): recovered = self.store.reconcile(attempt) - self.assertEqual(recovered.state, "success") + self.assertEqual(recovered.state, "completed") self.assertNotEqual(attempt_record.read_bytes(), before) self.assertEqual(load_web_validation(root).status, "failed") - self.assertEqual(self.store.status(run, self.manifest)["attempts"]["success"], 1) + self.assertEqual(self.store.status(run, self.manifest)["attempts"]["completed"], 1) def test_recovery_browser_start_failure_publishes_blocked(self): run, attempt = self._running_required_web(generated=True) @@ -1272,7 +1292,7 @@ class AttemptWebValidationTest(AttemptBase): ): with self.store.writer(run): recovered = self.store.reconcile(attempt) - self.assertEqual(recovered.state, "success") + self.assertEqual(recovered.state, "completed") web = load_web_validation(Path(attempt.root)) self.assertEqual(web.status, "blocked") self.assertFalse(web.record["screenshots"]) @@ -1345,7 +1365,7 @@ class AttemptWebValidationTest(AttemptBase): rejected() web_path.rmdir() web_path.write_bytes(saved["web"]) - self.assertEqual(self.store.status(run, self.manifest)["attempts"]["success"], 1) + self.assertEqual(self.store.status(run, self.manifest)["attempts"]["completed"], 1) class AttemptRecoveryTest(AttemptBase): @@ -1374,10 +1394,10 @@ class AttemptRecoveryTest(AttemptBase): self.assertTrue(alias.is_symlink()) with self.store.writer(run): recovered = self.store.reconcile(attempt) - self.assertEqual(recovered.state, "success") + self.assertEqual(recovered.state, "completed") self.assertFalse(os.path.lexists(alias)) - self.assertEqual(self.store.status(run, self.manifest)["attempts"]["success"], 1) - self.assertEqual(self.store.reconcile(recovered).state, "success") + self.assertEqual(self.store.status(run, self.manifest)["attempts"]["completed"], 1) + self.assertEqual(self.store.reconcile(recovered).state, "completed") def test_corrupt_terminal_variants_fail_closed_and_preserve_bytes(self): run, attempt = self._running_with_terminal() @@ -1578,14 +1598,29 @@ class AttemptRecoveryTest(AttemptBase): ("receipt-completed-after-end", "receipt", lambda raw: raw.__setitem__("completed_at", "2100-01-01T00:00:00+00:00")), ("receipt-completed-unparseable", "receipt", lambda raw: raw.__setitem__("completed_at", "not-a-timestamp")), ("result-submitted", "result", lambda raw: raw.__setitem__("submitted", False)), - ("result-exit-code", "result", lambda raw: raw.__setitem__("exit_code", 7)), + ( + "result-exit-code", + "result", + lambda raw: raw["process"].__setitem__("exit_code", 7), + ), ("events-cleared", "events", lambda events: []), ("events-missing-submitted", "events", _without("submitted")), + ("events-missing-caller-terminal", "events", _without("caller_terminal")), ("events-missing-finish", "events", _without("finish")), ("events-missing-idle", "events", _without("idle")), ("events-missing-quiet", "events", _without("quiet")), ("events-out-of-order", "events", _reordered), ("events-duplicate-finish", "events", _duplicated("finish")), + ( + "events-caller-terminal-detail", + "events", + lambda events: [ + {**event, "detail": "status=failed reason=caller_error"} + if event["kind"] == "caller_terminal" + else event + for event in events + ], + ), ) for name, record, mutate in cases: with self.subTest(case=name): @@ -1605,13 +1640,51 @@ class AttemptRecoveryTest(AttemptBase): with self.store.writer(run): recovered = self.store.reconcile(attempt) published = record.read_bytes() - self.assertEqual(recovered.state, "success") + self.assertEqual(recovered.state, "completed") self.assertNotEqual(running, published) with self.store.writer(run): - self.assertEqual(self.store.reconcile(recovered).state, "success") + self.assertEqual(self.store.reconcile(recovered).state, "completed") self.assertEqual(published, record.read_bytes()) self.assertEqual(len(self.store.attempts(run, Slot("a", 1))), 1) + def test_coherent_cleanup_failure_remains_a_terminal_independent_axis(self): + run, attempt = self._running_with_terminal() + root = Path(attempt.root) + result_path = root / "lifecycle-result.json" + journal_path = root / "lifecycle-journal.jsonl" + receipt_path = root / "control" / "cleanup-receipt.json" + + result = json.loads(result_path.read_text(encoding="utf-8")) + result["harness"].update({ + "status": "failed", + "reason": "cleanup_failed", + "cleanup_complete": False, + }) + result_path.write_text(json.dumps(result), encoding="utf-8") + + journal = [ + json.loads(line) + for line in journal_path.read_text(encoding="utf-8").splitlines() + ] + journal[-1]["harness"] = result["harness"] + journal_path.write_text( + "".join(json.dumps(line) + "\n" for line in journal), + encoding="utf-8", + ) + + receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + receipt.update({"reason": "cleanup_failed", "cleanup_complete": False}) + receipt_path.write_text(json.dumps(receipt), encoding="utf-8") + + with self.store.writer(run): + terminal = self.store.reconcile(attempt) + projection = self.store.attempt_outcomes(terminal) + self.assertEqual(terminal.state, "completed") + self.assertEqual( + (projection["product"], projection["harness"], projection["process"]), + ("succeeded", "failed", "exited"), + ) + def test_symlink_lifecycle_evidence_fails_closed(self): run, attempt = self._running_with_terminal() journal = Path(attempt.root) / "lifecycle-journal.jsonl" @@ -1636,7 +1709,7 @@ class AttemptRecoveryTest(AttemptBase): result = self.adapter("success", [])(current, started) path = Path(current.root) / "lifecycle-result.json" raw = json.loads(path.read_text(encoding="utf-8")) - raw["success"] = False + raw["product"]["status"] = "unknown" path.write_text(json.dumps(raw), encoding="utf-8") return result @@ -1832,7 +1905,7 @@ class AttemptRecoveryTest(AttemptBase): published = json.loads(attempt_record.read_text(encoding="utf-8")) self.assertEqual(published["state"], "interrupted") self.assertEqual( - published["lifecycle"]["terminal_reason"], + published["lifecycle"]["harness"]["reason"], REASON_CONTROLLER_LOST, ) self.assertTrue(alias.is_symlink()) @@ -1859,19 +1932,19 @@ class AttemptRecoveryTest(AttemptBase): ( "terminal-record-reason", attempt_record, - lambda raw: raw["lifecycle"].__setitem__( - "terminal_reason", "success" + lambda raw: raw["lifecycle"]["harness"].__setitem__( + "reason", "success" ), ), ( "terminal-record-state-success", attempt_record, - lambda raw: raw.__setitem__("state", "success"), + lambda raw: raw.__setitem__("state", "completed"), ), ( "terminal-record-state-failed", attempt_record, - lambda raw: raw.__setitem__("state", "failed"), + lambda raw: raw.__setitem__("state", "cancelled"), ), ) for name, target, tamper in terminal_tamper_cases: @@ -1979,8 +2052,8 @@ class AttemptRecoveryTest(AttemptBase): ) ) self.assertEqual(record["state"], "interrupted") - self.assertEqual(record["lifecycle"]["terminal_reason"], REASON_RECOVERED_STOP) - self.assertEqual(result["terminal_reason"], REASON_RECOVERED_STOP) + self.assertEqual(record["lifecycle"]["harness"]["reason"], REASON_RECOVERED_STOP) + self.assertEqual(result["harness"]["reason"], REASON_RECOVERED_STOP) self.assertEqual(receipt["reason"], REASON_RECOVERED_STOP) self.assertTrue(receipt["cleanup_complete"]) self.assertFalse(receipt["process_group_alive"]) @@ -2021,7 +2094,10 @@ class AttemptCliContractTest(AttemptBase): output = io.StringIO() with mock.patch.object(benchmark_cli, "_REPO_ROOT", self.root), contextlib.redirect_stdout(output): self.assertEqual(benchmark_cli.main(["status", "--manifest", str(self.manifest_path), "--run-id", run.run_id]), 0) - self.assertIn("'running': 0", output.getvalue()) + self.assertIn("running=0", output.getvalue()) + self.assertIn("product_succeeded=0", output.getvalue()) + self.assertIn("artifact_passed=0", output.getvalue()) + self.assertIn("unresolved=1", output.getvalue()) self.assertEqual( run_before, { diff --git a/scripts/agent_benchmark/claude_iop.py b/scripts/agent_benchmark/claude_iop.py index f5f777a1..3208e01b 100644 --- a/scripts/agent_benchmark/claude_iop.py +++ b/scripts/agent_benchmark/claude_iop.py @@ -23,8 +23,14 @@ from scripts.agent_benchmark.connectivity import ( make_result, ) from scripts.agent_benchmark.lifecycle import ( + CALLER_REASON_ERROR, + CALLER_REASON_SUCCESS, + CALLER_STATUS_FAILED, + CALLER_STATUS_SUCCEEDED, COMPLETION_EXIT_AFTER_IDLE, SUBMISSION_STDIN_ONCE, + CallerEvent, + CallerTerminal, InvocationSpec, LifecycleMetricError, ParsedMetric, @@ -251,7 +257,7 @@ class ClaudeStreamParser: self.claude_session_id = _required_string(event, "session_id") self._phase = "await_assistant" - def _consume_assistant(self, event: dict[str, Any]) -> str: + def _consume_assistant(self, event: dict[str, Any]) -> CallerEvent | None: if self._phase != "await_assistant": raise ClaudeIopProtocolError("duplicate or out-of-order Claude assistant") self._require_bound_session(event) @@ -304,7 +310,7 @@ class ClaudeStreamParser: self._phase = "await_tool_result" return None self._phase = "await_result" - return "finish" + return CallerEvent("finish") def _consume_user(self, event: dict[str, Any]) -> None: if self._phase not in ("await_assistant", "await_tool_result"): @@ -341,10 +347,11 @@ class ClaudeStreamParser: ): raise ClaudeIopProtocolError("invalid Claude API error terminal") self._phase = "complete" - # Do not emit finish/idle for an upstream API failure. The caller's - # non-zero exit is the lifecycle terminal; an unexpected zero exit - # still fails closed as missing terminal evidence. - return None + return ( + CallerTerminal(CALLER_STATUS_FAILED, CALLER_REASON_ERROR), + CallerEvent("finish"), + CallerEvent("idle"), + ) result_completes_active_message = False if self._phase == "await_assistant" and self._active_message_id is not None: if self._continuation_pending: @@ -357,8 +364,16 @@ class ClaudeStreamParser: if event.get("subtype") != "success" or event.get("is_error") is True: raise ClaudeIopProtocolError("invalid Claude result terminal") self._phase = "complete" - terminal = ("finish", "idle") if result_completes_active_message else ("idle",) - return (*self._observations(event), *terminal) + terminal = ( + (CallerEvent("finish"), CallerEvent("idle")) + if result_completes_active_message + else (CallerEvent("idle"),) + ) + return ( + *self._observations(event), + CallerTerminal(CALLER_STATUS_SUCCEEDED, CALLER_REASON_SUCCESS), + *terminal, + ) def _observations(self, event: dict[str, Any]) -> tuple[ParsedMetric, ...]: """Convert only allowlisted reported Claude values into observations.""" @@ -413,7 +428,7 @@ class ClaudeStreamParser: ): raise ClaudeIopProtocolError("invalid Claude usage observation") - def __call__(self, stream: str, raw_line: str) -> str | tuple[Any, ...] | None: + def __call__(self, stream: str, raw_line: str) -> Any: if stream != "stdout": return None event = _exact_object(raw_line) diff --git a/scripts/agent_benchmark/claude_iop_test.py b/scripts/agent_benchmark/claude_iop_test.py index 4393f846..d1ea5028 100644 --- a/scripts/agent_benchmark/claude_iop_test.py +++ b/scripts/agent_benchmark/claude_iop_test.py @@ -19,7 +19,10 @@ from scripts.agent_benchmark.claude_iop import ( parse_preflight_binding, redact_claude_event, ) -from scripts.agent_benchmark.lifecycle import REASON_PARSER_ERROR, REASON_SUCCESS, run_invocation +from scripts.agent_benchmark.lifecycle import ( + CallerEvent, CallerTerminal, ParsedMetric, + REASON_PARSER_ERROR, REASON_SUCCESS, run_invocation, +) from scripts.agent_benchmark.manifest import ExpectedBinding, IopCell, MatrixCell, Timeout from scripts.agent_benchmark.workspace import ( AttemptIdentity, @@ -182,8 +185,8 @@ class ClaudeIopTest(unittest.TestCase): lines = self._fixture_lines() parser = ClaudeStreamParser(self.cell, "session-fixture") parsed = [parser("stdout", line) for line in lines] - self.assertEqual(parsed[:2], [None, "finish"]) - self.assertEqual(parsed[2][-1], "idle") + self.assertEqual(parsed[:2], [None, CallerEvent("finish")]) + self.assertEqual(parsed[2][-1], CallerEvent("idle")) malformed = json.loads(lines[1]) del malformed["message"]["model"] @@ -219,7 +222,7 @@ class ClaudeIopTest(unittest.TestCase): for event in events: parser("stdout", event) parser = ClaudeStreamParser(self.cell, "session-fixture") - self.assertEqual([parser("stdout", event) for event in missing_result], [None, "finish"]) + self.assertEqual([parser("stdout", event) for event in missing_result], [None, CallerEvent("finish")]) def test_parser_accepts_partial_snapshots_and_tool_result_cycles(self) -> None: init, _, result = self._fixture_lines() @@ -240,10 +243,13 @@ class ClaudeIopTest(unittest.TestCase): }) self.assertEqual( [parser("stdout", line) for line in (init, partial, partial, user, final)], - [None, None, None, None, "finish"], + [None, None, None, None, CallerEvent("finish")], ) observations = parser("stdout", result) - metrics = {metric.name: metric.value for metric in observations[:-1]} + metrics = { + metric.name: metric.value for metric in observations + if isinstance(metric, ParsedMetric) + } self.assertEqual(metrics["model_calls"], 2) def test_parser_accepts_cumulative_message_ids_and_direct_result(self) -> None: @@ -272,9 +278,9 @@ class ClaudeIopTest(unittest.TestCase): ] self.assertEqual(parsed, [None] * len(parsed)) observations = parser("stdout", json.dumps(terminal)) - self.assertEqual(observations[-2:], ("finish", "idle")) + self.assertEqual(observations[-2:], (CallerEvent("finish"), CallerEvent("idle"))) self.assertEqual( - {metric.name: metric.value for metric in observations[:-2]}["model_calls"], 1 + {metric.name: metric.value for metric in observations if isinstance(metric, ParsedMetric)}["model_calls"], 1 ) def test_parser_classifies_synthetic_api_error_without_parser_failure(self) -> None: @@ -292,12 +298,16 @@ class ClaudeIopTest(unittest.TestCase): }) self.assertEqual( [parser("stdout", line) for line in (init, synthetic, terminal)], - [None, None, None], + [None, None, ( + CallerTerminal("failed", "caller_error"), + CallerEvent("finish"), CallerEvent("idle"), + )], ) outcome, _ = self._run_fake([init, synthetic, terminal], exit_code=1) - self.assertFalse(outcome.success) - self.assertNotIn(outcome.terminal_reason, ("malformed_event", "parser_error")) + self.assertFalse(outcome.product.status == "succeeded") + self.assertEqual(outcome.product.status, "failed") + self.assertEqual(outcome.harness.status, "passed") def test_lifecycle_accepts_result_direct_active_snapshot(self) -> None: init, _, result = self._fixture_lines() @@ -307,7 +317,7 @@ class ClaudeIopTest(unittest.TestCase): "stop_reason": None, "content": []}, }) outcome, _ = self._run_fake([init, partial, result]) - self.assertTrue(outcome.success, outcome) + self.assertTrue(outcome.product.status == "succeeded", outcome) kinds = [event.kind for event in outcome.events] self.assertLess(kinds.index("finish"), kinds.index("idle")) @@ -329,10 +339,10 @@ class ClaudeIopTest(unittest.TestCase): }) self.assertEqual( [parser("stdout", line) for line in (init, tool_use, user, final)], - [None, None, None, "finish"], + [None, None, None, CallerEvent("finish")], ) self.assertEqual( - {metric.name: metric.value for metric in parser("stdout", result)[:-1]}["model_calls"], + {metric.name: metric.value for metric in parser("stdout", result) if isinstance(metric, ParsedMetric)}["model_calls"], 2, ) duplicate = ClaudeStreamParser(self.cell, "session-fixture") @@ -348,8 +358,8 @@ class ClaudeIopTest(unittest.TestCase): parser("stdout", init) parser("stdout", assistant) parsed = parser("stdout", result) - self.assertEqual(parsed[-1], "idle") - observed = {metric.name: metric for metric in parsed[:-1]} + self.assertEqual(parsed[-1], CallerEvent("idle")) + observed = {metric.name: metric for metric in parsed if isinstance(metric, ParsedMetric)} self.assertEqual(observed["total_duration"].value, 1234 * 10 ** 6) self.assertFalse(observed["total_duration"].overlap) # The reported API duration is inside the reported total, so it is @@ -363,7 +373,9 @@ class ClaudeIopTest(unittest.TestCase): # The fixture omits cache creation, so that category stays unreported # rather than being reported as zero. self.assertNotIn("cache_write_tokens", observed) - for metric in parsed[:-1]: + for metric in parsed: + if not isinstance(metric, ParsedMetric): + continue self.assertEqual(metric.model, "claude-sonnet") self.assertEqual(metric.source, "caller_output") self.assertEqual(observed["input_tokens"].clock, "none") @@ -420,10 +432,10 @@ class ClaudeIopTest(unittest.TestCase): json.dumps(init), json.dumps(diagnostic), json.dumps(assistant), json.dumps(result), ]) result = outcome - self.assertTrue(result.success, result) - self.assertEqual(result.terminal_reason, REASON_SUCCESS) + self.assertTrue(result.product.status == "succeeded", result) + self.assertEqual(result.harness.reason, REASON_SUCCESS) self.assertTrue(result.submitted) - self.assertTrue(result.finish_then_idle_then_quiet) + self.assertTrue(result.harness.ordered_terminal) for sentinel in (*SENTINELS, *ARBITRARY_SENTINELS): self.assertNotIn(sentinel, durable) self.assertEqual( @@ -453,15 +465,15 @@ class ClaudeIopTest(unittest.TestCase): for name, lines in cases.items(): with self.subTest(name=name): outcome, durable = self._run_fake(lines) - self.assertFalse(outcome.success, outcome) + self.assertFalse(outcome.product.status == "succeeded", outcome) for sentinel in (*SENTINELS, *ARBITRARY_SENTINELS): self.assertNotIn(sentinel, durable) def test_metric_prefixed_malformed_output_is_redacted_before_durable_capture(self) -> None: metric_sentinel = "metric:" + SENTINELS[0] outcome, durable = self._run_fake([metric_sentinel]) - self.assertFalse(outcome.success, outcome) - self.assertEqual(outcome.terminal_reason, REASON_PARSER_ERROR) + self.assertFalse(outcome.product.status == "succeeded", outcome) + self.assertEqual(outcome.harness.reason, REASON_PARSER_ERROR) self.assertTrue(outcome.submitted) self.assertIn("invalid_claude_json", durable) for sentinel in (*SENTINELS, metric_sentinel, SENTINELS[0]): diff --git a/scripts/agent_benchmark/codex_iop.py b/scripts/agent_benchmark/codex_iop.py index 896e3388..46e2b949 100644 --- a/scripts/agent_benchmark/codex_iop.py +++ b/scripts/agent_benchmark/codex_iop.py @@ -29,8 +29,14 @@ if __package__ in (None, ""): sys.path.insert(0, str(Path(__file__).resolve().parents[2])) from scripts.agent_benchmark.lifecycle import ( + CALLER_REASON_ERROR, + CALLER_REASON_SUCCESS, + CALLER_STATUS_FAILED, + CALLER_STATUS_SUCCEEDED, COMPLETION_EXIT_AFTER_IDLE, SUBMISSION_STDIN_ONCE, + CallerEvent, + CallerTerminal, InvocationResult, InvocationSpec, LifecycleMetricError, @@ -311,7 +317,7 @@ class CodexJSONLParser: ) return make_result(self._cell, codex_capability(), binding, issues) - def parse(self, stream: str, line: str) -> str | ParsedMetric | tuple[Any, ...] | None: + def parse(self, stream: str, line: str) -> Any: if stream != "stdout": return None try: @@ -324,15 +330,22 @@ class CodexJSONLParser: if record["type"] == "turn.completed": status = record.get("status") if status is not None and status not in ("completed", "success"): - raise CodexJSONLError("unsuccessful Codex terminal turn") + return ( + CallerTerminal(CALLER_STATUS_FAILED, CALLER_REASON_ERROR), + CallerEvent("finish"), + ) self._turns += 1 - return (*self._turn_observations(record), "finish") + return ( + *self._turn_observations(record), + CallerTerminal(CALLER_STATUS_SUCCEEDED, CALLER_REASON_SUCCESS), + CallerEvent("finish"), + ) if record["type"] == _CODEX_ITEM_COMPLETED: return self._tool_interval(record) if record["type"] == _BRIDGE_IDLE_TYPE: if record != {"type": _BRIDGE_IDLE_TYPE, "adapter": _BRIDGE_ID, "nonce": self._idle_nonce, "child_exit": 0}: raise CodexJSONLError("unverified bridge idle marker") - return "idle" + return CallerEvent("idle") return None def _turn_observations(self, record: dict[str, Any]) -> tuple[ParsedMetric, ...]: diff --git a/scripts/agent_benchmark/codex_iop_test.py b/scripts/agent_benchmark/codex_iop_test.py index d64a6e33..0282737b 100644 --- a/scripts/agent_benchmark/codex_iop_test.py +++ b/scripts/agent_benchmark/codex_iop_test.py @@ -23,6 +23,8 @@ from scripts.agent_benchmark.codex_iop import ( runtime_from_environment, ) from scripts.agent_benchmark.lifecycle import ( + CallerEvent, + ParsedMetric, REASON_DUPLICATE_EVENT, REASON_NONZERO_EXIT, REASON_PARSER_ERROR, @@ -137,14 +139,14 @@ class CodexIOPTest(unittest.TestCase): parser = CodexJSONLParser(_cell(), "fixture-nonce-0001") fixture = Path("scripts/fixtures/agent-comparison-benchmark/codex-iop-stream.jsonl") events = [parser.parse("stdout", line) for line in fixture.read_text(encoding="utf-8").splitlines()] - self.assertEqual([events[0], events[-1]], [None, "idle"]) + self.assertEqual([events[0], events[-1]], [None, CallerEvent("idle")]) tool = events[1] self.assertEqual((tool.name, tool.value, tool.call_id), ("tool_duration", 7_250_000, "call-1")) # A tool interval is reported inside the turn, so it is published as an # overlapping interval instead of a subtractable slice. self.assertTrue(tool.overlap) - self.assertEqual(events[2][-1], "finish") - turn = {metric.name: metric.value for metric in events[2][:-1]} + self.assertEqual(events[2][-1], CallerEvent("finish")) + turn = {metric.name: metric.value for metric in events[2] if isinstance(metric, ParsedMetric)} self.assertEqual(turn, { "cache_write_tokens": 6, "cached_input_tokens": 8, "input_tokens": 31, "output_tokens": 12, "reasoning_tokens": 4, @@ -166,10 +168,10 @@ class CodexIOPTest(unittest.TestCase): fake = self._fake_codex(["TASK"]) invocation = build_codex_invocation(_cell(), self._prepared(), self._runtime(), _PROMPT, self._timeout(), codex_executable=(sys.executable, fake)) result = run_codex_invocation(invocation, lambda _: None) - self.assertTrue(result.lifecycle.success) + self.assertTrue(result.lifecycle.product.status == "succeeded") self.assertEqual([event.kind for event in result.lifecycle.events], [ "submitted", "first_output", "metric:model_calls", "metric:tool_calls", - "finish", "idle", "exited", "quiet", + "caller_terminal", "finish", "idle", "exited", "quiet", ]) capture = result.lifecycle.stdout.text self.assertNotIn(_PROMPT.decode(), capture) @@ -180,8 +182,8 @@ class CodexIOPTest(unittest.TestCase): def test_child_failure_never_synthesizes_idle(self) -> None: fake = self._fake_codex([{"type": "turn.completed", "status": "completed"}], exit_code=7) result = run_codex_invocation(build_codex_invocation(_cell(), self._prepared(), self._runtime(), _PROMPT, self._timeout(), codex_executable=(sys.executable, fake)), lambda _: None) - self.assertFalse(result.lifecycle.success) - self.assertEqual(result.lifecycle.terminal_reason, REASON_NONZERO_EXIT) + self.assertFalse(result.lifecycle.product.status == "succeeded") + self.assertEqual(result.lifecycle.harness.reason, REASON_NONZERO_EXIT) self.assertNotIn("idle", [event.kind for event in result.lifecycle.events]) def test_duplicate_malformed_and_unverified_idle_fail_closed(self) -> None: @@ -197,8 +199,8 @@ class CodexIOPTest(unittest.TestCase): prepared = self._prepared().__class__(**{**self._prepared().__dict__, "attempt_root": str(evidence)}) fake = self._fake_codex(records) result = run_codex_invocation(build_codex_invocation(_cell(), prepared, self._runtime(), _PROMPT, self._timeout(), codex_executable=(sys.executable, fake)), lambda _: None) - self.assertFalse(result.lifecycle.success) - self.assertEqual(result.lifecycle.terminal_reason, reason) + self.assertFalse(result.lifecycle.product.status == "succeeded") + self.assertEqual(result.lifecycle.harness.reason, reason) def test_tool_intervals_require_one_explicit_unique_pairing(self) -> None: parser = CodexJSONLParser(_cell(), "0123456789abcdef") @@ -225,7 +227,7 @@ class CodexIOPTest(unittest.TestCase): with self.assertRaises(CodexJSONLError): parser.parse("stdout", json.dumps({"type": "item.completed", "item": item})) turn = parser.parse("stdout", json.dumps({"type": "turn.completed"})) - counts = {metric.name: metric.value for metric in turn[:-1]} + counts = {metric.name: metric.value for metric in turn if isinstance(metric, ParsedMetric)} self.assertEqual(counts, {"model_calls": 1, "tool_calls": 2}) def test_unknown_or_fractional_turn_usage_fails_closed(self) -> None: diff --git a/scripts/agent_benchmark/connectivity_integration_test.py b/scripts/agent_benchmark/connectivity_integration_test.py index 1e2d2acb..96fbb5b9 100644 --- a/scripts/agent_benchmark/connectivity_integration_test.py +++ b/scripts/agent_benchmark/connectivity_integration_test.py @@ -52,6 +52,9 @@ from scripts.agent_benchmark.manifest import ( load_manifest, ) from scripts.agent_benchmark.lifecycle import ( + CALLER_REASON_ERROR, + CALLER_REASON_SUCCESS, + CALLER_STATUS_SUCCEEDED, CLOCK_HARNESS_MONOTONIC, COMPLETION_EXIT_AFTER_IDLE, METRIC_NAMES, @@ -60,10 +63,15 @@ from scripts.agent_benchmark.lifecycle import ( SUBMISSION_STDIN_ONCE, UNIT_NANOSECONDS, CaptureStream, + CallerEvent, + CallerTerminal, InvocationSpec, InvocationResult, + HarnessOutcome, LifecycleRecoveryError, ParsedMetric, + ProcessOutcome, + ProductOutcome, env_pairs, recover_invocation, run_invocation, @@ -89,6 +97,49 @@ from scripts.agent_benchmark.web_validation import ( ) +def _invocation_result( + *, + product_status: str = "succeeded", + harness_reason: str = "success", + exit_code: int = 0, + metrics: tuple[ParsedMetric, ...] = (), + spec_digest_value: str = "sha256:" + "a" * 64, + started_at: str = "2026-08-11T00:00:00+00:00", + ended_at: str = "2026-08-11T00:00:01+00:00", +) -> InvocationResult: + """Build an independent-axis lifecycle value for integration seams.""" + stream = CaptureStream("stdout", "", 0, 0, False) + product_reason = { + "succeeded": CALLER_REASON_SUCCESS, + "failed": CALLER_REASON_ERROR, + "unknown": "unavailable", + }[product_status] + harness_passed = harness_reason == "success" + return InvocationResult( + ProductOutcome(product_status, product_reason), + HarnessOutcome( + "passed" if harness_passed else "failed", + harness_reason, + harness_passed, + True, + ), + ProcessOutcome("exited", exit_code, None), + True, + False, + (), + stream, + replace(stream, stream="stderr"), + "", + "", + None, + spec_digest_value, + started_at, + ended_at, + 1, + metrics, + ) + + def _cell(cell_id: str, caller: str, model: str, effort: str) -> dict: return { "id": cell_id, @@ -387,8 +438,11 @@ class FakeAdapter: return run_invocation( spec, parse_event=lambda _stream, line: { - "FINISH": "finish", - "IDLE": "idle", + "FINISH": ( + CallerTerminal(CALLER_STATUS_SUCCEEDED, CALLER_REASON_SUCCESS), + CallerEvent("finish"), + ), + "IDLE": CallerEvent("idle"), }.get(line.strip()), on_started=lambda locator: on_started(locator, spec_digest(spec)), ) @@ -495,7 +549,9 @@ class ConnectivityIntegrationTest(unittest.TestCase): attempt.identity.attempt, caller, "sha256:" + "3" * 64, - "success", + ProductOutcome("succeeded", CALLER_REASON_SUCCESS), + HarnessOutcome("passed", "success", True, True), + ProcessOutcome("exited", 0, None), timeline, usage, WorkspaceWriteObservation( @@ -626,7 +682,25 @@ class ConnectivityIntegrationTest(unittest.TestCase): build_web_validation(manifest, workspace, measurement, render), ) return self.store.publish_terminal( - attempt, "success", result={"terminal_reason": "success"} + attempt, + "completed", + result={ + "product": { + "status": "succeeded", + "reason": CALLER_REASON_SUCCESS, + }, + "harness": { + "status": "passed", + "reason": "success", + "ordered_terminal": True, + "cleanup_complete": True, + }, + "process": { + "status": "exited", + "exit_code": 0, + "signal": None, + }, + }, ) def _run_live_scoring_mutation(self, case, mutate): @@ -651,14 +725,7 @@ class ConnectivityIntegrationTest(unittest.TestCase): def invoke(invocation, _on_started): mutate(invocation, secret, base_url) - stream = CaptureStream("stdout", "", 0, 0, False) - lifecycle = InvocationResult( - True, "success", 0, None, True, True, True, False, - (), stream, replace(stream, stream="stderr"), "", "", None, - "sha256:" + "a" * 64, - "2026-08-11T00:00:00+00:00", - "2026-08-11T00:00:01+00:00", 1, (), - ) + lifecycle = _invocation_result() binding = ( evaluator.iop.route_kind, evaluator.iop.route_id, @@ -804,11 +871,12 @@ class ConnectivityIntegrationTest(unittest.TestCase): digests = {state["spec_digest"], result["spec_digest"], header["spec_digest"]} self.assertEqual(len(digests), 1, attempt_root) self.assertRegex(digests.pop(), r"^sha256:[0-9a-f]{64}$") - self.assertEqual(state["state"], "success") - self.assertIs(result["success"], True) - self.assertEqual(result["terminal_reason"], "success") - self.assertIs(result["finish_then_idle_then_quiet"], True) - self.assertIs(result["cleanup_complete"], True) + self.assertEqual(state["state"], "completed") + self.assertEqual(result["product"]["status"], "succeeded") + self.assertEqual(result["harness"]["status"], "passed") + self.assertIs(result["harness"]["ordered_terminal"], True) + self.assertIs(result["harness"]["cleanup_complete"], True) + self.assertEqual(result["process"]["status"], "exited") self.assertIs(result["process_group_alive"], False) control_dir = Path(state["locator"]["control_dir"]) alias = control_dir.parent @@ -849,7 +917,7 @@ class ConnectivityIntegrationTest(unittest.TestCase): callers.add(caller) # This matrix binds the cell id to the caller name. self.assertEqual(measurement.cell_id, caller) - self.assertEqual(measurement.terminal_reason, "success") + self.assertEqual(measurement.harness.reason, "success") observed = { name for name, item in measurement.usage.items() if item.status == "observed" @@ -1040,9 +1108,9 @@ class ConnectivityIntegrationTest(unittest.TestCase): ): exit_code = benchmark_cli.main(["run", "--manifest", str(path)]) - self.assertEqual(exit_code, 0, stderr.getvalue()) - self.assertIn("ok: run run_id=", stdout.getvalue()) - self.assertEqual(stderr.getvalue(), "") + self.assertEqual(exit_code, 69) + self.assertEqual(stdout.getvalue(), "") + self.assertIn("artifact_failed=2", stderr.getvalue()) run_roots = list((self.root / manifest.output_root).glob("run-*")) self.assertEqual(len(run_roots), 1) preflight = json.loads( @@ -1106,8 +1174,10 @@ class ConnectivityIntegrationTest(unittest.TestCase): status = self.store.status(manifest=manifest, run=self.store.open( manifest, match.group(1) # type: ignore[union-attr] )) - self.assertEqual(status["attempts"]["failed"], 1) - self.assertEqual(status["attempts"]["success"], 1) + self.assertEqual(status["attempts"]["completed"], 2) + self.assertEqual(status["outcomes"]["product"]["unknown"], 1) + self.assertEqual(status["outcomes"]["product"]["succeeded"], 1) + self.assertEqual(status["outcomes"]["artifact"]["failed"], 2) self.assertEqual(status["attempts"]["running"], 0) def test_cli_resume_retries_append_only_and_status_is_read_only(self) -> None: @@ -1172,8 +1242,10 @@ class ConnectivityIntegrationTest(unittest.TestCase): ] ) - self.assertEqual(resume_exit, 0, resume_stderr.getvalue()) - self.assertIn("ok: resume", resume_stdout.getvalue()) + self.assertEqual(resume_exit, 69) + self.assertEqual(resume_stdout.getvalue(), "") + self.assertIn("product_succeeded=1", resume_stderr.getvalue()) + self.assertIn("artifact_failed=1", resume_stderr.getvalue()) self.assertEqual( old_bytes, { @@ -1184,7 +1256,7 @@ class ConnectivityIntegrationTest(unittest.TestCase): self.assertTrue( next(run_root.glob("cells/*/repetition-*/attempt-000002/attempt.json")) .read_text(encoding="utf-8") - .find('"state":"success"') + .find('"state":"completed"') >= 0 ) self.assertEqual(len(list((run_root / "preflight").glob("*.json"))), 2) @@ -1205,7 +1277,8 @@ class ConnectivityIntegrationTest(unittest.TestCase): ["status", "--manifest", str(path), "--run-id", run_id] ) self.assertEqual(status_exit, 0, status_stderr.getvalue()) - self.assertIn("'success': 1", status_stdout.getvalue()) + self.assertIn("product_succeeded=1", status_stdout.getvalue()) + self.assertIn("artifact_failed=1", status_stdout.getvalue()) self.assertEqual( before_status, { @@ -1428,14 +1501,7 @@ class ConnectivityIntegrationTest(unittest.TestCase): def invoke(invocation, _on_started): captured["spec"] = invocation.spec - stream = CaptureStream("stdout", "", 0, 0, False) - lifecycle = InvocationResult( - True, "success", 0, None, True, True, True, False, - (), stream, replace(stream, stream="stderr"), "", "", None, - "sha256:" + "a" * 64, - "2026-08-11T00:00:00+00:00", - "2026-08-11T00:00:01+00:00", 1, (), - ) + lifecycle = _invocation_result() return CodexInvocationResult( lifecycle, caller_binding["value"] ) @@ -1476,7 +1542,10 @@ class ConnectivityIntegrationTest(unittest.TestCase): self.manifest.timeout, lambda *_args: None, ) - self.assertTrue(result.success) + self.assertEqual( + (result.product, result.harness, result.process), + ("succeeded", "passed", "exited"), + ) expected_binding = ( evaluator.iop.route_kind, evaluator.iop.route_id, @@ -1509,8 +1578,8 @@ class ConnectivityIntegrationTest(unittest.TestCase): self.manifest.timeout, lambda *_args: None, ) - self.assertFalse(mismatch.success) - self.assertEqual(mismatch.terminal_reason, "binding_mismatch") + self.assertEqual(mismatch.harness, "failed") + self.assertEqual(mismatch.reason, "binding_mismatch") self.assertEqual( mismatch.effective_binding, caller_binding["value"] ) @@ -1562,14 +1631,7 @@ class ConnectivityIntegrationTest(unittest.TestCase): (evidence / "lifecycle-result.json").write_text( '{"record":"result"}\n', encoding="utf-8" ) - stream = CaptureStream("stdout", "", 0, 0, False) - lifecycle = InvocationResult( - True, "success", 0, None, True, True, True, False, - (), stream, replace(stream, stream="stderr"), "", "", None, - "sha256:" + "a" * 64, - "2026-08-11T00:00:00+00:00", - "2026-08-11T00:00:01+00:00", 1, (), - ) + lifecycle = _invocation_result() binding = ( evaluator.iop.route_kind, evaluator.iop.route_id, @@ -1605,7 +1667,10 @@ class ConnectivityIntegrationTest(unittest.TestCase): self.manifest.timeout, lambda *_args: None, ) - self.assertTrue(result.success) + self.assertEqual( + (result.product, result.harness, result.process), + ("succeeded", "passed", "exited"), + ) spec = captured["spec"] self.assertIsInstance(spec, InvocationSpec) assert isinstance(spec, InvocationSpec) @@ -1676,14 +1741,7 @@ class ConnectivityIntegrationTest(unittest.TestCase): (output / "diagnostic.txt").write_text( f"{base_url}\n{secret}\n", encoding="utf-8" ) - stream = CaptureStream("stdout", "", 0, 0, False) - lifecycle = InvocationResult( - True, "success", 0, None, True, True, True, False, - (), stream, replace(stream, stream="stderr"), "", "", None, - "sha256:" + "a" * 64, - "2026-08-11T00:00:00+00:00", - "2026-08-11T00:00:01+00:00", 1, (), - ) + lifecycle = _invocation_result() binding = ( evaluator.iop.route_kind, evaluator.iop.route_id, @@ -1721,7 +1779,10 @@ class ConnectivityIntegrationTest(unittest.TestCase): self.manifest.timeout, lambda *_args: None, ) - self.assertTrue(result.success) + self.assertEqual( + (result.product, result.harness, result.process), + ("succeeded", "passed", "exited"), + ) finalized = adapter.finalize_evidence(blind) self.assertEqual( (finalized.safe, finalized.reason), @@ -1886,14 +1947,7 @@ class ConnectivityIntegrationTest(unittest.TestCase): (sensitive_dir / "runtime.txt").write_text( secret + "\n" + base_url, encoding="utf-8" ) - stream = CaptureStream("stdout", "", 0, 0, False) - lifecycle = InvocationResult( - True, "success", 0, None, True, True, True, False, - (), stream, replace(stream, stream="stderr"), "", "", None, - "sha256:" + "a" * 64, - "2026-08-11T00:00:00+00:00", - "2026-08-11T00:00:01+00:00", 1, (), - ) + lifecycle = _invocation_result() binding = ( evaluator.iop.route_kind, evaluator.iop.route_id, @@ -1986,14 +2040,7 @@ class ConnectivityIntegrationTest(unittest.TestCase): if not first: self.assertTrue(workers) self.assertFalse(workers[0].is_alive()) - stream = CaptureStream("stdout", "", 0, 0, False) - lifecycle = InvocationResult( - True, "success", 0, None, True, True, True, False, - (), stream, replace(stream, stream="stderr"), "", "", None, - "sha256:" + "a" * 64, - "2026-08-11T00:00:00+00:00", - "2026-08-11T00:00:01+00:00", 1, (), - ) + lifecycle = _invocation_result() binding = ( evaluator.iop.route_kind, evaluator.iop.route_id, @@ -2098,7 +2145,7 @@ class ConnectivityIntegrationTest(unittest.TestCase): workers[0].join(5) self.assertFalse(workers[0].is_alive()) self.assertEqual(len(worker_results), 1) - self.assertTrue(worker_results[0].cleanup_complete) + self.assertTrue(worker_results[0].harness.cleanup_complete) self.assertFalse(worker_results[0].process_group_alive) receipt = json.loads( ( @@ -2116,7 +2163,10 @@ class ConnectivityIntegrationTest(unittest.TestCase): self.manifest.timeout, lambda *_args: None, ) - self.assertTrue(retry.success) + self.assertEqual( + (retry.product, retry.harness, retry.process), + ("succeeded", "passed", "exited"), + ) self.assertTrue(adapter.finalize_evidence(blind).safe) def test_catalog_only_never_creates_ready_binding(self) -> None: @@ -2270,7 +2320,6 @@ class ConnectivityIntegrationTest(unittest.TestCase): ) def invoke(_invocation, _on_started): - stream = CaptureStream("stdout", "", 0, 0, False) metric = ParsedMetric( "model_duration", 1, @@ -2289,13 +2338,7 @@ class ConnectivityIntegrationTest(unittest.TestCase): metric_model, "call-1", ) - lifecycle = InvocationResult( - True, "success", 0, None, True, True, True, False, - (), stream, replace(stream, stream="stderr"), "", "", None, - "sha256:" + "a" * 64, - "2026-08-11T00:00:00+00:00", - "2026-08-11T00:00:01+00:00", 1, (metric,), - ) + lifecycle = _invocation_result(metrics=(metric,)) binding = ( evaluator.iop.route_kind, evaluator.iop.route_id, @@ -2335,14 +2378,19 @@ class ConnectivityIntegrationTest(unittest.TestCase): finally: adapter.finalize_evidence(blind) - self.assertTrue(exercise(direct_manifest, "judge", "direct").success) - self.assertTrue( - exercise(preset_manifest, "judge-work", "preset-work").success + self.assertEqual( + exercise(direct_manifest, "judge", "direct").product, + "succeeded", ) - self.assertTrue( + self.assertEqual( + exercise(preset_manifest, "judge-work", "preset-work").product, + "succeeded", + ) + self.assertEqual( exercise( preset_manifest, "judge-plan", "preset-unqualified", metric_stage="" - ).success + ).product, + "succeeded", ) for stage, model in (("plan", "judge-work"), ("work", "judge-plan")): with self.subTest(stage=stage, model=model): @@ -2503,13 +2551,11 @@ class ConnectivityIntegrationTest(unittest.TestCase): ) self.assertEqual(raised.exception.issue_code, "stream_incompatible") - failed_stream = CaptureStream("stdout", "", 0, 0, False) - failed_lifecycle = InvocationResult( - False, "nonzero_exit", 1, None, True, False, True, False, - (), failed_stream, replace(failed_stream, stream="stderr"), "", "", None, - "sha256:" + "9" * 64, - "2026-08-11T00:00:00+00:00", - "2026-08-11T00:00:01+00:00", 1, (), + failed_lifecycle = _invocation_result( + product_status="unknown", + harness_reason="nonzero_exit", + exit_code=1, + spec_digest_value="sha256:" + "9" * 64, ) agy_adapter._invokers = live_iop._InvokerSeams( # type: ignore[attr-defined] live_iop._DEFAULT_INVOKERS.claude, @@ -2526,14 +2572,7 @@ class ConnectivityIntegrationTest(unittest.TestCase): ) self.assertIs(failed, failed_lifecycle) - stream = CaptureStream("stdout", "", 0, 0, False) - lifecycle = InvocationResult( - True, "success", 0, None, True, True, True, False, - (), stream, replace(stream, stream="stderr"), "", "", None, - "sha256:" + "a" * 64, - "2026-08-11T00:00:00+00:00", - "2026-08-11T00:00:01+00:00", 1, (), - ) + lifecycle = _invocation_result() invocation = CodexInvocation( InvocationSpec( argv=("codex",), cwd="/tmp", env=(), @@ -2618,27 +2657,9 @@ class ConnectivityIntegrationTest(unittest.TestCase): "agy-preset", ) - stream = CaptureStream("stdout", "", 0, 0, False) - lifecycle = InvocationResult( - True, - "success", - 0, - None, - True, - True, - True, - False, - (), - stream, - replace(stream, stream="stderr"), - "", - "", - None, - "sha256:" + "a" * 64, - "2026-08-12T00:00:00+00:00", - "2026-08-12T00:00:01+00:00", - 1, - (), + lifecycle = _invocation_result( + started_at="2026-08-12T00:00:00+00:00", + ended_at="2026-08-12T00:00:01+00:00", ) built: list[tuple[str, object]] = [] invoked: list[tuple[str, object]] = [] diff --git a/scripts/agent_benchmark/lifecycle.py b/scripts/agent_benchmark/lifecycle.py index d51615f6..ef6f44a8 100644 --- a/scripts/agent_benchmark/lifecycle.py +++ b/scripts/agent_benchmark/lifecycle.py @@ -61,6 +61,7 @@ EVENT_IDLE = "idle" EVENT_QUIET = "quiet" EVENT_EXITED = "exited" EVENT_TERMINAL = "terminal" +EVENT_CALLER_TERMINAL = "caller_terminal" PARSER_TERMINAL_KINDS = (EVENT_FINISH, EVENT_IDLE) METRIC_PREFIX = "metric:" @@ -122,6 +123,7 @@ REASON_CONTROLLER_LOST = "controller_lost" REASON_RECOVERED_STOP = "recovered_stop" REASON_CLEANUP_FAILED = "cleanup_failed" REASON_SUPERVISOR_ERROR = "supervisor_error" +REASON_INTERRUPTED = "interrupted" TERMINAL_REASONS = ( REASON_SUCCESS, REASON_START_CALLBACK_FAILED, @@ -139,6 +141,43 @@ TERMINAL_REASONS = ( REASON_RECOVERED_STOP, REASON_CLEANUP_FAILED, REASON_SUPERVISOR_ERROR, + REASON_INTERRUPTED, +) + +CALLER_STATUS_SUCCEEDED = "succeeded" +CALLER_STATUS_FAILED = "failed" +CALLER_STATUSES = (CALLER_STATUS_SUCCEEDED, CALLER_STATUS_FAILED) +CALLER_REASON_SUCCESS = "caller_success" +CALLER_REASON_ERROR = "caller_error" +CALLER_REASONS = (CALLER_REASON_SUCCESS, CALLER_REASON_ERROR) + +PRODUCT_STATUS_SUCCEEDED = "succeeded" +PRODUCT_STATUS_FAILED = "failed" +PRODUCT_STATUS_UNKNOWN = "unknown" +PRODUCT_STATUSES = ( + PRODUCT_STATUS_SUCCEEDED, PRODUCT_STATUS_FAILED, PRODUCT_STATUS_UNKNOWN, +) +PRODUCT_REASON_UNAVAILABLE = "unavailable" +PRODUCT_REASONS = ( + CALLER_REASON_SUCCESS, CALLER_REASON_ERROR, PRODUCT_REASON_UNAVAILABLE, +) + +HARNESS_STATUS_PASSED = "passed" +HARNESS_STATUS_FAILED = "failed" +HARNESS_STATUSES = (HARNESS_STATUS_PASSED, HARNESS_STATUS_FAILED) +HARNESS_REASONS = TERMINAL_REASONS + +PROCESS_STATUS_EXITED = "exited" +PROCESS_STATUS_SIGNALLED = "signalled" +PROCESS_STATUS_TIMED_OUT = "timed_out" +PROCESS_STATUS_CANCELLED = "cancelled" +PROCESS_STATUS_NOT_STARTED = "not_started" +PROCESS_STATUSES = ( + PROCESS_STATUS_EXITED, + PROCESS_STATUS_SIGNALLED, + PROCESS_STATUS_TIMED_OUT, + PROCESS_STATUS_CANCELLED, + PROCESS_STATUS_NOT_STARTED, ) FAULT_NONE = "" @@ -165,7 +204,7 @@ SUPERVISOR_ERR_FILENAME = "supervisor.err" REDACTED = "[redacted]" RECEIPT_VERSION = 1 -JOURNAL_VERSION = 1 +JOURNAL_VERSION = 2 MAX_TASK_PAYLOAD_BYTES = 1 << 20 MAX_CAPTURE_BYTES_LIMIT = 1 << 24 @@ -279,6 +318,36 @@ class ParsedMetric: overlap: bool = False +@dataclass(frozen=True) +class CallerEvent: + """One closed caller lifecycle observation.""" + + kind: str + + def __post_init__(self) -> None: + if self.kind not in PARSER_TERMINAL_KINDS: + raise LifecycleValidationError("caller event kind is invalid") + + +@dataclass(frozen=True) +class CallerTerminal: + """The caller-declared product outcome, independent of its process exit.""" + + status: str + reason: str + + def __post_init__(self) -> None: + expected = { + CALLER_STATUS_SUCCEEDED: CALLER_REASON_SUCCESS, + CALLER_STATUS_FAILED: CALLER_REASON_ERROR, + } + if self.status not in CALLER_STATUSES or self.reason != expected[self.status]: + raise LifecycleValidationError("caller terminal is invalid") + + +CallerObservation = CallerEvent | CallerTerminal | ParsedMetric + + @dataclass(frozen=True) class CaptureStream: stream: str @@ -310,16 +379,73 @@ class TerminalOutcome: @dataclass(frozen=True) -class InvocationResult: - """Terminal projection of one invocation, including typed observations.""" +class ProductOutcome: + status: str + reason: str - success: bool - terminal_reason: str + def __post_init__(self) -> None: + expected = { + PRODUCT_STATUS_SUCCEEDED: CALLER_REASON_SUCCESS, + PRODUCT_STATUS_FAILED: CALLER_REASON_ERROR, + PRODUCT_STATUS_UNKNOWN: PRODUCT_REASON_UNAVAILABLE, + } + if self.status not in PRODUCT_STATUSES or self.reason != expected[self.status]: + raise LifecycleValidationError("product outcome is invalid") + + +@dataclass(frozen=True) +class HarnessOutcome: + status: str + reason: str + ordered_terminal: bool + cleanup_complete: bool + + def __post_init__(self) -> None: + if ( + self.status not in HARNESS_STATUSES + or self.reason not in HARNESS_REASONS + or not isinstance(self.ordered_terminal, bool) + or not isinstance(self.cleanup_complete, bool) + or (self.status == HARNESS_STATUS_PASSED) != (self.reason == REASON_SUCCESS) + or (not self.cleanup_complete) != (self.reason == REASON_CLEANUP_FAILED) + or (self.status == HARNESS_STATUS_PASSED and not self.ordered_terminal) + ): + raise LifecycleValidationError("harness outcome is invalid") + + +@dataclass(frozen=True) +class ProcessOutcome: + status: str exit_code: Optional[int] signal: Optional[int] + + def __post_init__(self) -> None: + if self.status not in PROCESS_STATUSES: + raise LifecycleValidationError("process outcome is invalid") + if self.exit_code is not None and ( + not isinstance(self.exit_code, int) or isinstance(self.exit_code, bool) + ): + raise LifecycleValidationError("process exit code is invalid") + if self.signal is not None and ( + not isinstance(self.signal, int) or isinstance(self.signal, bool) + ): + raise LifecycleValidationError("process signal is invalid") + if self.status == PROCESS_STATUS_SIGNALLED and self.signal is None: + raise LifecycleValidationError("signalled process requires a signal") + if self.status in (PROCESS_STATUS_EXITED, PROCESS_STATUS_NOT_STARTED) and self.signal is not None: + raise LifecycleValidationError("process signal contradicts its status") + if self.status == PROCESS_STATUS_NOT_STARTED and self.exit_code is not None: + raise LifecycleValidationError("not-started process has an exit code") + + +@dataclass(frozen=True) +class InvocationResult: + """Independent product, harness, and process outcomes for one invocation.""" + + product: ProductOutcome + harness: HarnessOutcome + process: ProcessOutcome submitted: bool - finish_then_idle_then_quiet: bool - cleanup_complete: bool process_group_alive: bool events: tuple[LifecycleEvent, ...] stdout: CaptureStream @@ -978,7 +1104,11 @@ class _Supervisor: signal_num = -code if code < 0 else None group_alive = _group_alive(self.pgid) outcome = { - "reason": reason, + "reason": ( + reason + if not group_alive and io_complete and descendants_reaped + else REASON_CLEANUP_FAILED + ), "exit_code": exit_code, "signal": signal_num, "caller_launched": self.child is not None, @@ -1315,11 +1445,13 @@ class _Invocation: self.submitted = False self.finish_at: Optional[float] = None self.idle_at: Optional[float] = None + self.caller_terminal: Optional[CallerTerminal] = None self.last_output_at: Optional[float] = None self.quiet = False self.exited = False self.exit_code: Optional[int] = None self.signal: Optional[int] = None + self.process_status_hint: Optional[str] = None self.run_deadline = 0.0 self.control_dir: Optional[Path] = None self.owns_control_dir = False @@ -1558,7 +1690,7 @@ class _Invocation: def _apply_parsed( self, parsed: Any, stream: str, frame: dict[str, Any], redacted: str ) -> None: - """Apply one parser result: a terminal string, observations, or both.""" + """Apply one closed caller observation or a bounded tuple of them.""" if parsed is None: return items = parsed if isinstance(parsed, tuple) else (parsed,) @@ -1579,27 +1711,29 @@ class _Invocation: if isinstance(item, ParsedMetric): self._record_metric(item, stream, frame) return - if not isinstance(item, str) or not item: - self.reason = self.reason or REASON_MALFORMED_EVENT + if isinstance(item, CallerEvent): + self._apply_terminal_evidence(item.kind, stream, frame, redacted) return - if item.startswith(METRIC_PREFIX): - self._record_metric_label(item, stream, frame, redacted) + if isinstance(item, CallerTerminal): + self._apply_caller_terminal(item, stream, frame) return - if item not in PARSER_TERMINAL_KINDS: - self.reason = self.reason or REASON_MALFORMED_EVENT - return - self._apply_terminal_evidence(item, stream, frame, redacted) + self.reason = self.reason or REASON_MALFORMED_EVENT - def _record_metric_label( - self, parsed: str, stream: str, frame: dict[str, Any], redacted: str + def _apply_caller_terminal( + self, terminal: CallerTerminal, stream: str, frame: dict[str, Any] ) -> None: - """Record one untyped caller metric label with no numeric payload.""" - metric_kind = self._validate_metric_kind(parsed) - if metric_kind is None: - self.reason = self.reason or REASON_MALFORMED_EVENT - elif self.metric_events < MAX_METRIC_EVENTS: - self.metric_events += 1 - self._add_event(metric_kind, SOURCE_CALLER_OUTPUT, stream, frame, redacted) + if self.caller_terminal is not None: + self.reason = self.reason or REASON_DUPLICATE_EVENT + return + self.caller_terminal = terminal + self._add_event( + EVENT_CALLER_TERMINAL, + SOURCE_CALLER_OUTPUT, + stream, + frame, + f"status={terminal.status} reason={terminal.reason}", + safe=True, + ) def _record_metric( self, metric: ParsedMetric, stream: str, frame: dict[str, Any] @@ -1669,9 +1803,11 @@ class _Invocation: def _check_deadlines(self) -> None: now = time.monotonic() if _is_cancelled(self.cancellation): + self.process_status_hint = PROCESS_STATUS_CANCELLED self.reason = REASON_CANCELLED return if now >= self.run_deadline: + self.process_status_hint = PROCESS_STATUS_TIMED_OUT self.reason = REASON_TIMED_OUT return if ( @@ -1688,6 +1824,11 @@ class _Invocation: and len(self.stream_eof) == len(self.captures) and not self.quiet ): + if self.caller_terminal is not None and self.idle_at is not None: + # A caller-declared product failure commonly exits non-zero. + # Preserve that code on the process axis while still allowing + # the ordered terminal stream to reach quiet. + return if self.exit_code != 0: self.reason = REASON_NONZERO_EXIT elif self.idle_at is None: @@ -1702,10 +1843,18 @@ class _Invocation: self.quiet = True self._add_event(EVENT_QUIET, SOURCE_HARNESS, "", {"ns": time.monotonic_ns()}, "") if self.spec.completion_mode == COMPLETION_STOP_AFTER_IDLE: - self.reason = REASON_SUCCESS + self.reason = ( + REASON_SUCCESS + if self.caller_terminal is not None + else REASON_MISSING_IDLE + ) return if self.exited: - self.reason = REASON_SUCCESS if self.exit_code == 0 else REASON_NONZERO_EXIT + self.reason = ( + REASON_SUCCESS + if self.caller_terminal is not None + else (REASON_NONZERO_EXIT if self.exit_code != 0 else REASON_MISSING_IDLE) + ) # -- terminal handshake ------------------------------------------------ @@ -1814,16 +1963,48 @@ class _Invocation: cleanup_complete = bool(outcome.get("cleanup_complete")) group_alive = bool(outcome.get("process_group_alive")) ordered = bool(self.finish_at is not None and self.idle_at is not None and self.quiet) - success = reason == REASON_SUCCESS and cleanup_complete and not group_alive and ordered + harness_passed = ( + reason == REASON_SUCCESS + and cleanup_complete + and not group_alive + and ordered + and self.caller_terminal is not None + ) + product_evidence_valid = ( + self.caller_terminal is not None + and self.finish_at is not None + and self.idle_at is not None + and reason in {REASON_SUCCESS, REASON_CLEANUP_FAILED} + ) + if not product_evidence_valid: + product = ProductOutcome(PRODUCT_STATUS_UNKNOWN, PRODUCT_REASON_UNAVAILABLE) + elif self.caller_terminal.status == CALLER_STATUS_SUCCEEDED: + product = ProductOutcome(PRODUCT_STATUS_SUCCEEDED, CALLER_REASON_SUCCESS) + else: + product = ProductOutcome(PRODUCT_STATUS_FAILED, CALLER_REASON_ERROR) + exit_code = self.exit_code if self.exit_code is not None else outcome.get("exit_code") + process_signal = self.signal if self.signal is not None else outcome.get("signal") + if self.process_status_hint is not None: + process_status = self.process_status_hint + elif not self.submitted: + process_status = PROCESS_STATUS_NOT_STARTED + exit_code = None + process_signal = None + elif process_signal is not None: + process_status = PROCESS_STATUS_SIGNALLED + else: + process_status = PROCESS_STATUS_EXITED evidence_dir = Path(self.spec.evidence_dir) result = InvocationResult( - success=success, - terminal_reason=reason, - exit_code=self.exit_code if self.exit_code is not None else outcome.get("exit_code"), - signal=self.signal if self.signal is not None else outcome.get("signal"), + product=product, + harness=HarnessOutcome( + HARNESS_STATUS_PASSED if harness_passed else HARNESS_STATUS_FAILED, + REASON_SUCCESS if harness_passed else reason, + ordered, + cleanup_complete, + ), + process=ProcessOutcome(process_status, exit_code, process_signal), submitted=self.submitted, - finish_then_idle_then_quiet=ordered, - cleanup_complete=cleanup_complete, process_group_alive=group_alive, events=tuple(self.events), stdout=self.captures["stdout"].freeze(), @@ -1900,13 +2081,10 @@ def _event_record(event: LifecycleEvent) -> dict[str, Any]: def _result_record(result: InvocationResult, spec: InvocationSpec) -> dict[str, Any]: return { "record": "result", - "success": result.success, - "terminal_reason": result.terminal_reason, - "exit_code": result.exit_code, - "signal": result.signal, + "product": _product_record(result.product), + "harness": _harness_record(result.harness), + "process": _process_record(result.process), "submitted": result.submitted, - "finish_then_idle_then_quiet": result.finish_then_idle_then_quiet, - "cleanup_complete": result.cleanup_complete, "process_group_alive": result.process_group_alive, "submission_mode": spec.submission_mode, "completion_mode": spec.completion_mode, @@ -1921,6 +2099,27 @@ def _result_record(result: InvocationResult, spec: InvocationSpec) -> dict[str, } +def _product_record(outcome: ProductOutcome) -> dict[str, Any]: + return {"status": outcome.status, "reason": outcome.reason} + + +def _harness_record(outcome: HarnessOutcome) -> dict[str, Any]: + return { + "status": outcome.status, + "reason": outcome.reason, + "ordered_terminal": outcome.ordered_terminal, + "cleanup_complete": outcome.cleanup_complete, + } + + +def _process_record(outcome: ProcessOutcome) -> dict[str, Any]: + return { + "status": outcome.status, + "exit_code": outcome.exit_code, + "signal": outcome.signal, + } + + def _capture_record(capture: CaptureStream) -> dict[str, Any]: return { "stream": capture.stream, @@ -1943,9 +2142,9 @@ def _publish_evidence(result: InvocationResult, spec: InvocationSpec) -> None: } terminal = { "record": "terminal", - "terminal_reason": result.terminal_reason, - "success": result.success, - "cleanup_complete": result.cleanup_complete, + "product": _product_record(result.product), + "harness": _harness_record(result.harness), + "process": _process_record(result.process), "process_group_alive": result.process_group_alive, "ended_at": result.ended_at, } @@ -2101,9 +2300,9 @@ def run_invocation( Args: spec: Frozen invocation specification. parse_event: Adapter parser mapping ``(stream, line)`` to ``None``, - ``"finish"``, ``"idle"``, a ``"metric:"`` label, a typed - ``ParsedMetric``, or a bounded tuple of those items when one caller - line carries observations and terminal evidence together. + a typed ``CallerEvent``, ``CallerTerminal``, ``ParsedMetric``, or a + bounded tuple of those items when one caller line carries multiple + observations. on_started: Required durable locator commit callback. redact: Optional adapter redactor for exact secret values. cancellation: Optional cancellation token, event or predicate. diff --git a/scripts/agent_benchmark/lifecycle_test.py b/scripts/agent_benchmark/lifecycle_test.py index 413bd2de..63b8d6b1 100644 --- a/scripts/agent_benchmark/lifecycle_test.py +++ b/scripts/agent_benchmark/lifecycle_test.py @@ -16,6 +16,10 @@ from dataclasses import replace from pathlib import Path from scripts.agent_benchmark.lifecycle import ( + CALLER_REASON_ERROR, + CALLER_REASON_SUCCESS, + CALLER_STATUS_FAILED, + CALLER_STATUS_SUCCEEDED, COMPLETION_EXIT_AFTER_IDLE, COMPLETION_STOP_AFTER_IDLE, REASON_CANCELLED, @@ -26,6 +30,7 @@ from scripts.agent_benchmark.lifecycle import ( REASON_MISSING_IDLE, REASON_NONZERO_EXIT, REASON_OUT_OF_ORDER_EVENT, + REASON_PARSER_ERROR, REASON_READER_ERROR, REASON_RECOVERED_STOP, REASON_START_CALLBACK_FAILED, @@ -34,6 +39,9 @@ from scripts.agent_benchmark.lifecycle import ( SUBMISSION_ARGV_TASK, SUBMISSION_STDIN_ONCE, CancellationToken, + CallerEvent, + CallerTerminal, + HarnessOutcome, InvocationSpec, LifecycleError, LifecycleRecoveryError, @@ -51,8 +59,14 @@ from scripts.agent_benchmark.lifecycle import ( from scripts.agent_benchmark.manifest import Timeout -def _events(_: str, line: str) -> str | None: - return {"FINISH": "finish", "IDLE": "idle"}.get(line) +def _events(_: str, line: str): + return { + "FINISH": ( + CallerTerminal(CALLER_STATUS_SUCCEEDED, CALLER_REASON_SUCCESS), + CallerEvent("finish"), + ), + "IDLE": CallerEvent("idle"), + }.get(line) class LifecycleTest(unittest.TestCase): @@ -162,12 +176,13 @@ class LifecycleTest(unittest.TestCase): def test_exit_after_idle_publishes_ordered_atomic_evidence(self) -> None: result = self._run(self._spec("print('FINISH'); print('IDLE')")) - self.assertTrue(result.success) - self.assertTrue(result.cleanup_complete) + self.assertTrue(result.product.status == "succeeded") + self.assertTrue(result.harness.cleanup_complete) self.assertFalse(result.process_group_alive) - self.assertTrue(result.finish_then_idle_then_quiet) + self.assertTrue(result.harness.ordered_terminal) self.assertEqual([event.kind for event in result.events], [ - "submitted", "first_output", "finish", "idle", "exited", "quiet", + "submitted", "first_output", "caller_terminal", "finish", "idle", + "exited", "quiet", ]) self.assertTrue(Path(result.journal_path).is_file()) published = json.loads(Path(result.result_path).read_text(encoding="utf-8")) @@ -182,7 +197,7 @@ class LifecycleTest(unittest.TestCase): payload=b"single task payload", )) - self.assertTrue(result.success) + self.assertTrue(result.product.status == "succeeded") self.assertIn("single task payload", result.stdout.text) self.assertEqual(sum(event.kind == "submitted" for event in result.events), 1) @@ -196,10 +211,10 @@ class LifecycleTest(unittest.TestCase): )) self.assertLess(time.monotonic() - started, 6) - self.assertEqual(result.terminal_reason, REASON_TIMED_OUT) + self.assertEqual(result.harness.reason, REASON_TIMED_OUT) self.assertFalse(result.submitted) self.assertEqual(sum(event.kind == "submitted" for event in result.events), 0) - self.assertTrue(result.cleanup_complete) + self.assertTrue(result.harness.cleanup_complete) self.assertFalse(result.process_group_alive) def test_unterminated_final_idle_is_consumed_before_terminal(self) -> None: @@ -215,7 +230,7 @@ class LifecycleTest(unittest.TestCase): evidence_dir=str(evidence), )) - self.assertTrue(result.success) + self.assertTrue(result.product.status == "succeeded") kinds = [event.kind for event in result.events] self.assertLess(kinds.index("finish"), kinds.index("idle")) self.assertLess(kinds.index("idle"), kinds.index("quiet")) @@ -233,15 +248,15 @@ class LifecycleTest(unittest.TestCase): completion_mode=COMPLETION_STOP_AFTER_IDLE, )) - self.assertTrue(result.success) + self.assertTrue(result.product.status == "succeeded") self.assertLess(time.monotonic() - started, 8) - self.assertTrue(result.cleanup_complete) + self.assertTrue(result.harness.cleanup_complete) self.assertFalse(result.process_group_alive) def test_caller_output_cannot_synthesize_submission(self) -> None: result = self._run(self._spec("print('submitted'); print('FINISH'); print('IDLE')")) - self.assertTrue(result.success) + self.assertTrue(result.product.status == "succeeded") self.assertEqual(sum(event.kind == "submitted" for event in result.events), 1) self.assertEqual(result.events[0].source, "harness") @@ -257,9 +272,9 @@ class LifecycleTest(unittest.TestCase): evidence.mkdir() spec = replace(self._spec(source), evidence_dir=str(evidence)) result = self._run(spec) - self.assertFalse(result.success) - self.assertEqual(result.terminal_reason, reason) - self.assertTrue(result.cleanup_complete) + self.assertFalse(result.product.status == "succeeded") + self.assertEqual(result.harness.reason, reason) + self.assertTrue(result.harness.cleanup_complete) def test_malformed_parser_and_nonzero_exit_fail_closed(self) -> None: malformed = run_invocation( @@ -267,8 +282,8 @@ class LifecycleTest(unittest.TestCase): parse_event=lambda _stream, _line: "submitted", on_started=lambda _: None, ) - self.assertEqual(malformed.terminal_reason, REASON_MALFORMED_EVENT) - self.assertTrue(malformed.cleanup_complete) + self.assertEqual(malformed.harness.reason, REASON_MALFORMED_EVENT) + self.assertTrue(malformed.harness.cleanup_complete) evidence = self.root / "nonzero" evidence.mkdir() @@ -276,13 +291,111 @@ class LifecycleTest(unittest.TestCase): self._spec("print('FINISH'); print('IDLE'); raise SystemExit(7)"), evidence_dir=str(evidence), )) - self.assertEqual(failed.terminal_reason, REASON_NONZERO_EXIT) - self.assertTrue(failed.cleanup_complete) + self.assertEqual(failed.product.status, "succeeded") + self.assertEqual(failed.harness.status, "passed") + self.assertEqual(failed.process.exit_code, 7) + self.assertTrue(failed.harness.cleanup_complete) + + def test_product_error_can_have_clean_harness_and_process(self) -> None: + def parse_error(_stream: str, line: str): + return { + "ERROR": ( + CallerTerminal(CALLER_STATUS_FAILED, CALLER_REASON_ERROR), + CallerEvent("finish"), + ), + "IDLE": CallerEvent("idle"), + }.get(line) + + result = run_invocation( + self._spec("print('ERROR'); print('IDLE')"), + parse_event=parse_error, + on_started=lambda _: None, + ) + + self.assertEqual((result.product.status, result.product.reason), ( + "failed", CALLER_REASON_ERROR, + )) + self.assertEqual((result.harness.status, result.harness.reason), ( + "passed", "success", + )) + self.assertEqual((result.process.status, result.process.exit_code), ( + "exited", 0, + )) + + with self.assertRaises(LifecycleValidationError): + HarnessOutcome("failed", "success", True, True) + with self.assertRaises(LifecycleValidationError): + HarnessOutcome("passed", "success", False, True) + + def test_parser_failure_leaves_product_unknown(self) -> None: + def broken_parser(_stream: str, _line: str): + raise ValueError("synthetic parser failure") + + result = run_invocation( + self._spec("print('BROKEN')"), + parse_event=broken_parser, + on_started=lambda _: None, + ) + + self.assertEqual((result.product.status, result.product.reason), ( + "unknown", "unavailable", + )) + self.assertEqual((result.harness.status, result.harness.reason), ( + "failed", REASON_PARSER_ERROR, + )) + + def test_timeout_cancel_and_cleanup_do_not_fabricate_product(self) -> None: + timeout = self._run(self._spec("import time; time.sleep(30)", run_seconds=1)) + + token = CancellationToken() + timer = threading.Timer(0.2, token.cancel) + timer.start() + try: + cancel_root = self.root / "independent-cancel" + cancel_root.mkdir() + cancelled = self._run( + replace( + self._spec("import time; time.sleep(30)"), + evidence_dir=str(cancel_root), + ), + cancellation=token, + ) + finally: + timer.cancel() + + cleanup_root = self.root / "independent-cleanup" + cleanup_root.mkdir() + cleanup_control = self.root / "independent-cleanup-control" + + def collide_receipt(locator: SupervisorLocator) -> None: + (Path(locator.control_dir) / "cleanup-receipt.json").write_bytes( + b"collision" + ) + + cleanup = run_invocation( + replace( + self._spec("print('FINISH'); print('IDLE')"), + evidence_dir=str(cleanup_root), + control_dir=str(cleanup_control), + ), + parse_event=_events, + on_started=collide_receipt, + ) + + for result, product_status, process_status, reason in ( + (timeout, "unknown", "timed_out", REASON_TIMED_OUT), + (cancelled, "unknown", "cancelled", REASON_CANCELLED), + (cleanup, "succeeded", "exited", REASON_CLEANUP_FAILED), + ): + with self.subTest(reason=reason): + self.assertEqual(result.product.status, product_status) + self.assertEqual(result.harness.reason, reason) + self.assertEqual(result.process.status, process_status) def test_timeout_cancel_and_reader_error_all_cleanup(self) -> None: timeout = self._run(self._spec("import time; time.sleep(30)", run_seconds=1)) - self.assertEqual(timeout.terminal_reason, REASON_TIMED_OUT) - self.assertTrue(timeout.cleanup_complete) + self.assertEqual(timeout.harness.reason, REASON_TIMED_OUT) + self.assertTrue(timeout.harness.cleanup_complete) token = CancellationToken() timer = threading.Timer(0.2, token.cancel) @@ -296,8 +409,8 @@ class LifecycleTest(unittest.TestCase): ) finally: timer.cancel() - self.assertEqual(cancelled.terminal_reason, REASON_CANCELLED) - self.assertTrue(cancelled.cleanup_complete) + self.assertEqual(cancelled.harness.reason, REASON_CANCELLED) + self.assertTrue(cancelled.harness.cleanup_complete) evidence = self.root / "reader" evidence.mkdir() @@ -305,8 +418,8 @@ class LifecycleTest(unittest.TestCase): self._spec("print('FINISH'); print('IDLE')", fault_injection="reader_error"), evidence_dir=str(evidence), )) - self.assertEqual(reader_error.terminal_reason, REASON_READER_ERROR) - self.assertTrue(reader_error.cleanup_complete) + self.assertEqual(reader_error.harness.reason, REASON_READER_ERROR) + self.assertTrue(reader_error.harness.cleanup_complete) def test_redaction_and_capture_bounds_apply_before_publication(self) -> None: secret = "EXACT_SECRET_123456789" @@ -409,22 +522,22 @@ class LifecycleTest(unittest.TestCase): parse_event=_events, on_started=collide_receipt, ) - self.assertEqual(receipt_result.terminal_reason, REASON_CLEANUP_FAILED) - self.assertFalse(receipt_result.success) - self.assertFalse(receipt_result.cleanup_complete) + self.assertEqual(receipt_result.harness.reason, REASON_CLEANUP_FAILED) + self.assertEqual(receipt_result.product.status, "succeeded") + self.assertFalse(receipt_result.harness.cleanup_complete) self.assertFalse(receipt_result.process_group_alive) self.assertEqual( (receipt_control / "cleanup-receipt.json").read_bytes(), receipt_sentinel ) published = json.loads(Path(receipt_result.result_path).read_text(encoding="utf-8")) - self.assertEqual(published["terminal_reason"], REASON_CLEANUP_FAILED) + self.assertEqual(published["harness"]["reason"], REASON_CLEANUP_FAILED) journal = [ json.loads(line) for line in Path(receipt_result.journal_path).read_text(encoding="utf-8").splitlines() ] terminals = [record for record in journal if record.get("record") == "terminal"] self.assertEqual(len(terminals), 1) - self.assertEqual(terminals[0]["terminal_reason"], REASON_CLEANUP_FAILED) + self.assertEqual(terminals[0]["harness"]["reason"], REASON_CLEANUP_FAILED) def test_metric_kind_cannot_leak_secret(self) -> None: cases = ( @@ -452,7 +565,7 @@ class LifecycleTest(unittest.TestCase): on_started=lambda _: None, redact=redactor, ) - self.assertEqual(result.terminal_reason, REASON_MALFORMED_EVENT) + self.assertEqual(result.harness.reason, REASON_MALFORMED_EVENT) self.assertFalse(any(event.kind == f"metric:{metric_name}" for event in result.events)) persisted = "\n".join( path.read_text(encoding="utf-8") @@ -476,8 +589,9 @@ class LifecycleTest(unittest.TestCase): parse_event=parse_valid, on_started=lambda _: None, ) - self.assertTrue(valid.success) - self.assertIn("metric:duration_ms", [event.kind for event in valid.events]) + self.assertEqual(valid.product.status, "unknown") + self.assertEqual(valid.harness.reason, REASON_MALFORMED_EVENT) + self.assertNotIn("metric:duration_ms", [event.kind for event in valid.events]) def test_first_output_is_recorded_once_before_terminal_evidence(self) -> None: source = ( @@ -486,7 +600,7 @@ class LifecycleTest(unittest.TestCase): ) result = self._run(self._spec(source)) - self.assertTrue(result.success) + self.assertTrue(result.product.status == "succeeded") kinds = [event.kind for event in result.events] self.assertEqual(kinds.count("first_output"), 1) self.assertLess(kinds.index("submitted"), kinds.index("first_output")) @@ -499,7 +613,7 @@ class LifecycleTest(unittest.TestCase): def test_silent_caller_records_no_first_output(self) -> None: result = self._run(self._spec("import time; time.sleep(30)", run_seconds=1)) - self.assertEqual(result.terminal_reason, REASON_TIMED_OUT) + self.assertEqual(result.harness.reason, REASON_TIMED_OUT) self.assertNotIn("first_output", [event.kind for event in result.events]) def test_typed_observations_are_published_with_terminal_evidence(self) -> None: @@ -516,7 +630,7 @@ class LifecycleTest(unittest.TestCase): parse_event=parse_metric, on_started=lambda _: None, ) - self.assertTrue(result.success) + self.assertTrue(result.product.status == "succeeded") self.assertEqual( [(metric.name, metric.value) for metric in result.metrics], [("total_duration", 12_500_000), ("input_tokens", 11)], @@ -554,7 +668,7 @@ class LifecycleTest(unittest.TestCase): ), on_started=lambda _: None, ) - self.assertEqual(result.terminal_reason, REASON_MALFORMED_EVENT) + self.assertEqual(result.harness.reason, REASON_MALFORMED_EVENT) self.assertEqual(result.metrics, ()) def test_metric_event_overflow_is_malformed_after_retaining_the_bound(self) -> None: @@ -569,7 +683,7 @@ class LifecycleTest(unittest.TestCase): parse_event=parse_metric, on_started=lambda _: None, ) - self.assertEqual(result.terminal_reason, REASON_MALFORMED_EVENT) + self.assertEqual(result.harness.reason, REASON_MALFORMED_EVENT) self.assertEqual(len(result.metrics), MAX_METRIC_EVENTS) def test_adapter_redactor_cannot_corrupt_a_validated_observation(self) -> None: @@ -588,7 +702,7 @@ class LifecycleTest(unittest.TestCase): # rewrite a detail built from already validated closed fields. redact=lambda _line: "[structural]", ) - self.assertTrue(result.success) + self.assertTrue(result.product.status == "succeeded") detail = next( event.detail for event in result.events if event.kind == "metric:output_tokens" ) @@ -605,10 +719,10 @@ class LifecycleTest(unittest.TestCase): on_started=lambda _locator: (_ for _ in ()).throw(RuntimeError("durable write failed")), ) - self.assertEqual(result.terminal_reason, REASON_START_CALLBACK_FAILED) + self.assertEqual(result.harness.reason, REASON_START_CALLBACK_FAILED) self.assertFalse(result.submitted) self.assertFalse(marker.exists()) - self.assertTrue(result.cleanup_complete) + self.assertTrue(result.harness.cleanup_complete) def test_forged_live_locator_refuses_recovery(self) -> None: checked: list[SupervisorLocator] = [] @@ -625,7 +739,7 @@ class LifecycleTest(unittest.TestCase): on_started=verify, ) self.assertTrue(checked) - self.assertTrue(result.success) + self.assertTrue(result.product.status == "succeeded") def test_control_socket_bind_supports_short_symlink_alias(self) -> None: with tempfile.TemporaryDirectory( @@ -643,7 +757,7 @@ class LifecycleTest(unittest.TestCase): evidence_dir=str(evidence), control_dir=str(alias / "control"), )) - self.assertTrue(result.success) + self.assertTrue(result.product.status == "succeeded") self.assertTrue((target / "control" / "locator.json").is_file()) self.assertFalse((target / "control" / "control.sock").exists()) @@ -681,8 +795,8 @@ class LifecycleTest(unittest.TestCase): deadline = time.monotonic() + 2 while proc_path.exists() and time.monotonic() < deadline: time.sleep(.02) - self.assertTrue(result.success) - self.assertTrue(result.cleanup_complete) + self.assertTrue(result.product.status == "succeeded") + self.assertTrue(result.harness.cleanup_complete) self.assertFalse(result.process_group_alive) self.assertFalse(proc_path.exists()) receipt = json.loads( @@ -829,15 +943,15 @@ class LifecycleTest(unittest.TestCase): for line in Path(result.journal_path).read_text(encoding="utf-8").splitlines() ] terminals = [record for record in journal if record.get("record") == "terminal"] - self.assertEqual(result.terminal_reason, expected) + self.assertEqual(result.harness.reason, expected) self.assertEqual(receipt["reason"], expected) - self.assertEqual(published["terminal_reason"], expected) - self.assertEqual(receipt["exit_code"], published["exit_code"]) - self.assertEqual(receipt["signal"], published["signal"]) + self.assertEqual(published["harness"]["reason"], expected) + self.assertEqual(receipt["exit_code"], published["process"]["exit_code"]) + self.assertEqual(receipt["signal"], published["process"]["signal"]) self.assertEqual(len(terminals), 1) - self.assertEqual(terminals[0]["terminal_reason"], expected) + self.assertEqual(terminals[0]["harness"]["reason"], expected) self.assertEqual(journal[-1]["record"], "terminal") - self.assertTrue(result.cleanup_complete) + self.assertTrue(result.harness.cleanup_complete) self.assertFalse(result.process_group_alive) def test_controller_eof_routes_supervisor_through_cleanup(self) -> None: diff --git a/scripts/agent_benchmark/live_iop.py b/scripts/agent_benchmark/live_iop.py index 2a4c8435..a15aa89a 100644 --- a/scripts/agent_benchmark/live_iop.py +++ b/scripts/agent_benchmark/live_iop.py @@ -973,7 +973,10 @@ class _LiveAdapter: ) parser = AgyEventParser(cell, admitted) result = self._invokers.agy(spec, parser, agy_preflight, lambda locator: on_started(locator, spec_digest(spec))) - if not result.success: + if ( + result.product.status != "succeeded" + or result.harness.status != "passed" + ): return _bound_observations(result, admitted) observed = parser.observed_result(agy_preflight.capability, result) if observed.status != "ready" or observed.binding != admitted: @@ -1130,12 +1133,22 @@ class _LiveScoringAdapter: and result.effective_binding != expected ): return ScoringInvocationResult( - False, "binding_mismatch", result.effective_binding + result.lifecycle.product.status, + "failed", + result.lifecycle.process.status, + result.lifecycle.process.exit_code, + result.lifecycle.process.signal, + "binding_mismatch", + result.effective_binding, ) lifecycle = _bound_observations(result.lifecycle, admitted) return ScoringInvocationResult( - lifecycle.success, - lifecycle.terminal_reason, + lifecycle.product.status, + lifecycle.harness.status, + lifecycle.process.status, + lifecycle.process.exit_code, + lifecycle.process.signal, + lifecycle.harness.reason, expected, ) diff --git a/scripts/agent_benchmark/measurement.py b/scripts/agent_benchmark/measurement.py index 35ffe94b..8f119ff2 100644 --- a/scripts/agent_benchmark/measurement.py +++ b/scripts/agent_benchmark/measurement.py @@ -40,15 +40,19 @@ from scripts.agent_benchmark.lifecycle import ( SOURCE_WORKSPACE_POLL, UNIT_NANOSECONDS, InvocationResult, + HarnessOutcome, LifecycleMetricError, + LifecycleValidationError, ParsedMetric, + ProcessOutcome, + ProductOutcome, metric_record, publish_bytes_no_replace, validate_metric, ) MEASUREMENT_FILENAME = "attempt-measurement.json" -MEASUREMENT_VERSION = 1 +MEASUREMENT_VERSION = 2 MEASUREMENT_RECORD = "attempt_measurement" STATUS_OBSERVED = "observed" @@ -133,7 +137,9 @@ class AttemptMeasurement: attempt: int caller: str spec_digest: str - terminal_reason: str + product: ProductOutcome + harness: HarnessOutcome + process: ProcessOutcome timeline: dict[str, Observation] usage: dict[str, Observation] observer: WorkspaceWriteObservation @@ -490,7 +496,7 @@ def build_measurement( for number in (repetition, attempt) ): raise MeasurementError("measurement identity is invalid") - if not _is_digest(result.spec_digest) or not result.terminal_reason: + if not _is_digest(result.spec_digest): raise MeasurementError("measurement invocation identity is invalid") metrics = tuple(result.metrics) if len(metrics) > MAX_OBSERVATION_RECORDS: @@ -514,7 +520,9 @@ def build_measurement( attempt=attempt, caller=caller, spec_digest=result.spec_digest, - terminal_reason=result.terminal_reason, + product=result.product, + harness=result.harness, + process=result.process, timeline=_timeline(result, observation), usage=_usage(metrics), observer=observation, @@ -538,7 +546,21 @@ def measurement_record(measurement: AttemptMeasurement) -> dict[str, Any]: }, "caller": measurement.caller, "spec_digest": measurement.spec_digest, - "terminal_reason": measurement.terminal_reason, + "product": { + "status": measurement.product.status, + "reason": measurement.product.reason, + }, + "harness": { + "status": measurement.harness.status, + "reason": measurement.harness.reason, + "ordered_terminal": measurement.harness.ordered_terminal, + "cleanup_complete": measurement.harness.cleanup_complete, + }, + "process": { + "status": measurement.process.status, + "exit_code": measurement.process.exit_code, + "signal": measurement.process.signal, + }, "timeline": { name: observation_record(measurement.timeline[name]) for name in TIMELINE_NAMES @@ -682,7 +704,8 @@ def load_measurement(attempt_root: str | Path) -> AttemptMeasurement: raise MeasurementError("measurement is not canonical JSON") from exc fields = { "record", "measurement_version", "attempt", "caller", "spec_digest", - "terminal_reason", "timeline", "usage", "observer", "observations", + "product", "harness", "process", "timeline", "usage", "observer", + "observations", } if not isinstance(record, dict) or set(record) != fields: raise MeasurementError("measurement schema is invalid") @@ -691,11 +714,16 @@ def load_measurement(attempt_root: str | Path) -> AttemptMeasurement: or record["measurement_version"] != MEASUREMENT_VERSION or not isinstance(record["caller"], str) or not record["caller"] or not isinstance(record["spec_digest"], str) or not _is_digest(record["spec_digest"]) - or not isinstance(record["terminal_reason"], str) or not record["terminal_reason"] or not isinstance(record["observations"], list) ): raise MeasurementError("measurement identity is invalid") run_id, cell_id, repetition, attempt = _identity_from_record(record["attempt"]) + try: + product = ProductOutcome(**record["product"]) + harness = HarnessOutcome(**record["harness"]) + process = ProcessOutcome(**record["process"]) + except (TypeError, LifecycleValidationError, ValueError) as exc: + raise MeasurementError("measurement outcomes are invalid") from exc measurement = AttemptMeasurement( run_id=run_id, cell_id=cell_id, @@ -703,7 +731,9 @@ def load_measurement(attempt_root: str | Path) -> AttemptMeasurement: attempt=attempt, caller=record["caller"], spec_digest=record["spec_digest"], - terminal_reason=record["terminal_reason"], + product=product, + harness=harness, + process=process, timeline=_observation_map(record["timeline"], TIMELINE_NAMES, "timeline"), usage=_observation_map(record["usage"], METRIC_NAMES, "usage"), observer=_observer_from_record(record["observer"]), @@ -757,6 +787,25 @@ def validate_measurement_lifecycle_binding( """ if not isinstance(lifecycle, Mapping): raise MeasurementError("lifecycle evidence is invalid") + expected_outcomes = { + "product": { + "status": measurement.product.status, + "reason": measurement.product.reason, + }, + "harness": { + "status": measurement.harness.status, + "reason": measurement.harness.reason, + "ordered_terminal": measurement.harness.ordered_terminal, + "cleanup_complete": measurement.harness.cleanup_complete, + }, + "process": { + "status": measurement.process.status, + "exit_code": measurement.process.exit_code, + "signal": measurement.process.signal, + }, + } + if any(lifecycle.get(name) != value for name, value in expected_outcomes.items()): + raise MeasurementError("measurement outcomes do not match lifecycle evidence") events = lifecycle.get("events") if not isinstance(events, list): raise MeasurementError("lifecycle events are invalid") diff --git a/scripts/agent_benchmark/measurement_test.py b/scripts/agent_benchmark/measurement_test.py index 7f93f492..f3485cfa 100644 --- a/scripts/agent_benchmark/measurement_test.py +++ b/scripts/agent_benchmark/measurement_test.py @@ -26,6 +26,9 @@ from scripts.agent_benchmark.lifecycle import ( SOURCE_WORKSPACE_POLL, UNIT_NANOSECONDS, CaptureStream, + ProductOutcome, + HarnessOutcome, + ProcessOutcome, InvocationResult, LifecycleMetricError, LifecycleEvent, @@ -80,13 +83,18 @@ def _result( _event(METRIC_PREFIX + metric.name, 3_000, metric.source) for metric in metrics ) return InvocationResult( - success=terminal_reason == "success", - terminal_reason=terminal_reason, - exit_code=0, - signal=None, + product=ProductOutcome( + "succeeded" if terminal_reason == "success" else "unknown", + "caller_success" if terminal_reason == "success" else "unavailable", + ), + harness=HarnessOutcome( + "passed" if terminal_reason == "success" else "failed", + terminal_reason, + True, + True, + ), + process=ProcessOutcome("exited", 0, None), submitted=True, - finish_then_idle_then_quiet=True, - cleanup_complete=True, process_group_alive=False, events=events + published, stdout=_capture("stdout"), @@ -284,7 +292,7 @@ class MeasurementRecordTest(unittest.TestCase): for override in ( {"run_id": ""}, {"cell_id": None}, {"caller": ""}, {"repetition": 0}, {"attempt": True}, - {"result": _result(terminal_reason="")}, + {"result": object()}, {"observation": None}, ): with self.subTest(override=tuple(override)): @@ -572,7 +580,7 @@ class MeasurementSidecarTest(unittest.TestCase): record = measurement_record(self.measurement) cases = { "unknown-field": {**record, "extra": 1}, - "wrong-version": {**record, "measurement_version": 2}, + "wrong-version": {**record, "measurement_version": 1}, "forged-digest": {**record, "spec_digest": "sha256:not-a-digest"}, "zeroed-unavailable": { **record, diff --git a/scripts/agent_benchmark/reporting.py b/scripts/agent_benchmark/reporting.py index 407ddd8e..650e6e88 100644 --- a/scripts/agent_benchmark/reporting.py +++ b/scripts/agent_benchmark/reporting.py @@ -424,17 +424,20 @@ def render_report(projection: ReportProjection) -> bytes: else: lines.append("| — | unavailable | 0 |") - lines.extend(("", "## Attempt outcomes", "", "| cell | repetition | attempt | execution | terminal | web | scoring | total | rank |", "|---|---:|---:|---|---|---|---|---:|---:|")) + lines.extend(("", "## Attempt outcomes", "", "| cell | repetition | attempt | controller | product | harness | process | artifact | scoring | total | rank |", "|---|---:|---:|---|---|---|---|---|---|---:|---:|")) if not projection.attempts: - lines.append("| — | — | — | blocked | unavailable | unavailable | unavailable | — | — |") + lines.append("| — | — | — | blocked | unavailable | unavailable | unavailable | unavailable | unavailable | — | — |") for item in projection.attempts: - terminal = "unavailable" if item.measurement is None else item.measurement.terminal_reason - web = "unavailable" if item.web is None else item.web.status + product = "unavailable" if item.measurement is None else item.measurement.product.status + harness = "unavailable" if item.measurement is None else item.measurement.harness.status + process = "unavailable" if item.measurement is None else item.measurement.process.status + artifact = "unavailable" if item.web is None else item.web.status total = "—" if item.score.total is None else str(item.score.total) rank = "—" if item.score.rank is None else str(item.score.rank) lines.append( f"| {_markdown(item.cell.id)} | {item.attempt.identity.repetition} | {item.attempt.identity.attempt} | " - f"{_markdown(item.attempt.state)} | {_markdown(terminal)} | {_markdown(web)} | " + f"{_markdown(item.attempt.state)} | {_markdown(product)} | {_markdown(harness)} | " + f"{_markdown(process)} | {_markdown(artifact)} | " f"{_markdown(item.score.status)} | {total} | {rank} |" ) raw_paths.update(item.raw_paths) diff --git a/scripts/agent_benchmark/scoring.py b/scripts/agent_benchmark/scoring.py index 737da1f1..9d63b83a 100644 --- a/scripts/agent_benchmark/scoring.py +++ b/scripts/agent_benchmark/scoring.py @@ -34,6 +34,7 @@ from scripts.agent_benchmark.connectivity import ( ) from scripts.agent_benchmark.manifest import Manifest, MatrixCell, Timeout from scripts.agent_benchmark.lifecycle import ( + JOURNAL_VERSION, LifecycleRecoveryError, REASON_CONTROLLER_LOST, REASON_RECOVERED_STOP, @@ -57,7 +58,7 @@ from scripts.agent_benchmark.web_validation import ( ) -SCORING_VERSION = 1 +SCORING_VERSION = 2 SCORE_RE = re.compile(r"^score-([0-9]{6})$") BLIND_ID_RE = re.compile(r"^blind-[0-9a-f]{32}$") DIGEST_RE = re.compile(r"^sha256:[0-9a-f]{64}$") @@ -70,6 +71,9 @@ INPUT_FILENAME = "input.json" RESULT_FILENAME = "result.json" RUNNER_FILENAME = "runner.json" SCORING_STATUSES = ("scored", "unscored", "scoring_failed", "blocked") +SCORING_INVOCATION_REASONS = TERMINAL_REASONS + ( + "binding_mismatch", "evaluator_failed", +) _POST_CLEANUP_TIMEOUT_SECONDS = 2.0 _POST_CLEANUP_QUIET_SECONDS = 0.2 _POST_CLEANUP_POLL_SECONDS = 0.01 @@ -92,10 +96,46 @@ class BlindWorkspace: @dataclass(frozen=True) class ScoringInvocationResult: - success: bool - terminal_reason: str + product: str + harness: str + process: str + process_exit_code: int | None + process_signal: int | None + reason: str effective_binding: tuple[str, str, str, str] | None + def __post_init__(self) -> None: + if ( + self.product not in {"succeeded", "failed", "unknown"} + or self.harness not in {"passed", "failed"} + or self.process not in { + "exited", "signalled", "timed_out", "cancelled", "not_started" + } + or ( + self.process_exit_code is not None + and ( + not isinstance(self.process_exit_code, int) + or isinstance(self.process_exit_code, bool) + ) + ) + or ( + self.process_signal is not None + and ( + not isinstance(self.process_signal, int) + or isinstance(self.process_signal, bool) + ) + ) + or (self.process == "signalled" and self.process_signal is None) + or ( + self.process in {"exited", "not_started"} + and self.process_signal is not None + ) + or (self.process == "not_started" and self.process_exit_code is not None) + or self.reason not in SCORING_INVOCATION_REASONS + or (self.harness == "passed") != (self.reason == "success") + ): + raise ScoringError("scoring invocation result is invalid") + @dataclass(frozen=True) class ScoringEvidenceFinalization: @@ -693,7 +733,9 @@ def _validate_cleanup_receipt( or receipt["reason"] not in TERMINAL_REASONS or (expected_reason is not None and receipt["reason"] != expected_reason) or not isinstance(receipt["caller_launched"], bool) - or receipt["cleanup_complete"] is not True + or not isinstance(receipt["cleanup_complete"], bool) + or (not receipt["cleanup_complete"]) + != (receipt["reason"] == "cleanup_failed") or receipt["process_group_alive"] is not False or not isinstance(receipt["completed_at"], str) ): @@ -717,22 +759,66 @@ def _validate_lifecycle_binding( if not path.exists() and not path.is_symlink(): return None value = _load_json(path, "evaluator lifecycle") + product = value.get("product") + harness = value.get("harness") + process = value.get("process") if ( value.get("record") != "result" or value.get("spec_digest") != invocation_digest or value.get("locator") != _locator_public(locator) - or value.get("terminal_reason") not in TERMINAL_REASONS - or value.get("cleanup_complete") is not True + or not isinstance(product, dict) + or not isinstance(harness, dict) + or not isinstance(process, dict) + or set(product) != {"status", "reason"} + or product.get("status") not in {"succeeded", "failed", "unknown"} + or product.get("reason") + != { + "succeeded": "caller_success", + "failed": "caller_error", + "unknown": "unavailable", + }.get(product.get("status")) + or set(harness) + != {"status", "reason", "ordered_terminal", "cleanup_complete"} + or harness.get("status") not in {"passed", "failed"} + or harness.get("reason") not in TERMINAL_REASONS + or (harness.get("status") == "passed") + != (harness.get("reason") == "success") + or not isinstance(harness.get("ordered_terminal"), bool) + or (harness.get("status") == "passed") + and not harness.get("ordered_terminal") + or not isinstance(harness.get("cleanup_complete"), bool) + or (not harness.get("cleanup_complete")) + != (harness.get("reason") == "cleanup_failed") + or set(process) != {"status", "exit_code", "signal"} + or process.get("status") + not in {"exited", "signalled", "timed_out", "cancelled", "not_started"} + or any( + value is not None + and (not isinstance(value, int) or isinstance(value, bool)) + for value in (process.get("exit_code"), process.get("signal")) + ) + or (process.get("status") == "signalled" and process.get("signal") is None) + or ( + process.get("status") in {"exited", "not_started"} + and process.get("signal") is not None + ) + or ( + process.get("status") == "not_started" + and process.get("exit_code") is not None + ) or value.get("process_group_alive") is not False - or value.get("success") - is not (value.get("terminal_reason") == "success") ): raise ScoringError("evaluator lifecycle binding is invalid") - _validate_cleanup_receipt( + receipt, _ = _validate_cleanup_receipt( locator, - expected_reason=str(value["terminal_reason"]), + expected_reason=str(harness["reason"]), control_target=control_target, ) + if ( + receipt["exit_code"] != process["exit_code"] + or receipt["signal"] != process["signal"] + ): + raise ScoringError("evaluator lifecycle binding is invalid") journal = blind_root / "output" / "lifecycle-journal.jsonl" try: lines = _read_regular( @@ -745,11 +831,13 @@ def _validate_lifecycle_binding( if ( not isinstance(header, dict) or header.get("record") != "header" + or header.get("journal_version") != JOURNAL_VERSION or header.get("spec_digest") != invocation_digest or not isinstance(terminal, dict) or terminal.get("record") != "terminal" - or terminal.get("terminal_reason") != value["terminal_reason"] - or terminal.get("cleanup_complete") is not True + or terminal.get("product") != product + or terminal.get("harness") != harness + or terminal.get("process") != process or terminal.get("process_group_alive") is not False ): raise ScoringError("evaluator lifecycle journal is invalid") @@ -942,13 +1030,29 @@ def _release_runner_alias(runner: Mapping[str, Any]) -> None: def _eligibility(manifest: Manifest, attempt: Attempt) -> tuple[bool, tuple[str, ...]]: if attempt.state not in TERMINAL_STATES: return False, ("lifecycle_running",) - if attempt.state != "success": - return False, (f"lifecycle_{attempt.state}",) + try: + execution = _load_json(Path(attempt.root) / "attempt.json", "execution attempt") + except ScoringError: + raise + lifecycle = execution.get("lifecycle") + if not isinstance(lifecycle, dict): + raise ScoringError("execution eligibility evidence is invalid") + reasons: list[str] = [] + product = lifecycle.get("product") + harness = lifecycle.get("harness") + process = lifecycle.get("process") + if not isinstance(product, dict) or product.get("status") != "succeeded": + reasons.append("product_" + str((product or {}).get("status", "unknown"))) + if not isinstance(harness, dict) or harness.get("status") != "passed": + reasons.append("harness_" + str((harness or {}).get("status", "failed"))) + if not isinstance(process, dict) or process.get("status") != "exited": + reasons.append("process_" + str((process or {}).get("status", "not_started"))) + elif process.get("exit_code") != 0 or process.get("signal") is not None: + reasons.append("process_nonzero_exit") try: web = load_web_validation(attempt.root, manifest=manifest) except WebValidationError as exc: raise ScoringError("web eligibility evidence is invalid") from exc - reasons: list[str] = [] if web.status != "passed": reasons.append(f"web_{web.status}") if web.record["reason"]: @@ -1966,11 +2070,30 @@ def _score_one( lifecycle, runner_digest, receipt_digest, _ = _evidence_digests( score_root, blind_root, allocation, run, attempt ) + lifecycle_record = ( + None + if lifecycle is None + else _load_json( + blind_root / "output" / "lifecycle-result.json", + "evaluator lifecycle", + ) + ) if ( - not invocation.success - or invocation.terminal_reason != "success" + invocation.product != "succeeded" + or invocation.harness != "passed" + or invocation.process != "exited" + or invocation.process_exit_code != 0 + or invocation.process_signal is not None + or invocation.reason != "success" or invocation.effective_binding != expected_binding - or lifecycle is None + or lifecycle_record is None + or lifecycle_record["product"]["status"] != invocation.product + or lifecycle_record["harness"]["status"] != invocation.harness + or lifecycle_record["process"]["status"] != invocation.process + or lifecycle_record["process"]["exit_code"] + != invocation.process_exit_code + or lifecycle_record["process"]["signal"] != invocation.process_signal + or lifecycle_record["harness"]["reason"] != invocation.reason or runner_digest is None or receipt_digest is None ): diff --git a/scripts/agent_benchmark/scoring_test.py b/scripts/agent_benchmark/scoring_test.py index 0e5ecb0a..ab0cfe18 100644 --- a/scripts/agent_benchmark/scoring_test.py +++ b/scripts/agent_benchmark/scoring_test.py @@ -29,6 +29,8 @@ from scripts.agent_benchmark.connectivity import ( make_result, ) from scripts.agent_benchmark.lifecycle import ( + CALLER_REASON_ERROR, + CALLER_REASON_SUCCESS, COMPLETION_EXIT_AFTER_IDLE, CLOCK_HARNESS_MONOTONIC, METRIC_NAMES, @@ -37,6 +39,10 @@ from scripts.agent_benchmark.lifecycle import ( SUBMISSION_STDIN_ONCE, UNIT_NANOSECONDS, InvocationSpec, + JOURNAL_VERSION, + HarnessOutcome, + ProcessOutcome, + ProductOutcome, SupervisorLocator, env_pairs, recover_invocation, @@ -188,6 +194,7 @@ class FakeScoringAdapter: invocation_digest = "sha256:" + "4" * 64 on_started(locator, invocation_digest) terminal_reason = "nonzero_exit" if mode == "raise" else "success" + exit_code = 7 if mode == "process_nonzero" else (1 if mode == "raise" else 0) receipt = { "receipt_version": 1, "supervisor_pid": locator.supervisor_pid, @@ -195,7 +202,7 @@ class FakeScoringAdapter: locator.challenge.encode("utf-8") ).hexdigest(), "reason": terminal_reason, - "exit_code": 1 if mode == "raise" else 0, + "exit_code": exit_code, "signal": None, "caller_launched": True, "cleanup_complete": True, @@ -211,11 +218,27 @@ class FakeScoringAdapter: if key != "challenge" } public_locator["challenge_digest"] = receipt["challenge_digest"] + product = ( + {"status": "unknown", "reason": "unavailable"} + if mode == "raise" + else {"status": "succeeded", "reason": CALLER_REASON_SUCCESS} + ) + harness = { + "status": "failed" if mode == "raise" else "passed", + "reason": terminal_reason, + "ordered_terminal": mode != "raise", + "cleanup_complete": True, + } + process = { + "status": "exited", + "exit_code": exit_code, + "signal": None, + } lifecycle = { "record": "result", - "success": terminal_reason == "success", - "terminal_reason": terminal_reason, - "cleanup_complete": True, + "product": product, + "harness": harness, + "process": process, "process_group_alive": False, "spec_digest": invocation_digest, "locator": public_locator, @@ -231,14 +254,19 @@ class FakeScoringAdapter: ) journal = ( json.dumps( - {"record": "header", "spec_digest": invocation_digest} + { + "record": "header", + "journal_version": JOURNAL_VERSION, + "spec_digest": invocation_digest, + } ) + "\n" + json.dumps( { "record": "terminal", - "terminal_reason": terminal_reason, - "cleanup_complete": True, + "product": product, + "harness": harness, + "process": process, "process_group_alive": False, } ) @@ -279,7 +307,16 @@ class FakeScoringAdapter: ) if mode == "binding": binding = (binding[0], "substituted", binding[2], binding[3]) - return ScoringInvocationResult(mode not in {"failed", "binding"}, "success" if mode not in {"failed", "binding"} else "failed", binding) + passed = mode not in {"failed", "binding"} + return ScoringInvocationResult( + "succeeded" if passed else "failed", + "passed" if passed else "failed", + "exited", + 7 if mode == "process_nonzero" else 0, + None, + "success" if passed else "evaluator_failed", + binding, + ) def finalize_evidence(self, blind): leaked = False @@ -428,6 +465,7 @@ class ScoringTest(unittest.TestCase): name: unavailable(REASON_NOT_REPORTED, SOURCE_HARNESS) for name in METRIC_NAMES } + succeeded = terminal_reason == "success" return AttemptMeasurement( attempt.identity.run_id, attempt.identity.cell_id, @@ -435,7 +473,12 @@ class ScoringTest(unittest.TestCase): attempt.identity.attempt, "claude", "sha256:" + "3" * 64, - terminal_reason, + ProductOutcome( + "succeeded" if succeeded else "failed", + CALLER_REASON_SUCCESS if succeeded else CALLER_REASON_ERROR, + ), + HarnessOutcome("passed", "success", True, True), + ProcessOutcome("exited", 0 if succeeded else 1, None), timeline, usage, WorkspaceWriteObservation( @@ -526,7 +569,7 @@ class ScoringTest(unittest.TestCase): (workspace / "script.js").write_text( "document.body.dataset.ready='1';", encoding="utf-8" ) - terminal_reason = "success" if state == "success" else state + terminal_reason = "success" if state == "success" else "caller_error" measurement = self._measurement(attempt, terminal_reason) publish_measurement(attempt.root, measurement) if state == "success" and rendered: @@ -550,8 +593,28 @@ class ScoringTest(unittest.TestCase): self.manifest, workspace, measurement, render ) publish_web_validation(attempt.root, web) + succeeded = state == "success" + lifecycle = { + "product": { + "status": "succeeded" if succeeded else "failed", + "reason": ( + CALLER_REASON_SUCCESS if succeeded else CALLER_REASON_ERROR + ), + }, + "harness": { + "status": "passed", + "reason": "success", + "ordered_terminal": True, + "cleanup_complete": True, + }, + "process": { + "status": "exited", + "exit_code": 0 if succeeded else 1, + "signal": None, + }, + } terminal = self.store.publish_terminal( - attempt, state, result={"terminal_reason": terminal_reason} + attempt, "completed", result=lifecycle ) return terminal @@ -627,6 +690,18 @@ class ScoringTest(unittest.TestCase): self.assertEqual(len(adapter.invocations), 1) self.assertEqual(before, {path: path.read_bytes() for path in Path(attempt.root).rglob("*") if path.is_file()}) + def test_scoring_invocation_result_rejects_open_or_contradictory_reasons(self): + with self.assertRaises(ScoringError): + ScoringInvocationResult( + "succeeded", "passed", "exited", 0, None, + "arbitrary_reason", None, + ) + with self.assertRaises(ScoringError): + ScoringInvocationResult( + "succeeded", "passed", "exited", 0, None, + "evaluator_failed", None, + ) + def test_manifest_selected_rubric_drives_prompt_and_worksheet_validation(self): raw = json.loads(self.manifest_path.read_text(encoding="utf-8")) raw["rubric_version"] = ONE_SHOT_RUBRIC_VERSION @@ -707,7 +782,20 @@ class ScoringTest(unittest.TestCase): before = path.read_bytes() record = json.loads(before) self.assertEqual(record["status"], "unscored") - self.assertEqual(record["reasons"], ["lifecycle_failed"]) + self.assertEqual( + record["reasons"], + [ + "product_failed", + "process_nonzero_exit", + "web_failed", + "web_reason_render_not_run", + "gate_images", + "gate_network", + "gate_console", + "gate_responsive", + "gate_accessibility", + ], + ) self.assertFalse(set(record) & {"score", "total", "worksheet"}) score_run(self.store, self.run, self.manifest, adapter=adapter) self.assertEqual(path.read_bytes(), before) @@ -803,6 +891,23 @@ class ScoringTest(unittest.TestCase): self.assertEqual(result["status"], "scoring_failed") self.assertNotIn("worksheet", result) + self.run = self.store.create(self.manifest, self.manifest_path.read_bytes()) + process_attempt = self._attempt() + process = FakeScoringAdapter(modes=["process_nonzero"]) + summary = score_run(self.store, self.run, self.manifest, adapter=process) + self.assertEqual(summary.scoring_failed, 1) + result = json.loads( + ( + Path(process_attempt.root) + / "scoring" + / "score-000001" + / "result.json" + ).read_text() + ) + self.assertEqual((result["status"], result["reason"]), ( + "scoring_failed", "evaluator_failed", + )) + # A separate run proves an identity sentinel in retained page bytes is # rejected before the evaluator is invoked. self.run = self.store.create(self.manifest, self.manifest_path.read_bytes()) @@ -1132,23 +1237,38 @@ class ScoringTest(unittest.TestCase): key: value for key, value in locator_payload.items() if key != "challenge" } public_locator["challenge_digest"] = receipt["challenge_digest"] + product = {"status": "succeeded", "reason": CALLER_REASON_SUCCESS} + harness = { + "status": "passed", + "reason": "success", + "ordered_terminal": True, + "cleanup_complete": True, + } + process = {"status": "exited", "exit_code": 0, "signal": None} lifecycle = { "record": "result", - "success": True, - "terminal_reason": "success", - "cleanup_complete": True, + "product": product, + "harness": harness, + "process": process, "process_group_alive": False, "spec_digest": invocation_digest, "locator": public_locator, } journal = ( - json.dumps({"record": "header", "spec_digest": invocation_digest}) + json.dumps( + { + "record": "header", + "journal_version": JOURNAL_VERSION, + "spec_digest": invocation_digest, + } + ) + "\n" + json.dumps( { "record": "terminal", - "terminal_reason": "success", - "cleanup_complete": True, + "product": product, + "harness": harness, + "process": process, "process_group_alive": False, } ) @@ -1591,7 +1711,7 @@ class ScoringTest(unittest.TestCase): self.assertFalse(adapter.worker.is_alive()) # type: ignore[union-attr] self.assertIsNone(adapter.worker_error) self.assertIsNotNone(adapter.worker_result) - self.assertTrue(adapter.worker_result.cleanup_complete) + self.assertTrue(adapter.worker_result.harness.cleanup_complete) self.assertFalse(adapter.worker_result.process_group_alive) receipt = json.loads( ( diff --git a/scripts/agent_benchmark/skill_contract_test.py b/scripts/agent_benchmark/skill_contract_test.py index dbf50c99..dfe20300 100644 --- a/scripts/agent_benchmark/skill_contract_test.py +++ b/scripts/agent_benchmark/skill_contract_test.py @@ -305,6 +305,7 @@ class BenchmarkSkillContractTest(unittest.TestCase): self.assertIn("`one-shot-agent-comparison-v1`", procedure) self.assertIn("no substitute rubric or reinterpretation is permitted", procedure) self.assertIn("allocates a new score id and preserves every prior byte", procedure) + self.assertIn("failed product, harness, process, or required artifact gate", procedure) self.assertIn("`scoring_failed` used no fallback", validation) self.assertIn("Do not retry scoring implicitly", prohibitions) self.assertIn("convert `unscored`/`scoring_failed` to zero", prohibitions) @@ -339,6 +340,11 @@ class BenchmarkSkillContractTest(unittest.TestCase): self._assert_preflight_contract(skill_text) self._assert_scoring_contract(skill_text) self._assert_no_secret_operational_language(skill_text) + self.assertIn("product_succeeded=", skill_text) + self.assertIn("harness_passed=", skill_text) + self.assertIn("process_exited=", skill_text) + self.assertIn("artifact_passed=", skill_text) + self.assertIn("five-cell direct manifest", skill_text) # ------------------------------------------------------------------ # Template / frontmatter invariants diff --git a/scripts/agent_benchmark/web_validation.py b/scripts/agent_benchmark/web_validation.py index 6a8a7810..fe3447c4 100644 --- a/scripts/agent_benchmark/web_validation.py +++ b/scripts/agent_benchmark/web_validation.py @@ -31,7 +31,7 @@ from scripts.agent_benchmark.measurement import ( ) WEB_VALIDATION_FILENAME = "web-validation.json" -WEB_VALIDATION_VERSION = 1 +WEB_VALIDATION_VERSION = 2 WEB_STATUSES = ("passed", "failed", "blocked", "not_run") WEB_GATES = ( "generated_files", @@ -488,20 +488,12 @@ def build_web_validation( snapshot = _workspace_snapshot(workspace, manifest) generated_gate = _generated_gate(snapshot) static_gate = _static_gate(workspace, generated_gate, manifest) - terminal_reason = _reason_token( - getattr(measurement, "terminal_reason", "success"), "invalid_lifecycle" - ) - browser = {"status": "not_observed", "product": "", "origin": ""} requests: list[dict[str, Any]] = [] console: list[dict[str, Any]] = [] viewports: list[dict[str, Any]] = [] reason = "" - if terminal_reason != "success": - status = "not_run" - reason = f"lifecycle_{terminal_reason}" - gates = _not_observed_gates(reason, "lifecycle") - elif blocked: + if blocked: status = "blocked" reason = _reason_token(blocked, "browser_failure") gates = _not_observed_gates(reason, "browser") @@ -571,8 +563,6 @@ def validate_web_attempt( browser_binary: str = "chromium", ) -> WebValidation: workspace = Path(prepared.workspace_dir) - if getattr(measurement, "terminal_reason", "") != "success": - return build_web_validation(manifest, prepared, measurement, None) render = None blocked = "" generated_ready = all( diff --git a/scripts/agent_benchmark/web_validation_test.py b/scripts/agent_benchmark/web_validation_test.py index c25e35ab..e041ea06 100644 --- a/scripts/agent_benchmark/web_validation_test.py +++ b/scripts/agent_benchmark/web_validation_test.py @@ -87,7 +87,6 @@ class WebValidationTest(unittest.TestCase): cell_id="cell", repetition=1, attempt=1, - terminal_reason=reason, ) def _view(self, ident: str, width: int, *, suffix: str = "") -> ViewportObservation: @@ -294,15 +293,19 @@ class WebValidationTest(unittest.TestCase): gates = {item["id"]: item for item in record.record["gates"]} self.assertFalse(gates[gate]["passed"]) - def test_lifecycle_non_success_is_not_run_without_browser(self): + def test_lifecycle_non_success_still_validates_workspace(self): for reason in ("nonzero_exit", "timed_out", "cancelled", "controller_lost"): with self.subTest(reason=reason): record = build_web_validation( self._manifest(), self.workspace, self._measurement(reason), None ) - self.assertEqual(record.status, "not_run") - self.assertEqual(record.record["reason"], f"lifecycle_{reason}") - self.assertFalse(any(item["passed"] for item in record.record["gates"])) + self.assertEqual(record.status, "failed") + self.assertEqual(record.record["reason"], "render_not_run") + gates = {item["id"]: item for item in record.record["gates"]} + self.assertTrue(gates["generated_files"]["passed"]) + self.assertTrue(gates["static_safety"]["passed"]) + self.assertFalse(gates["images"]["passed"]) + self.assertFalse(gates["responsive"]["passed"]) def test_browser_discovery_or_start_failure_is_blocked(self): prepared = SimpleNamespace( diff --git a/scripts/agent_comparison_benchmark.py b/scripts/agent_comparison_benchmark.py index ac3c22fc..0d8eb4d4 100644 --- a/scripts/agent_comparison_benchmark.py +++ b/scripts/agent_comparison_benchmark.py @@ -126,7 +126,24 @@ def _cmd_state(args: argparse.Namespace) -> int: else: run = store.open(manifest, args.run_id, raw) if args.command == "status": - print("ok: " + str(store.status(run, manifest)["attempts"])) + status = store.status(run, manifest) + attempts = status["attempts"] + outcomes = status["outcomes"] + attempt_summary = " ".join( + f"{state}={attempts[state]}" + for state in ( + "completed", "timed_out", "cancelled", "interrupted", "running" + ) + ) + axes = " ".join( + f"{axis}_{name}={outcomes[axis][name]}" + for axis in ("product", "harness", "process", "artifact") + for name in outcomes[axis] + ) + print( + f"ok: status run_id={run.run_id} " + f"unresolved={outcomes['unresolved']} {attempt_summary} {axes}" + ) return EXIT_VALID completed = run_slots( @@ -158,17 +175,24 @@ def _cmd_state(args: argparse.Namespace) -> int: attempt_summary = " ".join( f"{state}={attempts[state]}" for state in ( - "success", "failed", "timed_out", "cancelled", "interrupted", "running" + "completed", "timed_out", "cancelled", "interrupted", "running" ) ) - unresolved = 0 - for slot in store.slots(manifest): - retained = store.attempts(run, slot) - if not retained or retained[-1].state != "success": - unresolved += 1 + outcomes = status["outcomes"] + unresolved = outcomes["unresolved"] + axes = " ".join( + f"{prefix}_{name}={counts[name]}" + for prefix, counts in ( + ("product", outcomes["product"]), + ("harness", outcomes["harness"]), + ("process", outcomes["process"]), + ("artifact", outcomes["artifact"]), + ) + for name in counts + ) summary = ( - f"run_id={run.run_id} completed={len(completed)} " - f"unresolved={unresolved} {attempt_summary}" + f"run_id={run.run_id} executed={len(completed)} " + f"unresolved={unresolved} {attempt_summary} {axes}" ) if unresolved: print("error: benchmark execution failed " + summary, file=sys.stderr) diff --git a/scripts/fixtures/agent-comparison-benchmark-report.expected.md b/scripts/fixtures/agent-comparison-benchmark-report.expected.md index e3f319b8..c2b0c11e 100644 --- a/scripts/fixtures/agent-comparison-benchmark-report.expected.md +++ b/scripts/fixtures/agent-comparison-benchmark-report.expected.md @@ -27,13 +27,13 @@ ## Attempt outcomes -| cell | repetition | attempt | execution | terminal | web | scoring | total | rank | -|---|---:|---:|---|---|---|---|---:|---:| -| cell-sentinel | 1 | 1 | success | success | passed | scored | 99 | 1 | -| cell-sentinel | 1 | 2 | success | success | passed | scored | 99 | 1 | -| cell-sentinel | 1 | 3 | failed | failed | not_run | unscored | — | — | -| cell-sentinel | 1 | 4 | success | success | passed | scoring_failed | — | — | -| cell-sentinel | 1 | 5 | success | success | passed | blocked | — | — | +| cell | repetition | attempt | controller | product | harness | process | artifact | scoring | total | rank | +|---|---:|---:|---|---|---|---|---|---|---:|---:| +| cell-sentinel | 1 | 1 | completed | succeeded | passed | exited | passed | scored | 99 | 1 | +| cell-sentinel | 1 | 2 | completed | succeeded | passed | exited | passed | scored | 99 | 1 | +| cell-sentinel | 1 | 3 | completed | failed | passed | exited | failed | unscored | — | — | +| cell-sentinel | 1 | 4 | completed | succeeded | passed | exited | passed | scoring_failed | — | — | +| cell-sentinel | 1 | 5 | completed | succeeded | passed | exited | passed | blocked | — | — | ## Quality score breakdown @@ -66,7 +66,7 @@ |---|---|---|---|---|---| | cell-sentinel/r1/a1 | generated_files=pass, static_safety=pass, images=pass, network=pass, console=pass, responsive=pass, accessibility=pass | screenshot-desktop.png, screenshot-mobile.png | score-000001 | codex/judge-route/judge-model/xhigh | recorded | | cell-sentinel/r1/a2 | generated_files=pass, static_safety=pass, images=pass, network=pass, console=pass, responsive=pass, accessibility=pass | screenshot-desktop.png, screenshot-mobile.png | score-000001 | codex/judge-route/judge-model/xhigh | recorded | -| cell-sentinel/r1/a3 | generated_files=fail, static_safety=fail, images=fail, network=fail, console=fail, responsive=fail, accessibility=fail | unavailable | — | unavailable | lifecycle_failed | +| cell-sentinel/r1/a3 | generated_files=pass, static_safety=pass, images=fail, network=fail, console=fail, responsive=fail, accessibility=fail | unavailable | — | unavailable | product_failed, process_nonzero_exit, web_failed, web_reason_render_not_run, gate_images, gate_network, gate_console, gate_responsive, gate_accessibility | | cell-sentinel/r1/a4 | generated_files=pass, static_safety=pass, images=pass, network=pass, console=pass, responsive=pass, accessibility=pass | screenshot-desktop.png, screenshot-mobile.png | score-000001 | codex/judge-route/judge-model/xhigh | invalid_worksheet | | cell-sentinel/r1/a5 | generated_files=pass, static_safety=pass, images=pass, network=pass, console=pass, responsive=pass, accessibility=pass | screenshot-desktop.png, screenshot-mobile.png | — | unavailable | evaluator_preflight_blocked |