From 73e6fb4df2e497319acb433c7716b6388178dc3c Mon Sep 17 00:00:00 2001 From: toki Date: Mon, 10 Aug 2026 02:29:55 +0900 Subject: [PATCH] =?UTF-8?q?chore(epic):=20agent-connectivity=20=EC=A4=80?= =?UTF-8?q?=EB=B9=84=20=EA=B2=B0=EA=B3=BC=EB=A5=BC=20=EA=B2=80=EC=A6=9D?= =?UTF-8?q?=ED=95=9C=EB=8B=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../CODE_REVIEW-cloud-G06.md | 129 +++++++++++++ .../PLAN-cloud-G06.md | 125 +++++++++++++ ...oud-G09.md => code_review_cloud_G09_0.log} | 0 ...PLAN-cloud-G09.md => plan_cloud_G09_0.log} | 0 .../CODE_REVIEW-cloud-G06.md | 137 ++++++++++++++ .../PLAN-local-G06.md | 117 ++++++++++++ .../CODE_REVIEW-cloud-G07.md | 8 +- .../PLAN-cloud-G07.md | 12 +- .../CODE_REVIEW-cloud-G07.md | 8 +- .../PLAN-cloud-G07.md | 12 +- .../CODE_REVIEW-cloud-G07.md | 8 +- .../PLAN-cloud-G07.md | 12 +- .../CODE_REVIEW-cloud-G10.md | 158 ++++++++++++++++ .../PLAN-cloud-G10.md | 147 +++++++++++++++ .../code_review_cloud_G10_0.log} | 0 .../plan_cloud_G10_0.log} | 0 .../CODE_REVIEW-cloud-G09.md | 157 ++++++++++++++++ .../PLAN-cloud-G09.md | 122 ++++++++++++ .../CODE_REVIEW-cloud-G06.md | 174 ++++++++++++++++++ .../PLAN-local-G06.md | 124 +++++++++++++ .../code_review_cloud_G06_0.log | 159 ++++++++++++++++ .../plan_local_G06_0.log | 111 +++++++++++ 22 files changed, 1690 insertions(+), 30 deletions(-) create mode 100644 agent-task/m-agent-comparison-benchmark-pipeline/06_connectivity_contract/CODE_REVIEW-cloud-G06.md create mode 100644 agent-task/m-agent-comparison-benchmark-pipeline/06_connectivity_contract/PLAN-cloud-G06.md rename agent-task/m-agent-comparison-benchmark-pipeline/06_connectivity_contract/{CODE_REVIEW-cloud-G09.md => code_review_cloud_G09_0.log} (100%) rename agent-task/m-agent-comparison-benchmark-pipeline/06_connectivity_contract/{PLAN-cloud-G09.md => plan_cloud_G09_0.log} (100%) create mode 100644 agent-task/m-agent-comparison-benchmark-pipeline/07_anthropic_effort_compatibility/CODE_REVIEW-cloud-G06.md create mode 100644 agent-task/m-agent-comparison-benchmark-pipeline/07_anthropic_effort_compatibility/PLAN-local-G06.md rename agent-task/m-agent-comparison-benchmark-pipeline/{07+06_claude_iop => 08+06_claude_iop}/CODE_REVIEW-cloud-G07.md (96%) rename agent-task/m-agent-comparison-benchmark-pipeline/{07+06_claude_iop => 08+06_claude_iop}/PLAN-cloud-G07.md (94%) rename agent-task/m-agent-comparison-benchmark-pipeline/{08+06_agy_iop => 09+06_agy_iop}/CODE_REVIEW-cloud-G07.md (96%) rename agent-task/m-agent-comparison-benchmark-pipeline/{08+06_agy_iop => 09+06_agy_iop}/PLAN-cloud-G07.md (94%) rename agent-task/m-agent-comparison-benchmark-pipeline/{09+06_codex_iop => 10+06_codex_iop}/CODE_REVIEW-cloud-G07.md (96%) rename agent-task/m-agent-comparison-benchmark-pipeline/{09+06_codex_iop => 10+06_codex_iop}/PLAN-cloud-G07.md (94%) create mode 100644 agent-task/m-agent-comparison-benchmark-pipeline/11+08,09,10_connectivity_preflight/CODE_REVIEW-cloud-G10.md create mode 100644 agent-task/m-agent-comparison-benchmark-pipeline/11+08,09,10_connectivity_preflight/PLAN-cloud-G10.md rename agent-task/m-agent-comparison-benchmark-pipeline/{10+07,08,09_connectivity_runner/CODE_REVIEW-cloud-G10.md => 11+08,09,10_connectivity_preflight/code_review_cloud_G10_0.log} (100%) rename agent-task/m-agent-comparison-benchmark-pipeline/{10+07,08,09_connectivity_runner/PLAN-cloud-G10.md => 11+08,09,10_connectivity_preflight/plan_cloud_G10_0.log} (100%) create mode 100644 agent-task/m-agent-comparison-benchmark-pipeline/12+11_connectivity_execution/CODE_REVIEW-cloud-G09.md create mode 100644 agent-task/m-agent-comparison-benchmark-pipeline/12+11_connectivity_execution/PLAN-cloud-G09.md create mode 100644 agent-task/m-agent-comparison-benchmark-pipeline/13+07,11_connectivity_live_evidence/CODE_REVIEW-cloud-G06.md create mode 100644 agent-task/m-agent-comparison-benchmark-pipeline/13+07,11_connectivity_live_evidence/PLAN-local-G06.md create mode 100644 agent-task/m-agent-comparison-benchmark-pipeline/13+07,11_connectivity_live_evidence/code_review_cloud_G06_0.log create mode 100644 agent-task/m-agent-comparison-benchmark-pipeline/13+07,11_connectivity_live_evidence/plan_local_G06_0.log diff --git a/agent-task/m-agent-comparison-benchmark-pipeline/06_connectivity_contract/CODE_REVIEW-cloud-G06.md b/agent-task/m-agent-comparison-benchmark-pipeline/06_connectivity_contract/CODE_REVIEW-cloud-G06.md new file mode 100644 index 00000000..4ecfd7ff --- /dev/null +++ b/agent-task/m-agent-comparison-benchmark-pipeline/06_connectivity_contract/CODE_REVIEW-cloud-G06.md @@ -0,0 +1,129 @@ + + +# Code Review Reference - API + +> **[IMPLEMENTING AGENT — READ FIRST] Filling in this file is the mandatory final step of implementation.** +> The task is NOT complete until every implementation-owned section below is filled in. +> Complete the `Implementation Checklist`; the final checklist item is mandatory before saving. +> Fill implementation-owned sections, then stop with active files in place and report ready for review. +> Execute the plan's selected root cause, scope, files, and dependency decisions as written. Do not choose another owner, narrow/expand the write boundary, or replace a fix with another verification attempt. +> If implementation is blocked, record the exact blocker, attempted commands/output, and resume condition only in implementation-owned evidence fields. +> Do not ask the user directly, present choices, call user-input tools, create control-plane stop files, or classify the next state. +> Finalization (`Code Review Result`, log rename, `complete.log`, archive moves, `Review-Only Checklist`) is review-agent-only, even after compaction/resume. +> Follow the ownership table at the bottom of this file for which sections you own. + +## Overview + +date=2026-08-10 +task=m-agent-comparison-benchmark-pipeline/06_connectivity_contract, plan=1, tag=API + +## For the Review Agent + +> **[REVIEW AGENT ONLY]** The finalization steps below are review-agent only. Implementing agents must not execute this section. + +Compare implementation of each item against source files and verify that output in `Verification Results` matches code. +Review completion means the following steps are finished: + +1. Append verdict and `review_rework_count` / `evidence_integrity_failure` routing signals. +2. Archive `CODE_REVIEW-cloud-G06.md` → `code_review_cloud_G06_1.log` and `PLAN-cloud-G06.md` → `plan_cloud_G06_1.log`. +3. If PASS, write `complete.log` and move active task directory to `agent-task/archive/YYYY/MM/m-agent-comparison-benchmark-pipeline/06_connectivity_contract/`. If WARN/FAIL, fully write the next filesystem state required by the code-review skill. +4. If PASS and task group is `m-`, preserve the first-line `milestone-task` metadata in `complete.log` and report it for the runtime aggregation event. Roadmap state evaluation belongs to `sync-milestone-workstate`. +5. Check applicable `Review-Only Checklist` items at the final `.log` location before reporting. + +--- + +## Implementation Item Completion + +| Item | Status | +|------|---------| +| API-1 Freeze route and effort preflight contracts | [ ] | +| API-2 Classify and persist secret-safe connection evidence | [ ] | + +## Implementation Checklist + +- [ ] Add frozen caller capability, requested/effective binding, issue and result contracts with exact no-substitution validation. +- [ ] Add the closed registration-versus-implementation classifier and secret-safe canonical no-overwrite evidence writer. +- [ ] Add deterministic unit coverage and run focused, aggregate and patch-integrity verification without caller/provider access. +- [ ] Fill implementation-owned sections in CODE_REVIEW-*-G??.md with actual implementation notes and verification output. + +## Review-Only Checklist + +> **[REVIEW AGENT ONLY]** This checklist is used only by the review agent. +> Implementing agents must not modify or check this section. + +- [ ] Append one verdict of `PASS`, `WARN`, or `FAIL` and verified `review_rework_count`, `evidence_integrity_failure` to `Code Review Result`. +- [ ] Verify that verdict, `Dimension Assessment`, and Required/Suggested/Nit classifications match. +- [ ] Archive active `CODE_REVIEW-*-G??.md` to `code_review_cloud_G06_1.log`. +- [ ] Archive active `PLAN-*-G??.md` to `plan_cloud_G06_1.log`. +- [ ] Verify that the Agent-Ops managed block in `.gitignore` unignores `agent-task/**/*.md` and `agent-task/**/*.log` and ignores `agent-roadmap/current.md`. +- [ ] If PASS, write `complete.log` based on `agent-ops/skills/common/code-review/templates/complete-log-template.md` and leave no active `.md` files. +- [ ] If PASS, move active task directory `agent-task/m-agent-comparison-benchmark-pipeline/06_connectivity_contract/` to `agent-task/archive/YYYY/MM/m-agent-comparison-benchmark-pipeline/06_connectivity_contract/` and update this checklist at the final archive path. +- [ ] If PASS and task group is `m-`, preserve and report `milestone-task` metadata for runtime aggregation, without modifying roadmap or directly calling `update-roadmap`. +- [ ] If PASS for split work, remove empty active parent `agent-task/m-agent-comparison-benchmark-pipeline/` or verify it was kept due to remaining siblings/files. +- [ ] If WARN/FAIL, write the next filesystem state matching code-review verdict and do not write `complete.log`. + +## Deviations from Plan + +_Record any deviations from the plan and the rationale here._ + +## Key Design Decisions + +_Record key design decisions here._ + +## Reviewer Checkpoints + +- Confirm every status, issue and binding field is closed/immutable and exact mismatch fails closed. +- Confirm registration blockers never become ready and `implementation_gap` remains the closed implementation-Plan candidate classification rather than registration. +- Confirm persisted bytes and returned errors exclude secret, private endpoint, prompt/tool content and unbounded caller output. +- Confirm no caller/provider/network invocation and no files outside the exact write set. + +## Verification Results + +Paste exact stdout/stderr and exit code for every command. + +### V1 Focused connectivity tests + +Command: `python3 -m unittest scripts.agent_benchmark.connectivity_test -v` + +```text + +``` +Exit code: `` + +### V2 Aggregate benchmark tests + +Command: `make test-agent-comparison-benchmark` + +```text + +``` +Exit code: `` + +### V3 Patch integrity + +Command: `git diff --check` + +```text + +``` +Exit code: `` + +--- + +> **[IMPLEMENTING AGENT — BEFORE SAVING] Have you filled in every implementation-owned section?** +> If anything is blank, go back and fill it in before saving this file. +> Leave review-agent-only sections unchanged. + +## Section Ownership + +| Section | Owner | Note | +|---------|-------|------| +| Header comment, Overview, Review Agent Instructions | Fixed at stub creation | Implementing agent must not modify or execute these (archive, complete.log, and task-directory archive move are review-agent only) | +| Archive Evidence Snapshot | Fixed at stub creation from plan when present | Implementing agent uses it as default prior-loop context; read only the specific archive files cited there when more detail is required | +| Implementation Item Completion (item names) | Fixed at stub creation | Implementing agent checks `[ ]` → `[x]` only | +| Implementation Checklist (item text/order) | Fixed at stub creation from plan | Implementing agent checks `[ ]` → `[x]` only | +| Review-Only Checklist | Review agent only | Implementing agent must not modify or check this section | +| Deviations from Plan, Key Design Decisions | Implementing agent | Replace placeholder text with actual content | +| Reviewer Checkpoints | Fixed at stub creation | Pre-filled from plan | +| Verification Results (section headings + commands) | Fixed at stub creation | Implementing agent fills in command output only; command changes require a `Deviations from Plan` entry | +| Code Review Result | Review agent appends | Not included in stub | diff --git a/agent-task/m-agent-comparison-benchmark-pipeline/06_connectivity_contract/PLAN-cloud-G06.md b/agent-task/m-agent-comparison-benchmark-pipeline/06_connectivity_contract/PLAN-cloud-G06.md new file mode 100644 index 00000000..4ac75f8a --- /dev/null +++ b/agent-task/m-agent-comparison-benchmark-pipeline/06_connectivity_contract/PLAN-cloud-G06.md @@ -0,0 +1,125 @@ + + +# Plan - API: IOP connectivity preflight contract + +## For the Implementing Agent + +공통 connectivity contract와 deterministic evidence만 구현한다. 모든 검증 결과와 구현 메모를 active `CODE_REVIEW-cloud-G06.md`의 구현 소유 섹션에 채운 뒤 review-ready로 보고한다. blocker가 있으면 exact command/output/resume condition만 기록한다. 사용자 질문, stop 파일, verdict, archive, `complete.log`는 만들지 않는다. + +## Background + +manifest는 requested route/model/effort와 expected binding을 보존하지만 caller 공통 preflight 결과, no-substitution oracle, registration-versus-implementation taxonomy, secret-safe durable evidence가 없다. 이 계약은 실제 IOP나 caller를 호출하지 않고도 독립적으로 완성되며, 현재 IOP가 특정 effort를 거부하는 상태도 `implementation_gap`으로 정확히 표현할 수 있어야 한다. + +## Analysis + +### Files Read + +- `AGENTS.md` +- `agent-ops/rules/project/rules.md` +- `agent-ops/rules/common/rules-roadmap.md` +- `agent-ops/rules/common/rules-agent-spec.md` +- `agent-ops/skills/common/router.md` +- `agent-ops/skills/common/plan/SKILL.md` +- `agent-ops/skills/common/refine-plans/SKILL.md` +- `agent-ops/skills/common/finalize-task-routing/SKILL.md` +- `agent-ops/skills/common/update-test/SKILL.md` +- `agent-roadmap/phase/knowledge-tool-optimization-extension/milestones/agent-comparison-benchmark-pipeline.md` +- `agent-roadmap/sdd/knowledge-tool-optimization-extension/agent-comparison-benchmark-pipeline/SDD.md` +- `scripts/agent_benchmark/manifest.py` +- `scripts/agent_benchmark/attempts.py` +- `scripts/agent_benchmark/lifecycle.py` +- `scripts/agent_benchmark/manifest_test.py` +- `scripts/agent_benchmark/attempts_test.py` +- `agent-test/local/rules.md` +- `agent-test/local/testing-smoke.md` +- `agent-ops/rules/project/domain/testing/rules.md` + +### SDD Criteria + +- SDD는 `[승인됨]`, 잠금은 `해제`다. +- milestone-task는 `effort-route,connection-gap`, Acceptance는 S09/S10이다. +- S09는 requested/effective route/model/effort의 exact matrix와 no-substitution evidence를 요구한다. +- S10은 credential/model 부재를 `registration_required`, endpoint/auth/protocol/stream incompatibility를 별도 implementation Plan 후보인 `implementation_gap`으로 분리하고 blocker를 성공으로 우회하지 않도록 요구한다. + +### Verification Context + +- 별도 verification handoff는 없었고 local test rules, Make target, 현재 source/test를 사용한다. +- current HEAD는 `62f4b9196bbcd7687810001e81129a5fcdc5037e`, pre-refine intent checkpoint는 `8c6a3c05ea766875b6625c5f3d58ceeee5558910`이다. 두 commit 사이에는 중앙 `prepare-epic-work-items` script/test만 바뀌었고 target source 구현은 시작되지 않았다. +- 이 child는 synthetic `MatrixCell`과 임시 디렉터리만 사용하고 caller/provider/network를 호출하지 않는다. + +### Test Coverage Gaps + +- requested/effective binding 불일치, registration 필요, implementation gap을 구분하는 closed contract가 없다. +- canonical ordering, sequence/no-overwrite, corruption/symlink 거부와 secret/private endpoint 비노출 검증이 없다. + +### Symbol References + +rename/remove symbol은 없다. 새 module은 기존 `MatrixCell`/`IopCell`을 import하고 08/09/10 caller adapter와 11 public preflight가 소비한다. + +### Split Judgment + +원본 API-1/API-2는 하나의 공통 contract/evidence boundary다. 원본 API-3의 Edge Anthropic effort 호환은 이 contract가 `implementation_gap`으로 표현할 수 있는 독립 production fix이므로 `07_anthropic_effort_compatibility`로 분리했다. 이 child는 Edge 변경 없이 독립 PASS할 수 있다. + +### Scope Rationale + +caller 명령/JSONL parser, public CLI wiring, skill 문서, Edge/provider 수정, live 호출은 제외한다. raw secret, private endpoint, prompt/tool content는 입력 검증과 persisted evidence 모두에서 금지한다. + +### Final Routing + +- evaluation_mode `isolated-reassessment`; finalizer `finalize-task-policy.sh pair`를 이 child에 정확히 한 번 실행했다. +- build closures 모두 true; scores `1/1/1/2/1`; base `local-fit`, risks `temporal_state,boundary_contract,structured_interpretation,variant_product` 4개로 `risk-boundary`; cloud G06; `PLAN-cloud-G06.md`; `worker/cloud/G06`. +- review closures 모두 true; scores `1/1/1/2/1`; `official-review`; cloud G06; `CODE_REVIEW-cloud-G06.md`; `review/cloud/G06`. +- large context false; rework 0; evidence integrity false; capability gap 없음. + +## Implementation Checklist + +- [ ] Add frozen caller capability, requested/effective binding, issue and result contracts with exact no-substitution validation. +- [ ] Add the closed registration-versus-implementation classifier and secret-safe canonical no-overwrite evidence writer. +- [ ] Add deterministic unit coverage and run focused, aggregate and patch-integrity verification without caller/provider access. +- [ ] Fill implementation-owned sections in CODE_REVIEW-*-G??.md with actual implementation notes and verification output. + +### [API-1] Freeze route and effort preflight contracts + +**Problem:** `manifest.py` stores requested values but there is no typed oracle proving a caller observed the exact route/model/effort without substitution. + +**Solution:** Add standard-library-only `connectivity.py` frozen dataclasses/protocols for caller capability, requested/effective binding, issue and per-cell result. Validate caller/cell identity, route kind/id, requested/effective model and effort, preset stage bindings, canonical order and closed statuses. Missing, extra, unsupported or substituted values fail closed. + +**Modified Files and Checklist:** + +- [ ] Create `scripts/agent_benchmark/connectivity.py` with frozen closed contracts and validators. +- [ ] Create `scripts/agent_benchmark/connectivity_test.py` with direct/preset, effort, alias, missing/extra field, order and immutability cases. + +**Test Strategy:** Use synthetic cells only; cover valid direct/preset results and every field-level no-substitution failure. + +**Verification:** focused connectivity unittest exits 0. + +### [API-2] Classify and persist secret-safe connection evidence + +**Problem:** S10 requires registration blockers and implementation gaps to remain separate, but no durable closed evidence format exists. + +**Solution:** Map only closed issue codes to `ready`, `registration_required`, or `implementation_gap`; treat `implementation_gap` as the explicit implementation-Plan candidate classification with a closed issue code and non-sensitive resume condition, without auto-creating a Plan. Never synthesize effective fields. Serialize canonical sorted JSON with public identifiers and hashed endpoint/config identity only. Reject raw credentials, authorization values, private endpoint strings, prompt/tool content, symlinks, existing targets and schema drift. + +**Modified Files and Checklist:** + +- [ ] Implement classifier and canonical evidence serializer in `scripts/agent_benchmark/connectivity.py`. +- [ ] Test precedence, redaction sentinels, canonical bytes, containment, symlink and no-overwrite behavior in `scripts/agent_benchmark/connectivity_test.py`. + +**Test Strategy:** Use temporary directories and sentinels; assert registration and implementation-Plan candidate routing stay distinct, sensitive bytes are absent from outputs/errors, and duplicate/corrupt evidence fails closed. + +**Verification:** focused and aggregate benchmark tests exit 0. + +## Modified Files Summary + +| File | Items | +|---|---| +| `scripts/agent_benchmark/connectivity.py` | API-1, API-2 | +| `scripts/agent_benchmark/connectivity_test.py` | API-1, API-2 | +| `agent-task/m-agent-comparison-benchmark-pipeline/06_connectivity_contract/CODE_REVIEW-cloud-G06.md` | API-1, API-2 evidence | + +## Final Verification + +1. `python3 -m unittest scripts.agent_benchmark.connectivity_test -v` +2. `make test-agent-comparison-benchmark` +3. `git diff --check` + +After completing all code changes, fill implementation-owned sections in `CODE_REVIEW-*-G??.md`. diff --git a/agent-task/m-agent-comparison-benchmark-pipeline/06_connectivity_contract/CODE_REVIEW-cloud-G09.md b/agent-task/m-agent-comparison-benchmark-pipeline/06_connectivity_contract/code_review_cloud_G09_0.log similarity index 100% rename from agent-task/m-agent-comparison-benchmark-pipeline/06_connectivity_contract/CODE_REVIEW-cloud-G09.md rename to agent-task/m-agent-comparison-benchmark-pipeline/06_connectivity_contract/code_review_cloud_G09_0.log diff --git a/agent-task/m-agent-comparison-benchmark-pipeline/06_connectivity_contract/PLAN-cloud-G09.md b/agent-task/m-agent-comparison-benchmark-pipeline/06_connectivity_contract/plan_cloud_G09_0.log similarity index 100% rename from agent-task/m-agent-comparison-benchmark-pipeline/06_connectivity_contract/PLAN-cloud-G09.md rename to agent-task/m-agent-comparison-benchmark-pipeline/06_connectivity_contract/plan_cloud_G09_0.log diff --git a/agent-task/m-agent-comparison-benchmark-pipeline/07_anthropic_effort_compatibility/CODE_REVIEW-cloud-G06.md b/agent-task/m-agent-comparison-benchmark-pipeline/07_anthropic_effort_compatibility/CODE_REVIEW-cloud-G06.md new file mode 100644 index 00000000..790acb46 --- /dev/null +++ b/agent-task/m-agent-comparison-benchmark-pipeline/07_anthropic_effort_compatibility/CODE_REVIEW-cloud-G06.md @@ -0,0 +1,137 @@ + + +# Code Review Reference - API + +> **[IMPLEMENTING AGENT — READ FIRST] Filling in this file is the mandatory final step of implementation.** +> The task is NOT complete until every implementation-owned section below is filled in. +> Complete the `Implementation Checklist`; the final checklist item is mandatory before saving. +> Fill implementation-owned sections, then stop with active files in place and report ready for review. +> Execute the plan's selected root cause, scope, files, and dependency decisions as written. Do not choose another owner, narrow/expand the write boundary, or replace a fix with another verification attempt. +> If implementation is blocked, record the exact blocker, attempted commands/output, and resume condition only in implementation-owned evidence fields. +> Do not ask the user directly, present choices, call user-input tools, create control-plane stop files, or classify the next state. +> Finalization (`Code Review Result`, log rename, `complete.log`, archive moves, `Review-Only Checklist`) is review-agent-only, even after compaction/resume. +> Follow the ownership table at the bottom of this file for which sections you own. + +## Overview + +date=2026-08-10 +task=m-agent-comparison-benchmark-pipeline/07_anthropic_effort_compatibility, plan=0, tag=API + +## For the Review Agent + +> **[REVIEW AGENT ONLY]** The finalization steps below are review-agent only. Implementing agents must not execute this section. + +Compare implementation of each item against source files and verify that output in `Verification Results` matches code. +Review completion means the following steps are finished: + +1. Append verdict and `review_rework_count` / `evidence_integrity_failure` routing signals. +2. Archive `CODE_REVIEW-cloud-G06.md` → `code_review_cloud_G06_0.log` and `PLAN-local-G06.md` → `plan_local_G06_0.log`. +3. If PASS, write `complete.log` and move active task directory to `agent-task/archive/YYYY/MM/m-agent-comparison-benchmark-pipeline/07_anthropic_effort_compatibility/`. If WARN/FAIL, fully write the next filesystem state required by the code-review skill. +4. If PASS and task group is `m-`, preserve the first-line `milestone-task` metadata in `complete.log` and report it for the runtime aggregation event. Roadmap state evaluation belongs to `sync-milestone-workstate`. +5. Check applicable `Review-Only Checklist` items at the final `.log` location before reporting. + +--- + +## Implementation Item Completion + +| Item | Status | +|------|---------| +| API-1 Preserve Claude Code high-tier effort through IOP | [ ] | + +## Implementation Checklist + +- [ ] Accept and preserve only `low|medium|high|xhigh|max` across Anthropic native and Chat-bridge routes without substitution. +- [ ] Update the Anthropic outer contract and current implementation spec with exact high-tier semantics. +- [ ] Add deterministic Go coverage and run focused, scoped, complete-Go-or-blocker and patch-integrity verification. +- [ ] Fill implementation-owned sections in CODE_REVIEW-*-G??.md with actual implementation notes and verification output. + +## Review-Only Checklist + +> **[REVIEW AGENT ONLY]** This checklist is used only by the review agent. +> Implementing agents must not modify or check this section. + +- [ ] Append one verdict of `PASS`, `WARN`, or `FAIL` and verified `review_rework_count`, `evidence_integrity_failure` to `Code Review Result`. +- [ ] Verify that verdict, `Dimension Assessment`, and Required/Suggested/Nit classifications match. +- [ ] Archive active `CODE_REVIEW-*-G??.md` to `code_review_cloud_G06_0.log`. +- [ ] Archive active `PLAN-*-G??.md` to `plan_local_G06_0.log`. +- [ ] Verify that the Agent-Ops managed block in `.gitignore` unignores `agent-task/**/*.md` and `agent-task/**/*.log` and ignores `agent-roadmap/current.md`. +- [ ] If PASS, write `complete.log` based on `agent-ops/skills/common/code-review/templates/complete-log-template.md` and leave no active `.md` files. +- [ ] If PASS, move active task directory `agent-task/m-agent-comparison-benchmark-pipeline/07_anthropic_effort_compatibility/` to `agent-task/archive/YYYY/MM/m-agent-comparison-benchmark-pipeline/07_anthropic_effort_compatibility/` and update this checklist at the final archive path. +- [ ] If PASS and task group is `m-`, preserve and report `milestone-task` metadata for runtime aggregation, without modifying roadmap or directly calling `update-roadmap`. +- [ ] If PASS for split work, remove empty active parent `agent-task/m-agent-comparison-benchmark-pipeline/` or verify it was kept due to remaining siblings/files. +- [ ] If WARN/FAIL, write the next filesystem state matching code-review verdict and do not write `complete.log`. + +## Deviations from Plan + +_Record any deviations from the plan and the rationale here._ + +## Key Design Decisions + +_Record key design decisions here._ + +## Reviewer Checkpoints + +- Confirm `xhigh|max` are exact, never aliased/downshifted, and unknown effort fails before provider wire. +- Confirm native preservation and bridge mapping tests exercise actual HTTP/tunnel paths. +- Confirm outer contract/spec match implementation and no OpenAI-general effort contract changed. +- Confirm only exact write-set files changed and this child does not claim live readiness. + +## Verification Results + +Paste exact stdout/stderr and exit code. Preserve the ignored user artifact if the full suite is blocked. + +### V1 Focused Anthropic compatibility tests + +Command: `go test ./apps/edge/internal/openai -run 'TestAnthropic(ChatBridge|Native)' -count=1` + +```text + +``` +Exit code: `` + +### V2 Scoped Edge regression + +Command: `go test ./apps/edge/... -count=1` + +```text + +``` +Exit code: `` + +### V3 Complete Go regression or blocker + +Command: `if [ -e build/r14-remote-anthropic_handler.go ]; then echo 'BLOCKED: ignored build artifact shadows iop/build' >&2; exit 69; fi; go test ./... -count=1` + +```text + +``` +Exit code: `` + +### V4 Patch integrity + +Command: `git diff --check` + +```text + +``` +Exit code: `` + +--- + +> **[IMPLEMENTING AGENT — BEFORE SAVING] Have you filled in every implementation-owned section?** +> If anything is blank, go back and fill it in before saving this file. +> Leave review-agent-only sections unchanged. + +## Section Ownership + +| Section | Owner | Note | +|---------|-------|------| +| Header comment, Overview, Review Agent Instructions | Fixed at stub creation | Implementing agent must not modify or execute these (archive, complete.log, and task-directory archive move are review-agent only) | +| Archive Evidence Snapshot | Fixed at stub creation from plan when present | Implementing agent uses it as default prior-loop context; read only the specific archive files cited there when more detail is required | +| Implementation Item Completion (item names) | Fixed at stub creation | Implementing agent checks `[ ]` → `[x]` only | +| Implementation Checklist (item text/order) | Fixed at stub creation from plan | Implementing agent checks `[ ]` → `[x]` only | +| Review-Only Checklist | Review agent only | Implementing agent must not modify or check this section | +| Deviations from Plan, Key Design Decisions | Implementing agent | Replace placeholder text with actual content | +| Reviewer Checkpoints | Fixed at stub creation | Pre-filled from plan | +| Verification Results (section headings + commands) | Fixed at stub creation | Implementing agent fills in command output only; command changes require a `Deviations from Plan` entry | +| Code Review Result | Review agent appends | Not included in stub | diff --git a/agent-task/m-agent-comparison-benchmark-pipeline/07_anthropic_effort_compatibility/PLAN-local-G06.md b/agent-task/m-agent-comparison-benchmark-pipeline/07_anthropic_effort_compatibility/PLAN-local-G06.md new file mode 100644 index 00000000..f77fd85f --- /dev/null +++ b/agent-task/m-agent-comparison-benchmark-pipeline/07_anthropic_effort_compatibility/PLAN-local-G06.md @@ -0,0 +1,117 @@ + + +# Plan - API: Anthropic high-tier effort compatibility + +## For the Implementing Agent + +IOP Anthropic-compatible surface의 high-tier effort 보존만 구현한다. 검증 결과와 구현 메모를 active `CODE_REVIEW-cloud-G06.md`에 채우고 review-ready로 보고한다. 사용자 질문, stop 파일, verdict, archive, `complete.log`는 만들지 않는다. + +## Background + +Claude Code 2.1.223의 최고 effort는 `max`이며 `xhigh`도 지원하지만, 현재 `decodeAnthropicMessageRequest`와 외부 계약은 `low|medium|high`만 허용한다. 이 repository-fixable protocol gap은 공통 preflight contract와 독립적으로 구현·검증할 수 있고, live Claude matrix 전에 닫혀야 한다. + +## Analysis + +### Files Read + +- `AGENTS.md` +- `agent-ops/rules/project/rules.md` +- `agent-ops/rules/common/rules-roadmap.md` +- `agent-ops/rules/common/rules-agent-spec.md` +- `agent-ops/skills/common/router.md` +- `agent-ops/skills/common/plan/SKILL.md` +- `agent-ops/skills/common/refine-plans/SKILL.md` +- `agent-ops/skills/common/finalize-task-routing/SKILL.md` +- `agent-roadmap/phase/knowledge-tool-optimization-extension/milestones/agent-comparison-benchmark-pipeline.md` +- `agent-roadmap/sdd/knowledge-tool-optimization-extension/agent-comparison-benchmark-pipeline/SDD.md` +- `agent-spec/index.md` +- `agent-spec/input/openai-compatible-surface.md` +- `agent-contract/index.md` +- `agent-contract/outer/anthropic-compatible-api.md` +- `apps/edge/internal/openai/anthropic_types.go` +- `apps/edge/internal/openai/anthropic_bridge.go` +- `apps/edge/internal/openai/anthropic_bridge_test.go` +- `apps/edge/internal/openai/anthropic_native_test.go` +- `agent-test/local/rules.md` +- `agent-test/local/edge-smoke.md` +- `agent-ops/rules/project/domain/testing/rules.md` +- `agent-ops/rules/project/domain/edge/rules.md` + +### SDD Criteria + +- milestone-task는 `effort-route`, Acceptance는 S09다. +- S09는 Sonnet 최고 effort, Gemini high, GPT xhigh와 requested/effective no-substitution evidence를 요구한다. +- D07에 따라 확인된 protocol incompatibility는 registration blocker와 섞지 않고 별도 implementation unit으로 닫는다. + +### Verification Context + +- current HEAD는 `62f4b9196bbcd7687810001e81129a5fcdc5037e`, pre-refine intent checkpoint는 `8c6a3c05ea766875b6625c5f3d58ceeee5558910`이다. 두 commit 사이의 변경은 target source 밖 중앙 Epic 준비 script/test에 한정된다. +- `claude --help`는 `low|medium|high|xhigh|max`를 노출한다. 현재 source/contract는 `low|medium|high`만 허용하고 bridge는 허용된 값을 `reasoning_effort`로 그대로 전달한다. +- provider 호출은 하지 않으며 fake provider tunnel로 native/bridge wire를 검증한다. + +### Test Coverage Gaps + +- `xhigh|max` strict 수용과 exact bridge mapping, native raw preservation, unknown effort의 pre-wire rejection이 없다. + +### Symbol References + +rename/remove symbol은 없다. `output_config.effort` validation만 확장하며 existing bridge/native ownership을 유지한다. + +### Split Judgment + +원본 API-3은 공통 Python contract와 write set, verification, 완료 조건을 공유하지 않는 독립 behavior slice다. 이 child는 단일 Edge compatibility boundary로 더 나누지 않는다. + +### Scope Rationale + +허용 집합은 `low|medium|high|xhigh|max`로 닫고 lowercase/alias/cap/downshift를 금지한다. OpenAI Chat/Responses 일반 effort 계약, provider capability 정책, caller adapter와 live 실행은 제외한다. + +### Final Routing + +- evaluation_mode `isolated-reassessment`; finalizer를 이 child에 정확히 한 번 실행했다. +- build closures 모두 true; scores `1/0/2/1/2`; base/route `local-fit`; local G06; `PLAN-local-G06.md`; `worker/local/G06`. Risks `boundary_contract,variant_product` 2개. +- review closures 모두 true; 같은 scores; `official-review`; cloud G06; `CODE_REVIEW-cloud-G06.md`; `review/cloud/G06`. +- large context false; rework 0; evidence integrity false; capability gap 없음. + +## Implementation Checklist + +- [ ] Accept and preserve only `low|medium|high|xhigh|max` across Anthropic native and Chat-bridge routes without substitution. +- [ ] Update the Anthropic outer contract and current implementation spec with exact high-tier semantics. +- [ ] Add deterministic Go coverage and run focused, scoped, complete-Go-or-blocker and patch-integrity verification. +- [ ] Fill implementation-owned sections in CODE_REVIEW-*-G??.md with actual implementation notes and verification output. + +### [API-1] Preserve Claude Code high-tier effort through IOP + +**Problem:** `decodeAnthropicMessageRequest` rejects `output_config.effort=xhigh|max` before provider dispatch even though the benchmark requires Claude→GPT xhigh and Claude→Sonnet max. + +**Solution:** Extend only the Anthropic compatibility effort enum. Native Messages retains request bytes except model rewrite; Chat bridge maps the exact value to outbound `reasoning_effort`. Unknown values remain a pre-wire `invalid_request_error`; upstream capability rejection is not converted to success. + +**Modified Files and Checklist:** + +- [ ] Update `apps/edge/internal/openai/anthropic_types.go` validation and error text. +- [ ] Add exact `xhigh|max` bridge and invalid-value tests in `apps/edge/internal/openai/anthropic_bridge_test.go`. +- [ ] Add `max` request-byte preservation evidence in `apps/edge/internal/openai/anthropic_native_test.go`. +- [ ] Update `agent-contract/outer/anthropic-compatible-api.md` and `agent-spec/input/openai-compatible-surface.md`. + +**Test Strategy:** Drive the HTTP surface with fake provider tunnels; assert exact tokens reach the selected route and unknown values create zero provider requests. + +**Verification:** focused and package-level Edge tests exit 0; live readiness remains downstream. + +## Modified Files Summary + +| File | Items | +|---|---| +| `apps/edge/internal/openai/anthropic_types.go` | API-1 | +| `apps/edge/internal/openai/anthropic_bridge_test.go` | API-1 | +| `apps/edge/internal/openai/anthropic_native_test.go` | API-1 | +| `agent-contract/outer/anthropic-compatible-api.md` | API-1 | +| `agent-spec/input/openai-compatible-surface.md` | API-1 | +| `agent-task/m-agent-comparison-benchmark-pipeline/07_anthropic_effort_compatibility/CODE_REVIEW-cloud-G06.md` | API-1 evidence | + +## Final Verification + +1. `go test ./apps/edge/internal/openai -run 'TestAnthropic(ChatBridge|Native)' -count=1` +2. `go test ./apps/edge/... -count=1` +3. `if [ -e build/r14-remote-anthropic_handler.go ]; then echo 'BLOCKED: ignored build artifact shadows iop/build' >&2; exit 69; fi; go test ./... -count=1` +4. `git diff --check` + +After completing all code changes, fill implementation-owned sections in `CODE_REVIEW-*-G??.md`. diff --git a/agent-task/m-agent-comparison-benchmark-pipeline/07+06_claude_iop/CODE_REVIEW-cloud-G07.md b/agent-task/m-agent-comparison-benchmark-pipeline/08+06_claude_iop/CODE_REVIEW-cloud-G07.md similarity index 96% rename from agent-task/m-agent-comparison-benchmark-pipeline/07+06_claude_iop/CODE_REVIEW-cloud-G07.md rename to agent-task/m-agent-comparison-benchmark-pipeline/08+06_claude_iop/CODE_REVIEW-cloud-G07.md index 8a3a758b..142902ab 100644 --- a/agent-task/m-agent-comparison-benchmark-pipeline/07+06_claude_iop/CODE_REVIEW-cloud-G07.md +++ b/agent-task/m-agent-comparison-benchmark-pipeline/08+06_claude_iop/CODE_REVIEW-cloud-G07.md @@ -1,4 +1,4 @@ - + # Code Review Reference - API @@ -15,7 +15,7 @@ ## Overview date=2026-08-10 -task=m-agent-comparison-benchmark-pipeline/07+06_claude_iop, plan=0, tag=API +task=m-agent-comparison-benchmark-pipeline/08+06_claude_iop, plan=0, tag=API @@ -28,7 +28,7 @@ Review completion means the following steps are finished: 1. Append verdict and `review_rework_count` / `evidence_integrity_failure` routing signals. 2. Archive `CODE_REVIEW-cloud-G07.md` → `code_review_cloud_G07_0.log` and `PLAN-cloud-G07.md` → `plan_cloud_G07_0.log`. -3. If PASS, write `complete.log` and move active task directory to `agent-task/archive/YYYY/MM/m-agent-comparison-benchmark-pipeline/07+06_claude_iop/`. If WARN/FAIL, fully write the next filesystem state required by the code-review skill. +3. If PASS, write `complete.log` and move active task directory to `agent-task/archive/YYYY/MM/m-agent-comparison-benchmark-pipeline/08+06_claude_iop/`. If WARN/FAIL, fully write the next filesystem state required by the code-review skill. 4. If PASS and task group is `m-`, preserve the first-line `milestone-task` metadata in `complete.log` and report it for the runtime aggregation event. Roadmap state evaluation belongs to `sync-milestone-workstate`. 5. Check applicable `Review-Only Checklist` items at the final `.log` location before reporting. @@ -59,7 +59,7 @@ Review completion means the following steps are finished: - [ ] Archive active `PLAN-*-G??.md` to `plan_cloud_G07_0.log`. - [ ] Verify that the Agent-Ops managed block in `.gitignore` unignores `agent-task/**/*.md` and `agent-task/**/*.log` and ignores `agent-roadmap/current.md`. - [ ] If PASS, write `complete.log` based on `agent-ops/skills/common/code-review/templates/complete-log-template.md` and leave no active `.md` files. -- [ ] If PASS, move active task directory `agent-task/m-agent-comparison-benchmark-pipeline/07+06_claude_iop/` to `agent-task/archive/YYYY/MM/m-agent-comparison-benchmark-pipeline/07+06_claude_iop/` and update this checklist at the final archive path. +- [ ] If PASS, move active task directory `agent-task/m-agent-comparison-benchmark-pipeline/08+06_claude_iop/` to `agent-task/archive/YYYY/MM/m-agent-comparison-benchmark-pipeline/08+06_claude_iop/` and update this checklist at the final archive path. - [ ] If PASS and task group is `m-`, preserve and report `milestone-task` metadata for runtime aggregation, without modifying roadmap or directly calling `update-roadmap`. - [ ] If PASS for split work, remove empty active parent `agent-task/m-agent-comparison-benchmark-pipeline/` or verify it was kept due to remaining siblings/files. - [ ] If WARN/FAIL, write the next filesystem state matching code-review verdict and do not write `complete.log`. diff --git a/agent-task/m-agent-comparison-benchmark-pipeline/07+06_claude_iop/PLAN-cloud-G07.md b/agent-task/m-agent-comparison-benchmark-pipeline/08+06_claude_iop/PLAN-cloud-G07.md similarity index 94% rename from agent-task/m-agent-comparison-benchmark-pipeline/07+06_claude_iop/PLAN-cloud-G07.md rename to agent-task/m-agent-comparison-benchmark-pipeline/08+06_claude_iop/PLAN-cloud-G07.md index 7089813e..6a112850 100644 --- a/agent-task/m-agent-comparison-benchmark-pipeline/07+06_claude_iop/PLAN-cloud-G07.md +++ b/agent-task/m-agent-comparison-benchmark-pipeline/08+06_claude_iop/PLAN-cloud-G07.md @@ -1,4 +1,4 @@ - + # Plan - API: Claude Code IOP adapter @@ -52,12 +52,12 @@ generic lifecycle은 caller별 argv, environment, JSONL parser와 redactor를 - SDD는 `[승인됨]`, 잠금 `해제`; milestone-task `claude-iop`; Acceptance `S06`; Evidence Map S06. - S06은 Sonnet/Gemini/GPT direct auth/model/stream/terminal과 arbitrary preset adapter 계약을 요구하고, evidence row는 redacted Claude Code→IOP preflight를 요구한다. -- 이 packet은 adapter 계약과 credential-free stream fixture를 완성한다. 실제 dev preflight와 route readiness evidence는 세 adapter를 연결하는 ordered consumer `10+07,08,09_connectivity_runner`가 수행한다. +- 이 packet은 adapter 계약과 credential-free stream fixture를 완성한다. 실제 dev preflight와 route readiness evidence는 세 adapter를 연결하는 ordered consumer `11+08,09,10_connectivity_preflight`와 live evidence child가 수행한다. ### Verification Context - 별도 verification handoff는 없고 local rules, current CLI help/version, source/tests/contracts로 보완했다. -- runner: `/config/workspace/iop-s0`; branch `feature/agent-comparison-benchmark-pipeline`; HEAD `df0ecdbf1d6cbf8878c84a9b154bc4d022ab035d`; Linux aarch64; Python 3.12.3. 요청 시작 HEAD 이후 두 커밋은 중앙 관리 Agent-Ops 파일만 바꿨고 대상 benchmark source/test는 동일하다. tracked worktree는 materialization 전에 clean이었으며 현재 활성 변경은 이 Epic의 plan/review pair뿐이다. +- runner: `/config/workspace/iop-s0`; branch `feature/agent-comparison-benchmark-pipeline`; current HEAD `62f4b9196bbcd7687810001e81129a5fcdc5037e`, pre-refine intent checkpoint `8c6a3c05ea766875b6625c5f3d58ceeee5558910`이다. 사이 변경은 target source 밖 중앙 Epic 준비 script/test뿐이다. Linux aarch64, Python 3.12.3 환경이며 구현 source는 아직 시작되지 않았다. - Claude binary: `/config/.local/bin/claude`, version `2.1.223`. `claude --help`에서 `--bare`, `--print`, `--input-format`, `--output-format=stream-json`, `--model`, `--effort`, `--no-session-persistence`, `--tools`를 확인했다. - caller config/credential은 repo 밖 사용자 관리 상태이며 원문을 읽지 않았다. adapter는 base URL 값과 secret env-var 이름을 런타임 입력으로만 받고 evidence에는 digest/존재 여부만 남겨야 한다. 로컬 포트는 필요 없고 external host는 redacted dev IOP Edge다. - dev inventory는 Gemini direct exact match를 보였지만 Sonnet 5와 GPT-5.6 luna는 미등록이다. network/provider call은 NOT_RUN; 현재 성공으로 간주하지 않는다. @@ -75,7 +75,7 @@ rename/remove symbol은 없다. 새 adapter는 `InvocationSpec`, `run_invocation ### Split Judgment -Claude adapter는 agy/Codex와 파일을 공유하지 않고 fixture-driven PASS가 가능하다. `07+06`은 predecessor `06_connectivity_contract`만 요구한다. 현재 `agent-task/m-agent-comparison-benchmark-pipeline/06_connectivity_contract/complete.log`는 아직 없으며 runtime은 해당 active 또는 archived exact `complete.log`가 생기기 전 구현을 시작하면 안 된다. +Claude adapter는 agy/Codex와 파일을 공유하지 않고 fixture-driven PASS가 가능하다. `08+06`은 predecessor `06_connectivity_contract`만 요구한다. 현재 `agent-task/m-agent-comparison-benchmark-pipeline/06_connectivity_contract/complete.log`는 아직 없으며 runtime은 해당 active 또는 archived exact `complete.log`가 생기기 전 구현을 시작하면 안 된다. ### Scope Rationale @@ -160,7 +160,7 @@ redact_claude_event(raw_line, sensitive_values) -> canonical_redacted_json ## Dependencies and Execution Order -- `06_connectivity_contract` must have exactly one active or archived `complete.log` before implementation. The dependency is encoded by `07+06_claude_iop`; it is currently missing during planning. +- `06_connectivity_contract` must have exactly one active or archived `complete.log` before implementation. The dependency is encoded by `08+06_claude_iop`; it is currently missing during planning. - This packet does not wait on 08/09 and does not perform the downstream live preflight. ## Modified Files Summary @@ -170,7 +170,7 @@ redact_claude_event(raw_line, sensitive_values) -> canonical_redacted_json | `scripts/agent_benchmark/claude_iop.py` | API-1, API-2 | | `scripts/agent_benchmark/claude_iop_test.py` | API-1, API-2 | | `scripts/fixtures/agent-comparison-benchmark/claude-iop-stream.jsonl` | API-2 | -| `agent-task/m-agent-comparison-benchmark-pipeline/07+06_claude_iop/CODE_REVIEW-cloud-G07.md` | API-1, API-2 evidence | +| `agent-task/m-agent-comparison-benchmark-pipeline/08+06_claude_iop/CODE_REVIEW-cloud-G07.md` | API-1, API-2 evidence | ## Final Verification diff --git a/agent-task/m-agent-comparison-benchmark-pipeline/08+06_agy_iop/CODE_REVIEW-cloud-G07.md b/agent-task/m-agent-comparison-benchmark-pipeline/09+06_agy_iop/CODE_REVIEW-cloud-G07.md similarity index 96% rename from agent-task/m-agent-comparison-benchmark-pipeline/08+06_agy_iop/CODE_REVIEW-cloud-G07.md rename to agent-task/m-agent-comparison-benchmark-pipeline/09+06_agy_iop/CODE_REVIEW-cloud-G07.md index 6c3529d4..f756bda5 100644 --- a/agent-task/m-agent-comparison-benchmark-pipeline/08+06_agy_iop/CODE_REVIEW-cloud-G07.md +++ b/agent-task/m-agent-comparison-benchmark-pipeline/09+06_agy_iop/CODE_REVIEW-cloud-G07.md @@ -1,4 +1,4 @@ - + # Code Review Reference - API @@ -15,7 +15,7 @@ ## Overview date=2026-08-10 -task=m-agent-comparison-benchmark-pipeline/08+06_agy_iop, plan=0, tag=API +task=m-agent-comparison-benchmark-pipeline/09+06_agy_iop, plan=0, tag=API @@ -28,7 +28,7 @@ Review completion means the following steps are finished: 1. Append verdict and `review_rework_count` / `evidence_integrity_failure` routing signals. 2. Archive `CODE_REVIEW-cloud-G07.md` → `code_review_cloud_G07_0.log` and `PLAN-cloud-G07.md` → `plan_cloud_G07_0.log`. -3. If PASS, write `complete.log` and move active task directory to `agent-task/archive/YYYY/MM/m-agent-comparison-benchmark-pipeline/08+06_agy_iop/`. If WARN/FAIL, fully write the next filesystem state required by the code-review skill. +3. If PASS, write `complete.log` and move active task directory to `agent-task/archive/YYYY/MM/m-agent-comparison-benchmark-pipeline/09+06_agy_iop/`. If WARN/FAIL, fully write the next filesystem state required by the code-review skill. 4. If PASS and task group is `m-`, preserve the first-line `milestone-task` metadata in `complete.log` and report it for the runtime aggregation event. Roadmap state evaluation belongs to `sync-milestone-workstate`. 5. Check applicable `Review-Only Checklist` items at the final `.log` location before reporting. @@ -59,7 +59,7 @@ Review completion means the following steps are finished: - [ ] Archive active `PLAN-*-G??.md` to `plan_cloud_G07_0.log`. - [ ] Verify that the Agent-Ops managed block in `.gitignore` unignores `agent-task/**/*.md` and `agent-task/**/*.log` and ignores `agent-roadmap/current.md`. - [ ] If PASS, write `complete.log` based on `agent-ops/skills/common/code-review/templates/complete-log-template.md` and leave no active `.md` files. -- [ ] If PASS, move active task directory `agent-task/m-agent-comparison-benchmark-pipeline/08+06_agy_iop/` to `agent-task/archive/YYYY/MM/m-agent-comparison-benchmark-pipeline/08+06_agy_iop/` and update this checklist at the final archive path. +- [ ] If PASS, move active task directory `agent-task/m-agent-comparison-benchmark-pipeline/09+06_agy_iop/` to `agent-task/archive/YYYY/MM/m-agent-comparison-benchmark-pipeline/09+06_agy_iop/` and update this checklist at the final archive path. - [ ] If PASS and task group is `m-`, preserve and report `milestone-task` metadata for runtime aggregation, without modifying roadmap or directly calling `update-roadmap`. - [ ] If PASS for split work, remove empty active parent `agent-task/m-agent-comparison-benchmark-pipeline/` or verify it was kept due to remaining siblings/files. - [ ] If WARN/FAIL, write the next filesystem state matching code-review verdict and do not write `complete.log`. diff --git a/agent-task/m-agent-comparison-benchmark-pipeline/08+06_agy_iop/PLAN-cloud-G07.md b/agent-task/m-agent-comparison-benchmark-pipeline/09+06_agy_iop/PLAN-cloud-G07.md similarity index 94% rename from agent-task/m-agent-comparison-benchmark-pipeline/08+06_agy_iop/PLAN-cloud-G07.md rename to agent-task/m-agent-comparison-benchmark-pipeline/09+06_agy_iop/PLAN-cloud-G07.md index a784700a..d35c5a1a 100644 --- a/agent-task/m-agent-comparison-benchmark-pipeline/08+06_agy_iop/PLAN-cloud-G07.md +++ b/agent-task/m-agent-comparison-benchmark-pipeline/09+06_agy_iop/PLAN-cloud-G07.md @@ -1,4 +1,4 @@ - + # Plan - API: agy IOP adapter @@ -52,12 +52,12 @@ agy는 현재 CLI에 model/effort/stream-json은 노출하지만 IOP endpoint/au - SDD `[승인됨]`, 잠금 `해제`; milestone-task `agy-iop`; Acceptance `S07`; Evidence Map S07. - S07은 Gemini direct preflight와 generic preset fixture에서 IOP 경유를 입증하거나 exact compatibility gap을 기록하도록 한다. D07/D08은 gap을 별도 Plan 후보로 분류하고 dispatcher/ambient provider fallback을 금지한다. -- adapter와 fixture/parser unit evidence는 이 packet, 실제 redacted dev preflight 또는 exact blocker는 downstream `10+07,08,09_connectivity_runner`가 소유한다. +- adapter와 fixture/parser unit evidence는 이 packet, 실제 redacted dev preflight 또는 exact blocker는 downstream `11+08,09,10_connectivity_preflight`와 live evidence child가 소유한다. ### Verification Context - 별도 verification handoff 없음; local rules, repo source/tests/contracts, CLI read-only probes로 보완했다. -- runner `/config/workspace/iop-s0`; branch `feature/agent-comparison-benchmark-pipeline`; HEAD `df0ecdbf1d6cbf8878c84a9b154bc4d022ab035d`; Linux aarch64; Python 3.12.3. 요청 시작 HEAD 이후 두 커밋은 중앙 관리 Agent-Ops 파일만 바꿨고 대상 benchmark source/test는 동일하다. tracked worktree는 materialization 전에 clean이었으며 현재 활성 변경은 이 Epic의 plan/review pair뿐이다. +- runner `/config/workspace/iop-s0`; branch `feature/agent-comparison-benchmark-pipeline`; current HEAD `62f4b9196bbcd7687810001e81129a5fcdc5037e`, pre-refine intent checkpoint `8c6a3c05ea766875b6625c5f3d58ceeee5558910`이다. 사이 변경은 target source 밖 중앙 Epic 준비 script/test뿐이다. Linux aarch64, Python 3.12.3 환경이며 구현 source는 아직 시작되지 않았다. - agy wrapper `/config/.local/bin/agy`, version `1.1.11`; underlying `/config/.local/bin/agy-real`. Help에서 `--print`, `--output-format stream-json`, `--model`, `--effort low|medium|high`, `--mode`, `--sandbox`를 확인했으나 public base-url/provider override option은 확인되지 않았다. - agent inventory는 agy를 configured지만 quota-blocked로 기록한다. dev `gemini-3.6-flash` route는 exact active match다. provider/network probe는 NOT_RUN이며 이 조합을 ready로 단정하지 않는다. - caller config/auth path와 private endpoint/secret 원문은 읽지 않았다. 로컬 포트 요구 없음; external host는 redacted dev IOP Edge. @@ -75,7 +75,7 @@ rename/remove symbol은 없다. 새 adapter는 06 contract와 기존 `Invocation ### Split Judgment -agy adapter는 독립 file/fixture로 credential-free PASS가 가능하다. `08+06`의 유일한 predecessor는 `06_connectivity_contract`; 현재 exact `complete.log`는 missing이며 runtime은 active/archive completion 전 시작하면 안 된다. Claude/Codex packet과 직접 dependency는 없다. +agy adapter는 독립 file/fixture로 credential-free PASS가 가능하다. `09+06`의 유일한 predecessor는 `06_connectivity_contract`; 현재 exact `complete.log`는 missing이며 runtime은 active/archive completion 전 시작하면 안 된다. Claude/Codex packet과 직접 dependency는 없다. ### Scope Rationale @@ -159,7 +159,7 @@ redact_agy_event(raw_line, sensitive_values) -> canonical_redacted_json ## Dependencies and Execution Order -- `06_connectivity_contract` must produce exactly one active/archive `complete.log`; encoded by `08+06_agy_iop` and currently missing. +- `06_connectivity_contract` must produce exactly one active/archive `complete.log`; encoded by `09+06_agy_iop` and currently missing. - Live readiness is a downstream consumer check, not a hidden predecessor of this adapter unit packet. ## Modified Files Summary @@ -169,7 +169,7 @@ redact_agy_event(raw_line, sensitive_values) -> canonical_redacted_json | `scripts/agent_benchmark/agy_iop.py` | API-1, API-2 | | `scripts/agent_benchmark/agy_iop_test.py` | API-1, API-2 | | `scripts/fixtures/agent-comparison-benchmark/agy-iop-stream.jsonl` | API-2 | -| `agent-task/m-agent-comparison-benchmark-pipeline/08+06_agy_iop/CODE_REVIEW-cloud-G07.md` | API-1, API-2 evidence | +| `agent-task/m-agent-comparison-benchmark-pipeline/09+06_agy_iop/CODE_REVIEW-cloud-G07.md` | API-1, API-2 evidence | ## Final Verification diff --git a/agent-task/m-agent-comparison-benchmark-pipeline/09+06_codex_iop/CODE_REVIEW-cloud-G07.md b/agent-task/m-agent-comparison-benchmark-pipeline/10+06_codex_iop/CODE_REVIEW-cloud-G07.md similarity index 96% rename from agent-task/m-agent-comparison-benchmark-pipeline/09+06_codex_iop/CODE_REVIEW-cloud-G07.md rename to agent-task/m-agent-comparison-benchmark-pipeline/10+06_codex_iop/CODE_REVIEW-cloud-G07.md index 5349c881..73b8f2c2 100644 --- a/agent-task/m-agent-comparison-benchmark-pipeline/09+06_codex_iop/CODE_REVIEW-cloud-G07.md +++ b/agent-task/m-agent-comparison-benchmark-pipeline/10+06_codex_iop/CODE_REVIEW-cloud-G07.md @@ -1,4 +1,4 @@ - + # Code Review Reference - API @@ -15,7 +15,7 @@ ## Overview date=2026-08-10 -task=m-agent-comparison-benchmark-pipeline/09+06_codex_iop, plan=0, tag=API +task=m-agent-comparison-benchmark-pipeline/10+06_codex_iop, plan=0, tag=API @@ -28,7 +28,7 @@ Review completion means the following steps are finished: 1. Append verdict and `review_rework_count` / `evidence_integrity_failure` routing signals. 2. Archive `CODE_REVIEW-cloud-G07.md` → `code_review_cloud_G07_0.log` and `PLAN-cloud-G07.md` → `plan_cloud_G07_0.log`. -3. If PASS, write `complete.log` and move active task directory to `agent-task/archive/YYYY/MM/m-agent-comparison-benchmark-pipeline/09+06_codex_iop/`. If WARN/FAIL, fully write the next filesystem state required by the code-review skill. +3. If PASS, write `complete.log` and move active task directory to `agent-task/archive/YYYY/MM/m-agent-comparison-benchmark-pipeline/10+06_codex_iop/`. If WARN/FAIL, fully write the next filesystem state required by the code-review skill. 4. If PASS and task group is `m-`, preserve the first-line `milestone-task` metadata in `complete.log` and report it for the runtime aggregation event. Roadmap state evaluation belongs to `sync-milestone-workstate`. 5. Check applicable `Review-Only Checklist` items at the final `.log` location before reporting. @@ -59,7 +59,7 @@ Review completion means the following steps are finished: - [ ] Archive active `PLAN-*-G??.md` to `plan_cloud_G07_0.log`. - [ ] Verify that the Agent-Ops managed block in `.gitignore` unignores `agent-task/**/*.md` and `agent-task/**/*.log` and ignores `agent-roadmap/current.md`. - [ ] If PASS, write `complete.log` based on `agent-ops/skills/common/code-review/templates/complete-log-template.md` and leave no active `.md` files. -- [ ] If PASS, move active task directory `agent-task/m-agent-comparison-benchmark-pipeline/09+06_codex_iop/` to `agent-task/archive/YYYY/MM/m-agent-comparison-benchmark-pipeline/09+06_codex_iop/` and update this checklist at the final archive path. +- [ ] If PASS, move active task directory `agent-task/m-agent-comparison-benchmark-pipeline/10+06_codex_iop/` to `agent-task/archive/YYYY/MM/m-agent-comparison-benchmark-pipeline/10+06_codex_iop/` and update this checklist at the final archive path. - [ ] If PASS and task group is `m-`, preserve and report `milestone-task` metadata for runtime aggregation, without modifying roadmap or directly calling `update-roadmap`. - [ ] If PASS for split work, remove empty active parent `agent-task/m-agent-comparison-benchmark-pipeline/` or verify it was kept due to remaining siblings/files. - [ ] If WARN/FAIL, write the next filesystem state matching code-review verdict and do not write `complete.log`. diff --git a/agent-task/m-agent-comparison-benchmark-pipeline/09+06_codex_iop/PLAN-cloud-G07.md b/agent-task/m-agent-comparison-benchmark-pipeline/10+06_codex_iop/PLAN-cloud-G07.md similarity index 94% rename from agent-task/m-agent-comparison-benchmark-pipeline/09+06_codex_iop/PLAN-cloud-G07.md rename to agent-task/m-agent-comparison-benchmark-pipeline/10+06_codex_iop/PLAN-cloud-G07.md index 19f24c6d..bd132951 100644 --- a/agent-task/m-agent-comparison-benchmark-pipeline/09+06_codex_iop/PLAN-cloud-G07.md +++ b/agent-task/m-agent-comparison-benchmark-pipeline/10+06_codex_iop/PLAN-cloud-G07.md @@ -1,4 +1,4 @@ - + # Plan - API: Codex IOP adapter @@ -53,12 +53,12 @@ Codex exec는 custom model provider와 JSONL 실행을 제공하지만 현재 be - SDD `[승인됨]`, 잠금 `해제`; milestone-task `codex-iop`; Acceptance `S08`; Evidence Map S08. - S08은 GPT direct preflight/generic preset fixture에서 IOP 경유를 입증하거나 endpoint/auth/protocol/stream exact blocker를 남기도록 한다. S09는 xhigh no-substitution을 요구한다. -- adapter unit evidence는 이 packet, 실제 redacted dev preflight 또는 exact blocker는 downstream `10+07,08,09_connectivity_runner`가 수행한다. +- adapter unit evidence는 이 packet, 실제 redacted dev preflight 또는 exact blocker는 downstream `11+08,09,10_connectivity_preflight`와 live evidence child가 수행한다. ### Verification Context - verification handoff 없음; local rules, repository source/tests/contracts와 installed CLI read-only probes로 보완했다. -- runner `/config/workspace/iop-s0`; branch `feature/agent-comparison-benchmark-pipeline`; HEAD `df0ecdbf1d6cbf8878c84a9b154bc4d022ab035d`; Linux aarch64; Python 3.12.3. 요청 시작 HEAD 이후 두 커밋은 중앙 관리 Agent-Ops 파일만 바꿨고 대상 benchmark source/test는 동일하다. tracked worktree는 materialization 전에 clean이었으며 현재 활성 변경은 이 Epic의 plan/review pair뿐이다. +- runner `/config/workspace/iop-s0`; branch `feature/agent-comparison-benchmark-pipeline`; current HEAD `62f4b9196bbcd7687810001e81129a5fcdc5037e`, pre-refine intent checkpoint `8c6a3c05ea766875b6625c5f3d58ceeee5558910`이다. 사이 변경은 target source 밖 중앙 Epic 준비 script/test뿐이다. Linux aarch64, Python 3.12.3 환경이며 구현 source는 아직 시작되지 않았다. - Codex binary `/config/.npm-global/bin/codex`, version `codex-cli 0.147.0`. `codex exec --help`에서 `--json`, `--ephemeral`, `--ignore-user-config`, `--strict-config`, `-m/--model`, `-c/--config`, `-C/--cd`, sandbox options와 stdin prompt form을 확인했다. - current agent inventory의 기본 GPT profile은 luna가 아니며 dev `gpt-5.6-luna` exact route도 미등록이다. external network/provider is NOT_RUN; readiness로 간주하지 않는다. - auth/config/private endpoint 원문은 읽지 않았다. adapter는 caller-supplied runtime env를 child에만 전달하고 config/argv/capture에는 raw secret을 남기지 않는다. local port 없음; external host는 redacted dev IOP Edge. @@ -76,7 +76,7 @@ rename/remove는 없다. 새 adapter는 06 contract와 `InvocationSpec`, `run_in ### Split Judgment -Codex adapter는 독립 fixture와 fake binary로 PASS할 수 있다. `09+06` predecessor는 `06_connectivity_contract` 하나이며 현재 exact completion은 missing이다. runtime은 active/archive `complete.log` 전 구현을 시작하지 않는다; Claude/agy와 직접 dependency는 없다. +Codex adapter는 독립 fixture와 fake binary로 PASS할 수 있다. `10+06` predecessor는 `06_connectivity_contract` 하나이며 현재 exact completion은 missing이다. runtime은 active/archive `complete.log` 전 구현을 시작하지 않는다; Claude/agy와 직접 dependency는 없다. ### Scope Rationale @@ -160,7 +160,7 @@ verified child exit + stdout EOF -> adapter.idle -> idle ## Dependencies and Execution Order -- `06_connectivity_contract` must have one exact active/archive `complete.log`; dependency encoded by `09+06_codex_iop`, currently missing. +- `06_connectivity_contract` must have one exact active/archive `complete.log`; dependency encoded by `10+06_codex_iop`, currently missing. - Downstream integration owns live dev preflight and is not a predecessor of this fixture-driven adapter packet. ## Modified Files Summary @@ -170,7 +170,7 @@ verified child exit + stdout EOF -> adapter.idle -> idle | `scripts/agent_benchmark/codex_iop.py` | API-1, API-2 | | `scripts/agent_benchmark/codex_iop_test.py` | API-1, API-2 | | `scripts/fixtures/agent-comparison-benchmark/codex-iop-stream.jsonl` | API-2 | -| `agent-task/m-agent-comparison-benchmark-pipeline/09+06_codex_iop/CODE_REVIEW-cloud-G07.md` | API-1, API-2 evidence | +| `agent-task/m-agent-comparison-benchmark-pipeline/10+06_codex_iop/CODE_REVIEW-cloud-G07.md` | API-1, API-2 evidence | ## Final Verification diff --git a/agent-task/m-agent-comparison-benchmark-pipeline/11+08,09,10_connectivity_preflight/CODE_REVIEW-cloud-G10.md b/agent-task/m-agent-comparison-benchmark-pipeline/11+08,09,10_connectivity_preflight/CODE_REVIEW-cloud-G10.md new file mode 100644 index 00000000..70c441f7 --- /dev/null +++ b/agent-task/m-agent-comparison-benchmark-pipeline/11+08,09,10_connectivity_preflight/CODE_REVIEW-cloud-G10.md @@ -0,0 +1,158 @@ + + +# Code Review Reference - API + +> **[IMPLEMENTING AGENT — READ FIRST] Filling in this file is the mandatory final step of implementation.** +> The task is NOT complete until every implementation-owned section below is filled in. +> Complete the `Implementation Checklist`; the final checklist item is mandatory before saving. +> Fill implementation-owned sections, then stop with active files in place and report ready for review. +> Execute the plan's selected root cause, scope, files, and dependency decisions as written. Do not choose another owner, narrow/expand the write boundary, or replace a fix with another verification attempt. +> If implementation is blocked, record the exact blocker, attempted commands/output, and resume condition only in implementation-owned evidence fields. +> Do not ask the user directly, present choices, call user-input tools, create control-plane stop files, or classify the next state. +> Finalization (`Code Review Result`, log rename, `complete.log`, archive moves, `Review-Only Checklist`) is review-agent-only, even after compaction/resume. +> Follow the ownership table at the bottom of this file for which sections you own. + +## Overview + +date=2026-08-10 +task=m-agent-comparison-benchmark-pipeline/11+08,09,10_connectivity_preflight, plan=1, tag=API + +## For the Review Agent + +> **[REVIEW AGENT ONLY]** The finalization steps below are review-agent only. Implementing agents must not execute this section. + +Compare implementation of each item against source files and verify that output in `Verification Results` matches code. +Review completion means the following steps are finished: + +1. Append verdict and `review_rework_count` / `evidence_integrity_failure` routing signals. +2. Archive `CODE_REVIEW-cloud-G10.md` → `code_review_cloud_G10_1.log` and `PLAN-cloud-G10.md` → `plan_cloud_G10_1.log`. +3. If PASS, write `complete.log` and move active task directory to `agent-task/archive/YYYY/MM/m-agent-comparison-benchmark-pipeline/11+08,09,10_connectivity_preflight/`. If WARN/FAIL, fully write the next filesystem state required by the code-review skill. +4. If PASS and task group is `m-`, preserve the first-line `milestone-task` metadata in `complete.log` and report it for the runtime aggregation event. Roadmap state evaluation belongs to `sync-milestone-workstate`. +5. Check applicable `Review-Only Checklist` items at the final `.log` location before reporting. + +--- + +## Implementation Item Completion + +| Item | Status | +|------|---------| +| API-1 Add append-only public route preflight | [ ] | +| API-2 Publish the preflight-only skill and fixtures | [ ] | + +## Implementation Checklist + +- [ ] Add the three-adapter registry and public `preflight`; persist canonical append-only results and expose only redacted closed summaries. +- [ ] Keep run/resume unavailable and prove every preflight blocker prevents attempt allocation. +- [ ] Publish the preflight-only CLI/skill and direct-live versus generic-preset fixture contract with deterministic tests. +- [ ] Run predecessor, focused, CLI, fixture, aggregate and patch-integrity verification without live calls. +- [ ] Fill implementation-owned sections in CODE_REVIEW-*-G??.md with actual implementation notes and verification output. + +## Review-Only Checklist + +> **[REVIEW AGENT ONLY]** This checklist is used only by the review agent. +> Implementing agents must not modify or check this section. + +- [ ] Append one verdict of `PASS`, `WARN`, or `FAIL` and verified `review_rework_count`, `evidence_integrity_failure` to `Code Review Result`. +- [ ] Verify that verdict, `Dimension Assessment`, and Required/Suggested/Nit classifications match. +- [ ] Archive active `CODE_REVIEW-*-G??.md` to `code_review_cloud_G10_1.log`. +- [ ] Archive active `PLAN-*-G??.md` to `plan_cloud_G10_1.log`. +- [ ] Verify that the Agent-Ops managed block in `.gitignore` unignores `agent-task/**/*.md` and `agent-task/**/*.log` and ignores `agent-roadmap/current.md`. +- [ ] If PASS, write `complete.log` based on `agent-ops/skills/common/code-review/templates/complete-log-template.md` and leave no active `.md` files. +- [ ] If PASS, move active task directory `agent-task/m-agent-comparison-benchmark-pipeline/11+08,09,10_connectivity_preflight/` to `agent-task/archive/YYYY/MM/m-agent-comparison-benchmark-pipeline/11+08,09,10_connectivity_preflight/` and update this checklist at the final archive path. +- [ ] If PASS and task group is `m-`, preserve and report `milestone-task` metadata for runtime aggregation, without modifying roadmap or directly calling `update-roadmap`. +- [ ] If PASS for split work, remove empty active parent `agent-task/m-agent-comparison-benchmark-pipeline/` or verify it was kept due to remaining siblings/files. +- [ ] If WARN/FAIL, write the next filesystem state matching code-review verdict and do not write `complete.log`. + +## Deviations from Plan + +_Record any deviations from the plan and the rationale here._ + +## Key Design Decisions + +_Record key design decisions here._ + +## Reviewer Checkpoints + +- Confirm preflight records are writer-owned, append-only, schema-validated and never scored attempts. +- Confirm every blocker prevents allocation, registration remains distinct from the `implementation_gap` Plan-candidate class, and no model/effort/provider fallback can produce ready. +- Confirm run/resume remain explicitly unavailable after this child and CLI/skill/tests agree. +- Confirm generic preset is fixture-only, direct manifest has exactly five target cells, and no live call occurred. +- Confirm diff/evidence has no secret, private endpoint, prompt/tool content or user config mutation. + +## Verification Results + +Paste exact stdout/stderr and exit code for every command. + +### V1 Predecessors + +Command: `python3 -c 'from pathlib import Path; g="m-agent-comparison-benchmark-pipeline"; ids=("08","09","10"); a=Path("agent-task")/g; r=Path("agent-task/archive"); f={i:sorted([*a.glob(f"{i}_*/complete.log"),*a.glob(f"{i}+*/complete.log"),*r.glob(f"*/*/{g}/{i}_*/complete.log"),*r.glob(f"*/*/{g}/{i}+*/complete.log")],key=str) for i in ids}; bad={i:[str(x) for x in p] for i,p in f.items() if len(p)!=1}; assert not bad,bad; print("\n".join(str(f[i][0]) for i in ids))'` + +```text + +``` +Exit code: `` + +### V2 Focused preflight integration + +Command: `python3 -m unittest scripts.agent_benchmark.attempts_test scripts.agent_benchmark.connectivity_integration_test scripts.agent_benchmark.skill_contract_test scripts.agent_benchmark.manifest_test -v` + +```text + +``` +Exit code: `` + +### V3 CLI surface + +Command: `python3 scripts/agent_comparison_benchmark.py --help && python3 scripts/agent_comparison_benchmark.py validate --help && python3 scripts/agent_comparison_benchmark.py preflight --help && python3 scripts/agent_comparison_benchmark.py run --help && python3 scripts/agent_comparison_benchmark.py resume --help && python3 scripts/agent_comparison_benchmark.py status --help` + +```text + +``` +Exit code: `` + +### V4 Fixture validation + +Command: `python3 scripts/agent_comparison_benchmark.py validate --manifest scripts/fixtures/agent-comparison-benchmark-manifest.example.json && python3 scripts/agent_comparison_benchmark.py validate --manifest scripts/fixtures/agent-comparison-benchmark-direct-preflight.example.json` + +```text + +``` +Exit code: `` + +### V5 Aggregate benchmark tests + +Command: `make test-agent-comparison-benchmark` + +```text + +``` +Exit code: `` + +### V6 Patch integrity + +Command: `git diff --check` + +```text + +``` +Exit code: `` + +--- + +> **[IMPLEMENTING AGENT — BEFORE SAVING] Have you filled in every implementation-owned section?** +> If anything is blank, go back and fill it in before saving this file. +> Leave review-agent-only sections unchanged. + +## Section Ownership + +| Section | Owner | Note | +|---------|-------|------| +| Header comment, Overview, Review Agent Instructions | Fixed at stub creation | Implementing agent must not modify or execute these (archive, complete.log, and task-directory archive move are review-agent only) | +| Archive Evidence Snapshot | Fixed at stub creation from plan when present | Implementing agent uses it as default prior-loop context; read only the specific archive files cited there when more detail is required | +| Implementation Item Completion (item names) | Fixed at stub creation | Implementing agent checks `[ ]` → `[x]` only | +| Implementation Checklist (item text/order) | Fixed at stub creation from plan | Implementing agent checks `[ ]` → `[x]` only | +| Review-Only Checklist | Review agent only | Implementing agent must not modify or check this section | +| Deviations from Plan, Key Design Decisions | Implementing agent | Replace placeholder text with actual content | +| Reviewer Checkpoints | Fixed at stub creation | Pre-filled from plan | +| Verification Results (section headings + commands) | Fixed at stub creation | Implementing agent fills in command output only; command changes require a `Deviations from Plan` entry | +| Code Review Result | Review agent appends | Not included in stub | diff --git a/agent-task/m-agent-comparison-benchmark-pipeline/11+08,09,10_connectivity_preflight/PLAN-cloud-G10.md b/agent-task/m-agent-comparison-benchmark-pipeline/11+08,09,10_connectivity_preflight/PLAN-cloud-G10.md new file mode 100644 index 00000000..8f7befb6 --- /dev/null +++ b/agent-task/m-agent-comparison-benchmark-pipeline/11+08,09,10_connectivity_preflight/PLAN-cloud-G10.md @@ -0,0 +1,147 @@ + + +# Plan - API: public connectivity preflight integration + +## For the Implementing Agent + +완료된 08/09/10 adapter를 append-only public `preflight`에만 연결한다. run/resume의 기존 capability-unavailable 동작은 유지한다. 검증 결과와 구현 메모를 active `CODE_REVIEW-cloud-G10.md`에 채우고 review-ready로 보고한다. 사용자 질문, live provider 호출, verdict, archive, `complete.log`는 하지 않는다. + +## Background + +현재 CLI는 validate/run/resume/status만 제공하고 valid run/resume는 caller-adapter unavailable로 종료한다. 세 adapter를 등록해 direct route readiness와 exact binding을 scored attempt 전에 독립적으로 확인하고, blocker에서도 append-only redacted setup evidence를 남기는 안정 경계가 먼저 필요하다. + +## Analysis + +### Files Read + +- `AGENTS.md` +- `agent-ops/rules/project/rules.md` +- `agent-ops/rules/common/rules-roadmap.md` +- `agent-ops/skills/common/router.md` +- `agent-ops/skills/common/plan/SKILL.md` +- `agent-ops/skills/common/refine-plans/SKILL.md` +- `agent-ops/skills/common/finalize-task-routing/SKILL.md` +- `agent-roadmap/phase/knowledge-tool-optimization-extension/milestones/agent-comparison-benchmark-pipeline.md` +- `agent-roadmap/sdd/knowledge-tool-optimization-extension/agent-comparison-benchmark-pipeline/SDD.md` +- `scripts/agent_comparison_benchmark.py` +- `scripts/agent_benchmark/__init__.py` +- `scripts/agent_benchmark/manifest.py` +- `scripts/agent_benchmark/attempts.py` +- `scripts/agent_benchmark/attempts_test.py` +- `scripts/agent_benchmark/manifest_test.py` +- `scripts/agent_benchmark/skill_contract_test.py` +- `scripts/fixtures/agent-comparison-benchmark-manifest.schema.json` +- `scripts/fixtures/agent-comparison-benchmark-manifest.example.json` +- `agent-ops/skills/project/iop-agent-comparison-benchmark/SKILL.md` +- `agent-test/local/rules.md` +- `agent-test/local/testing-smoke.md` +- `agent-ops/rules/project/domain/testing/rules.md` + +### SDD Criteria + +- milestone-task는 다섯 allowed id 전체이며 Acceptance S06-S10에 기여한다. +- S06-S08은 caller별 IOP preflight 또는 exact compatibility blocker, S09는 no-substitution matrix, S10은 registration/implementation gap 분리를 요구한다. +- D01은 direct live와 generic preset fixture를 분리하고, D02/D08은 dispatcher나 ambient provider fallback을 금지한다. + +### Verification Context + +- current HEAD는 `62f4b9196bbcd7687810001e81129a5fcdc5037e`, pre-refine intent checkpoint는 `8c6a3c05ea766875b6625c5f3d58ceeee5558910`이다. 사이 변경은 target source 밖 중앙 Epic 준비 script/test뿐이고 source implementation은 미착수다. +- caller binary help/version은 read-only로 확인했다. config/credential 원문과 provider/network는 이 child에서 읽거나 호출하지 않는다. +- 08/09/10의 exact completion이 선행되어야 하며 현재 모두 missing이다. + +### Test Coverage Gaps + +- 세 adapter registry, public preflight, append-only preflight sequence/read validation, blocker에서 zero-attempt 보장이 없다. +- CLI/skill/fixture가 direct-live와 generic-preset contract를 분리해 설명하는 검증이 없다. + +### Symbol References + +rename/remove symbol은 없다. `CapabilityUnavailable("capability-unavailable: caller-adapter")`는 run/resume에 유지하고 public preflight path만 새로 연결한다. `run_slots` signature는 이 child에서 바꾸지 않는다. + +### Split Judgment + +원본 API-1과 preflight에 필요한 API-3 subset은 public preflight라는 독립 안정 계약이다. run/resume activation은 `12+11_connectivity_execution`, authorized live probe는 `13+07,11_connectivity_live_evidence`로 분리했다. 이 child는 network 없이 PASS하고 다음 child가 소비할 append-only state/API를 제공한다. + +### Scope Rationale + +attempt 실행, run/resume 활성화, external inventory/runtime/preflight, report/timing/web/scoring, hybrid preset live readiness는 제외한다. Project skill은 preflight만 available로 바꾸고 run/resume unavailable을 유지한다. + +### Final Routing + +- evaluation_mode `isolated-reassessment`; finalizer를 이 child에 정확히 한 번 실행했다. +- build/review closures 모두 true; scores `2/2/2/2/2`; build `grade-boundary`, cloud G10, `PLAN-cloud-G10.md`, `worker/cloud/G10`; review `official-review`, cloud G10, `CODE_REVIEW-cloud-G10.md`, `review/cloud/G10`. +- risks 다섯 개; large context false; rework 0; integrity false; capability gap 없음. + +## Implementation Checklist + +- [ ] Add the three-adapter registry and public `preflight`; persist canonical append-only results and expose only redacted closed summaries. +- [ ] Keep run/resume unavailable and prove every preflight blocker prevents attempt allocation. +- [ ] Publish the preflight-only CLI/skill and direct-live versus generic-preset fixture contract with deterministic tests. +- [ ] Run predecessor, focused, CLI, fixture, aggregate and patch-integrity verification without live calls. +- [ ] Fill implementation-owned sections in CODE_REVIEW-*-G??.md with actual implementation notes and verification output. + +### [API-1] Add append-only public route preflight + +**Problem:** there is no operator command to collect S06-S10 setup evidence separately from scored attempts. + +**Solution:** Add `preflight --manifest`, build the exact registry from the three adapter modules, verify callers before mutation, create a run identity, and append `preflight/preflight-NNNNNN.json` under the run writer. Return 0 only when every direct cell is ready; return 69 with a closed redacted summary that preserves `registration_required` versus the explicit implementation-Plan candidate class `implementation_gap`. Generic preset cells are local contract validation only. + +**Modified Files and Checklist:** + +- [ ] Update `scripts/agent_comparison_benchmark.py` parser, registry, sanitized output and preflight handler while retaining run/resume unavailable. +- [ ] Update `scripts/agent_benchmark/attempts.py` with writer-owned append-only preflight sequencing/read validation and status projection. +- [ ] Export completed connectivity/adapters through `scripts/agent_benchmark/__init__.py`. +- [ ] Add state/order/corruption/secret and fake three-caller tests to `attempts_test.py` and new `connectivity_integration_test.py`. + +**Test Strategy:** fake every caller/runtime input; cover ready, each blocker class, mixed blockers, missing adapter, duplicate/concurrent preflight, corruption/symlink, read-only status and zero attempt directories. + +**Verification:** focused preflight integration tests pass with no network. + +### [API-2] Publish the preflight-only skill and fixtures + +**Problem:** project skill currently fixes exactly four commands and prohibits all adapter/provider execution, while fixtures do not fully separate generic preset contract from direct readiness. + +**Solution:** Publish deterministic preflight delegation only; prohibit ad-hoc caller/provider calls and preserve run/resume unavailable until child 12. Update semantic/mutation tests, add all three generic preset contracts to the non-live example, and add a direct-preflight manifest with the five required route/model/effort cells and only public aliases. + +**Modified Files and Checklist:** + +- [ ] Update `agent-ops/skills/project/iop-agent-comparison-benchmark/SKILL.md` for preflight-only availability. +- [ ] Update `scripts/agent_benchmark/skill_contract_test.py` for five-command parity, preflight delegation, blocker safety and unchanged run/resume/report/prepare constraints. +- [ ] Update `scripts/fixtures/agent-comparison-benchmark-manifest.example.json` and add `scripts/fixtures/agent-comparison-benchmark-direct-preflight.example.json`. +- [ ] Update `scripts/agent_benchmark/manifest_test.py` for exact fixture scopes and D01 separation. + +**Test Strategy:** help/fake adapter/fixture tests only; mutations for fallback, preset live claim, substitution, secret output and blocker bypass must fail. + +**Verification:** CLI help, both fixture validations and aggregate benchmark target pass. + +## Dependencies and Execution Order + +- `08+06_claude_iop`, `09+06_agy_iop`, `10+06_codex_iop` must each have exactly one active/archive `complete.log` before implementation. +- 06 is transitive through those adapters and is not an extra direct predecessor. + +## Modified Files Summary + +| File | Items | +|---|---| +| `scripts/agent_comparison_benchmark.py` | API-1 | +| `scripts/agent_benchmark/__init__.py` | API-1 | +| `scripts/agent_benchmark/attempts.py` | API-1 | +| `scripts/agent_benchmark/attempts_test.py` | API-1 | +| `scripts/agent_benchmark/connectivity_integration_test.py` | API-1 | +| `scripts/agent_benchmark/skill_contract_test.py` | API-2 | +| `scripts/agent_benchmark/manifest_test.py` | API-2 | +| `agent-ops/skills/project/iop-agent-comparison-benchmark/SKILL.md` | API-2 | +| `scripts/fixtures/agent-comparison-benchmark-manifest.example.json` | API-2 | +| `scripts/fixtures/agent-comparison-benchmark-direct-preflight.example.json` | API-2 | +| `agent-task/m-agent-comparison-benchmark-pipeline/11+08,09,10_connectivity_preflight/CODE_REVIEW-cloud-G10.md` | API-1, API-2 evidence | + +## Final Verification + +1. `python3 -c 'from pathlib import Path; g="m-agent-comparison-benchmark-pipeline"; ids=("08","09","10"); a=Path("agent-task")/g; r=Path("agent-task/archive"); f={i:sorted([*a.glob(f"{i}_*/complete.log"),*a.glob(f"{i}+*/complete.log"),*r.glob(f"*/*/{g}/{i}_*/complete.log"),*r.glob(f"*/*/{g}/{i}+*/complete.log")],key=str) for i in ids}; bad={i:[str(x) for x in p] for i,p in f.items() if len(p)!=1}; assert not bad,bad; print("\n".join(str(f[i][0]) for i in ids))'` +2. `python3 -m unittest scripts.agent_benchmark.attempts_test scripts.agent_benchmark.connectivity_integration_test scripts.agent_benchmark.skill_contract_test scripts.agent_benchmark.manifest_test -v` +3. `python3 scripts/agent_comparison_benchmark.py --help && python3 scripts/agent_comparison_benchmark.py validate --help && python3 scripts/agent_comparison_benchmark.py preflight --help && python3 scripts/agent_comparison_benchmark.py run --help && python3 scripts/agent_comparison_benchmark.py resume --help && python3 scripts/agent_comparison_benchmark.py status --help` +4. `python3 scripts/agent_comparison_benchmark.py validate --manifest scripts/fixtures/agent-comparison-benchmark-manifest.example.json && python3 scripts/agent_comparison_benchmark.py validate --manifest scripts/fixtures/agent-comparison-benchmark-direct-preflight.example.json` +5. `make test-agent-comparison-benchmark` +6. `git diff --check` + +After completing all code changes, fill implementation-owned sections in `CODE_REVIEW-*-G??.md`. diff --git a/agent-task/m-agent-comparison-benchmark-pipeline/10+07,08,09_connectivity_runner/CODE_REVIEW-cloud-G10.md b/agent-task/m-agent-comparison-benchmark-pipeline/11+08,09,10_connectivity_preflight/code_review_cloud_G10_0.log similarity index 100% rename from agent-task/m-agent-comparison-benchmark-pipeline/10+07,08,09_connectivity_runner/CODE_REVIEW-cloud-G10.md rename to agent-task/m-agent-comparison-benchmark-pipeline/11+08,09,10_connectivity_preflight/code_review_cloud_G10_0.log diff --git a/agent-task/m-agent-comparison-benchmark-pipeline/10+07,08,09_connectivity_runner/PLAN-cloud-G10.md b/agent-task/m-agent-comparison-benchmark-pipeline/11+08,09,10_connectivity_preflight/plan_cloud_G10_0.log similarity index 100% rename from agent-task/m-agent-comparison-benchmark-pipeline/10+07,08,09_connectivity_runner/PLAN-cloud-G10.md rename to agent-task/m-agent-comparison-benchmark-pipeline/11+08,09,10_connectivity_preflight/plan_cloud_G10_0.log diff --git a/agent-task/m-agent-comparison-benchmark-pipeline/12+11_connectivity_execution/CODE_REVIEW-cloud-G09.md b/agent-task/m-agent-comparison-benchmark-pipeline/12+11_connectivity_execution/CODE_REVIEW-cloud-G09.md new file mode 100644 index 00000000..46396219 --- /dev/null +++ b/agent-task/m-agent-comparison-benchmark-pipeline/12+11_connectivity_execution/CODE_REVIEW-cloud-G09.md @@ -0,0 +1,157 @@ + + +# Code Review Reference - API + +> **[IMPLEMENTING AGENT — READ FIRST] Filling in this file is the mandatory final step of implementation.** +> The task is NOT complete until every implementation-owned section below is filled in. +> Complete the `Implementation Checklist`; the final checklist item is mandatory before saving. +> Fill implementation-owned sections, then stop with active files in place and report ready for review. +> Execute the plan's selected root cause, scope, files, and dependency decisions as written. Do not choose another owner, narrow/expand the write boundary, or replace a fix with another verification attempt. +> If implementation is blocked, record the exact blocker, attempted commands/output, and resume condition only in implementation-owned evidence fields. +> Do not ask the user directly, present choices, call user-input tools, create control-plane stop files, or classify the next state. +> Finalization (`Code Review Result`, log rename, `complete.log`, archive moves, `Review-Only Checklist`) is review-agent-only, even after compaction/resume. +> Follow the ownership table at the bottom of this file for which sections you own. + +## Overview + +date=2026-08-10 +task=m-agent-comparison-benchmark-pipeline/12+11_connectivity_execution, plan=0, tag=API + +## For the Review Agent + +> **[REVIEW AGENT ONLY]** The finalization steps below are review-agent only. Implementing agents must not execute this section. + +Compare implementation of each item against source files and verify that output in `Verification Results` matches code. +Review completion means the following steps are finished: + +1. Append verdict and `review_rework_count` / `evidence_integrity_failure` routing signals. +2. Archive `CODE_REVIEW-cloud-G09.md` → `code_review_cloud_G09_0.log` and `PLAN-cloud-G09.md` → `plan_cloud_G09_0.log`. +3. If PASS, write `complete.log` and move active task directory to `agent-task/archive/YYYY/MM/m-agent-comparison-benchmark-pipeline/12+11_connectivity_execution/`. If WARN/FAIL, fully write the next filesystem state required by the code-review skill. +4. If PASS and task group is `m-`, preserve the first-line `milestone-task` metadata in `complete.log` and report it for the runtime aggregation event. Roadmap state evaluation belongs to `sync-milestone-workstate`. +5. Check applicable `Review-Only Checklist` items at the final `.log` location before reporting. + +--- + +## Implementation Item Completion + +| Item | Status | +|------|---------| +| API-1 Wire ready runs and append-only resume attempts | [ ] | + +## Implementation Checklist + +- [ ] Evolve `run_slots` to bind typed adapters, exact cells and prepared workspaces under the existing writer lifecycle. +- [ ] Enable run/resume to append a fresh preflight before allocation, stop on blockers, and preserve append-only retry/state semantics. +- [ ] Update skill/contract/integration tests from preflight-only to available run/resume while retaining report unavailable and no public prepare. +- [ ] Run predecessor, focused, CLI, aggregate, complete-Go-or-blocker and patch-integrity verification without live calls. +- [ ] Fill implementation-owned sections in CODE_REVIEW-*-G??.md with actual implementation notes and verification output. + +## Review-Only Checklist + +> **[REVIEW AGENT ONLY]** This checklist is used only by the review agent. +> Implementing agents must not modify or check this section. + +- [ ] Append one verdict of `PASS`, `WARN`, or `FAIL` and verified `review_rework_count`, `evidence_integrity_failure` to `Code Review Result`. +- [ ] Verify that verdict, `Dimension Assessment`, and Required/Suggested/Nit classifications match. +- [ ] Archive active `CODE_REVIEW-*-G??.md` to `code_review_cloud_G09_0.log`. +- [ ] Archive active `PLAN-*-G??.md` to `plan_cloud_G09_0.log`. +- [ ] Verify that the Agent-Ops managed block in `.gitignore` unignores `agent-task/**/*.md` and `agent-task/**/*.log` and ignores `agent-roadmap/current.md`. +- [ ] If PASS, write `complete.log` based on `agent-ops/skills/common/code-review/templates/complete-log-template.md` and leave no active `.md` files. +- [ ] If PASS, move active task directory `agent-task/m-agent-comparison-benchmark-pipeline/12+11_connectivity_execution/` to `agent-task/archive/YYYY/MM/m-agent-comparison-benchmark-pipeline/12+11_connectivity_execution/` and update this checklist at the final archive path. +- [ ] If PASS and task group is `m-`, preserve and report `milestone-task` metadata for runtime aggregation, without modifying roadmap or directly calling `update-roadmap`. +- [ ] If PASS for split work, remove empty active parent `agent-task/m-agent-comparison-benchmark-pipeline/` or verify it was kept due to remaining siblings/files. +- [ ] If WARN/FAIL, write the next filesystem state matching code-review verdict and do not write `complete.log`. + +## Deviations from Plan + +_Record any deviations from the plan and the rationale here._ + +## Key Design Decisions + +_Record key design decisions here._ + +## Reviewer Checkpoints + +- Confirm each run/resume appends preflight before allocation and blockers allocate nothing. +- Confirm typed adapter/cell/prepared identity cannot drift and exactly one task submission occurs in a fresh workspace/session. +- Confirm retry preserves prior evidence, status is read-only and no nested writer can overwrite state. +- Confirm CLI, skill and tests expose run/resume while report remains unavailable and public prepare absent. +- Confirm no live provider call or write outside the exact set. + +## Verification Results + +Paste exact stdout/stderr and exit code. + +### V1 Predecessor + +Command: `python3 -c 'from pathlib import Path; g="m-agent-comparison-benchmark-pipeline"; i="11"; a=Path("agent-task")/g; r=Path("agent-task/archive"); p=sorted([*a.glob(f"{i}_*/complete.log"),*a.glob(f"{i}+*/complete.log"),*r.glob(f"*/*/{g}/{i}_*/complete.log"),*r.glob(f"*/*/{g}/{i}+*/complete.log")],key=str); assert len(p)==1,[str(x) for x in p]; print(p[0])'` + +```text + +``` +Exit code: `` + +### V2 Focused execution integration + +Command: `python3 -m unittest scripts.agent_benchmark.attempts_test scripts.agent_benchmark.connectivity_integration_test scripts.agent_benchmark.skill_contract_test -v` + +```text + +``` +Exit code: `` + +### V3 CLI surface + +Command: `python3 scripts/agent_comparison_benchmark.py --help && python3 scripts/agent_comparison_benchmark.py run --help && python3 scripts/agent_comparison_benchmark.py resume --help && python3 scripts/agent_comparison_benchmark.py status --help` + +```text + +``` +Exit code: `` + +### V4 Aggregate benchmark tests + +Command: `make test-agent-comparison-benchmark` + +```text + +``` +Exit code: `` + +### V5 Complete Go regression or blocker + +Command: `if [ -e build/r14-remote-anthropic_handler.go ]; then echo 'BLOCKED: ignored build artifact shadows iop/build' >&2; exit 69; fi; go test ./... -count=1` + +```text + +``` +Exit code: `` + +### V6 Patch integrity + +Command: `git diff --check` + +```text + +``` +Exit code: `` + +--- + +> **[IMPLEMENTING AGENT — BEFORE SAVING] Have you filled in every implementation-owned section?** +> If anything is blank, go back and fill it in before saving this file. +> Leave review-agent-only sections unchanged. + +## Section Ownership + +| Section | Owner | Note | +|---------|-------|------| +| Header comment, Overview, Review Agent Instructions | Fixed at stub creation | Implementing agent must not modify or execute these (archive, complete.log, and task-directory archive move are review-agent only) | +| Archive Evidence Snapshot | Fixed at stub creation from plan when present | Implementing agent uses it as default prior-loop context; read only the specific archive files cited there when more detail is required | +| Implementation Item Completion (item names) | Fixed at stub creation | Implementing agent checks `[ ]` → `[x]` only | +| Implementation Checklist (item text/order) | Fixed at stub creation from plan | Implementing agent checks `[ ]` → `[x]` only | +| Review-Only Checklist | Review agent only | Implementing agent must not modify or check this section | +| Deviations from Plan, Key Design Decisions | Implementing agent | Replace placeholder text with actual content | +| Reviewer Checkpoints | Fixed at stub creation | Pre-filled from plan | +| Verification Results (section headings + commands) | Fixed at stub creation | Implementing agent fills in command output only; command changes require a `Deviations from Plan` entry | +| Code Review Result | Review agent appends | Not included in stub | diff --git a/agent-task/m-agent-comparison-benchmark-pipeline/12+11_connectivity_execution/PLAN-cloud-G09.md b/agent-task/m-agent-comparison-benchmark-pipeline/12+11_connectivity_execution/PLAN-cloud-G09.md new file mode 100644 index 00000000..245da177 --- /dev/null +++ b/agent-task/m-agent-comparison-benchmark-pipeline/12+11_connectivity_execution/PLAN-cloud-G09.md @@ -0,0 +1,122 @@ + + +# Plan - API: connectivity run and resume execution + +## For the Implementing Agent + +11의 stable preflight/state contract 위에서 run/resume execution만 활성화한다. 실제 notes/output은 active `CODE_REVIEW-cloud-G09.md`에 기록하고 review-ready로 보고한다. live provider 검증, 사용자 질문, verdict, archive, `complete.log`는 하지 않는다. + +## Background + +public preflight가 scored attempt와 분리되어도 run/resume는 아직 capability-unavailable다. Ready preflight와 exact caller/cell/prepared workspace identity를 기존 append-only attempt lifecycle에 결합하고, blocker에서는 attempt가 절대 할당되지 않도록 해야 한다. + +## Analysis + +### Files Read + +- `AGENTS.md` +- `agent-ops/rules/project/rules.md` +- `agent-ops/rules/common/rules-roadmap.md` +- `agent-ops/skills/common/router.md` +- `agent-ops/skills/common/plan/SKILL.md` +- `agent-ops/skills/common/refine-plans/SKILL.md` +- `agent-ops/skills/common/finalize-task-routing/SKILL.md` +- `agent-roadmap/phase/knowledge-tool-optimization-extension/milestones/agent-comparison-benchmark-pipeline.md` +- `agent-roadmap/sdd/knowledge-tool-optimization-extension/agent-comparison-benchmark-pipeline/SDD.md` +- `scripts/agent_comparison_benchmark.py` +- `scripts/agent_benchmark/attempts.py` +- `scripts/agent_benchmark/attempts_test.py` +- `scripts/agent_benchmark/workspace.py` +- `scripts/agent_benchmark/lifecycle.py` +- `scripts/agent_benchmark/skill_contract_test.py` +- `agent-ops/skills/project/iop-agent-comparison-benchmark/SKILL.md` +- `agent-test/local/rules.md` +- `agent-test/local/testing-smoke.md` +- `agent-ops/rules/project/domain/testing/rules.md` + +### SDD Criteria + +- milestone-task는 `claude-iop,agy-iop,codex-iop`이고 S06-S08의 direct runner 요구에 기여한다. +- D02/D08은 직접 CLI entrypoint와 IOP-only 호출을 요구하고 dispatcher를 금지한다. +- D04/D10은 one user task, fresh session/workspace, finish→idle과 append-only retry를 요구한다. + +### Verification Context + +- current HEAD는 `62f4b9196bbcd7687810001e81129a5fcdc5037e`, pre-refine intent checkpoint는 `8c6a3c05ea766875b6625c5f3d58ceeee5558910`이다. 사이 변경은 target source 밖 중앙 Epic 준비 script/test뿐이며 source implementation은 미착수다. +- current `run_slots`는 plain callables를 받고 CLI run/resume는 이 path에 도달하지 않는다. 11이 public preflight와 adapter registry/state schema를 제공한 뒤에만 이 child를 시작한다. +- fake adapters/workspaces만 사용하며 external provider/network는 검증하지 않는다. + +### Test Coverage Gaps + +- run/resume가 매번 preflight를 append하고 blocker에서 zero allocation을 보장하는 integration test가 없다. +- ready run, crash/retry, old-byte preservation, fresh workspace/session, exact one-submission, status read-only를 함께 검증하지 않는다. + +### Symbol References + +`run_slots` signature/contract를 typed adapter/cell/prepared identity로 변경한다. 현재 call sites는 `attempts_test.py`와 새 CLI integration이며 모두 같은 write set에서 갱신한다. + +### Split Judgment + +원본 API-2와 run/resume availability에 필요한 API-3 subset은 하나의 execution state boundary다. 11의 public preflight API를 stable predecessor로 소비하고 live readiness와 독립적으로 fake integration PASS가 가능하다. + +### Scope Rationale + +preflight schema/fixture 신설, caller adapter 내부, live inventory/runtime, provider product 수정, report/timing/web/scoring은 제외한다. 기존 preflight evidence는 scored output으로 재사용하지 않는다. + +### Final Routing + +- evaluation_mode `isolated-reassessment`; finalizer를 이 child에 정확히 한 번 실행했다. +- build/review closures 모두 true; scores `2/2/2/1/2`; build `grade-boundary`, cloud G09, `PLAN-cloud-G09.md`, `worker/cloud/G09`; review `official-review`, cloud G09, `CODE_REVIEW-cloud-G09.md`, `review/cloud/G09`. +- risks `temporal_state,concurrent_consistency,boundary_contract,variant_product`; large context false; rework 0; integrity false; capability gap 없음. + +## Implementation Checklist + +- [ ] Evolve `run_slots` to bind typed adapters, exact cells and prepared workspaces under the existing writer lifecycle. +- [ ] Enable run/resume to append a fresh preflight before allocation, stop on blockers, and preserve append-only retry/state semantics. +- [ ] Update skill/contract/integration tests from preflight-only to available run/resume while retaining report unavailable and no public prepare. +- [ ] Run predecessor, focused, CLI, aggregate, complete-Go-or-blocker and patch-integrity verification without live calls. +- [ ] Fill implementation-owned sections in CODE_REVIEW-*-G??.md with actual implementation notes and verification output. + +### [API-1] Wire ready runs and append-only resume attempts + +**Problem:** `run_slots` accepts plain callables while caller/cell/prepared workspace and preflight binding are not coupled; CLI run never creates/opens a run and invokes it. + +**Solution:** Make `run_slots` consume typed adapters, find the exact cell, prepare one clean workspace, and invoke the adapter once with cell+prepared+attempt identity. Run calls `RunStore.create`, resume calls `open`; each appends a fresh preflight under the same writer before any allocation. Any blocker stops without an attempt. Retry allocates a new attempt and never overwrites prior bytes. + +**Modified Files and Checklist:** + +- [ ] Update `scripts/agent_benchmark/attempts.py` orchestration and durable validation. +- [ ] Wire run/resume/status in `scripts/agent_comparison_benchmark.py` without nested writers. +- [ ] Update all `run_slots` call sites and ready/blocker/retry/crash coverage in `attempts_test.py` and `connectivity_integration_test.py`. +- [ ] Update `agent-ops/skills/project/iop-agent-comparison-benchmark/SKILL.md` and `skill_contract_test.py` for available deterministic run/resume. + +**Test Strategy:** temp repo/testbed, fake adapters, deterministic clock/token. Assert preflight-before-allocation, one task submission, fresh workspace, blocker zero-run, crash/retry byte preservation and read-only status. + +**Verification:** focused integration, CLI and aggregate benchmark tests pass. + +## Dependencies and Execution Order + +- `11+08,09,10_connectivity_preflight` must have exactly one active/archive `complete.log` before implementation. + +## Modified Files Summary + +| File | Items | +|---|---| +| `scripts/agent_comparison_benchmark.py` | API-1 | +| `scripts/agent_benchmark/attempts.py` | API-1 | +| `scripts/agent_benchmark/attempts_test.py` | API-1 | +| `scripts/agent_benchmark/connectivity_integration_test.py` | API-1 | +| `scripts/agent_benchmark/skill_contract_test.py` | API-1 | +| `agent-ops/skills/project/iop-agent-comparison-benchmark/SKILL.md` | API-1 | +| `agent-task/m-agent-comparison-benchmark-pipeline/12+11_connectivity_execution/CODE_REVIEW-cloud-G09.md` | API-1 evidence | + +## Final Verification + +1. `python3 -c 'from pathlib import Path; g="m-agent-comparison-benchmark-pipeline"; i="11"; a=Path("agent-task")/g; r=Path("agent-task/archive"); p=sorted([*a.glob(f"{i}_*/complete.log"),*a.glob(f"{i}+*/complete.log"),*r.glob(f"*/*/{g}/{i}_*/complete.log"),*r.glob(f"*/*/{g}/{i}+*/complete.log")],key=str); assert len(p)==1,[str(x) for x in p]; print(p[0])'` +2. `python3 -m unittest scripts.agent_benchmark.attempts_test scripts.agent_benchmark.connectivity_integration_test scripts.agent_benchmark.skill_contract_test -v` +3. `python3 scripts/agent_comparison_benchmark.py --help && python3 scripts/agent_comparison_benchmark.py run --help && python3 scripts/agent_comparison_benchmark.py resume --help && python3 scripts/agent_comparison_benchmark.py status --help` +4. `make test-agent-comparison-benchmark` +5. `if [ -e build/r14-remote-anthropic_handler.go ]; then echo 'BLOCKED: ignored build artifact shadows iop/build' >&2; exit 69; fi; go test ./... -count=1` +6. `git diff --check` + +After completing all code changes, fill implementation-owned sections in `CODE_REVIEW-*-G??.md`. diff --git a/agent-task/m-agent-comparison-benchmark-pipeline/13+07,11_connectivity_live_evidence/CODE_REVIEW-cloud-G06.md b/agent-task/m-agent-comparison-benchmark-pipeline/13+07,11_connectivity_live_evidence/CODE_REVIEW-cloud-G06.md new file mode 100644 index 00000000..dfb187d6 --- /dev/null +++ b/agent-task/m-agent-comparison-benchmark-pipeline/13+07,11_connectivity_live_evidence/CODE_REVIEW-cloud-G06.md @@ -0,0 +1,174 @@ + + +# Code Review Reference - TEST + +> **[IMPLEMENTING AGENT — READ FIRST] Filling in this file is the mandatory final step of implementation.** +> The task is NOT complete until every implementation-owned section below is filled in. +> Complete the `Implementation Checklist`; the final checklist item is mandatory before saving. +> Fill implementation-owned sections, then stop with active files in place and report ready for review. +> Execute the plan's selected root cause, scope, files, and dependency decisions as written. Do not choose another owner, narrow/expand the write boundary, or replace a fix with another verification attempt. +> If implementation is blocked, record the exact blocker, attempted commands/output, and resume condition only in implementation-owned evidence fields. +> Do not ask the user directly, present choices, call user-input tools, create control-plane stop files, or classify the next state. +> Finalization (`Code Review Result`, log rename, `complete.log`, archive moves, `Review-Only Checklist`) is review-agent-only, even after compaction/resume. +> Follow the ownership table at the bottom of this file for which sections you own. + +## Overview + +date=2026-08-10 +task=m-agent-comparison-benchmark-pipeline/13+07,11_connectivity_live_evidence, plan=1, tag=TEST + + + +## For the Review Agent + +> **[REVIEW AGENT ONLY]** The finalization steps below are review-agent only. Implementing agents must not execute this section. + +Compare implementation of each item against source files and verify that output in `Verification Results` matches code. +Review completion means the following steps are finished: + +1. Append verdict and `review_rework_count` / `evidence_integrity_failure` routing signals. +2. Archive `CODE_REVIEW-cloud-G06.md` → `code_review_cloud_G06_1.log` and `PLAN-local-G06.md` → `plan_local_G06_1.log`. +3. If PASS, write `complete.log` and move active task directory to `agent-task/archive/YYYY/MM/m-agent-comparison-benchmark-pipeline/13+07,11_connectivity_live_evidence/`. If WARN/FAIL, fully write the next filesystem state required by the code-review skill. +4. If PASS and task group is `m-`, preserve the first-line `milestone-task` metadata in `complete.log` and report it for the runtime aggregation event. Roadmap state evaluation belongs to `sync-milestone-workstate`. +5. Check applicable `Review-Only Checklist` items at the final `.log` location before reporting. + +--- + +## Implementation Item Completion + +| Item | Status | +|------|---------| +| TEST-1 Capture direct dev evidence or exact blockers | [ ] | + +## Implementation Checklist + +- [ ] Verify exact predecessor completions, caller binaries, testbed provenance, dev route inventory and only the presence of named runtime inputs without exposing values. +- [ ] Run one authorized direct-only preflight with no fallback or substitution after every external precondition is ready. +- [ ] Apply the caller-specific closure oracle: Claude three-cell exact-ready; agy/Codex exact-ready or closed implementation gap classified as an implementation Plan candidate; registration/quota/unknown outcomes remain blockers. +- [ ] Record exact command/output, exit code, durable redacted evidence path, authorization state and resume condition; confirm no sensitive tracked bytes. +- [ ] Fill implementation-owned sections in CODE_REVIEW-*-G??.md and report review-ready only when every assigned S06-S10 criterion is closed. + +## Review-Only Checklist + +> **[REVIEW AGENT ONLY]** This checklist is used only by the review agent. +> Implementing agents must not modify or check this section. + +- [ ] Append one verdict of `PASS`, `WARN`, or `FAIL` and verified `review_rework_count`, `evidence_integrity_failure` to `Code Review Result`. +- [ ] Verify that verdict, `Dimension Assessment`, and Required/Suggested/Nit classifications match. +- [ ] Archive active `CODE_REVIEW-*-G??.md` to `code_review_cloud_G06_1.log`. +- [ ] Archive active `PLAN-*-G??.md` to `plan_local_G06_1.log`. +- [ ] Verify that the Agent-Ops managed block in `.gitignore` unignores `agent-task/**/*.md` and `agent-task/**/*.log` and ignores `agent-roadmap/current.md`. +- [ ] If PASS, write `complete.log` based on `agent-ops/skills/common/code-review/templates/complete-log-template.md` and leave no active `.md` files. +- [ ] If PASS, move active task directory `agent-task/m-agent-comparison-benchmark-pipeline/13+07,11_connectivity_live_evidence/` to `agent-task/archive/YYYY/MM/m-agent-comparison-benchmark-pipeline/13+07,11_connectivity_live_evidence/` and update this checklist at the final archive path. +- [ ] If PASS and task group is `m-`, preserve and report `milestone-task` metadata for runtime aggregation, without modifying roadmap or directly calling `update-roadmap`. +- [ ] If PASS for split work, remove empty active parent `agent-task/m-agent-comparison-benchmark-pipeline/` or verify it was kept due to remaining siblings/files. +- [ ] If WARN/FAIL, write the next filesystem state matching code-review verdict and do not write `complete.log`. + +## Deviations from Plan + +_Record any deviations from the plan and the rationale here._ + +## Key Design Decisions + +_Record key design decisions here._ + +## Reviewer Checkpoints + +- Confirm one authorized direct-only preflight covered exactly the required five cells. +- Confirm all three Claude cells are exact-ready and agy/Codex each are exact-ready or a closed compatibility gap with `effective=unavailable`, exact owner/resume evidence and implementation-Plan candidate classification. +- Confirm registration, quota, unknown/unclassified gaps, substitution, missing evidence and redaction failures never become PASS. +- Confirm requested/effective route/model/effort and finish→idle evidence satisfy the S06-S10 caller-specific oracle. +- Confirm no hybrid preset, scored run, fallback, config mutation, raw secret/private endpoint/prompt/tool output. +- Confirm durable evidence path exists and bytes are redacted/canonical. + +## Verification Results + +Paste actual stdout/stderr and exit code for every command; blockers require an exact resume condition. Never paste secrets or private endpoints. + +### V1 Predecessors + +Command: `python3 -c 'from pathlib import Path; g="m-agent-comparison-benchmark-pipeline"; ids=("07","11"); a=Path("agent-task")/g; r=Path("agent-task/archive"); f={i:sorted([*a.glob(f"{i}_*/complete.log"),*a.glob(f"{i}+*/complete.log"),*r.glob(f"*/*/{g}/{i}_*/complete.log"),*r.glob(f"*/*/{g}/{i}+*/complete.log")],key=str) for i in ids}; bad={i:[str(x) for x in p] for i,p in f.items() if len(p)!=1}; assert not bad,bad; print("\n".join(str(f[i][0]) for i in ids))'` + +```text + +``` +Exit code: `` + +### V2 Caller binaries + +Command: `command -v claude && claude --version && command -v agy && agy --version && command -v codex && codex --version` + +```text + +``` +Exit code: `` + +### V3 Testbed provenance + +Command: `git -C ../iop-s2 status --porcelain=v1 --untracked-files=all && git -C ../iop-s2 branch --show-current && git -C ../iop-s2 rev-parse HEAD` + +```text + +``` +Exit code: `` + +### V4 Route inventory + +Command: `go run ./scripts/inventory-query --env dev --model claude-sonnet-5 && go run ./scripts/inventory-query --env dev --model gemini-3.6-flash && go run ./scripts/inventory-query --env dev --model gpt-5.6-luna` + +```text + +``` +Exit code: `` + +### V5 Runtime input presence + +Command: `python3 -c 'import os; pairs=(("IOP_BENCH_CLAUDE_BASE_URL","IOP_BENCH_CLAUDE_SECRET_ENV"),("IOP_BENCH_AGY_BASE_URL","IOP_BENCH_AGY_SECRET_ENV"),("IOP_BENCH_CODEX_BASE_URL","IOP_BENCH_CODEX_SECRET_ENV")); missing=[]; [(missing.append(base) if not os.environ.get(base) else None, missing.append(ref) if not os.environ.get(ref) else (missing.append(ref+"->secret") if not os.environ.get(os.environ[ref]) else None)) for base,ref in pairs]; assert not missing,missing; print("ok: caller runtime inputs present")'` + +```text + +``` +Exit code: `` + +### V6 Authorized direct dev preflight + +Command: `python3 scripts/agent_comparison_benchmark.py preflight --manifest scripts/fixtures/agent-comparison-benchmark-direct-preflight.example.json` + +```text + +``` +Exit code: `` + +Durable redacted evidence path: `` +Authorization/runtime state: `` +Per-cell closure: `` +Resume condition: `` + +### V7 Patch integrity + +Command: `git diff --check` + +```text + +``` +Exit code: `` + +--- + +> **[IMPLEMENTING AGENT — BEFORE SAVING] Have you filled in every implementation-owned section?** +> If anything is blank, go back and fill it in before saving this file. +> Leave review-agent-only sections unchanged. + +## Section Ownership + +| Section | Owner | Note | +|---------|-------|------| +| Header comment, Overview, Review Agent Instructions | Fixed at stub creation | Implementing agent must not modify or execute these (archive, complete.log, and task-directory archive move are review-agent only) | +| Archive Evidence Snapshot | Fixed at stub creation from plan when present | Implementing agent uses it as default prior-loop context; read only the specific archive files cited there when more detail is required | +| Implementation Item Completion (item names) | Fixed at stub creation | Implementing agent checks `[ ]` → `[x]` only | +| Implementation Checklist (item text/order) | Fixed at stub creation from plan | Implementing agent checks `[ ]` → `[x]` only | +| Review-Only Checklist | Review agent only | Implementing agent must not modify or check this section | +| Deviations from Plan, Key Design Decisions | Implementing agent | Replace placeholder text with actual content | +| Reviewer Checkpoints | Fixed at stub creation | Pre-filled from plan | +| Verification Results (section headings + commands) | Fixed at stub creation | Implementing agent fills in command output only; command changes require a `Deviations from Plan` entry | +| Code Review Result | Review agent appends | Not included in stub | diff --git a/agent-task/m-agent-comparison-benchmark-pipeline/13+07,11_connectivity_live_evidence/PLAN-local-G06.md b/agent-task/m-agent-comparison-benchmark-pipeline/13+07,11_connectivity_live_evidence/PLAN-local-G06.md new file mode 100644 index 00000000..b15f4383 --- /dev/null +++ b/agent-task/m-agent-comparison-benchmark-pipeline/13+07,11_connectivity_live_evidence/PLAN-local-G06.md @@ -0,0 +1,124 @@ + + +# Plan - TEST: authorized direct connectivity evidence + +## For the Implementing Agent + +07과 11 완료 뒤 authorized direct dev preflight만 실행해 redacted evidence를 active `CODE_REVIEW-cloud-G06.md`에 기록한다. 아래 caller별 closure oracle을 충족할 때만 review-ready로 보고한다. 외부 readiness 부족은 exact blocker와 resume condition을 구현 evidence에 기록하고 blocked로 보고한다. config/credential 수정, fallback, 사용자 질문, verdict, archive, `complete.log`는 하지 않는다. + +## Background + +fixture/fake tests는 caller→IOP→provider의 실제 auth/model/effort/stream terminal을 증명하지 못한다. S06-S10 closure에는 direct-only manifest의 다섯 cell을 한 번 authorized preflight하여 exact effective binding 또는 SDD가 허용한 caller compatibility gap을 durable redacted evidence로 남기는 별도 외부 gate가 필요하다. 단순 exit 0/69가 아니라 caller별 acceptance를 판정해야 registration/quota blocker를 compatibility evidence로 오인하지 않는다. + +## Analysis + +### Files Read + +- `AGENTS.md` +- `agent-ops/rules/project/rules.md` +- `agent-ops/rules/common/rules-roadmap.md` +- `agent-ops/rules/common/rules-agent-spec.md` +- `agent-ops/skills/common/router.md` +- `agent-ops/skills/common/plan/SKILL.md` +- `agent-ops/skills/common/plan/templates/review-stub-template.md` +- `agent-ops/skills/common/refine-plans/SKILL.md` +- `agent-ops/skills/common/finalize-task-routing/SKILL.md` +- `agent-roadmap/current.md` +- `agent-roadmap/phase/knowledge-tool-optimization-extension/PHASE.md` +- `agent-roadmap/phase/knowledge-tool-optimization-extension/milestones/agent-comparison-benchmark-pipeline.md` +- `agent-roadmap/sdd/knowledge-tool-optimization-extension/agent-comparison-benchmark-pipeline/SDD.md` +- `agent-task/m-agent-comparison-benchmark-pipeline/07_anthropic_effort_compatibility/PLAN-local-G06.md` +- `agent-task/m-agent-comparison-benchmark-pipeline/11+08,09,10_connectivity_preflight/PLAN-cloud-G10.md` +- `scripts/agent_comparison_benchmark.py` +- `scripts/agent_benchmark/manifest.py` +- `scripts/agent_benchmark/lifecycle.py` +- `scripts/agent_benchmark/attempts.py` +- `agent-test/local/rules.md` +- `agent-test/local/testing-smoke.md` +- `agent-ops/rules/project/domain/testing/rules.md` + +### SDD Criteria + +- SDD는 `[승인됨]`, 잠금은 `해제`; milestone-task는 allowed id 다섯 개 전체다. +- S06은 Claude의 Sonnet/Gemini/GPT direct auth/model/stream readiness를 요구한다. S07/S08은 agy/Codex가 supported이면 exact-ready, 아니면 endpoint/auth/protocol/stream의 정확한 compatibility gap을 허용한다. +- S09는 requested/effective route/model/effort의 no-substitution evidence를 요구한다. 허용된 compatibility gap에서는 requested 값과 `effective=unavailable` 및 closed issue가 함께 남아야 한다. +- S10은 credential/model 등록·quota 같은 미충족 precondition과 implementation gap을 구분하고, closed `implementation_gap`을 별도 implementation Plan 후보로 분류하며 어떤 blocker도 우회 PASS로 만들지 않도록 요구한다. +- D01에 따라 generic preset은 fixture-only이며 이 child는 direct five-cell preflight만 수행한다. + +### Verification Context + +- 별도 verification handoff는 없다. repository-native rules/source, 설치 CLI help/version, bounded inventory query와 safe environment-presence probe로 보완했다. +- runner는 `/config/workspace/iop-s0`, branch `feature/agent-comparison-benchmark-pipeline`, current HEAD `62f4b9196bbcd7687810001e81129a5fcdc5037e`다. pre-refine intent checkpoint `8c6a3c05ea766875b6625c5f3d58ceeee5558910` 이후 target source 변경은 없고 중앙 `prepare-epic-work-items` script/test만 변경됐다. worktree 변경은 이 Epic의 미착수 pair materialization/refinement이며 source implementation은 시작되지 않았다. +- host는 Linux 6.10 aarch64, Python 3.12.3, Go 1.26.2다. caller는 `/config/.local/bin/claude` 2.1.223, `/config/.local/bin/agy` 1.1.11, `/config/.npm-global/bin/codex` 0.147.0이다. +- dev testbed는 `/config/workspace/iop-s2`, branch `dev`, HEAD `1f2f7f1066fcf165a9e469bae77203b569b6f772`, clean이다. source sync는 현재 checkout의 manifest-declared provenance gate가 다시 판정한다. +- bounded inventory 결과는 `claude-sonnet-5` missing, `gemini-3.6-flash` exact active, `gpt-5.6-luna` missing이다. named caller runtime inputs 여섯 개는 모두 absent다. provider/network call은 NOT_RUN이다. +- runtime identity는 caller-supplied redacted dev IOP Edge이고 local listening port는 없다. config/credential은 repo 밖 사용자 소유이며 원문과 private path를 읽거나 출력하지 않는다. +- 현재 resume condition은 predecessor 07/11 completion, Sonnet/GPT exact route registration, required named runtime inputs, agy quota/runtime readiness다. 이 조건이 충족되기 전에는 V5를 실행하지 않고 blocked evidence만 기록한다. Confidence는 local oracle high, current live readiness low다. + +### Test Coverage Gaps + +- 다섯 direct cell의 실제 IOP auth, exact model/effort, finish→idle과 durable redaction evidence가 없다. +- 기존 refined pair는 S07/S08이 허용한 exact compatibility gap도 무조건 실패로 처리해 SDD acceptance를 과도하게 좁혔다. + +### Symbol References + +source symbol 변경은 없다. 11이 제공할 public preflight command와 direct fixture만 소비한다. 해당 future files는 현재 checkout에 없어 `Files Read` 근거로 주장하지 않는다. + +### Split Judgment + +이 replacement는 production 변경 PASS에 필요하지 않은 하나의 external closure boundary다. caller별 결과는 같은 authorized preflight와 동일 secret-safe evidence를 공유하므로 더 나누면 one-run matrix consistency를 잃는다. `07_anthropic_effort_compatibility`와 `11+08,09,10_connectivity_preflight`가 stable predecessors이며 12 run/resume는 필요하지 않다. 교체 후 `refine-plans`를 한 번 적용한 결과 유지가 타당하다. + +### Scope Rationale + +source/config/credential/route 등록 변경, hybrid preset, scored run, retry, report/timing/web/scoring은 제외한다. 모델·effort 치환이나 ambient direct provider 호출은 금지한다. Manifest-validated `agent-test/runs///` evidence는 pipeline runtime이 동적으로 소유하며, 구현자가 편집하는 workspace write set은 active review 하나뿐이다. 생성된 exact run/evidence path는 review에 기록한다. + +### Final Routing + +- evaluation_mode `isolated-reassessment`; finalizer `finalize-task-policy.sh pair`를 changed packet에 정확히 한 번 실행했다. +- build closures 모두 true; scores `1/0/1/2/2`; base/route `local-fit`; local G06; `PLAN-local-G06.md`; `worker/local/G06`. +- review closures 모두 true; scores `1/0/1/2/2`; `official-review`; cloud G06; `CODE_REVIEW-cloud-G06.md`; `review/cloud/G06`. +- large context false; risks `boundary_contract,variant_product` 2개; rework 0; evidence integrity false; capability gap 없음. + +## Implementation Checklist + +- [ ] Verify exact predecessor completions, caller binaries, testbed provenance, dev route inventory and only the presence of named runtime inputs without exposing values. +- [ ] Run one authorized direct-only preflight with no fallback or substitution after every external precondition is ready. +- [ ] Apply the caller-specific closure oracle: Claude three-cell exact-ready; agy/Codex exact-ready or closed implementation gap classified as an implementation Plan candidate; registration/quota/unknown outcomes remain blockers. +- [ ] Record exact command/output, exit code, durable redacted evidence path, authorization state and resume condition; confirm no sensitive tracked bytes. +- [ ] Fill implementation-owned sections in CODE_REVIEW-*-G??.md and report review-ready only when every assigned S06-S10 criterion is closed. + +### [TEST-1] Capture direct dev evidence or exact blockers + +**Problem:** local fixtures cannot satisfy S06-S09 actual caller→IOP→provider evidence. The prior all-five-ready oracle also rejected S07/S08's valid exact compatibility-gap outcome, while a broad exit-69 acceptance would incorrectly pass missing registration or quota. + +**Solution:** Verify public binaries, testbed provenance, exact inventory aliases and named input presence, then invoke the direct-preflight manifest once. Claude's Sonnet/Gemini/GPT cells must all be exact-ready. agy and Codex each close with exact-ready or a closed endpoint/auth/protocol/stream `implementation_gap` carrying requested values, `effective=unavailable`, no substitution, exact owner/resume evidence, and the implementation-Plan candidate classification defined by the common contract. `registration_required`, quota, unknown/unclassified gaps, missing durable evidence, or redaction failure block completion. Aggregate exit 69 is neither automatic PASS nor automatic failure; the closed per-cell evidence is authoritative. + +**Modified Files and Checklist:** + +- [ ] Record exact external evidence only in `agent-task/m-agent-comparison-benchmark-pipeline/13+07,11_connectivity_live_evidence/CODE_REVIEW-cloud-G06.md`. + +**Test Strategy:** Run one authorized preflight after read-only preconditions pass. Do not retry blindly, invoke a hybrid preset/scored run, mutate config, or copy raw provider output. Assert all five cell records satisfy the caller-specific oracle and every durable/error byte is redacted. + +**Verification:** Claude's three cells are exact-ready; agy/Codex are exact-ready or closed implementation gaps routed to a Plan. Any registration/quota/unknown blocker stops with exact resume evidence. + +## Dependencies and Execution Order + +- `07_anthropic_effort_compatibility` and `11+08,09,10_connectivity_preflight` must each have exactly one active/archive `complete.log` before execution. + +## Modified Files Summary + +| File | Items | +|---|---| +| `agent-task/m-agent-comparison-benchmark-pipeline/13+07,11_connectivity_live_evidence/CODE_REVIEW-cloud-G06.md` | TEST-1 evidence only | + +## Final Verification + +1. `python3 -c 'from pathlib import Path; g="m-agent-comparison-benchmark-pipeline"; ids=("07","11"); a=Path("agent-task")/g; r=Path("agent-task/archive"); f={i:sorted([*a.glob(f"{i}_*/complete.log"),*a.glob(f"{i}+*/complete.log"),*r.glob(f"*/*/{g}/{i}_*/complete.log"),*r.glob(f"*/*/{g}/{i}+*/complete.log")],key=str) for i in ids}; bad={i:[str(x) for x in p] for i,p in f.items() if len(p)!=1}; assert not bad,bad; print("\n".join(str(f[i][0]) for i in ids))'` +2. `command -v claude && claude --version && command -v agy && agy --version && command -v codex && codex --version` +3. `git -C ../iop-s2 status --porcelain=v1 --untracked-files=all && git -C ../iop-s2 branch --show-current && git -C ../iop-s2 rev-parse HEAD` +4. `go run ./scripts/inventory-query --env dev --model claude-sonnet-5 && go run ./scripts/inventory-query --env dev --model gemini-3.6-flash && go run ./scripts/inventory-query --env dev --model gpt-5.6-luna` +5. `python3 -c 'import os; pairs=(("IOP_BENCH_CLAUDE_BASE_URL","IOP_BENCH_CLAUDE_SECRET_ENV"),("IOP_BENCH_AGY_BASE_URL","IOP_BENCH_AGY_SECRET_ENV"),("IOP_BENCH_CODEX_BASE_URL","IOP_BENCH_CODEX_SECRET_ENV")); missing=[]; [(missing.append(base) if not os.environ.get(base) else None, missing.append(ref) if not os.environ.get(ref) else (missing.append(ref+"->secret") if not os.environ.get(os.environ[ref]) else None)) for base,ref in pairs]; assert not missing,missing; print("ok: caller runtime inputs present")'` +6. `python3 scripts/agent_comparison_benchmark.py preflight --manifest scripts/fixtures/agent-comparison-benchmark-direct-preflight.example.json` +7. `git diff --check` + +V1-V5 must pass before V6. V6 completion is decided from the durable per-cell evidence under the caller-specific oracle, not its aggregate exit code alone. After completing all verification, fill implementation-owned sections in `CODE_REVIEW-*-G??.md`. diff --git a/agent-task/m-agent-comparison-benchmark-pipeline/13+07,11_connectivity_live_evidence/code_review_cloud_G06_0.log b/agent-task/m-agent-comparison-benchmark-pipeline/13+07,11_connectivity_live_evidence/code_review_cloud_G06_0.log new file mode 100644 index 00000000..9e32b0f3 --- /dev/null +++ b/agent-task/m-agent-comparison-benchmark-pipeline/13+07,11_connectivity_live_evidence/code_review_cloud_G06_0.log @@ -0,0 +1,159 @@ + + +# Code Review Reference - API + +> **[IMPLEMENTING AGENT — READ FIRST] Filling in this file is the mandatory final step of implementation.** +> Complete every implementation-owned section and report review-ready only when all five direct cells are exact-ready. +> On external blocker, record exact redacted evidence and resume condition, then report blocked without checking completion items. +> Do not mutate config/credentials/routes, retry blindly, ask the user, append a verdict, archive, or write `complete.log`. + +## Overview + +date=2026-08-10 +task=m-agent-comparison-benchmark-pipeline/13+07,11_connectivity_live_evidence, plan=0, tag=API + +## For the Review Agent + +> **[REVIEW AGENT ONLY]** Implementing agents must not execute this section. + +1. Compare the exact external evidence with the plan and durable redacted record. +2. Append verdict and routing signals only when review is actually requested. +3. Archive `CODE_REVIEW-cloud-G06.md` → `code_review_cloud_G06_0.log` and `PLAN-local-G06.md` → `plan_local_G06_0.log`. +4. PASS requires all five exact-ready cells; then write `complete.log`, preserve metadata and move the directory to the dated archive. A blocker gets no PASS/complete log. + +--- + +## Implementation Item Completion + +| Item | Status | +|---|---| +| API-1 Capture direct dev evidence or exact blockers | [ ] | + +## Implementation Checklist + +- [ ] Verify exact caller binaries, dev route inventory and only the presence of named runtime inputs without exposing values. +- [ ] Run one authorized direct-only preflight with no fallback or substitution. +- [ ] Record exact command/output, exit code, durable redacted evidence path, authorization state and resume condition. +- [ ] Confirm tracked diff/evidence contains no raw credential, private endpoint, prompt/tool content or private config path. +- [ ] Fill implementation-owned sections in CODE_REVIEW-*-G??.md; report blocked unless all five cells are exact-ready. + +## Review-Only Checklist + +> **[REVIEW AGENT ONLY]** Implementing agents must not modify this section. + +- [ ] Verify all five cells are ready with exact bindings before considering PASS. +- [ ] Append verdict/routing signals and verify findings/dimensions. +- [ ] Archive active review/plan to suffix `_0.log` and verify `.gitignore` managed rules. +- [ ] If PASS, write `complete.log`, preserve milestone-task metadata, archive directory and leave no active pair. +- [ ] If blocked/WARN/FAIL, write the correct next state and no `complete.log`. + +## Deviations from Plan + +_Record deviations and rationale here._ + +## Key Design Decisions + +_Record key design decisions here._ + +## Reviewer Checkpoints + +- Confirm one authorized direct-only preflight covered exactly the required five cells. +- Confirm requested/effective route/model/effort match exactly and finish→idle evidence is ordered. +- Confirm registration/quota/endpoint/auth/protocol/stream blockers never became ready or PASS. +- Confirm no hybrid preset, scored run, fallback, config mutation, raw secret/private endpoint/prompt/tool output. +- Confirm durable evidence path exists and bytes are redacted/canonical. + +## Verification Results + +Paste exact redacted stdout/stderr and exit codes. Do not paste secrets or private endpoints. + +### V1 Predecessors + +Command: `python3 -c 'from pathlib import Path; g="m-agent-comparison-benchmark-pipeline"; ids=("07","11"); a=Path("agent-task")/g; r=Path("agent-task/archive"); f={i:sorted([*a.glob(f"{i}_*/complete.log"),*a.glob(f"{i}+*/complete.log"),*r.glob(f"*/*/{g}/{i}_*/complete.log"),*r.glob(f"*/*/{g}/{i}+*/complete.log")],key=str) for i in ids}; bad={i:[str(x) for x in p] for i,p in f.items() if len(p)!=1}; assert not bad,bad; print("\n".join(str(f[i][0]) for i in ids))'` + +```text + +``` +Exit code: `` + +### V2 Caller binaries + +Command: `command -v claude && claude --version && command -v agy && agy --version && command -v codex && codex --version` + +```text + +``` +Exit code: `` + +### V3A Sonnet inventory + +Command: `go run ./scripts/inventory-query --env dev --model claude-sonnet-5` + +```text + +``` +Exit code: `` + +### V3B Gemini inventory + +Command: `go run ./scripts/inventory-query --env dev --model gemini-3.6-flash` + +```text + +``` +Exit code: `` + +### V3C GPT inventory + +Command: `go run ./scripts/inventory-query --env dev --model gpt-5.6-luna` + +```text + +``` +Exit code: `` + +### V4 Runtime input presence + +Command: `python3 -c 'import os; pairs=(("IOP_BENCH_CLAUDE_BASE_URL","IOP_BENCH_CLAUDE_SECRET_ENV"),("IOP_BENCH_AGY_BASE_URL","IOP_BENCH_AGY_SECRET_ENV"),("IOP_BENCH_CODEX_BASE_URL","IOP_BENCH_CODEX_SECRET_ENV")); missing=[]; [(missing.append(base) if not os.environ.get(base) else None, missing.append(ref) if not os.environ.get(ref) else (missing.append(ref+"->secret") if not os.environ.get(os.environ[ref]) else None)) for base,ref in pairs]; assert not missing,missing; print("ok: caller runtime inputs present")'` + +```text + +``` +Exit code: `` + +### V5 Authorized direct dev preflight + +Command: `python3 scripts/agent_comparison_benchmark.py preflight --manifest scripts/fixtures/agent-comparison-benchmark-direct-preflight.example.json` + +```text + +``` +Exit code: `` + +Durable redacted evidence path: `` +Authorization/runtime state: `` +Resume condition: `` + +### V6 Patch integrity + +Command: `git diff --check` + +```text + +``` +Exit code: `` + +--- + +> **[IMPLEMENTING AGENT — BEFORE SAVING] Have you filled in every implementation-owned section?** + +## Section Ownership + +| Section | Owner | Note | +|---|---|---| +| Header, Overview, Review instructions | Fixed | Implementing agent must not modify | +| Implementation items/checklist | Fixed | Implementing agent changes checkboxes only after exact-ready | +| Review-Only Checklist | Review agent only | Implementing agent must not modify | +| Deviations, Key Design Decisions | Implementing agent | Replace placeholders | +| Verification Results headings/commands | Fixed | Implementing agent fills redacted outputs only | +| Code Review Result | Review agent appends | Not included in stub | diff --git a/agent-task/m-agent-comparison-benchmark-pipeline/13+07,11_connectivity_live_evidence/plan_local_G06_0.log b/agent-task/m-agent-comparison-benchmark-pipeline/13+07,11_connectivity_live_evidence/plan_local_G06_0.log new file mode 100644 index 00000000..396cc66f --- /dev/null +++ b/agent-task/m-agent-comparison-benchmark-pipeline/13+07,11_connectivity_live_evidence/plan_local_G06_0.log @@ -0,0 +1,111 @@ + + +# Plan - API: authorized direct connectivity evidence + +## For the Implementing Agent + +07과 11 완료 뒤 authorized direct dev preflight만 실행해 redacted evidence를 active `CODE_REVIEW-cloud-G06.md`에 기록한다. 다섯 cell이 모두 exact-ready일 때만 review-ready로 보고한다. exit 69나 외부 readiness 부족은 exact blocker와 resume condition을 기록하고 blocked로 보고한다. config/credential 수정, fallback, 사용자 질문, verdict, archive, `complete.log`는 하지 않는다. + +## Background + +fixture/fake tests는 caller→IOP→provider의 실제 auth/model/effort/stream terminal을 증명하지 못한다. S06-S10 closure에는 direct-only manifest의 다섯 cell을 한 번 authorized preflight하여 exact effective binding과 blocker taxonomy를 durable redacted evidence로 남기는 별도 외부 gate가 필요하다. + +## Analysis + +### Files Read + +- `AGENTS.md` +- `agent-ops/rules/project/rules.md` +- `agent-ops/rules/common/rules-roadmap.md` +- `agent-ops/skills/common/router.md` +- `agent-ops/skills/common/plan/SKILL.md` +- `agent-ops/skills/common/refine-plans/SKILL.md` +- `agent-ops/skills/common/finalize-task-routing/SKILL.md` +- `agent-roadmap/phase/knowledge-tool-optimization-extension/milestones/agent-comparison-benchmark-pipeline.md` +- `agent-roadmap/sdd/knowledge-tool-optimization-extension/agent-comparison-benchmark-pipeline/SDD.md` +- `scripts/agent_comparison_benchmark.py` +- `scripts/fixtures/agent-comparison-benchmark-direct-preflight.example.json` +- `agent-test/inventory.yaml` +- `agent-test/inventory-agent.yaml` +- `agent-test/local/rules.md` +- `agent-test/local/testing-smoke.md` +- `agent-ops/rules/project/domain/testing/rules.md` + +### SDD Criteria + +- milestone-task는 allowed id 다섯 개 전체이며 Acceptance S06-S10의 actual environment gate다. +- S06-S08은 redacted caller→IOP auth/model/stream evidence, S09는 exact requested/effective effort matrix, S10은 blocker의 no-bypass classification을 요구한다. +- D01에 따라 generic preset은 fixture-only이며 이 child는 direct five-cell preflight만 수행한다. + +### Verification Context + +- HEAD/pre-refine checkpoint는 `8c6a3c05ea766875b6625c5f3d58ceeee5558910`이다. +- planning 당시 dev inventory는 Gemini exact active, Sonnet/GPT missing이고 agy는 quota-blocked였다. 이는 예상 blocker일 뿐 실행 결과나 PASS가 아니다. +- caller config/credential은 사용자 관리 외부 상태이며 값은 읽거나 출력하지 않는다. 고정 env-var 존재 여부와 redacted evidence path만 기록한다. + +### Test Coverage Gaps + +- 다섯 direct cell의 실제 IOP auth, exact model/effort, finish→idle과 durable redaction evidence가 없다. + +### Symbol References + +source symbol 변경은 없다. 11이 제공한 public preflight command와 direct fixture만 소비한다. + +### Split Judgment + +원본 API-4는 production 변경 PASS에 필요하지 않은 추가 external closure evidence이므로 별도 test-only child가 될 수 있다. 07 high-tier effort compatibility와 11 public preflight가 stable predecessors다. 12 run/resume는 이 direct-preflight evidence에 필요하지 않다. + +### Scope Rationale + +source/config/credential/route 등록 변경, hybrid preset, scored run, retry, report/timing/web/scoring은 제외한다. 모델·effort 치환이나 ambient direct provider 호출은 금지한다. + +### Final Routing + +- evaluation_mode `isolated-reassessment`; finalizer를 이 child에 정확히 한 번 실행했다. +- build/review closures 모두 true; scores `1/0/1/2/2`; build `local-fit`, local G06, `PLAN-local-G06.md`, `worker/local/G06`; review `official-review`, cloud G06, `CODE_REVIEW-cloud-G06.md`, `review/cloud/G06`. +- risks `boundary_contract,variant_product`; large context false; rework 0; integrity false; capability gap 없음. + +## Implementation Checklist + +- [ ] Verify exact caller binaries, dev route inventory and only the presence of named runtime inputs without exposing values. +- [ ] Run one authorized direct-only preflight with no fallback or substitution. +- [ ] Record exact command/output, exit code, durable redacted evidence path, authorization state and resume condition. +- [ ] Confirm tracked diff/evidence contains no raw credential, private endpoint, prompt/tool content or private config path. +- [ ] Fill implementation-owned sections in CODE_REVIEW-*-G??.md; report blocked unless all five cells are exact-ready. + +### [API-1] Capture direct dev evidence or exact blockers + +**Problem:** local fixtures cannot satisfy S06-S09 actual caller→IOP→provider evidence, and current readiness indicators must not be promoted to success. + +**Solution:** Verify public binaries and exact inventory aliases, check only configured env names, then invoke the direct-preflight manifest once. Completion requires five ready cells with exact requested/effective route/model/effort and ordered terminal evidence. Exit 69 records closed registration/quota/compatibility blockers and resume condition but does not complete this child or S06-S09. + +**Modified Files and Checklist:** + +- [ ] Record exact external evidence only in `agent-task/m-agent-comparison-benchmark-pipeline/13+07,11_connectivity_live_evidence/CODE_REVIEW-cloud-G06.md`. + +**Test Strategy:** one authorized preflight; no blind retry, hybrid preset, scored run, fallback, config mutation or raw output copying. + +**Verification:** preflight exits 0 with five exact-ready cells, otherwise the implementation stops blocked with exit-69 evidence. + +## Dependencies and Execution Order + +- `07_anthropic_effort_compatibility` and `11+08,09,10_connectivity_preflight` must each have exactly one active/archive `complete.log` before execution. + +## Modified Files Summary + +| File | Items | +|---|---| +| `agent-task/m-agent-comparison-benchmark-pipeline/13+07,11_connectivity_live_evidence/CODE_REVIEW-cloud-G06.md` | API-1 evidence only | + +## Final Verification + +1. `python3 -c 'from pathlib import Path; g="m-agent-comparison-benchmark-pipeline"; ids=("07","11"); a=Path("agent-task")/g; r=Path("agent-task/archive"); f={i:sorted([*a.glob(f"{i}_*/complete.log"),*a.glob(f"{i}+*/complete.log"),*r.glob(f"*/*/{g}/{i}_*/complete.log"),*r.glob(f"*/*/{g}/{i}+*/complete.log")],key=str) for i in ids}; bad={i:[str(x) for x in p] for i,p in f.items() if len(p)!=1}; assert not bad,bad; print("\n".join(str(f[i][0]) for i in ids))'` +2. `command -v claude && claude --version && command -v agy && agy --version && command -v codex && codex --version` +3. `go run ./scripts/inventory-query --env dev --model claude-sonnet-5` +4. `go run ./scripts/inventory-query --env dev --model gemini-3.6-flash` +5. `go run ./scripts/inventory-query --env dev --model gpt-5.6-luna` +6. `python3 -c 'import os; pairs=(("IOP_BENCH_CLAUDE_BASE_URL","IOP_BENCH_CLAUDE_SECRET_ENV"),("IOP_BENCH_AGY_BASE_URL","IOP_BENCH_AGY_SECRET_ENV"),("IOP_BENCH_CODEX_BASE_URL","IOP_BENCH_CODEX_SECRET_ENV")); missing=[]; [(missing.append(base) if not os.environ.get(base) else None, missing.append(ref) if not os.environ.get(ref) else (missing.append(ref+"->secret") if not os.environ.get(os.environ[ref]) else None)) for base,ref in pairs]; assert not missing,missing; print("ok: caller runtime inputs present")'` +7. `python3 scripts/agent_comparison_benchmark.py preflight --manifest scripts/fixtures/agent-comparison-benchmark-direct-preflight.example.json` +8. `git diff --check` + +After completing all verification, fill implementation-owned sections in `CODE_REVIEW-*-G??.md`.