toki
0be1a3dcce
Epic 3 준비 전에 caller별 IOP direct preflight와 attempt recovery의 검증된 완료 상태를 원격 checkpoint로 보존한다.
83 lines
2.4 KiB
JSON
83 lines
2.4 KiB
JSON
{
|
|
"pipeline_version": "1",
|
|
"environment": "dev",
|
|
"testbed": "../iop-s2",
|
|
"repetitions": 1,
|
|
"session_policy": "fresh",
|
|
"setup_cache_policy": "isolated",
|
|
"timeout": {
|
|
"run_seconds": 300,
|
|
"idle_seconds": 30,
|
|
"quiet_seconds": 10,
|
|
"cleanup_grace_seconds": 5
|
|
},
|
|
"viewports": [
|
|
{"id": "desktop_1080", "width": 1920, "height": 1080},
|
|
{"id": "mobile_375", "width": 375, "height": 812}
|
|
],
|
|
"rubric_version": "v1.0",
|
|
"output_root": "agent-test/runs/bench-01-supported-direct",
|
|
"fixture": {
|
|
"version": "v1.0",
|
|
"prompt": "scripts/fixtures/agent-comparison-benchmark/prompt.md",
|
|
"assets": [
|
|
{"source": "scripts/fixtures/agent-comparison-benchmark/prompt.md", "workspace_path": "workspace/prompt.md"},
|
|
{"source": "scripts/fixtures/agent-comparison-benchmark/reference.txt", "workspace_path": "workspace/reference.txt"}
|
|
],
|
|
"checksum": "sha256:f87b1a06dcd60687f2964a7c8e48227847acda195d3e5808710fe3f0e8149108"
|
|
},
|
|
"matrix": [
|
|
{
|
|
"id": "claude-sonnet-direct",
|
|
"caller": "claude",
|
|
"iop": {
|
|
"request_model": "claude-sonnet-5",
|
|
"requested_effort": "max",
|
|
"route_kind": "direct",
|
|
"route_id": "claude-sonnet-5",
|
|
"expected_bindings": [
|
|
{"stage": "request", "model": "claude-sonnet-5", "effort": "max"}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"id": "claude-gemini-direct",
|
|
"caller": "claude",
|
|
"iop": {
|
|
"request_model": "gemini-3.6-flash",
|
|
"requested_effort": "high",
|
|
"route_kind": "direct",
|
|
"route_id": "gemini-3.6-flash",
|
|
"expected_bindings": [
|
|
{"stage": "request", "model": "gemini-3.6-flash", "effort": "high"}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"id": "claude-gpt-direct",
|
|
"caller": "claude",
|
|
"iop": {
|
|
"request_model": "gpt-5.6-luna",
|
|
"requested_effort": "xhigh",
|
|
"route_kind": "direct",
|
|
"route_id": "gpt-5.6-luna",
|
|
"expected_bindings": [
|
|
{"stage": "request", "model": "gpt-5.6-luna", "effort": "xhigh"}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"id": "codex-gpt-direct",
|
|
"caller": "codex",
|
|
"iop": {
|
|
"request_model": "gpt-5.6-luna",
|
|
"requested_effort": "xhigh",
|
|
"route_kind": "direct",
|
|
"route_id": "gpt-5.6-luna",
|
|
"expected_bindings": [
|
|
{"stage": "request", "model": "gpt-5.6-luna", "effort": "xhigh"}
|
|
]
|
|
}
|
|
}
|
|
]
|
|
}
|