80 lines
2.6 KiB
JSON
80 lines
2.6 KiB
JSON
{
|
|
"pipeline_version": "1",
|
|
"environment": "dev",
|
|
"testbed": "../iop-s2",
|
|
"repetitions": 1,
|
|
"session_policy": "fresh",
|
|
"setup_cache_policy": "isolated",
|
|
"timeout": {
|
|
"run_seconds": 300,
|
|
"idle_seconds": 30,
|
|
"quiet_seconds": 10,
|
|
"cleanup_grace_seconds": 5
|
|
},
|
|
"viewports": [
|
|
{"id": "desktop_1080", "width": 1920, "height": 1080},
|
|
{"id": "mobile_375", "width": 375, "height": 812}
|
|
],
|
|
"rubric_version": "v1.0",
|
|
"output_root": "agent-test/runs/bench-01",
|
|
"fixture": {
|
|
"version": "landing-v1",
|
|
"prompt": "scripts/fixtures/agent-comparison-benchmark/prompt.md",
|
|
"assets": [
|
|
{"source": "scripts/fixtures/agent-comparison-benchmark/reference.txt", "workspace_path": "brief/reference.txt"},
|
|
{"source": "scripts/fixtures/agent-comparison-benchmark/aurora-grid.svg", "workspace_path": "assets/aurora-grid.svg"},
|
|
{"source": "scripts/fixtures/agent-comparison-benchmark/orbit-rings.svg", "workspace_path": "assets/orbit-rings.svg"}
|
|
],
|
|
"checksum": "sha256:7dc1be6ed4a9f2f873016b708b99d827b0249c74f2ac287e1fcf8deade8dcd98"
|
|
},
|
|
"matrix": [
|
|
{
|
|
"id": "claude-generic-preset",
|
|
"caller": "claude",
|
|
"iop": {
|
|
"request_model": "claude-sonnet-5",
|
|
"requested_effort": "high",
|
|
"route_kind": "execution_preset",
|
|
"route_id": "claude-generic",
|
|
"expected_bindings": [
|
|
{"stage": "selector", "model": "claude-sonnet-5"},
|
|
{"stage": "plan", "model": "claude-sonnet-5"},
|
|
{"stage": "work", "model": "claude-sonnet-5"},
|
|
{"stage": "review", "model": "claude-sonnet-5"}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"id": "agy-generic-preset",
|
|
"caller": "agy",
|
|
"iop": {
|
|
"request_model": "gemini-3.6-flash",
|
|
"requested_effort": "high",
|
|
"route_kind": "execution_preset",
|
|
"route_id": "agy-generic",
|
|
"expected_bindings": [
|
|
{"stage": "selector", "model": "gemini-3.6-flash"},
|
|
{"stage": "plan", "model": "gemini-3.6-flash"},
|
|
{"stage": "work", "model": "gemini-3.6-flash"},
|
|
{"stage": "review", "model": "gemini-3.6-flash"}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"id": "codex-generic-preset",
|
|
"caller": "codex",
|
|
"iop": {
|
|
"request_model": "gpt-5.6-luna",
|
|
"requested_effort": "xhigh",
|
|
"route_kind": "execution_preset",
|
|
"route_id": "codex-generic",
|
|
"expected_bindings": [
|
|
{"stage": "selector", "model": "gpt-5.6-luna"},
|
|
{"stage": "plan", "model": "gpt-5.6-luna"},
|
|
{"stage": "work", "model": "gpt-5.6-luna"},
|
|
{"stage": "review", "model": "gpt-5.6-luna"}
|
|
]
|
|
}
|
|
}
|
|
]
|
|
}
|