96 lines
2.8 KiB
JSON
96 lines
2.8 KiB
JSON
{
|
|
"pipeline_version": "2",
|
|
"environment": "dev",
|
|
"testbed": "../iop-s2",
|
|
"repetitions": 1,
|
|
"session_policy": "fresh",
|
|
"setup_cache_policy": "isolated",
|
|
"timeout": {
|
|
"run_seconds": 300,
|
|
"idle_seconds": 30,
|
|
"quiet_seconds": 10,
|
|
"cleanup_grace_seconds": 5
|
|
},
|
|
"viewports": [
|
|
{"id": "desktop_1080", "width": 1920, "height": 1080},
|
|
{"id": "mobile_375", "width": 375, "height": 812}
|
|
],
|
|
"rubric_version": "landing-quality-v1",
|
|
"evaluator": {
|
|
"caller": "codex",
|
|
"iop": {
|
|
"request_model": "gpt-5.6-luna",
|
|
"requested_effort": "xhigh",
|
|
"route_kind": "direct",
|
|
"route_id": "gpt-5.6-luna",
|
|
"expected_bindings": [
|
|
{"stage": "request", "model": "gpt-5.6-luna", "effort": "xhigh"}
|
|
]
|
|
}
|
|
},
|
|
"output_root": "agent-test/runs/bench-01-supported-direct",
|
|
"fixture": {
|
|
"version": "landing-v1",
|
|
"prompt": "scripts/fixtures/agent-comparison-benchmark/prompt.md",
|
|
"assets": [
|
|
{"source": "scripts/fixtures/agent-comparison-benchmark/reference.txt", "workspace_path": "brief/reference.txt"},
|
|
{"source": "scripts/fixtures/agent-comparison-benchmark/aurora-grid.svg", "workspace_path": "assets/aurora-grid.svg"},
|
|
{"source": "scripts/fixtures/agent-comparison-benchmark/orbit-rings.svg", "workspace_path": "assets/orbit-rings.svg"}
|
|
],
|
|
"checksum": "sha256:7dc1be6ed4a9f2f873016b708b99d827b0249c74f2ac287e1fcf8deade8dcd98"
|
|
},
|
|
"matrix": [
|
|
{
|
|
"id": "claude-sonnet-direct",
|
|
"caller": "claude",
|
|
"iop": {
|
|
"request_model": "claude-sonnet-5",
|
|
"requested_effort": "max",
|
|
"route_kind": "direct",
|
|
"route_id": "claude-sonnet-5",
|
|
"expected_bindings": [
|
|
{"stage": "request", "model": "claude-sonnet-5", "effort": "max"}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"id": "claude-gemini-direct",
|
|
"caller": "claude",
|
|
"iop": {
|
|
"request_model": "gemini-3.6-flash",
|
|
"requested_effort": "high",
|
|
"route_kind": "direct",
|
|
"route_id": "gemini-3.6-flash",
|
|
"expected_bindings": [
|
|
{"stage": "request", "model": "gemini-3.6-flash", "effort": "high"}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"id": "claude-gpt-direct",
|
|
"caller": "claude",
|
|
"iop": {
|
|
"request_model": "gpt-5.6-luna",
|
|
"requested_effort": "xhigh",
|
|
"route_kind": "direct",
|
|
"route_id": "gpt-5.6-luna",
|
|
"expected_bindings": [
|
|
{"stage": "request", "model": "gpt-5.6-luna", "effort": "xhigh"}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"id": "codex-gpt-direct",
|
|
"caller": "codex",
|
|
"iop": {
|
|
"request_model": "gpt-5.6-luna",
|
|
"requested_effort": "xhigh",
|
|
"route_kind": "direct",
|
|
"route_id": "gpt-5.6-luna",
|
|
"expected_bindings": [
|
|
{"stage": "request", "model": "gpt-5.6-luna", "effort": "xhigh"}
|
|
]
|
|
}
|
|
}
|
|
]
|
|
}
|