toki
de4d8f4ff8
세 Agent의 direct route를 동일한 fail-closed preflight와 격리 실행 경계에서 비교하고, 관측되지 않은 preset 셀이 실행되는 것을 막기 위해 연결 계약과 증거 수집 흐름을 고정한다.
96 lines
2.8 KiB
JSON
96 lines
2.8 KiB
JSON
{
|
|
"pipeline_version": "1",
|
|
"environment": "dev",
|
|
"testbed": "../iop-s2",
|
|
"repetitions": 1,
|
|
"session_policy": "fresh",
|
|
"setup_cache_policy": "isolated",
|
|
"timeout": {
|
|
"run_seconds": 300,
|
|
"idle_seconds": 30,
|
|
"quiet_seconds": 10,
|
|
"cleanup_grace_seconds": 5
|
|
},
|
|
"viewports": [
|
|
{"id": "desktop_1080", "width": 1920, "height": 1080},
|
|
{"id": "mobile_375", "width": 375, "height": 812}
|
|
],
|
|
"rubric_version": "v1.0",
|
|
"output_root": "agent-test/runs/bench-01-direct-preflight",
|
|
"fixture": {
|
|
"version": "v1.0",
|
|
"prompt": "scripts/fixtures/agent-comparison-benchmark/prompt.md",
|
|
"assets": [
|
|
{"source": "scripts/fixtures/agent-comparison-benchmark/prompt.md", "workspace_path": "workspace/prompt.md"},
|
|
{"source": "scripts/fixtures/agent-comparison-benchmark/reference.txt", "workspace_path": "workspace/reference.txt"}
|
|
],
|
|
"checksum": "sha256:f87b1a06dcd60687f2964a7c8e48227847acda195d3e5808710fe3f0e8149108"
|
|
},
|
|
"matrix": [
|
|
{
|
|
"id": "claude-sonnet-direct",
|
|
"caller": "claude",
|
|
"iop": {
|
|
"request_model": "claude-sonnet-5",
|
|
"requested_effort": "max",
|
|
"route_kind": "direct",
|
|
"route_id": "claude-sonnet-direct",
|
|
"expected_bindings": [
|
|
{"stage": "request", "model": "claude-sonnet-5", "effort": "max"}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"id": "claude-gemini-direct",
|
|
"caller": "claude",
|
|
"iop": {
|
|
"request_model": "gemini-3.6-flash",
|
|
"requested_effort": "high",
|
|
"route_kind": "direct",
|
|
"route_id": "claude-gemini-direct",
|
|
"expected_bindings": [
|
|
{"stage": "request", "model": "gemini-3.6-flash", "effort": "high"}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"id": "claude-gpt-direct",
|
|
"caller": "claude",
|
|
"iop": {
|
|
"request_model": "gpt-5.6-luna",
|
|
"requested_effort": "xhigh",
|
|
"route_kind": "direct",
|
|
"route_id": "claude-gpt-direct",
|
|
"expected_bindings": [
|
|
{"stage": "request", "model": "gpt-5.6-luna", "effort": "xhigh"}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"id": "agy-gemini-direct",
|
|
"caller": "agy",
|
|
"iop": {
|
|
"request_model": "gemini-3.6-flash",
|
|
"requested_effort": "high",
|
|
"route_kind": "direct",
|
|
"route_id": "agy-gemini-direct",
|
|
"expected_bindings": [
|
|
{"stage": "request", "model": "gemini-3.6-flash", "effort": "high"}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"id": "codex-gpt-direct",
|
|
"caller": "codex",
|
|
"iop": {
|
|
"request_model": "gpt-5.6-luna",
|
|
"requested_effort": "xhigh",
|
|
"route_kind": "direct",
|
|
"route_id": "codex-gpt-direct",
|
|
"expected_bindings": [
|
|
{"stage": "request", "model": "gpt-5.6-luna", "effort": "xhigh"}
|
|
]
|
|
}
|
|
}
|
|
]
|
|
}
|