iop/scripts/fixtures/agent-comparison-benchmark-iop-one-shot.json

{
  "pipeline_version": "2",
  "environment": "dev",
  "testbed": "../iop-s2",
  "execution_order_seed": "bench-02-c01-c09-v1",
  "repetitions": 1,
  "session_policy": "fresh",
  "setup_cache_policy": "isolated",
  "timeout": {
    "run_seconds": 300,
    "idle_seconds": 30,
    "quiet_seconds": 10,
    "cleanup_grace_seconds": 5
  },
  "viewports": [
    {"id": "desktop_1080", "width": 1920, "height": 1080},
    {"id": "mobile_375", "width": 375, "height": 812}
  ],
  "rubric_version": "one-shot-agent-comparison-v1",
  "evaluator": {
    "caller": "codex",
    "iop": {
      "request_model": "gpt-5.6-luna",
      "requested_effort": "xhigh",
      "route_kind": "direct",
      "route_id": "gpt-5.6-luna",
      "expected_bindings": [
        {"stage": "request", "model": "gpt-5.6-luna", "effort": "xhigh"}
      ]
    }
  },
  "output_root": "agent-test/runs/bench-02",
  "fixture": {
    "version": "product-card-v2",
    "prompt": "scripts/fixtures/agent-comparison-benchmark/prompt.md",
    "assets": [
      {"source": "scripts/fixtures/agent-comparison-benchmark/reference.txt", "workspace_path": "brief/reference.txt"},
      {"source": "scripts/fixtures/agent-comparison-benchmark/aurora-grid.svg", "workspace_path": "assets/aurora-grid.svg"},
      {"source": "scripts/fixtures/agent-comparison-benchmark/orbit-rings.svg", "workspace_path": "assets/orbit-rings.svg"}
    ],
    "checksum": "sha256:fb16198fd4c3576f880f047ed7de54dddc160b0f70c2ba55b435cf078c61828e"
  },
  "matrix": [
    {
      "id": "c01-claude-sonnet-direct",
      "caller": "claude",
      "iop": {
        "request_model": "claude-sonnet-5",
        "requested_effort": "max",
        "route_kind": "direct",
        "route_id": "claude-sonnet-5",
        "expected_bindings": [
          {"stage": "request", "model": "claude-sonnet-5", "effort": "max"}
        ]
      }
    },
    {
      "id": "c02-claude-gemini-direct",
      "caller": "claude",
      "iop": {
        "request_model": "gemini-3.6-flash",
        "requested_effort": "high",
        "route_kind": "direct",
        "route_id": "gemini-3.6-flash",
        "expected_bindings": [
          {"stage": "request", "model": "gemini-3.6-flash", "effort": "high"}
        ]
      }
    },
    {
      "id": "c03-agy-gemini-direct",
      "caller": "agy",
      "iop": {
        "request_model": "gemini-3.6-flash",
        "requested_effort": "high",
        "route_kind": "direct",
        "route_id": "gemini-3.6-flash",
        "expected_bindings": [
          {"stage": "request", "model": "gemini-3.6-flash", "effort": "high"}
        ]
      }
    },
    {
      "id": "c04-claude-gpt-direct",
      "caller": "claude",
      "iop": {
        "request_model": "gpt-5.6-luna",
        "requested_effort": "xhigh",
        "route_kind": "direct",
        "route_id": "gpt-5.6-luna",
        "expected_bindings": [
          {"stage": "request", "model": "gpt-5.6-luna", "effort": "xhigh"}
        ]
      }
    },
    {
      "id": "c05-codex-gpt-direct",
      "caller": "codex",
      "iop": {
        "request_model": "gpt-5.6-luna",
        "requested_effort": "xhigh",
        "route_kind": "direct",
        "route_id": "gpt-5.6-luna",
        "expected_bindings": [
          {"stage": "request", "model": "gpt-5.6-luna", "effort": "xhigh"}
        ]
      }
    },
    {
      "id": "c06-claude-gemini-hybrid",
      "caller": "claude",
      "iop": {
        "request_model": "gemini-hybrid",
        "requested_effort": "high",
        "route_kind": "execution_preset",
        "route_id": "gemini-hybrid",
        "expected_bindings": [
          {"stage": "selector", "model": "gemini-3.6-flash", "effort": "high"},
          {"stage": "plan", "model": "gemini-3.6-flash", "effort": "high"},
          {"stage": "work", "model": "ornith-fast"},
          {"stage": "review", "model": "gemini-3.6-flash", "effort": "high"},
          {"stage": "repair", "model": "gemini-3.6-flash", "effort": "high"}
        ]
      }
    },
    {
      "id": "c07-agy-gemini-hybrid",
      "caller": "agy",
      "iop": {
        "request_model": "gemini-hybrid",
        "requested_effort": "high",
        "route_kind": "execution_preset",
        "route_id": "gemini-hybrid",
        "expected_bindings": [
          {"stage": "selector", "model": "gemini-3.6-flash", "effort": "high"},
          {"stage": "plan", "model": "gemini-3.6-flash", "effort": "high"},
          {"stage": "work", "model": "ornith-fast"},
          {"stage": "review", "model": "gemini-3.6-flash", "effort": "high"},
          {"stage": "repair", "model": "gemini-3.6-flash", "effort": "high"}
        ]
      }
    },
    {
      "id": "c08-claude-gpt-hybrid",
      "caller": "claude",
      "iop": {
        "request_model": "gpt-hybrid",
        "requested_effort": "xhigh",
        "route_kind": "execution_preset",
        "route_id": "gpt-hybrid",
        "expected_bindings": [
          {"stage": "selector", "model": "gpt-5.6-terra", "effort": "high"},
          {"stage": "plan", "model": "gpt-5.6-terra", "effort": "high"},
          {"stage": "work", "model": "ornith-fast"},
          {"stage": "review", "model": "gpt-5.6-terra", "effort": "high"},
          {"stage": "repair", "model": "gpt-5.6-terra", "effort": "high"}
        ]
      }
    },
    {
      "id": "c09-codex-gpt-hybrid",
      "caller": "codex",
      "iop": {
        "request_model": "gpt-hybrid",
        "requested_effort": "xhigh",
        "route_kind": "execution_preset",
        "route_id": "gpt-hybrid",
        "expected_bindings": [
          {"stage": "selector", "model": "gpt-5.6-terra", "effort": "high"},
          {"stage": "plan", "model": "gpt-5.6-terra", "effort": "high"},
          {"stage": "work", "model": "ornith-fast"},
          {"stage": "review", "model": "gpt-5.6-terra", "effort": "high"},
          {"stage": "repair", "model": "gpt-5.6-terra", "effort": "high"}
        ]
      }
    }
  ]
}