iop/scripts/fixtures/agent-comparison-benchmark-manifest.schema.json

{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://iop.local/schemas/agent-comparison-benchmark-manifest.schema.json",
  "title": "Agent comparison benchmark manifest",
  "description": "Closed Draft 2020-12 schema for the benchmark manifest. Every object rejects unknown members. The loader accepts only the subset used by this packet.",
  "type": "object",
  "additionalProperties": false,
  "required": [
    "pipeline_version",
    "environment",
    "testbed",
    "fixture",
    "matrix",
    "session_policy",
    "setup_cache_policy",
    "timeout",
    "viewports",
    "rubric_version",
    "output_root"
  ],
  "properties": {
    "pipeline_version": { "const": "1" },
    "environment": { "const": "dev" },
    "testbed": { "const": "../iop-s2" },
    "repetitions": {
      "type": "integer",
      "minimum": 1,
      "default": 1
    },
    "session_policy": { "const": "fresh" },
    "setup_cache_policy": { "const": "isolated" },
    "timeout": { "$ref": "#/$defs/timeout" },
    "viewports": { "$ref": "#/$defs/viewports" },
    "rubric_version": { "$ref": "#/$defs/bounded_token" },
    "output_root": { "$ref": "#/$defs/output_root" },
    "fixture": { "$ref": "#/$defs/fixture" },
    "matrix": { "$ref": "#/$defs/matrix" }
  },
  "$defs": {
    "bounded_token": {
      "type": "string",
      "pattern": "^[a-z0-9][a-z0-9_.+-]{0,31}$"
    },
    "timeout": {
      "type": "object",
      "additionalProperties": false,
      "required": ["run_seconds", "idle_seconds", "quiet_seconds", "cleanup_grace_seconds"],
      "properties": {
        "run_seconds": { "type": "integer", "minimum": 1, "maximum": 86400 },
        "idle_seconds": { "type": "integer", "minimum": 1, "maximum": 600 },
        "quiet_seconds": { "type": "integer", "minimum": 1, "maximum": 60 },
        "cleanup_grace_seconds": { "type": "integer", "minimum": 1, "maximum": 60 }
      }
    },
    "viewport_record": {
      "type": "object",
      "additionalProperties": false,
      "required": ["id", "width", "height"],
      "properties": {
        "id": { "$ref": "#/$defs/bounded_token" },
        "width": { "type": "integer", "minimum": 1, "maximum": 8192 },
        "height": { "type": "integer", "minimum": 1, "maximum": 8192 }
      }
    },
    "viewports": {
      "type": "array",
      "minItems": 1,
      "items": { "$ref": "#/$defs/viewport_record" },
      "uniqueItems": true
    },
    "output_root": {
      "type": "string",
      "pattern": "^agent-test/runs/[^/]+$"
    },
    "asset_mapping": {
      "type": "object",
      "additionalProperties": false,
      "required": ["source", "workspace_path"],
      "properties": {
        "source": {
          "type": "string",
          "pattern": "^[^/][^:]*$"
        },
        "workspace_path": {
          "type": "string",
          "pattern": "^[^/][^:]*$"
        }
      }
    },
    "fixture": {
      "type": "object",
      "additionalProperties": false,
      "required": ["version", "prompt", "assets", "checksum"],
      "properties": {
        "version": { "$ref": "#/$defs/bounded_token" },
        "prompt": {
          "type": "string",
          "pattern": "^[^/][^:]*$"
        },
        "assets": {
          "type": "array",
          "minItems": 1,
          "items": { "$ref": "#/$defs/asset_mapping" }
        },
        "checksum": {
          "type": "string",
          "pattern": "^sha256:[0-9a-f]{64}$"
        }
      }
    },
    "binding": {
      "type": "object",
      "additionalProperties": false,
      "required": ["stage", "model"],
      "properties": {
        "stage": {
          "type": "string",
          "enum": ["request", "selector", "plan", "work", "review", "repair"]
        },
        "model": { "$ref": "#/$defs/bounded_token" },
        "effort": { "$ref": "#/$defs/bounded_token" }
      }
    },
    "expected_binding": {
      "type": "object",
      "additionalProperties": false,
      "required": ["stage", "model"],
      "properties": {
        "stage": {
          "type": "string",
          "enum": ["request", "selector", "plan", "work", "review", "repair"]
        },
        "model": { "$ref": "#/$defs/bounded_token" },
        "effort": { "$ref": "#/$defs/bounded_token" }
      }
    },
    "iop_cell": {
      "type": "object",
      "additionalProperties": false,
      "required": ["request_model", "requested_effort", "route_kind", "route_id", "expected_bindings"],
      "properties": {
        "request_model": { "$ref": "#/$defs/bounded_token" },
        "requested_effort": { "$ref": "#/$defs/bounded_token" },
        "route_kind": { "type": "string", "enum": ["direct", "execution_preset"] },
        "route_id": { "$ref": "#/$defs/bounded_token" },
        "expected_bindings": {
          "type": "array",
          "minItems": 1,
          "items": { "$ref": "#/$defs/expected_binding" },
          "uniqueItems": true
        }
      },
      "allOf": [
        {
          "if": {
            "properties": { "route_kind": { "const": "direct" } }
          },
          "then": {
            "properties": {
              "expected_bindings": {
                "type": "array",
                "minItems": 1,
                "maxItems": 1,
                "items": {
                  "type": "object",
                  "properties": {
                    "stage": { "const": "request" }
                  }
                }
              }
            }
          }
        },
        {
          "if": {
            "properties": { "route_kind": { "const": "execution_preset" } }
          },
          "then": {
            "properties": {
              "expected_bindings": {
                "type": "array",
                "minItems": 4,
                "maxItems": 5,
                "items": {
                  "type": "object",
                  "properties": {
                    "stage": { "enum": ["selector", "plan", "work", "review", "repair"] }
                  }
                },
                "allOf": [
                  { "contains": { "properties": { "stage": { "const": "selector" } }, "required": ["stage"] }, "minContains": 1, "maxContains": 1 },
                  { "contains": { "properties": { "stage": { "const": "plan" } }, "required": ["stage"] }, "minContains": 1, "maxContains": 1 },
                  { "contains": { "properties": { "stage": { "const": "work" } }, "required": ["stage"] }, "minContains": 1, "maxContains": 1 },
                  { "contains": { "properties": { "stage": { "const": "review" } }, "required": ["stage"] }, "minContains": 1, "maxContains": 1 },
                  { "contains": { "properties": { "stage": { "const": "repair" } }, "required": ["stage"] }, "minContains": 0, "maxContains": 1 }
                ]
              }
            }
          }
        }
      ]
    },
    "cell": {
      "type": "object",
      "additionalProperties": false,
      "required": ["id", "caller", "iop"],
      "properties": {
        "id": {
          "type": "string",
          "pattern": "^[a-z0-9][a-z0-9_-]{0,63}$"
        },
        "caller": { "type": "string", "enum": ["claude", "agy", "codex"] },
        "iop": { "$ref": "#/$defs/iop_cell" }
      }
    },
    "matrix": {
      "type": "array",
      "minItems": 1,
      "items": { "$ref": "#/$defs/cell" },
      "uniqueItems": true
    }
  }
}