{
  "aggregates": {
    "case": {
      "image-chart": {
        "attempts": 2,
        "latency_p50_seconds": 0.43507399999999996,
        "latency_p95_seconds": 0.459055,
        "pass_rate": 1.0,
        "passed": 2
      },
      "image-ocr": {
        "attempts": 2,
        "latency_p50_seconds": 0.3187095,
        "latency_p95_seconds": 0.347228,
        "pass_rate": 1.0,
        "passed": 2
      },
      "image-scene": {
        "attempts": 2,
        "latency_p50_seconds": 1.6547425,
        "latency_p95_seconds": 1.727975,
        "pass_rate": 1.0,
        "passed": 2
      },
      "image-spatial-count": {
        "attempts": 2,
        "latency_p50_seconds": 0.2449,
        "latency_p95_seconds": 0.283704,
        "pass_rate": 1.0,
        "passed": 2
      },
      "image-ui": {
        "attempts": 2,
        "latency_p50_seconds": 0.349082,
        "latency_p95_seconds": 0.37139,
        "pass_rate": 1.0,
        "passed": 2
      },
      "multi-image-comparison": {
        "attempts": 2,
        "latency_p50_seconds": 0.585097,
        "latency_p95_seconds": 0.783601,
        "pass_rate": 1.0,
        "passed": 2
      }
    },
    "modality": {
      "image": {
        "attempts": 12,
        "latency_p50_seconds": 0.37899150000000004,
        "latency_p95_seconds": 1.727975,
        "pass_rate": 1.0,
        "passed": 12
      }
    }
  },
  "attempts": [
    {
      "case_id": "image-chart",
      "failure": null,
      "finish_reason": "stop",
      "media": [
        {
          "bytes": 12635,
          "kind": "image",
          "mime": "image/png",
          "path": "chart.png",
          "sha256": "c905e453c4e16eb274e46992d1a901bbced8f2314b47b9d408971bb176ab2f3e"
        }
      ],
      "modality": "image",
      "output": "Here are the quarterly values in ascending quarter order:\n\n- **Q1: 10**\n- **Q2: 20**\n- **Q3: 30**",
      "passed": true,
      "reasoning": {
        "chars": 0,
        "field": null,
        "tokens": null
      },
      "repetition": 1,
      "sampling": null,
      "seconds": 0.459055,
      "usage": {
        "completion_tokens": 35,
        "prompt_tokens": 329,
        "prompt_tokens_details": {
          "cached_tokens": 0,
          "image_tokens": 299
        },
        "reasoning_tokens": 0,
        "total_tokens": 364
      },
      "validation": {
        "assertions": [
          {
            "passed": true,
            "type": "ordered_casefold",
            "values": [
              "Q1",
              "10",
              "Q2",
              "20",
              "Q3",
              "30"
            ]
          }
        ],
        "finish_reason_allowed": true,
        "visible_answer": true
      }
    },
    {
      "case_id": "image-chart",
      "failure": null,
      "finish_reason": "stop",
      "media": [
        {
          "bytes": 12635,
          "kind": "image",
          "mime": "image/png",
          "path": "chart.png",
          "sha256": "c905e453c4e16eb274e46992d1a901bbced8f2314b47b9d408971bb176ab2f3e"
        }
      ],
      "modality": "image",
      "output": "Here are the quarterly values in ascending quarter order:\n\n- **Q1: 10**\n- **Q2: 20**\n- **Q3: 30**",
      "passed": true,
      "reasoning": {
        "chars": 0,
        "field": null,
        "tokens": null
      },
      "repetition": 2,
      "sampling": null,
      "seconds": 0.411093,
      "usage": {
        "completion_tokens": 35,
        "prompt_tokens": 329,
        "prompt_tokens_details": {
          "cached_tokens": 320,
          "image_tokens": 299
        },
        "reasoning_tokens": 0,
        "total_tokens": 364
      },
      "validation": {
        "assertions": [
          {
            "passed": true,
            "type": "ordered_casefold",
            "values": [
              "Q1",
              "10",
              "Q2",
              "20",
              "Q3",
              "30"
            ]
          }
        ],
        "finish_reason_allowed": true,
        "visible_answer": true
      }
    },
    {
      "case_id": "image-ocr",
      "failure": null,
      "finish_reason": "stop",
      "media": [
        {
          "bytes": 15916,
          "kind": "image",
          "mime": "image/png",
          "path": "ocr.png",
          "sha256": "a8d5158019fb2b08ee9f6a153860ab2342384644f7c53ab58fdd45658124e83e"
        }
      ],
      "modality": "image",
      "output": "ANVIL READY\nCODE 42917\nTEMPERATURE 42 C",
      "passed": true,
      "reasoning": {
        "chars": 0,
        "field": null,
        "tokens": null
      },
      "repetition": 1,
      "sampling": null,
      "seconds": 0.347228,
      "usage": {
        "completion_tokens": 18,
        "prompt_tokens": 325,
        "prompt_tokens_details": {
          "cached_tokens": 0,
          "image_tokens": 299
        },
        "reasoning_tokens": 0,
        "total_tokens": 343
      },
      "validation": {
        "assertions": [
          {
            "passed": true,
            "type": "contains_casefold",
            "value": "ANVIL READY"
          },
          {
            "passed": true,
            "type": "contains_casefold",
            "value": "42917"
          },
          {
            "passed": true,
            "type": "contains_casefold",
            "value": "42"
          }
        ],
        "finish_reason_allowed": true,
        "visible_answer": true
      }
    },
    {
      "case_id": "image-ocr",
      "failure": null,
      "finish_reason": "stop",
      "media": [
        {
          "bytes": 15916,
          "kind": "image",
          "mime": "image/png",
          "path": "ocr.png",
          "sha256": "a8d5158019fb2b08ee9f6a153860ab2342384644f7c53ab58fdd45658124e83e"
        }
      ],
      "modality": "image",
      "output": "ANVIL READY\nCODE 42917\nTEMPERATURE 42 C",
      "passed": true,
      "reasoning": {
        "chars": 0,
        "field": null,
        "tokens": null
      },
      "repetition": 2,
      "sampling": null,
      "seconds": 0.290191,
      "usage": {
        "completion_tokens": 18,
        "prompt_tokens": 325,
        "prompt_tokens_details": {
          "cached_tokens": 320,
          "image_tokens": 299
        },
        "reasoning_tokens": 0,
        "total_tokens": 343
      },
      "validation": {
        "assertions": [
          {
            "passed": true,
            "type": "contains_casefold",
            "value": "ANVIL READY"
          },
          {
            "passed": true,
            "type": "contains_casefold",
            "value": "42917"
          },
          {
            "passed": true,
            "type": "contains_casefold",
            "value": "42"
          }
        ],
        "finish_reason_allowed": true,
        "visible_answer": true
      }
    },
    {
      "case_id": "image-scene",
      "failure": null,
      "finish_reason": "stop",
      "media": [
        {
          "bytes": 13187,
          "kind": "image",
          "mime": "image/png",
          "path": "scene.png",
          "sha256": "30a1f11e40bf4e0a8595d39828e0aa362e8b28ac683c404a1068cc20bf7f7bda"
        }
      ],
      "modality": "image",
      "output": "# Scene Description\n\nThe image shows a simple graphic scene with a dark navy blue background representing the sky.\n\n## Prominent Labels (verbatim):\n\n1. **SUNNY DAY** \u2014 displayed in bright yellow text at the top of the image\n2. **RED CAR** \u2014 displayed in white text in the upper-left area\n3. **GREEN PARK** \u2014 displayed in white text on the right side of the green area\n\n## Visual Elements:\n\n- A **red rectangle** (representing the car) sits in the left portion of the image, positioned above the green area\n- A **green horizontal band** (representing the park/ground) spans the lower portion of the image\n- The dark blue background fills the rest of the scene, suggesting a clear sky on a sunny day\n\nThe scene appears to be a minimalist illustration of a red car parked in a green park on a sunny day.",
      "passed": true,
      "reasoning": {
        "chars": 0,
        "field": null,
        "tokens": null
      },
      "repetition": 1,
      "sampling": null,
      "seconds": 1.727975,
      "usage": {
        "completion_tokens": 184,
        "prompt_tokens": 329,
        "prompt_tokens_details": {
          "cached_tokens": 0,
          "image_tokens": 299
        },
        "reasoning_tokens": 0,
        "total_tokens": 513
      },
      "validation": {
        "assertions": [
          {
            "passed": true,
            "type": "contains_casefold",
            "value": "red car"
          },
          {
            "passed": true,
            "type": "contains_casefold",
            "value": "green park"
          },
          {
            "passed": true,
            "type": "contains_casefold",
            "value": "sunny day"
          }
        ],
        "finish_reason_allowed": true,
        "visible_answer": true
      }
    },
    {
      "case_id": "image-scene",
      "failure": null,
      "finish_reason": "stop",
      "media": [
        {
          "bytes": 13187,
          "kind": "image",
          "mime": "image/png",
          "path": "scene.png",
          "sha256": "30a1f11e40bf4e0a8595d39828e0aa362e8b28ac683c404a1068cc20bf7f7bda"
        }
      ],
      "modality": "image",
      "output": "# Scene Description\n\nThe image shows a simple graphic scene with a dark navy blue background. The scene contains:\n\n- **A red rectangular shape** positioned in the upper-left area, sitting on top of a green surface\n- **A green horizontal band** at the bottom of the image, representing a park or grassy area\n\n# Prominent Labels\n\nThe following text labels appear in the image:\n\n1. **\"SUNNY DAY\"** \u2014 displayed in bold yellow text at the top center of the image\n2. **\"RED CAR\"** \u2014 displayed in bold white text on the left side, above the red rectangle\n3. **\"GREEN PARK\"** \u2014 displayed in bold white text on the right side, within the green band\n\nThe labels appear to identify the elements in the scene: the red rectangle represents a car, the green band represents a park, and the overall scene depicts a sunny day.",
      "passed": true,
      "reasoning": {
        "chars": 0,
        "field": null,
        "tokens": null
      },
      "repetition": 2,
      "sampling": null,
      "seconds": 1.58151,
      "usage": {
        "completion_tokens": 187,
        "prompt_tokens": 329,
        "prompt_tokens_details": {
          "cached_tokens": 320,
          "image_tokens": 299
        },
        "reasoning_tokens": 0,
        "total_tokens": 516
      },
      "validation": {
        "assertions": [
          {
            "passed": true,
            "type": "contains_casefold",
            "value": "red car"
          },
          {
            "passed": true,
            "type": "contains_casefold",
            "value": "green park"
          },
          {
            "passed": true,
            "type": "contains_casefold",
            "value": "sunny day"
          }
        ],
        "finish_reason_allowed": true,
        "visible_answer": true
      }
    },
    {
      "case_id": "image-spatial-count",
      "failure": null,
      "finish_reason": "stop",
      "media": [
        {
          "bytes": 8785,
          "kind": "image",
          "mime": "image/png",
          "path": "spatial.png",
          "sha256": "d48f47cfba9d65524d6a9ca37d593550e10400c1b5fa160bec8aea24e0164051"
        }
      ],
      "modality": "image",
      "output": "three",
      "passed": true,
      "reasoning": {
        "chars": 0,
        "field": null,
        "tokens": null
      },
      "repetition": 1,
      "sampling": null,
      "seconds": 0.283704,
      "usage": {
        "completion_tokens": 2,
        "prompt_tokens": 329,
        "prompt_tokens_details": {
          "cached_tokens": 0,
          "image_tokens": 299
        },
        "reasoning_tokens": 0,
        "total_tokens": 331
      },
      "validation": {
        "assertions": [
          {
            "passed": true,
            "type": "contains_casefold",
            "value": "three"
          }
        ],
        "finish_reason_allowed": true,
        "visible_answer": true
      }
    },
    {
      "case_id": "image-spatial-count",
      "failure": null,
      "finish_reason": "stop",
      "media": [
        {
          "bytes": 8785,
          "kind": "image",
          "mime": "image/png",
          "path": "spatial.png",
          "sha256": "d48f47cfba9d65524d6a9ca37d593550e10400c1b5fa160bec8aea24e0164051"
        }
      ],
      "modality": "image",
      "output": "three",
      "passed": true,
      "reasoning": {
        "chars": 0,
        "field": null,
        "tokens": null
      },
      "repetition": 2,
      "sampling": null,
      "seconds": 0.206096,
      "usage": {
        "completion_tokens": 2,
        "prompt_tokens": 329,
        "prompt_tokens_details": {
          "cached_tokens": 320,
          "image_tokens": 299
        },
        "reasoning_tokens": 0,
        "total_tokens": 331
      },
      "validation": {
        "assertions": [
          {
            "passed": true,
            "type": "contains_casefold",
            "value": "three"
          }
        ],
        "finish_reason_allowed": true,
        "visible_answer": true
      }
    },
    {
      "case_id": "image-ui",
      "failure": null,
      "finish_reason": "stop",
      "media": [
        {
          "bytes": 16940,
          "kind": "image",
          "mime": "image/png",
          "path": "ui.png",
          "sha256": "03fd14d8b68973c915de2848736d22eb6f67ba6bdb5da2a7a563085ac894e804"
        }
      ],
      "modality": "image",
      "output": "**Application Name:** ANVIL SERVING\n**Status:** READY\n**GPU:** 42 PERCENT",
      "passed": true,
      "reasoning": {
        "chars": 0,
        "field": null,
        "tokens": null
      },
      "repetition": 1,
      "sampling": null,
      "seconds": 0.37139,
      "usage": {
        "completion_tokens": 24,
        "prompt_tokens": 325,
        "prompt_tokens_details": {
          "cached_tokens": 0,
          "image_tokens": 299
        },
        "reasoning_tokens": 0,
        "total_tokens": 349
      },
      "validation": {
        "assertions": [
          {
            "passed": true,
            "type": "contains_casefold",
            "value": "ANVIL SERVING"
          },
          {
            "passed": true,
            "type": "contains_casefold",
            "value": "READY"
          },
          {
            "passed": true,
            "type": "contains_casefold",
            "value": "42"
          }
        ],
        "finish_reason_allowed": true,
        "visible_answer": true
      }
    },
    {
      "case_id": "image-ui",
      "failure": null,
      "finish_reason": "stop",
      "media": [
        {
          "bytes": 16940,
          "kind": "image",
          "mime": "image/png",
          "path": "ui.png",
          "sha256": "03fd14d8b68973c915de2848736d22eb6f67ba6bdb5da2a7a563085ac894e804"
        }
      ],
      "modality": "image",
      "output": "**Application Name:** ANVIL SERVING\n\n**Status:** READY\n\n**GPU Percentage:** 42%",
      "passed": true,
      "reasoning": {
        "chars": 0,
        "field": null,
        "tokens": null
      },
      "repetition": 2,
      "sampling": null,
      "seconds": 0.326774,
      "usage": {
        "completion_tokens": 24,
        "prompt_tokens": 325,
        "prompt_tokens_details": {
          "cached_tokens": 320,
          "image_tokens": 299
        },
        "reasoning_tokens": 0,
        "total_tokens": 349
      },
      "validation": {
        "assertions": [
          {
            "passed": true,
            "type": "contains_casefold",
            "value": "ANVIL SERVING"
          },
          {
            "passed": true,
            "type": "contains_casefold",
            "value": "READY"
          },
          {
            "passed": true,
            "type": "contains_casefold",
            "value": "42"
          }
        ],
        "finish_reason_allowed": true,
        "visible_answer": true
      }
    },
    {
      "case_id": "multi-image-comparison",
      "failure": null,
      "finish_reason": "stop",
      "media": [
        {
          "bytes": 8000,
          "kind": "image",
          "mime": "image/png",
          "path": "compare-a.png",
          "sha256": "7dac4f31d62c88d765150aa6051efea9ddaaa8c64312080be4c5b8f6ad96322a"
        },
        {
          "bytes": 7679,
          "kind": "image",
          "mime": "image/png",
          "path": "compare-b.png",
          "sha256": "8f7cdfd733f1ee9731de6160e77af6aa50189edec6eaf9ddd7654a0c8300a43e"
        }
      ],
      "modality": "image",
      "output": "Based on the images provided:\n\n**Meter B is higher** at **75 percent**, compared to Meter A, which is at 25 percent.\n\nMeter B's fill level is three times greater than Meter A's (75% vs. 25%), a difference of 50 percentage points.",
      "passed": true,
      "reasoning": {
        "chars": 0,
        "field": null,
        "tokens": null
      },
      "repetition": 1,
      "sampling": null,
      "seconds": 0.783601,
      "usage": {
        "completion_tokens": 59,
        "prompt_tokens": 630,
        "prompt_tokens_details": {
          "cached_tokens": 0,
          "image_tokens": 598
        },
        "reasoning_tokens": 0,
        "total_tokens": 689
      },
      "validation": {
        "assertions": [
          {
            "passed": true,
            "type": "contains_casefold",
            "value": "meter b"
          },
          {
            "passed": true,
            "type": "contains_casefold",
            "value": "75"
          }
        ],
        "finish_reason_allowed": true,
        "visible_answer": true
      }
    },
    {
      "case_id": "multi-image-comparison",
      "failure": null,
      "finish_reason": "stop",
      "media": [
        {
          "bytes": 8000,
          "kind": "image",
          "mime": "image/png",
          "path": "compare-a.png",
          "sha256": "7dac4f31d62c88d765150aa6051efea9ddaaa8c64312080be4c5b8f6ad96322a"
        },
        {
          "bytes": 7679,
          "kind": "image",
          "mime": "image/png",
          "path": "compare-b.png",
          "sha256": "8f7cdfd733f1ee9731de6160e77af6aa50189edec6eaf9ddd7654a0c8300a43e"
        }
      ],
      "modality": "image",
      "output": "**Meter B is higher** at **75 percent**, compared to Meter A at 25 percent.",
      "passed": true,
      "reasoning": {
        "chars": 0,
        "field": null,
        "tokens": null
      },
      "repetition": 2,
      "sampling": null,
      "seconds": 0.386593,
      "usage": {
        "completion_tokens": 21,
        "prompt_tokens": 630,
        "prompt_tokens_details": {
          "cached_tokens": 576,
          "image_tokens": 598
        },
        "reasoning_tokens": 0,
        "total_tokens": 651
      },
      "validation": {
        "assertions": [
          {
            "passed": true,
            "type": "contains_casefold",
            "value": "meter b"
          },
          {
            "passed": true,
            "type": "contains_casefold",
            "value": "75"
          }
        ],
        "finish_reason_allowed": true,
        "visible_answer": true
      }
    }
  ],
  "configuration": {
    "allowed_finish_reasons": [
      "stop"
    ],
    "chat_template_kwargs": {
      "enable_thinking": false
    },
    "concurrency": 1,
    "max_images_per_request": 4,
    "max_tokens": 2048,
    "max_videos_per_request": 1,
    "reasoning_effort": null,
    "thinking_mode": "disabled",
    "timeout_seconds": 3600.0
  },
  "corpus": {
    "cases": [
      {
        "assertions": [
          {
            "type": "contains_casefold",
            "value": "red car"
          },
          {
            "type": "contains_casefold",
            "value": "green park"
          },
          {
            "type": "contains_casefold",
            "value": "sunny day"
          }
        ],
        "id": "image-scene",
        "media": [
          {
            "bytes": 13187,
            "kind": "image",
            "mime": "image/png",
            "path": "scene.png",
            "sha256": "30a1f11e40bf4e0a8595d39828e0aa362e8b28ac683c404a1068cc20bf7f7bda"
          }
        ],
        "modality": "image",
        "prompt": "Describe the scene and report every prominent label. Use the labels verbatim.",
        "repetitions": 2,
        "sampling": null
      },
      {
        "assertions": [
          {
            "type": "contains_casefold",
            "value": "ANVIL READY"
          },
          {
            "type": "contains_casefold",
            "value": "42917"
          },
          {
            "type": "contains_casefold",
            "value": "42"
          }
        ],
        "id": "image-ocr",
        "media": [
          {
            "bytes": 15916,
            "kind": "image",
            "mime": "image/png",
            "path": "ocr.png",
            "sha256": "a8d5158019fb2b08ee9f6a153860ab2342384644f7c53ab58fdd45658124e83e"
          }
        ],
        "modality": "image",
        "prompt": "Transcribe all visible text exactly. Preserve every number.",
        "repetitions": 2,
        "sampling": null
      },
      {
        "assertions": [
          {
            "type": "ordered_casefold",
            "values": [
              "Q1",
              "10",
              "Q2",
              "20",
              "Q3",
              "30"
            ]
          }
        ],
        "id": "image-chart",
        "media": [
          {
            "bytes": 12635,
            "kind": "image",
            "mime": "image/png",
            "path": "chart.png",
            "sha256": "c905e453c4e16eb274e46992d1a901bbced8f2314b47b9d408971bb176ab2f3e"
          }
        ],
        "modality": "image",
        "prompt": "Read the chart and report each quarter with its value in ascending quarter order.",
        "repetitions": 2,
        "sampling": null
      },
      {
        "assertions": [
          {
            "type": "contains_casefold",
            "value": "ANVIL SERVING"
          },
          {
            "type": "contains_casefold",
            "value": "READY"
          },
          {
            "type": "contains_casefold",
            "value": "42"
          }
        ],
        "id": "image-ui",
        "media": [
          {
            "bytes": 16940,
            "kind": "image",
            "mime": "image/png",
            "path": "ui.png",
            "sha256": "03fd14d8b68973c915de2848736d22eb6f67ba6bdb5da2a7a563085ac894e804"
          }
        ],
        "modality": "image",
        "prompt": "Report the application name, status, and GPU percentage.",
        "repetitions": 2,
        "sampling": null
      },
      {
        "assertions": [
          {
            "type": "contains_casefold",
            "value": "three"
          }
        ],
        "id": "image-spatial-count",
        "media": [
          {
            "bytes": 8785,
            "kind": "image",
            "mime": "image/png",
            "path": "spatial.png",
            "sha256": "d48f47cfba9d65524d6a9ca37d593550e10400c1b5fa160bec8aea24e0164051"
          }
        ],
        "modality": "image",
        "prompt": "Count the blue boxes. Reply with the count written as an English word.",
        "repetitions": 2,
        "sampling": null
      },
      {
        "assertions": [
          {
            "type": "contains_casefold",
            "value": "meter b"
          },
          {
            "type": "contains_casefold",
            "value": "75"
          }
        ],
        "id": "multi-image-comparison",
        "media": [
          {
            "bytes": 8000,
            "kind": "image",
            "mime": "image/png",
            "path": "compare-a.png",
            "sha256": "7dac4f31d62c88d765150aa6051efea9ddaaa8c64312080be4c5b8f6ad96322a"
          },
          {
            "bytes": 7679,
            "kind": "image",
            "mime": "image/png",
            "path": "compare-b.png",
            "sha256": "8f7cdfd733f1ee9731de6160e77af6aa50189edec6eaf9ddd7654a0c8300a43e"
          }
        ],
        "modality": "image",
        "prompt": "Compare the two meters. Identify which meter is higher and report its percentage.",
        "repetitions": 2,
        "sampling": null
      }
    ],
    "path": "tests/fixtures/eval-data/agents-a1-v1/image-corpus.json",
    "provenance": {
      "derived_from": "corpus.json",
      "observed_date": "2026-08-29",
      "scope": "Deterministic image-only qualification subset for servers that intentionally disable video",
      "synthetic": {
        "generator": "generate.py",
        "license": "CC0-1.0"
      }
    },
    "schema": "multimodal-corpus/v1",
    "sha256": "a514ae146b5aab3bcb55ae9fb27f55a41791a20863304d1bb95947e93931cd12"
  },
  "failed_attempts": 0,
  "finished_at": "2026-09-02T11:39:51Z",
  "identity": {
    "base_url": "http://127.0.0.1:39118/v1",
    "engine_build_ref": null,
    "engine_revision": "4c2c169b53dbf362f0cd95111f4ae275cd0167c1",
    "hardware": "2x NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition TP2 PCIe WSL2",
    "model": "glm53-flash-ormandj-sglang-sm120-tp2-240k-c1-adaptive-mtp",
    "model_revision": "c3cbb9891b67c741bcbf6b176dd7af9265b069db",
    "runtime_image": "ghcr.io/ormandj/sglang-glm53-flash-sm120@sha256:0c0637959c3931829f05154087bbefd2c50003fb9b2010200ce0ec82f4d71a53",
    "served_models": [
      "glm53-flash-ormandj-sglang-sm120-tp2-240k-c1-adaptive-mtp"
    ]
  },
  "passed": true,
  "passed_attempts": 12,
  "requests": 12,
  "run_id": "multimodal-20260902T113944Z",
  "schema": "multimodal-benchmark-evidence/v1",
  "started_at": "2026-09-02T11:39:44Z"
}