{ "aggregates": { "case": { "cc-video-cat-opening-door": { "attempts": 2, "latency_p50_seconds": 0.0189265, "latency_p95_seconds": 0.019093, "pass_rate": 0.0, "passed": 0 }, "cc-video-chemical-traffic-light": { "attempts": 2, "latency_p50_seconds": 0.0131725, "latency_p95_seconds": 0.018809, "pass_rate": 0.0, "passed": 0 }, "image-chart": { "attempts": 2, "latency_p50_seconds": 0.515316, "latency_p95_seconds": 0.515939, "pass_rate": 1.0, "passed": 2 }, "image-ocr": { "attempts": 2, "latency_p50_seconds": 0.3469535, "latency_p95_seconds": 0.353771, "pass_rate": 1.0, "passed": 2 }, "image-scene": { "attempts": 2, "latency_p50_seconds": 2.8757015, "latency_p95_seconds": 3.05106, "pass_rate": 1.0, "passed": 2 }, "image-spatial-count": { "attempts": 2, "latency_p50_seconds": 0.098841, "latency_p95_seconds": 0.099516, "pass_rate": 1.0, "passed": 2 }, "image-ui": { "attempts": 2, "latency_p50_seconds": 0.3060705, "latency_p95_seconds": 0.312426, "pass_rate": 1.0, "passed": 2 }, "mixed-video-four-images": { "attempts": 2, "latency_p50_seconds": 0.0092645, "latency_p95_seconds": 0.014209, "pass_rate": 0.0, "passed": 0 }, "mixed-video-one-image": { "attempts": 2, "latency_p50_seconds": 0.0133255, "latency_p95_seconds": 0.013817, "pass_rate": 0.0, "passed": 0 }, "multi-image-comparison": { "attempts": 2, "latency_p50_seconds": 1.181329, "latency_p95_seconds": 1.202455, "pass_rate": 1.0, "passed": 2 }, "video-continuity-120s": { "attempts": 2, "latency_p50_seconds": 0.0079465, "latency_p95_seconds": 0.01269, "pass_rate": 0.0, "passed": 0 }, "video-event-localization-60s": { "attempts": 2, "latency_p50_seconds": 0.012969999999999999, "latency_p95_seconds": 0.013019, "pass_rate": 0.0, "passed": 0 }, "video-ocr-30s": { "attempts": 2, "latency_p50_seconds": 0.0083885, "latency_p95_seconds": 0.013648, "pass_rate": 0.0, "passed": 0 }, "video-state-change-30s": { "attempts": 2, "latency_p50_seconds": 0.0131545, "latency_p95_seconds": 0.013574, "pass_rate": 0.0, "passed": 0 }, "video-temporal-order-10s": { "attempts": 2, "latency_p50_seconds": 0.013818, "latency_p95_seconds": 0.014608, "pass_rate": 0.0, "passed": 0 } }, "modality": { "image": { "attempts": 12, "latency_p50_seconds": 0.43423199999999995, "latency_p95_seconds": 3.05106, "pass_rate": 1.0, "passed": 12 }, "mixed": { "attempts": 4, "latency_p50_seconds": 0.0133255, "latency_p95_seconds": 0.014209, "pass_rate": 0.0, "passed": 0 }, "video": { "attempts": 14, "latency_p50_seconds": 0.0130235, "latency_p95_seconds": 0.019093, "pass_rate": 0.0, "passed": 0 } } }, "attempts": [ { "case_id": "cc-video-cat-opening-door", "failure": { "message": "HTTP 400: {\"error\":{\"message\":\"Could not open video stream\",\"type\":\"BadRequestError\",\"param\":null,\"code\":400}}", "type": "RuntimeError" }, "finish_reason": null, "media": [ { "bytes": 490171, "kind": "video", "mime": "video/mp4", "path": "cc/cat-opening-door-normalized.mp4", "sha256": "b01ffe7800a3f74eead304a06bf35ece0e8917e107e1ce1e25837f3b5611cb61" } ], "modality": "video", "output": "", "passed": false, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 1, "sampling": { "num_frames": 16 }, "seconds": 0.019093, "usage": null, "validation": { "assertions": [], "finish_reason_allowed": false, "visible_answer": false } }, { "case_id": "cc-video-cat-opening-door", "failure": { "message": "HTTP 400: {\"error\":{\"message\":\"Could not open video stream\",\"type\":\"BadRequestError\",\"param\":null,\"code\":400}}", "type": "RuntimeError" }, "finish_reason": null, "media": [ { "bytes": 490171, "kind": "video", "mime": "video/mp4", "path": "cc/cat-opening-door-normalized.mp4", "sha256": "b01ffe7800a3f74eead304a06bf35ece0e8917e107e1ce1e25837f3b5611cb61" } ], "modality": "video", "output": "", "passed": false, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 2, "sampling": { "num_frames": 16 }, "seconds": 0.01876, "usage": null, "validation": { "assertions": [], "finish_reason_allowed": false, "visible_answer": false } }, { "case_id": "cc-video-chemical-traffic-light", "failure": { "message": "HTTP 400: {\"error\":{\"message\":\"Could not open video stream\",\"type\":\"BadRequestError\",\"param\":null,\"code\":400}}", "type": "RuntimeError" }, "finish_reason": null, "media": [ { "bytes": 459852, "kind": "video", "mime": "video/mp4", "path": "cc/chemical-traffic-light-normalized.mp4", "sha256": "1ab998a0339eaedb8494392be2c11f02bf9ce2f632fb01cf51b7089990d18f9c" } ], "modality": "video", "output": "", "passed": false, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 1, "sampling": { "num_frames": 24 }, "seconds": 0.007536, "usage": null, "validation": { "assertions": [], "finish_reason_allowed": false, "visible_answer": false } }, { "case_id": "cc-video-chemical-traffic-light", "failure": { "message": "HTTP 400: {\"error\":{\"message\":\"Could not open video stream\",\"type\":\"BadRequestError\",\"param\":null,\"code\":400}}", "type": "RuntimeError" }, "finish_reason": null, "media": [ { "bytes": 459852, "kind": "video", "mime": "video/mp4", "path": "cc/chemical-traffic-light-normalized.mp4", "sha256": "1ab998a0339eaedb8494392be2c11f02bf9ce2f632fb01cf51b7089990d18f9c" } ], "modality": "video", "output": "", "passed": false, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 2, "sampling": { "num_frames": 24 }, "seconds": 0.018809, "usage": null, "validation": { "assertions": [], "finish_reason_allowed": false, "visible_answer": false } }, { "case_id": "image-chart", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 12635, "kind": "image", "mime": "image/png", "path": "chart.png", "sha256": "c905e453c4e16eb274e46992d1a901bbced8f2314b47b9d408971bb176ab2f3e" } ], "modality": "image", "output": "Here are the quarterly units in ascending quarter order:\n\n- Q1: 10 \n- Q2: 20 \n- Q3: 30", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 1, "sampling": null, "seconds": 0.515939, "usage": { "completion_tokens": 35, "prompt_tokens": 249, "prompt_tokens_details": null, "total_tokens": 284 }, "validation": { "assertions": [ { "passed": true, "type": "ordered_casefold", "values": [ "Q1", "10", "Q2", "20", "Q3", "30" ] } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "image-chart", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 12635, "kind": "image", "mime": "image/png", "path": "chart.png", "sha256": "c905e453c4e16eb274e46992d1a901bbced8f2314b47b9d408971bb176ab2f3e" } ], "modality": "image", "output": "Here are the quarterly units in ascending quarter order:\n\n- Q1: 10 \n- Q2: 20 \n- Q3: 30", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 2, "sampling": null, "seconds": 0.514693, "usage": { "completion_tokens": 35, "prompt_tokens": 249, "prompt_tokens_details": null, "total_tokens": 284 }, "validation": { "assertions": [ { "passed": true, "type": "ordered_casefold", "values": [ "Q1", "10", "Q2", "20", "Q3", "30" ] } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "image-ocr", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 15916, "kind": "image", "mime": "image/png", "path": "ocr.png", "sha256": "a8d5158019fb2b08ee9f6a153860ab2342384644f7c53ab58fdd45658124e83e" } ], "modality": "image", "output": "ANVIL READY\nCODE 42917\nTEMPERATURE 42 C", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 1, "sampling": null, "seconds": 0.353771, "usage": { "completion_tokens": 21, "prompt_tokens": 245, "prompt_tokens_details": null, "total_tokens": 266 }, "validation": { "assertions": [ { "passed": true, "type": "contains_casefold", "value": "ANVIL READY" }, { "passed": true, "type": "contains_casefold", "value": "42917" }, { "passed": true, "type": "contains_casefold", "value": "42" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "image-ocr", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 15916, "kind": "image", "mime": "image/png", "path": "ocr.png", "sha256": "a8d5158019fb2b08ee9f6a153860ab2342384644f7c53ab58fdd45658124e83e" } ], "modality": "image", "output": "ANVIL READY\nCODE 42917\nTEMPERATURE 42 C", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 2, "sampling": null, "seconds": 0.340136, "usage": { "completion_tokens": 21, "prompt_tokens": 245, "prompt_tokens_details": null, "total_tokens": 266 }, "validation": { "assertions": [ { "passed": true, "type": "contains_casefold", "value": "ANVIL READY" }, { "passed": true, "type": "contains_casefold", "value": "42917" }, { "passed": true, "type": "contains_casefold", "value": "42" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "image-scene", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 13187, "kind": "image", "mime": "image/png", "path": "scene.png", "sha256": "30a1f11e40bf4e0a8595d39828e0aa362e8b28ac683c404a1068cc20bf7f7bda" } ], "modality": "image", "output": "The image is a stylized, minimalist graphic with a dark navy blue background. It features simple geometric shapes and bold text labels.\n\nAt the top center, in large yellow capital letters, it reads: \n**SUNNY DAY**\n\nBelow that, on the left side, in white capital letters, it says: \n**RED CAR**\n\nBeneath this label is a solid red rectangle, representing the \u201cred car.\u201d\n\nAt the bottom of the image, there is a wide horizontal green rectangle, representing a park or grassy area. Inside this green shape, on the right side, in white capital letters, it reads: \n**GREEN PARK**\n\nThe red rectangle (car) sits partially overlapping the top edge of the green rectangle (park), suggesting the car is parked at or near the park.\n\nNo other objects, people, or details are present \u2014 the scene is purely symbolic and text-driven.\n\nProminent labels (verbatim): \n- SUNNY DAY \n- RED CAR \n- GREEN PARK", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 1, "sampling": null, "seconds": 3.05106, "usage": { "completion_tokens": 208, "prompt_tokens": 249, "prompt_tokens_details": null, "total_tokens": 457 }, "validation": { "assertions": [ { "passed": true, "type": "contains_casefold", "value": "red car" }, { "passed": true, "type": "contains_casefold", "value": "green park" }, { "passed": true, "type": "contains_casefold", "value": "sunny day" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "image-scene", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 13187, "kind": "image", "mime": "image/png", "path": "scene.png", "sha256": "30a1f11e40bf4e0a8595d39828e0aa362e8b28ac683c404a1068cc20bf7f7bda" } ], "modality": "image", "output": "The image is a stylized, minimalist graphic with a dark navy blue background. It features simple geometric shapes and bold text labels.\n\nAt the top center, in large yellow capital letters, it reads: \n**SUNNY DAY**\n\nBelow that, on the left side, in white capital letters, it says: \n**RED CAR**\n\nBeneath this label is a solid red rectangle, representing the \u201cred car.\u201d\n\nAt the bottom of the image, there is a wide horizontal green rectangle, representing a park or grassy area. Inside this green shape, on the right side, in white capital letters, it reads: \n**GREEN PARK**\n\nThe red rectangle (car) sits partially overlapping the top edge of the green rectangle (park), suggesting the car is parked at or near the park.\n\nNo other objects, people, or details are present \u2014 the scene is purely symbolic and text-driven.\n\nProminent labels (verbatim): \n- SUNNY DAY \n- RED CAR \n- GREEN PARK", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 2, "sampling": null, "seconds": 2.700343, "usage": { "completion_tokens": 208, "prompt_tokens": 249, "prompt_tokens_details": null, "total_tokens": 457 }, "validation": { "assertions": [ { "passed": true, "type": "contains_casefold", "value": "red car" }, { "passed": true, "type": "contains_casefold", "value": "green park" }, { "passed": true, "type": "contains_casefold", "value": "sunny day" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "image-spatial-count", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 8785, "kind": "image", "mime": "image/png", "path": "spatial.png", "sha256": "d48f47cfba9d65524d6a9ca37d593550e10400c1b5fa160bec8aea24e0164051" } ], "modality": "image", "output": "three", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 1, "sampling": null, "seconds": 0.099516, "usage": { "completion_tokens": 2, "prompt_tokens": 249, "prompt_tokens_details": null, "total_tokens": 251 }, "validation": { "assertions": [ { "passed": true, "type": "contains_casefold", "value": "three" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "image-spatial-count", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 8785, "kind": "image", "mime": "image/png", "path": "spatial.png", "sha256": "d48f47cfba9d65524d6a9ca37d593550e10400c1b5fa160bec8aea24e0164051" } ], "modality": "image", "output": "three", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 2, "sampling": null, "seconds": 0.098166, "usage": { "completion_tokens": 2, "prompt_tokens": 249, "prompt_tokens_details": null, "total_tokens": 251 }, "validation": { "assertions": [ { "passed": true, "type": "contains_casefold", "value": "three" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "image-ui", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 16940, "kind": "image", "mime": "image/png", "path": "ui.png", "sha256": "03fd14d8b68973c915de2848736d22eb6f67ba6bdb5da2a7a563085ac894e804" } ], "modality": "image", "output": "Application: Anvil Serving \nStatus: Ready \nGPU Usage: 42%", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 1, "sampling": null, "seconds": 0.312426, "usage": { "completion_tokens": 18, "prompt_tokens": 245, "prompt_tokens_details": null, "total_tokens": 263 }, "validation": { "assertions": [ { "passed": true, "type": "contains_casefold", "value": "ANVIL SERVING" }, { "passed": true, "type": "contains_casefold", "value": "READY" }, { "passed": true, "type": "contains_casefold", "value": "42" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "image-ui", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 16940, "kind": "image", "mime": "image/png", "path": "ui.png", "sha256": "03fd14d8b68973c915de2848736d22eb6f67ba6bdb5da2a7a563085ac894e804" } ], "modality": "image", "output": "Application: Anvil Serving \nStatus: Ready \nGPU Usage: 42%", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 2, "sampling": null, "seconds": 0.299715, "usage": { "completion_tokens": 18, "prompt_tokens": 245, "prompt_tokens_details": null, "total_tokens": 263 }, "validation": { "assertions": [ { "passed": true, "type": "contains_casefold", "value": "ANVIL SERVING" }, { "passed": true, "type": "contains_casefold", "value": "READY" }, { "passed": true, "type": "contains_casefold", "value": "42" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "mixed-video-four-images", "failure": { "message": "HTTP 400: {\"error\":{\"message\":\"Could not open video stream\",\"type\":\"BadRequestError\",\"param\":null,\"code\":400}}", "type": "RuntimeError" }, "finish_reason": null, "media": [ { "bytes": 14731, "kind": "video", "mime": "video/mp4", "path": "video-ocr-30s.mp4", "sha256": "05c6cd1b1a9b4939f513944717f06d02dd5be54dd18b27870dccf2449b457e34" }, { "bytes": 13187, "kind": "image", "mime": "image/png", "path": "scene.png", "sha256": "30a1f11e40bf4e0a8595d39828e0aa362e8b28ac683c404a1068cc20bf7f7bda" }, { "bytes": 15916, "kind": "image", "mime": "image/png", "path": "ocr.png", "sha256": "a8d5158019fb2b08ee9f6a153860ab2342384644f7c53ab58fdd45658124e83e" }, { "bytes": 12635, "kind": "image", "mime": "image/png", "path": "chart.png", "sha256": "c905e453c4e16eb274e46992d1a901bbced8f2314b47b9d408971bb176ab2f3e" }, { "bytes": 16940, "kind": "image", "mime": "image/png", "path": "ui.png", "sha256": "03fd14d8b68973c915de2848736d22eb6f67ba6bdb5da2a7a563085ac894e804" } ], "modality": "mixed", "output": "", "passed": false, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 1, "sampling": { "num_frames": 16 }, "seconds": 0.014209, "usage": null, "validation": { "assertions": [], "finish_reason_allowed": false, "visible_answer": false } }, { "case_id": "mixed-video-four-images", "failure": { "message": "HTTP 400: {\"error\":{\"message\":\"Could not open video stream\",\"type\":\"BadRequestError\",\"param\":null,\"code\":400}}", "type": "RuntimeError" }, "finish_reason": null, "media": [ { "bytes": 14731, "kind": "video", "mime": "video/mp4", "path": "video-ocr-30s.mp4", "sha256": "05c6cd1b1a9b4939f513944717f06d02dd5be54dd18b27870dccf2449b457e34" }, { "bytes": 13187, "kind": "image", "mime": "image/png", "path": "scene.png", "sha256": "30a1f11e40bf4e0a8595d39828e0aa362e8b28ac683c404a1068cc20bf7f7bda" }, { "bytes": 15916, "kind": "image", "mime": "image/png", "path": "ocr.png", "sha256": "a8d5158019fb2b08ee9f6a153860ab2342384644f7c53ab58fdd45658124e83e" }, { "bytes": 12635, "kind": "image", "mime": "image/png", "path": "chart.png", "sha256": "c905e453c4e16eb274e46992d1a901bbced8f2314b47b9d408971bb176ab2f3e" }, { "bytes": 16940, "kind": "image", "mime": "image/png", "path": "ui.png", "sha256": "03fd14d8b68973c915de2848736d22eb6f67ba6bdb5da2a7a563085ac894e804" } ], "modality": "mixed", "output": "", "passed": false, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 2, "sampling": { "num_frames": 16 }, "seconds": 0.00432, "usage": null, "validation": { "assertions": [], "finish_reason_allowed": false, "visible_answer": false } }, { "case_id": "mixed-video-one-image", "failure": { "message": "HTTP 400: {\"error\":{\"message\":\"Could not open video stream\",\"type\":\"BadRequestError\",\"param\":null,\"code\":400}}", "type": "RuntimeError" }, "finish_reason": null, "media": [ { "bytes": 11627, "kind": "video", "mime": "video/mp4", "path": "temporal-order-10s.mp4", "sha256": "4d82c50cd2e500fc58df6d4930efec12649a0eaf236bee53d0d7c7ca84e35980" }, { "bytes": 16940, "kind": "image", "mime": "image/png", "path": "ui.png", "sha256": "03fd14d8b68973c915de2848736d22eb6f67ba6bdb5da2a7a563085ac894e804" } ], "modality": "mixed", "output": "", "passed": false, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 1, "sampling": { "fps": 1.0 }, "seconds": 0.013817, "usage": null, "validation": { "assertions": [], "finish_reason_allowed": false, "visible_answer": false } }, { "case_id": "mixed-video-one-image", "failure": { "message": "HTTP 400: {\"error\":{\"message\":\"Could not open video stream\",\"type\":\"BadRequestError\",\"param\":null,\"code\":400}}", "type": "RuntimeError" }, "finish_reason": null, "media": [ { "bytes": 11627, "kind": "video", "mime": "video/mp4", "path": "temporal-order-10s.mp4", "sha256": "4d82c50cd2e500fc58df6d4930efec12649a0eaf236bee53d0d7c7ca84e35980" }, { "bytes": 16940, "kind": "image", "mime": "image/png", "path": "ui.png", "sha256": "03fd14d8b68973c915de2848736d22eb6f67ba6bdb5da2a7a563085ac894e804" } ], "modality": "mixed", "output": "", "passed": false, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 2, "sampling": { "fps": 1.0 }, "seconds": 0.012834, "usage": null, "validation": { "assertions": [], "finish_reason_allowed": false, "visible_answer": false } }, { "case_id": "multi-image-comparison", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 8000, "kind": "image", "mime": "image/png", "path": "compare-a.png", "sha256": "7dac4f31d62c88d765150aa6051efea9ddaaa8c64312080be4c5b8f6ad96322a" }, { "bytes": 7679, "kind": "image", "mime": "image/png", "path": "compare-b.png", "sha256": "8f7cdfd733f1ee9731de6160e77af6aa50189edec6eaf9ddd7654a0c8300a43e" } ], "modality": "image", "output": "Let's compare the two meters:\n\n- **Meter A** shows **25 percent**, represented by an orange bar that fills one-quarter of the total length.\n- **Meter B** shows **75 percent**, represented by a green bar that fills three-quarters of the total length.\n\nClearly, **Meter B is higher**.\n\n\u2705 **Final Answer: Meter B is higher at 75 percent.**", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 1, "sampling": null, "seconds": 1.202455, "usage": { "completion_tokens": 85, "prompt_tokens": 471, "prompt_tokens_details": null, "total_tokens": 556 }, "validation": { "assertions": [ { "passed": true, "type": "contains_casefold", "value": "meter b" }, { "passed": true, "type": "contains_casefold", "value": "75" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "multi-image-comparison", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 8000, "kind": "image", "mime": "image/png", "path": "compare-a.png", "sha256": "7dac4f31d62c88d765150aa6051efea9ddaaa8c64312080be4c5b8f6ad96322a" }, { "bytes": 7679, "kind": "image", "mime": "image/png", "path": "compare-b.png", "sha256": "8f7cdfd733f1ee9731de6160e77af6aa50189edec6eaf9ddd7654a0c8300a43e" } ], "modality": "image", "output": "Let's compare the two meters:\n\n- **Meter A** shows **25 percent**, represented by an orange bar that fills one-quarter of the total length.\n- **Meter B** shows **75 percent**, represented by a green bar that fills three-quarters of the total length.\n\nClearly, **Meter B is higher**.\n\n\u2705 **Final Answer: Meter B is higher at 75 percent.**", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 2, "sampling": null, "seconds": 1.160203, "usage": { "completion_tokens": 85, "prompt_tokens": 471, "prompt_tokens_details": null, "total_tokens": 556 }, "validation": { "assertions": [ { "passed": true, "type": "contains_casefold", "value": "meter b" }, { "passed": true, "type": "contains_casefold", "value": "75" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "video-continuity-120s", "failure": { "message": "HTTP 400: {\"error\":{\"message\":\"Could not open video stream\",\"type\":\"BadRequestError\",\"param\":null,\"code\":400}}", "type": "RuntimeError" }, "finish_reason": null, "media": [ { "bytes": 23622, "kind": "video", "mime": "video/mp4", "path": "continuity-120s.mp4", "sha256": "1ebc5459ed8c158ffb8294a0317005a13198748c529a49b66a2f76ff1c982b95" } ], "modality": "video", "output": "", "passed": false, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 1, "sampling": { "num_frames": 32 }, "seconds": 0.01269, "usage": null, "validation": { "assertions": [], "finish_reason_allowed": false, "visible_answer": false } }, { "case_id": "video-continuity-120s", "failure": { "message": "HTTP 400: {\"error\":{\"message\":\"Could not open video stream\",\"type\":\"BadRequestError\",\"param\":null,\"code\":400}}", "type": "RuntimeError" }, "finish_reason": null, "media": [ { "bytes": 23622, "kind": "video", "mime": "video/mp4", "path": "continuity-120s.mp4", "sha256": "1ebc5459ed8c158ffb8294a0317005a13198748c529a49b66a2f76ff1c982b95" } ], "modality": "video", "output": "", "passed": false, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 2, "sampling": { "num_frames": 32 }, "seconds": 0.003203, "usage": null, "validation": { "assertions": [], "finish_reason_allowed": false, "visible_answer": false } }, { "case_id": "video-event-localization-60s", "failure": { "message": "HTTP 400: {\"error\":{\"message\":\"Could not open video stream\",\"type\":\"BadRequestError\",\"param\":null,\"code\":400}}", "type": "RuntimeError" }, "finish_reason": null, "media": [ { "bytes": 13829, "kind": "video", "mime": "video/mp4", "path": "event-localization-60s.mp4", "sha256": "5e0f40b3760e009f8ea0035b96bb4a8654fea58e0bb8a31a838eb78b224e7cbf" } ], "modality": "video", "output": "", "passed": false, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 1, "sampling": { "fps": 1.0 }, "seconds": 0.012921, "usage": null, "validation": { "assertions": [], "finish_reason_allowed": false, "visible_answer": false } }, { "case_id": "video-event-localization-60s", "failure": { "message": "HTTP 400: {\"error\":{\"message\":\"Could not open video stream\",\"type\":\"BadRequestError\",\"param\":null,\"code\":400}}", "type": "RuntimeError" }, "finish_reason": null, "media": [ { "bytes": 13829, "kind": "video", "mime": "video/mp4", "path": "event-localization-60s.mp4", "sha256": "5e0f40b3760e009f8ea0035b96bb4a8654fea58e0bb8a31a838eb78b224e7cbf" } ], "modality": "video", "output": "", "passed": false, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 2, "sampling": { "fps": 1.0 }, "seconds": 0.013019, "usage": null, "validation": { "assertions": [], "finish_reason_allowed": false, "visible_answer": false } }, { "case_id": "video-ocr-30s", "failure": { "message": "HTTP 400: {\"error\":{\"message\":\"Could not open video stream\",\"type\":\"BadRequestError\",\"param\":null,\"code\":400}}", "type": "RuntimeError" }, "finish_reason": null, "media": [ { "bytes": 14731, "kind": "video", "mime": "video/mp4", "path": "video-ocr-30s.mp4", "sha256": "05c6cd1b1a9b4939f513944717f06d02dd5be54dd18b27870dccf2449b457e34" } ], "modality": "video", "output": "", "passed": false, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 1, "sampling": { "num_frames": 16 }, "seconds": 0.013648, "usage": null, "validation": { "assertions": [], "finish_reason_allowed": false, "visible_answer": false } }, { "case_id": "video-ocr-30s", "failure": { "message": "HTTP 400: {\"error\":{\"message\":\"Could not open video stream\",\"type\":\"BadRequestError\",\"param\":null,\"code\":400}}", "type": "RuntimeError" }, "finish_reason": null, "media": [ { "bytes": 14731, "kind": "video", "mime": "video/mp4", "path": "video-ocr-30s.mp4", "sha256": "05c6cd1b1a9b4939f513944717f06d02dd5be54dd18b27870dccf2449b457e34" } ], "modality": "video", "output": "", "passed": false, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 2, "sampling": { "num_frames": 16 }, "seconds": 0.003129, "usage": null, "validation": { "assertions": [], "finish_reason_allowed": false, "visible_answer": false } }, { "case_id": "video-state-change-30s", "failure": { "message": "HTTP 400: {\"error\":{\"message\":\"Could not open video stream\",\"type\":\"BadRequestError\",\"param\":null,\"code\":400}}", "type": "RuntimeError" }, "finish_reason": null, "media": [ { "bytes": 14552, "kind": "video", "mime": "video/mp4", "path": "state-change-30s.mp4", "sha256": "4d720486425afff4a7450f0f7f3ce6e3eea591e063b66e48fc953415b9172e62" } ], "modality": "video", "output": "", "passed": false, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 1, "sampling": { "fps": 1.0 }, "seconds": 0.013574, "usage": null, "validation": { "assertions": [], "finish_reason_allowed": false, "visible_answer": false } }, { "case_id": "video-state-change-30s", "failure": { "message": "HTTP 400: {\"error\":{\"message\":\"Could not open video stream\",\"type\":\"BadRequestError\",\"param\":null,\"code\":400}}", "type": "RuntimeError" }, "finish_reason": null, "media": [ { "bytes": 14552, "kind": "video", "mime": "video/mp4", "path": "state-change-30s.mp4", "sha256": "4d720486425afff4a7450f0f7f3ce6e3eea591e063b66e48fc953415b9172e62" } ], "modality": "video", "output": "", "passed": false, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 2, "sampling": { "fps": 1.0 }, "seconds": 0.012735, "usage": null, "validation": { "assertions": [], "finish_reason_allowed": false, "visible_answer": false } }, { "case_id": "video-temporal-order-10s", "failure": { "message": "HTTP 400: {\"error\":{\"message\":\"Could not open video stream\",\"type\":\"BadRequestError\",\"param\":null,\"code\":400}}", "type": "RuntimeError" }, "finish_reason": null, "media": [ { "bytes": 11627, "kind": "video", "mime": "video/mp4", "path": "temporal-order-10s.mp4", "sha256": "4d82c50cd2e500fc58df6d4930efec12649a0eaf236bee53d0d7c7ca84e35980" } ], "modality": "video", "output": "", "passed": false, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 1, "sampling": { "fps": 1.0 }, "seconds": 0.014608, "usage": null, "validation": { "assertions": [], "finish_reason_allowed": false, "visible_answer": false } }, { "case_id": "video-temporal-order-10s", "failure": { "message": "HTTP 400: {\"error\":{\"message\":\"Could not open video stream\",\"type\":\"BadRequestError\",\"param\":null,\"code\":400}}", "type": "RuntimeError" }, "finish_reason": null, "media": [ { "bytes": 11627, "kind": "video", "mime": "video/mp4", "path": "temporal-order-10s.mp4", "sha256": "4d82c50cd2e500fc58df6d4930efec12649a0eaf236bee53d0d7c7ca84e35980" } ], "modality": "video", "output": "", "passed": false, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 2, "sampling": { "fps": 1.0 }, "seconds": 0.013028, "usage": null, "validation": { "assertions": [], "finish_reason_allowed": false, "visible_answer": false } } ], "configuration": { "allowed_finish_reasons": [ "stop" ], "chat_template_kwargs": { "enable_thinking": false }, "concurrency": 1, "max_tokens": 1024, "reasoning_effort": null, "thinking_mode": "disabled", "timeout_seconds": 900.0 }, "corpus": { "cases": [ { "assertions": [ { "type": "contains_casefold", "value": "red car" }, { "type": "contains_casefold", "value": "green park" }, { "type": "contains_casefold", "value": "sunny day" } ], "id": "image-scene", "media": [ { "bytes": 13187, "kind": "image", "mime": "image/png", "path": "scene.png", "sha256": "30a1f11e40bf4e0a8595d39828e0aa362e8b28ac683c404a1068cc20bf7f7bda" } ], "modality": "image", "prompt": "Describe the scene and report every prominent label. Use the labels verbatim.", "repetitions": 2, "sampling": null }, { "assertions": [ { "type": "contains_casefold", "value": "ANVIL READY" }, { "type": "contains_casefold", "value": "42917" }, { "type": "contains_casefold", "value": "42" } ], "id": "image-ocr", "media": [ { "bytes": 15916, "kind": "image", "mime": "image/png", "path": "ocr.png", "sha256": "a8d5158019fb2b08ee9f6a153860ab2342384644f7c53ab58fdd45658124e83e" } ], "modality": "image", "prompt": "Transcribe all visible text exactly. Preserve every number.", "repetitions": 2, "sampling": null }, { "assertions": [ { "type": "ordered_casefold", "values": [ "Q1", "10", "Q2", "20", "Q3", "30" ] } ], "id": "image-chart", "media": [ { "bytes": 12635, "kind": "image", "mime": "image/png", "path": "chart.png", "sha256": "c905e453c4e16eb274e46992d1a901bbced8f2314b47b9d408971bb176ab2f3e" } ], "modality": "image", "prompt": "Read the chart and report each quarter with its value in ascending quarter order.", "repetitions": 2, "sampling": null }, { "assertions": [ { "type": "contains_casefold", "value": "ANVIL SERVING" }, { "type": "contains_casefold", "value": "READY" }, { "type": "contains_casefold", "value": "42" } ], "id": "image-ui", "media": [ { "bytes": 16940, "kind": "image", "mime": "image/png", "path": "ui.png", "sha256": "03fd14d8b68973c915de2848736d22eb6f67ba6bdb5da2a7a563085ac894e804" } ], "modality": "image", "prompt": "Report the application name, status, and GPU percentage.", "repetitions": 2, "sampling": null }, { "assertions": [ { "type": "contains_casefold", "value": "three" } ], "id": "image-spatial-count", "media": [ { "bytes": 8785, "kind": "image", "mime": "image/png", "path": "spatial.png", "sha256": "d48f47cfba9d65524d6a9ca37d593550e10400c1b5fa160bec8aea24e0164051" } ], "modality": "image", "prompt": "Count the blue boxes. Reply with the count written as an English word.", "repetitions": 2, "sampling": null }, { "assertions": [ { "type": "contains_casefold", "value": "meter b" }, { "type": "contains_casefold", "value": "75" } ], "id": "multi-image-comparison", "media": [ { "bytes": 8000, "kind": "image", "mime": "image/png", "path": "compare-a.png", "sha256": "7dac4f31d62c88d765150aa6051efea9ddaaa8c64312080be4c5b8f6ad96322a" }, { "bytes": 7679, "kind": "image", "mime": "image/png", "path": "compare-b.png", "sha256": "8f7cdfd733f1ee9731de6160e77af6aa50189edec6eaf9ddd7654a0c8300a43e" } ], "modality": "image", "prompt": "Compare the two meters. Identify which meter is higher and report its percentage.", "repetitions": 2, "sampling": null }, { "assertions": [ { "type": "ordered_casefold", "values": [ "red", "green" ] } ], "id": "video-temporal-order-10s", "media": [ { "bytes": 11627, "kind": "video", "mime": "video/mp4", "path": "temporal-order-10s.mp4", "sha256": "4d82c50cd2e500fc58df6d4930efec12649a0eaf236bee53d0d7c7ca84e35980" } ], "modality": "video", "prompt": "Report the two states in the order they appear.", "repetitions": 2, "sampling": { "fps": 1.0 } }, { "assertions": [ { "type": "ordered_casefold", "values": [ "closed", "open" ] } ], "id": "video-state-change-30s", "media": [ { "bytes": 14552, "kind": "video", "mime": "video/mp4", "path": "state-change-30s.mp4", "sha256": "4d720486425afff4a7450f0f7f3ce6e3eea591e063b66e48fc953415b9172e62" } ], "modality": "video", "prompt": "Describe the state change in temporal order.", "repetitions": 2, "sampling": { "fps": 1.0 } }, { "assertions": [ { "type": "contains_casefold", "value": "alert" }, { "type": "contains_casefold", "value": "42" } ], "id": "video-event-localization-60s", "media": [ { "bytes": 13829, "kind": "video", "mime": "video/mp4", "path": "event-localization-60s.mp4", "sha256": "5e0f40b3760e009f8ea0035b96bb4a8654fea58e0bb8a31a838eb78b224e7cbf" } ], "modality": "video", "prompt": "Identify the localized event and report its timestamp in seconds.", "repetitions": 2, "sampling": { "fps": 1.0 } }, { "assertions": [ { "type": "contains_casefold", "value": "ANVIL VIDEO OCR" }, { "type": "contains_casefold", "value": "7291" } ], "id": "video-ocr-30s", "media": [ { "bytes": 14731, "kind": "video", "mime": "video/mp4", "path": "video-ocr-30s.mp4", "sha256": "05c6cd1b1a9b4939f513944717f06d02dd5be54dd18b27870dccf2449b457e34" } ], "modality": "video", "prompt": "Transcribe every stable text label and preserve the numeric code.", "repetitions": 2, "sampling": { "num_frames": 16 } }, { "assertions": [ { "type": "ordered_casefold", "values": [ "alpha", "beta", "gamma" ] } ], "id": "video-continuity-120s", "media": [ { "bytes": 23622, "kind": "video", "mime": "video/mp4", "path": "continuity-120s.mp4", "sha256": "1ebc5459ed8c158ffb8294a0317005a13198748c529a49b66a2f76ff1c982b95" } ], "modality": "video", "prompt": "Report all phases in the order they appear across the full video.", "repetitions": 2, "sampling": { "num_frames": 32 } }, { "assertions": [ { "type": "ordered_casefold", "values": [ "red", "green" ] }, { "type": "contains_casefold", "value": "ready" } ], "id": "mixed-video-one-image", "media": [ { "bytes": 11627, "kind": "video", "mime": "video/mp4", "path": "temporal-order-10s.mp4", "sha256": "4d82c50cd2e500fc58df6d4930efec12649a0eaf236bee53d0d7c7ca84e35980" }, { "bytes": 16940, "kind": "image", "mime": "image/png", "path": "ui.png", "sha256": "03fd14d8b68973c915de2848736d22eb6f67ba6bdb5da2a7a563085ac894e804" } ], "modality": "mixed", "prompt": "Report the video's ordered colors and the dashboard's status.", "repetitions": 2, "sampling": { "fps": 1.0 } }, { "assertions": [ { "type": "contains_casefold", "value": "7291" }, { "type": "contains_casefold", "value": "42917" }, { "type": "contains_casefold", "value": "red car" }, { "type": "contains_casefold", "value": "Q3" }, { "type": "contains_casefold", "value": "ready" } ], "id": "mixed-video-four-images", "media": [ { "bytes": 14731, "kind": "video", "mime": "video/mp4", "path": "video-ocr-30s.mp4", "sha256": "05c6cd1b1a9b4939f513944717f06d02dd5be54dd18b27870dccf2449b457e34" }, { "bytes": 13187, "kind": "image", "mime": "image/png", "path": "scene.png", "sha256": "30a1f11e40bf4e0a8595d39828e0aa362e8b28ac683c404a1068cc20bf7f7bda" }, { "bytes": 15916, "kind": "image", "mime": "image/png", "path": "ocr.png", "sha256": "a8d5158019fb2b08ee9f6a153860ab2342384644f7c53ab58fdd45658124e83e" }, { "bytes": 12635, "kind": "image", "mime": "image/png", "path": "chart.png", "sha256": "c905e453c4e16eb274e46992d1a901bbced8f2314b47b9d408971bb176ab2f3e" }, { "bytes": 16940, "kind": "image", "mime": "image/png", "path": "ui.png", "sha256": "03fd14d8b68973c915de2848736d22eb6f67ba6bdb5da2a7a563085ac894e804" } ], "modality": "mixed", "prompt": "Inspect all five media inputs. Report both numeric codes, the red object, the highest chart quarter, and the dashboard status.", "repetitions": 2, "sampling": { "num_frames": 16 } }, { "assertions": [ { "type": "contains_casefold", "value": "color" } ], "id": "cc-video-chemical-traffic-light", "media": [ { "bytes": 459852, "kind": "video", "mime": "video/mp4", "path": "cc/chemical-traffic-light-normalized.mp4", "sha256": "1ab998a0339eaedb8494392be2c11f02bf9ce2f632fb01cf51b7089990d18f9c" } ], "modality": "video", "prompt": "Describe the experiment and the visible color changes.", "repetitions": 2, "sampling": { "num_frames": 24 } }, { "assertions": [ { "type": "contains_casefold", "value": "cat" }, { "type": "contains_casefold", "value": "door" } ], "id": "cc-video-cat-opening-door", "media": [ { "bytes": 490171, "kind": "video", "mime": "video/mp4", "path": "cc/cat-opening-door-normalized.mp4", "sha256": "b01ffe7800a3f74eead304a06bf35ece0e8917e107e1ce1e25837f3b5611cb61" } ], "modality": "video", "prompt": "Identify the animal and describe the main action it completes.", "repetitions": 2, "sampling": { "num_frames": 16 } } ], "path": "C:\\Users\\operator\\ai-code\\anvil-serving-wt-agents-a1-qwen-head-to-head\\benchmarks\\corpora\\agents-a1-v1\\corpus.json", "provenance": { "creative_commons": { "attribution": "cc/README.md", "role": "supplementary generalization evidence" }, "observed_date": "2026-07-28", "synthetic": { "generator": "generate.py", "license": "CC0-1.0", "runtime_image": "vllm/vllm-openai:nightly-f25953cc59f9b4ba9b04b16228d2b86dcfbcbdb1" } }, "schema": "multimodal-corpus/v1", "sha256": "ebff9dcc87a7fd13f801fc19eeea7271aec01a99fe560d721be99c1c9becad49" }, "failed_attempts": 18, "finished_at": "2026-07-29T10:33:43Z", "identity": { "base_url": "http://127.0.0.1:30010/v1", "engine_build_ref": "f6c161e872eacdbcdd16fdae647bd5318a12f49b", "engine_revision": null, "hardware": "rtx-pro-6000-sm120", "model": "qwen35-122b-a10b-nvfp4-video-262k", "model_revision": "98915d837c4e7c87ac8296d02e89de19b3207e6d", "runtime_image": "nvcr.io/nvidia/vllm:26.06-py3", "served_models": [ "qwen35-122b-a10b-nvfp4-video-262k" ] }, "passed": false, "passed_attempts": 12, "requests": 30, "run_id": "multimodal-20260729T103332Z", "schema": "multimodal-benchmark-evidence/v1", "started_at": "2026-07-29T10:33:32Z" }