{ "aggregates": { "case": { "cc-video-cat-opening-door": { "attempts": 2, "latency_p50_seconds": 2.812173, "latency_p95_seconds": 3.67815, "pass_rate": 1.0, "passed": 2 }, "cc-video-chemical-traffic-light": { "attempts": 2, "latency_p50_seconds": 2.616948, "latency_p95_seconds": 3.713631, "pass_rate": 0.5, "passed": 1 }, "image-chart": { "attempts": 2, "latency_p50_seconds": 0.6749695, "latency_p95_seconds": 0.681617, "pass_rate": 1.0, "passed": 2 }, "image-ocr": { "attempts": 2, "latency_p50_seconds": 0.265099, "latency_p95_seconds": 0.265438, "pass_rate": 1.0, "passed": 2 }, "image-scene": { "attempts": 2, "latency_p50_seconds": 0.857885, "latency_p95_seconds": 0.863628, "pass_rate": 1.0, "passed": 2 }, "image-spatial-count": { "attempts": 2, "latency_p50_seconds": 0.6990225000000001, "latency_p95_seconds": 1.199518, "pass_rate": 1.0, "passed": 2 }, "image-ui": { "attempts": 2, "latency_p50_seconds": 0.6206785, "latency_p95_seconds": 0.642092, "pass_rate": 1.0, "passed": 2 }, "mixed-video-four-images": { "attempts": 2, "latency_p50_seconds": 11.419781, "latency_p95_seconds": 20.75047, "pass_rate": 1.0, "passed": 2 }, "mixed-video-one-image": { "attempts": 2, "latency_p50_seconds": 1.3696325, "latency_p95_seconds": 1.818276, "pass_rate": 1.0, "passed": 2 }, "multi-image-comparison": { "attempts": 2, "latency_p50_seconds": 0.9590945, "latency_p95_seconds": 1.019792, "pass_rate": 1.0, "passed": 2 }, "video-continuity-120s": { "attempts": 2, "latency_p50_seconds": 1.1216665, "latency_p95_seconds": 1.303399, "pass_rate": 1.0, "passed": 2 }, "video-event-localization-60s": { "attempts": 2, "latency_p50_seconds": 0.536057, "latency_p95_seconds": 0.587466, "pass_rate": 0.5, "passed": 1 }, "video-ocr-30s": { "attempts": 2, "latency_p50_seconds": 0.38192349999999997, "latency_p95_seconds": 0.428313, "pass_rate": 1.0, "passed": 2 }, "video-state-change-30s": { "attempts": 2, "latency_p50_seconds": 1.7367855, "latency_p95_seconds": 2.107069, "pass_rate": 1.0, "passed": 2 }, "video-temporal-order-10s": { "attempts": 2, "latency_p50_seconds": 0.964112, "latency_p95_seconds": 1.097537, "pass_rate": 1.0, "passed": 2 } }, "modality": { "image": { "attempts": 12, "latency_p50_seconds": 0.6749695, "latency_p95_seconds": 1.199518, "pass_rate": 1.0, "passed": 12 }, "mixed": { "attempts": 4, "latency_p50_seconds": 1.953684, "latency_p95_seconds": 20.75047, "pass_rate": 1.0, "passed": 4 }, "video": { "attempts": 14, "latency_p50_seconds": 1.2004679999999999, "latency_p95_seconds": 3.713631, "pass_rate": 0.8571428571428571, "passed": 12 } } }, "attempts": [ { "case_id": "cc-video-cat-opening-door", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 490171, "kind": "video", "mime": "video/mp4", "path": "cc/cat-opening-door-normalized.mp4", "sha256": "b01ffe7800a3f74eead304a06bf35ece0e8917e107e1ce1e25837f3b5611cb61" } ], "modality": "video", "output": "The animal is a cat, and the main action it completes is opening a door.\n\nHere's a breakdown of how the cat accomplishes this:\n- The orange tabby cat first approaches the closed door and looks up at the handle.\n- It then jumps up, placing its front paws on the door to get leverage.\n- The cat drops to the floor and begins to push against the bottom edge of the door with its head and body.\n- After several attempts of pushing and wiggling, the door finally swings open, revealing a closet with items inside.", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 1, "sampling": { "num_frames": 16 }, "seconds": 3.67815, "usage": { "completion_tokens": 119, "prompt_tokens": 11808, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 11927 }, "validation": { "assertions": [ { "passed": true, "type": "contains_casefold", "value": "cat" }, { "passed": true, "type": "contains_casefold", "value": "door" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "cc-video-cat-opening-door", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 490171, "kind": "video", "mime": "video/mp4", "path": "cc/cat-opening-door-normalized.mp4", "sha256": "b01ffe7800a3f74eead304a06bf35ece0e8917e107e1ce1e25837f3b5611cb61" } ], "modality": "video", "output": "The animal in the video is a cat, and its main action is opening a door.\n\nHere's a breakdown of how it accomplishes this:\n- The cat first jumps up and pushes against the door with its paws, causing it to crack open slightly.\n- It then drops to the floor and uses its head and body to push the bottom of the door, widening the gap.\n- Finally, it jumps up again to push the upper part of the door, swinging it open enough to walk through the doorway.", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 2, "sampling": { "num_frames": 16 }, "seconds": 1.946196, "usage": { "completion_tokens": 108, "prompt_tokens": 11808, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 11916 }, "validation": { "assertions": [ { "passed": true, "type": "contains_casefold", "value": "cat" }, { "passed": true, "type": "contains_casefold", "value": "door" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "cc-video-chemical-traffic-light", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 459852, "kind": "video", "mime": "video/mp4", "path": "cc/chemical-traffic-light-normalized.mp4", "sha256": "1ab998a0339eaedb8494392be2c11f02bf9ce2f632fb01cf51b7089990d18f9c" } ], "modality": "video", "output": "The experiment depicted in the video is called the \"Chemical Traffic Light.\" It involves a large, clear bottle filled with a yellow liquid. The bottle is placed on a black table in what appears to be a laboratory setting. The video begins by showing the bottle with the text \"Chemical Traffic Light\" displayed over it.\n\nAs the video progresses, the following sequence of events occurs:\n\n1. **Initial State**: The liquid in the bottle is yellow.\n2. **Shaking the Bottle**: A hand enters the frame and shakes the bottle vigorously. As the bottle is shaken, the liquid changes from yellow to green.\n3. **Color Change Explanation**: The text \"Green is the oxidized state\" appears on the screen, indicating that the green color represents the oxidized state of the chemical reaction.\n4. **Settling Down**: After shaking, the bottle is set back on the table, and the liquid gradually changes from green to orange, and finally back to yellow.\n\nThe color changes in the liquid are as follows:\n- **Yellow** (initial state)\n- **Green** (after shaking, representing the oxidized state)\n- **Orange** (as the liquid settles)\n- **Yellow** (final state, returning to the initial color)\n\nThis sequence of color changes mimics the appearance of a traffic light, hence the name \"Chemical Traffic Light.\" The experiment demonstrates a reversible chemical reaction where the color of the solution changes in response to being shaken and then allowed to settle.", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 1, "sampling": { "num_frames": 24 }, "seconds": 3.713631, "usage": { "completion_tokens": 313, "prompt_tokens": 12221, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 12534 }, "validation": { "assertions": [ { "passed": true, "type": "contains_casefold", "value": "color" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "cc-video-chemical-traffic-light", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 459852, "kind": "video", "mime": "video/mp4", "path": "cc/chemical-traffic-light-normalized.mp4", "sha256": "1ab998a0339eaedb8494392be2c11f02bf9ce2f632fb01cf51b7089990d18f9c" } ], "modality": "video", "output": "The experiment is called the \"Chemical Traffic Light.\" It involves a bottle containing a yellow liquid, which is a mixture of NaOH, dextrose (glucose), and 1% indigo carmine. When the bottle is shaken, the liquid turns green, indicating the oxidized state. As the bottle sits, the liquid gradually changes back to yellow, passing through shades of orange and red along the way.", "passed": false, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 2, "sampling": { "num_frames": 24 }, "seconds": 1.520265, "usage": { "completion_tokens": 86, "prompt_tokens": 12221, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 12307 }, "validation": { "assertions": [ { "passed": false, "type": "contains_casefold", "value": "color" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "image-chart", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 12635, "kind": "image", "mime": "image/png", "path": "chart.png", "sha256": "c905e453c4e16eb274e46992d1a901bbced8f2314b47b9d408971bb176ab2f3e" } ], "modality": "image", "output": "Based on the provided bar chart titled \"QUARTERLY UNITS\", here are the quarters and their corresponding values in ascending quarter order:\n\n- **Q1**: 10 units \n- **Q2**: 20 units \n- **Q3**: 30 units \n\nThe chart shows a clear upward trend in units sold or produced each quarter, with Q3 having the highest value at 30 units.", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 1, "sampling": null, "seconds": 0.681617, "usage": { "completion_tokens": 88, "prompt_tokens": 249, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 337 }, "validation": { "assertions": [ { "passed": true, "type": "ordered_casefold", "values": [ "Q1", "10", "Q2", "20", "Q3", "30" ] } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "image-chart", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 12635, "kind": "image", "mime": "image/png", "path": "chart.png", "sha256": "c905e453c4e16eb274e46992d1a901bbced8f2314b47b9d408971bb176ab2f3e" } ], "modality": "image", "output": "Based on the provided bar chart titled \"QUARTERLY UNITS\", here are the quarters and their corresponding values in ascending quarter order:\n\n- **Q1**: 10 units \n- **Q2**: 20 units \n- **Q3**: 30 units \n\n*(Note: The chart only displays data for Q1, Q2, and Q3. No Q4 is shown.)*", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 2, "sampling": null, "seconds": 0.668322, "usage": { "completion_tokens": 86, "prompt_tokens": 249, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 335 }, "validation": { "assertions": [ { "passed": true, "type": "ordered_casefold", "values": [ "Q1", "10", "Q2", "20", "Q3", "30" ] } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "image-ocr", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 15916, "kind": "image", "mime": "image/png", "path": "ocr.png", "sha256": "a8d5158019fb2b08ee9f6a153860ab2342384644f7c53ab58fdd45658124e83e" } ], "modality": "image", "output": "ANVIL READY\nCODE 42917\nTEMPERATURE 42 C", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 1, "sampling": null, "seconds": 0.265438, "usage": { "completion_tokens": 21, "prompt_tokens": 245, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 266 }, "validation": { "assertions": [ { "passed": true, "type": "contains_casefold", "value": "ANVIL READY" }, { "passed": true, "type": "contains_casefold", "value": "42917" }, { "passed": true, "type": "contains_casefold", "value": "42" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "image-ocr", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 15916, "kind": "image", "mime": "image/png", "path": "ocr.png", "sha256": "a8d5158019fb2b08ee9f6a153860ab2342384644f7c53ab58fdd45658124e83e" } ], "modality": "image", "output": "ANVIL READY\nCODE 42917\nTEMPERATURE 42 C", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 2, "sampling": null, "seconds": 0.26476, "usage": { "completion_tokens": 21, "prompt_tokens": 245, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 266 }, "validation": { "assertions": [ { "passed": true, "type": "contains_casefold", "value": "ANVIL READY" }, { "passed": true, "type": "contains_casefold", "value": "42917" }, { "passed": true, "type": "contains_casefold", "value": "42" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "image-scene", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 13187, "kind": "image", "mime": "image/png", "path": "scene.png", "sha256": "30a1f11e40bf4e0a8595d39828e0aa362e8b28ac683c404a1068cc20bf7f7bda" } ], "modality": "image", "output": "The scene depicts a simple, stylized illustration with a dark navy blue background. At the top, in bold yellow capital letters, is the label \"SUNNY DAY\". Below that, on the left side, there\u2019s a red rectangle labeled \"RED CAR\" in white uppercase text above it. Beneath the red car and extending to the right is a larger green rectangle labeled \"GREEN PARK\" in white uppercase text on its right side.\n\nProminent labels (verbatim):\n- SUNNY DAY\n- RED CAR\n- GREEN PARK", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 1, "sampling": null, "seconds": 0.863628, "usage": { "completion_tokens": 110, "prompt_tokens": 249, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 359 }, "validation": { "assertions": [ { "passed": true, "type": "contains_casefold", "value": "red car" }, { "passed": true, "type": "contains_casefold", "value": "green park" }, { "passed": true, "type": "contains_casefold", "value": "sunny day" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "image-scene", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 13187, "kind": "image", "mime": "image/png", "path": "scene.png", "sha256": "30a1f11e40bf4e0a8595d39828e0aa362e8b28ac683c404a1068cc20bf7f7bda" } ], "modality": "image", "output": "The scene depicts a simple, stylized illustration with a dark navy blue background. At the top, in bold yellow capital letters, is the label \"SUNNY DAY\". Below that, on the left side, there\u2019s a red rectangle labeled \"RED CAR\" in white uppercase text above it. Beneath the red car and extending to the right is a larger green rectangle labeled \"GREEN PARK\" in white uppercase text on its right side.\n\nProminent labels (verbatim):\n- SUNNY DAY\n- RED CAR\n- GREEN PARK", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 2, "sampling": null, "seconds": 0.852142, "usage": { "completion_tokens": 110, "prompt_tokens": 249, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 359 }, "validation": { "assertions": [ { "passed": true, "type": "contains_casefold", "value": "red car" }, { "passed": true, "type": "contains_casefold", "value": "green park" }, { "passed": true, "type": "contains_casefold", "value": "sunny day" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "image-spatial-count", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 8785, "kind": "image", "mime": "image/png", "path": "spatial.png", "sha256": "d48f47cfba9d65524d6a9ca37d593550e10400c1b5fa160bec8aea24e0164051" } ], "modality": "image", "output": "three", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 1, "sampling": null, "seconds": 0.198527, "usage": { "completion_tokens": 2, "prompt_tokens": 249, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 251 }, "validation": { "assertions": [ { "passed": true, "type": "contains_casefold", "value": "three" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "image-spatial-count", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 8785, "kind": "image", "mime": "image/png", "path": "spatial.png", "sha256": "d48f47cfba9d65524d6a9ca37d593550e10400c1b5fa160bec8aea24e0164051" } ], "modality": "image", "output": "three", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 2, "sampling": null, "seconds": 1.199518, "usage": { "completion_tokens": 2, "prompt_tokens": 249, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 251 }, "validation": { "assertions": [ { "passed": true, "type": "contains_casefold", "value": "three" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "image-ui", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 16940, "kind": "image", "mime": "image/png", "path": "ui.png", "sha256": "03fd14d8b68973c915de2848736d22eb6f67ba6bdb5da2a7a563085ac894e804" } ], "modality": "image", "output": "Based on the image provided:\n\n- **Application Name**: ANVIL SERVING \n- **Status**: READY (displayed in green) \n- **GPU Usage**: 42 PERCENT (displayed in blue)\n\nThis appears to be a status dashboard or monitoring interface for an application called \u201cAnvil Serving,\u201d indicating it is currently ready to handle requests and is using 42% of GPU resources.", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 1, "sampling": null, "seconds": 0.642092, "usage": { "completion_tokens": 86, "prompt_tokens": 245, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 331 }, "validation": { "assertions": [ { "passed": true, "type": "contains_casefold", "value": "ANVIL SERVING" }, { "passed": true, "type": "contains_casefold", "value": "READY" }, { "passed": true, "type": "contains_casefold", "value": "42" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "image-ui", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 16940, "kind": "image", "mime": "image/png", "path": "ui.png", "sha256": "03fd14d8b68973c915de2848736d22eb6f67ba6bdb5da2a7a563085ac894e804" } ], "modality": "image", "output": "Based on the image provided:\n\n- **Application Name**: ANVIL SERVING \n- **Status**: READY \n- **GPU Percentage**: 42%\n\nThis appears to be a status dashboard or monitoring display for an application called \u201cAnvil Serving,\u201d indicating it is currently ready to serve requests and is using 42% of GPU resources.", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 2, "sampling": null, "seconds": 0.599265, "usage": { "completion_tokens": 73, "prompt_tokens": 245, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 318 }, "validation": { "assertions": [ { "passed": true, "type": "contains_casefold", "value": "ANVIL SERVING" }, { "passed": true, "type": "contains_casefold", "value": "READY" }, { "passed": true, "type": "contains_casefold", "value": "42" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "mixed-video-four-images", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 14731, "kind": "video", "mime": "video/mp4", "path": "video-ocr-30s.mp4", "sha256": "05c6cd1b1a9b4939f513944717f06d02dd5be54dd18b27870dccf2449b457e34" }, { "bytes": 13187, "kind": "image", "mime": "image/png", "path": "scene.png", "sha256": "30a1f11e40bf4e0a8595d39828e0aa362e8b28ac683c404a1068cc20bf7f7bda" }, { "bytes": 15916, "kind": "image", "mime": "image/png", "path": "ocr.png", "sha256": "a8d5158019fb2b08ee9f6a153860ab2342384644f7c53ab58fdd45658124e83e" }, { "bytes": 12635, "kind": "image", "mime": "image/png", "path": "chart.png", "sha256": "c905e453c4e16eb274e46992d1a901bbced8f2314b47b9d408971bb176ab2f3e" }, { "bytes": 16940, "kind": "image", "mime": "image/png", "path": "ui.png", "sha256": "03fd14d8b68973c915de2848736d22eb6f67ba6bdb5da2a7a563085ac894e804" } ], "modality": "mixed", "output": "Based on the five media inputs provided, here is the requested information:\n\n- **Numeric Codes:** The two numeric codes found are `7291` (from the first image) and `42917` (from the third image).\n- **Red Object:** The red object is a rectangle labeled \"RED CAR\" in the second image.\n- **Highest Chart Quarter:** In the bar chart from the fourth image, the highest quarter is Q3, with a value of 30.\n- **Dashboard Status:** The status shown on the dashboard in the fifth image is \"STATUS READY\".", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 1, "sampling": { "num_frames": 16 }, "seconds": 20.75047, "usage": { "completion_tokens": 127, "prompt_tokens": 2737, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 2864 }, "validation": { "assertions": [ { "passed": true, "type": "contains_casefold", "value": "7291" }, { "passed": true, "type": "contains_casefold", "value": "42917" }, { "passed": true, "type": "contains_casefold", "value": "red car" }, { "passed": true, "type": "contains_casefold", "value": "Q3" }, { "passed": true, "type": "contains_casefold", "value": "ready" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "mixed-video-four-images", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 14731, "kind": "video", "mime": "video/mp4", "path": "video-ocr-30s.mp4", "sha256": "05c6cd1b1a9b4939f513944717f06d02dd5be54dd18b27870dccf2449b457e34" }, { "bytes": 13187, "kind": "image", "mime": "image/png", "path": "scene.png", "sha256": "30a1f11e40bf4e0a8595d39828e0aa362e8b28ac683c404a1068cc20bf7f7bda" }, { "bytes": 15916, "kind": "image", "mime": "image/png", "path": "ocr.png", "sha256": "a8d5158019fb2b08ee9f6a153860ab2342384644f7c53ab58fdd45658124e83e" }, { "bytes": 12635, "kind": "image", "mime": "image/png", "path": "chart.png", "sha256": "c905e453c4e16eb274e46992d1a901bbced8f2314b47b9d408971bb176ab2f3e" }, { "bytes": 16940, "kind": "image", "mime": "image/png", "path": "ui.png", "sha256": "03fd14d8b68973c915de2848736d22eb6f67ba6bdb5da2a7a563085ac894e804" } ], "modality": "mixed", "output": "Based on the provided media inputs, here is the requested information:\n\n- **Numeric Codes:** The two numeric codes found are `7291` (from the first image) and `42917` (from the third image).\n- **Red Object:** The red object is a rectangle labeled \"RED CAR\" in the second image.\n- **Highest Chart Quarter:** In the bar chart from the fourth image, the highest quarter is Q3, with a value of 30.\n- **Dashboard Status:** The status shown on the dashboard in the fifth image is \"STATUS READY\".", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 2, "sampling": { "num_frames": 16 }, "seconds": 2.089092, "usage": { "completion_tokens": 126, "prompt_tokens": 2737, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 2863 }, "validation": { "assertions": [ { "passed": true, "type": "contains_casefold", "value": "7291" }, { "passed": true, "type": "contains_casefold", "value": "42917" }, { "passed": true, "type": "contains_casefold", "value": "red car" }, { "passed": true, "type": "contains_casefold", "value": "Q3" }, { "passed": true, "type": "contains_casefold", "value": "ready" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "mixed-video-one-image", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 11627, "kind": "video", "mime": "video/mp4", "path": "temporal-order-10s.mp4", "sha256": "4d82c50cd2e500fc58df6d4930efec12649a0eaf236bee53d0d7c7ca84e35980" }, { "bytes": 16940, "kind": "image", "mime": "image/png", "path": "ui.png", "sha256": "03fd14d8b68973c915de2848736d22eb6f67ba6bdb5da2a7a563085ac894e804" } ], "modality": "mixed", "output": "Based on the video provided, here is a report of the ordered colors and the dashboard's status:\n\n- **Ordered Colors:** The video first displays the text \"FIRST RED\" in red, followed by the text \"THEN GREEN\" in green. Therefore, the ordered colors are red, then green.\n- **Dashboard Status:** The final frame shows a dashboard titled \"ANVIL SERVING\". The status on this dashboard is \"STATUS READY\" and the GPU usage is \"GPU 42 PERCENT\".", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 1, "sampling": { "fps": 1.0 }, "seconds": 1.818276, "usage": { "completion_tokens": 105, "prompt_tokens": 848, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 953 }, "validation": { "assertions": [ { "passed": true, "type": "ordered_casefold", "values": [ "red", "green" ] }, { "passed": true, "type": "contains_casefold", "value": "ready" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "mixed-video-one-image", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 11627, "kind": "video", "mime": "video/mp4", "path": "temporal-order-10s.mp4", "sha256": "4d82c50cd2e500fc58df6d4930efec12649a0eaf236bee53d0d7c7ca84e35980" }, { "bytes": 16940, "kind": "image", "mime": "image/png", "path": "ui.png", "sha256": "03fd14d8b68973c915de2848736d22eb6f67ba6bdb5da2a7a563085ac894e804" } ], "modality": "mixed", "output": "Based on the video provided, here is a report of the ordered colors and the dashboard's status:\n\n**Ordered Colors:**\nThe video displays text in the following order:\n1. **Red:** The text \"FIRST RED\" appears in red.\n2. **Green:** The text \"THEN GREEN\" appears in green.\n\n**Dashboard Status:**\nThe final frame shows a dashboard with the following information:\n* **Title:** ANVIL SERVING\n* **Status:** STATUS READY (displayed in green)\n* **GPU Usage:** GPU 42 PERCENT (displayed in blue)", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 2, "sampling": { "fps": 1.0 }, "seconds": 0.920989, "usage": { "completion_tokens": 131, "prompt_tokens": 848, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 979 }, "validation": { "assertions": [ { "passed": true, "type": "ordered_casefold", "values": [ "red", "green" ] }, { "passed": true, "type": "contains_casefold", "value": "ready" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "multi-image-comparison", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 8000, "kind": "image", "mime": "image/png", "path": "compare-a.png", "sha256": "7dac4f31d62c88d765150aa6051efea9ddaaa8c64312080be4c5b8f6ad96322a" }, { "bytes": 7679, "kind": "image", "mime": "image/png", "path": "compare-b.png", "sha256": "8f7cdfd733f1ee9731de6160e77af6aa50189edec6eaf9ddd7654a0c8300a43e" } ], "modality": "image", "output": "Based on the visual information provided in the two images, here is a comparison of the meters:\n\n- **Meter A** shows a value of 25 percent. This is indicated by the orange bar filling one-quarter of the total bar length and the text \"25 PERCENT\" below it.\n- **Meter B** shows a value of 75 percent. This is indicated by the green bar filling three-quarters of the total bar length and the text \"75 PERCENT\" below it.\n\n---\n\nComparing the two values, $75\\%$ is greater than $25\\%$.\n\nThe meter that is higher is **Meter B**, with a percentage of **75%**.", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 1, "sampling": null, "seconds": 1.019792, "usage": { "completion_tokens": 148, "prompt_tokens": 471, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 619 }, "validation": { "assertions": [ { "passed": true, "type": "contains_casefold", "value": "meter b" }, { "passed": true, "type": "contains_casefold", "value": "75" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "multi-image-comparison", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 8000, "kind": "image", "mime": "image/png", "path": "compare-a.png", "sha256": "7dac4f31d62c88d765150aa6051efea9ddaaa8c64312080be4c5b8f6ad96322a" }, { "bytes": 7679, "kind": "image", "mime": "image/png", "path": "compare-b.png", "sha256": "8f7cdfd733f1ee9731de6160e77af6aa50189edec6eaf9ddd7654a0c8300a43e" } ], "modality": "image", "output": "Based on the visual information provided in the two images, we can compare the meters as follows:\n\n- **Meter A** shows a value of 25 percent. The orange bar fills one-quarter of the total track length.\n- **Meter B** shows a value of 75 percent. The green bar fills three-quarters of the total track length.\n\nBy comparing the two values, $75\\%$ is greater than $25\\%$.\n\n---\n\nThe meter that is higher is **Meter B**, and its percentage is **75 percent**.", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 2, "sampling": null, "seconds": 0.898397, "usage": { "completion_tokens": 118, "prompt_tokens": 471, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 589 }, "validation": { "assertions": [ { "passed": true, "type": "contains_casefold", "value": "meter b" }, { "passed": true, "type": "contains_casefold", "value": "75" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "video-continuity-120s", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 23622, "kind": "video", "mime": "video/mp4", "path": "continuity-120s.mp4", "sha256": "1ebc5459ed8c158ffb8294a0317005a13198748c529a49b66a2f76ff1c982b95" } ], "modality": "video", "output": "Based on the video frames provided, here are the phases in the order they appear:\n\n1. **PHASE ALPHA** (appears first, in red text)\n2. **PHASE BETA** (appears second, in orange text)\n3. **PHASE GAMMA** (appears third, in green text)", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 1, "sampling": { "num_frames": 32 }, "seconds": 1.303399, "usage": { "completion_tokens": 74, "prompt_tokens": 7292, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 7366 }, "validation": { "assertions": [ { "passed": true, "type": "ordered_casefold", "values": [ "alpha", "beta", "gamma" ] } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "video-continuity-120s", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 23622, "kind": "video", "mime": "video/mp4", "path": "continuity-120s.mp4", "sha256": "1ebc5459ed8c158ffb8294a0317005a13198748c529a49b66a2f76ff1c982b95" } ], "modality": "video", "output": "The phases appear in the following order:\n\n1. **Phase Alpha** (0:00 - 0:39)\n2. **Phase Beta** (0:40 - 1:19)\n3. **Phase Gamma** (1:20 - 1:58)", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 2, "sampling": { "num_frames": 32 }, "seconds": 0.939934, "usage": { "completion_tokens": 66, "prompt_tokens": 7292, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 7358 }, "validation": { "assertions": [ { "passed": true, "type": "ordered_casefold", "values": [ "alpha", "beta", "gamma" ] } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "video-event-localization-60s", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 13829, "kind": "video", "mime": "video/mp4", "path": "event-localization-60s.mp4", "sha256": "5e0f40b3760e009f8ea0035b96bb4a8654fea58e0bb8a31a838eb78b224e7cbf" } ], "modality": "video", "output": "The localized event occurs between 42 and 47 seconds.", "passed": false, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 1, "sampling": { "fps": 1.0 }, "seconds": 0.484648, "usage": { "completion_tokens": 15, "prompt_tokens": 3651, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 3666 }, "validation": { "assertions": [ { "passed": false, "type": "contains_casefold", "value": "alert" }, { "passed": true, "type": "contains_casefold", "value": "42" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "video-event-localization-60s", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 13829, "kind": "video", "mime": "video/mp4", "path": "event-localization-60s.mp4", "sha256": "5e0f40b3760e009f8ea0035b96bb4a8654fea58e0bb8a31a838eb78b224e7cbf" } ], "modality": "video", "output": "The localized event is 'ALERT AT 42 SECONDS' text appears, occurring between 41 and 47 seconds.", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 2, "sampling": { "fps": 1.0 }, "seconds": 0.587466, "usage": { "completion_tokens": 29, "prompt_tokens": 3651, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 3680 }, "validation": { "assertions": [ { "passed": true, "type": "contains_casefold", "value": "alert" }, { "passed": true, "type": "contains_casefold", "value": "42" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "video-ocr-30s", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 14731, "kind": "video", "mime": "video/mp4", "path": "video-ocr-30s.mp4", "sha256": "05c6cd1b1a9b4939f513944717f06d02dd5be54dd18b27870dccf2449b457e34" } ], "modality": "video", "output": "ANVIL VIDEO OCR\nCODE 7291", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 1, "sampling": { "num_frames": 16 }, "seconds": 0.428313, "usage": { "completion_tokens": 13, "prompt_tokens": 1836, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 1849 }, "validation": { "assertions": [ { "passed": true, "type": "contains_casefold", "value": "ANVIL VIDEO OCR" }, { "passed": true, "type": "contains_casefold", "value": "7291" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "video-ocr-30s", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 14731, "kind": "video", "mime": "video/mp4", "path": "video-ocr-30s.mp4", "sha256": "05c6cd1b1a9b4939f513944717f06d02dd5be54dd18b27870dccf2449b457e34" } ], "modality": "video", "output": "ANVIL VIDEO OCR\nCODE 7291", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 2, "sampling": { "num_frames": 16 }, "seconds": 0.335534, "usage": { "completion_tokens": 13, "prompt_tokens": 1836, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 1849 }, "validation": { "assertions": [ { "passed": true, "type": "contains_casefold", "value": "ANVIL VIDEO OCR" }, { "passed": true, "type": "contains_casefold", "value": "7291" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "video-state-change-30s", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 14552, "kind": "video", "mime": "video/mp4", "path": "state-change-30s.mp4", "sha256": "4d720486425afff4a7450f0f7f3ce6e3eea591e063b66e48fc953415b9172e62" } ], "modality": "video", "output": "Based on the visual information provided in the video frames, the state change occurs in the following temporal order:\n\n1. **Initial State (00:00 - 00:14):** The video begins with a dark blue background displaying the text \"DOOR CLOSED\" in white, capital letters. This state remains static for the first 14 seconds.\n2. **Transition (around 00:15):** The text on the screen changes.\n3. **Final State (00:15 - 00:28):** The text changes to \"DOOR OPEN\" and its color changes from white to bright green. This new state remains static until the end of the provided frames.\n\nIn summary, the state changes from **\"DOOR CLOSED\" (in white text)** to **\"DOOR OPEN\" (in green text)**.", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 1, "sampling": { "fps": 1.0 }, "seconds": 1.366502, "usage": { "completion_tokens": 181, "prompt_tokens": 1832, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 2013 }, "validation": { "assertions": [ { "passed": true, "type": "ordered_casefold", "values": [ "closed", "open" ] } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "video-state-change-30s", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 14552, "kind": "video", "mime": "video/mp4", "path": "state-change-30s.mp4", "sha256": "4d720486425afff4a7450f0f7f3ce6e3eea591e063b66e48fc953415b9172e62" } ], "modality": "video", "output": "Based on the video frames provided, here is the state change described in temporal order:\n\n* **00:00 - 00:14:** The video begins with a static display showing the text \"DOOR CLOSED\" in white letters against a dark blue background. This indicates the initial state of the door is closed.\n* **00:15 - 00:28:** At the 15-second mark, the display changes. The text now reads \"DOOR OPEN\" in bright green letters on the same dark blue background. This indicates a transition where the door has changed from a closed state to an open state.", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 2, "sampling": { "fps": 1.0 }, "seconds": 2.107069, "usage": { "completion_tokens": 137, "prompt_tokens": 1832, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 1969 }, "validation": { "assertions": [ { "passed": true, "type": "ordered_casefold", "values": [ "closed", "open" ] } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "video-temporal-order-10s", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 11627, "kind": "video", "mime": "video/mp4", "path": "temporal-order-10s.mp4", "sha256": "4d82c50cd2e500fc58df6d4930efec12649a0eaf236bee53d0d7c7ca84e35980" } ], "modality": "video", "output": "The video begins with a dark blue background and the words \"FIRST RED\" written in red text. This initial state is maintained for several seconds, from 00:00 to approximately 00:03. Then, around the 00:04 mark, the text changes to \"THEN GREEN,\" which is displayed in green text on the same dark blue background. This second state persists until the end of the video at 00:08. Therefore, the two states in the order they appear are: first, \"FIRST RED\" in red text, and second, \"THEN GREEN\" in green text.", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 1, "sampling": { "fps": 1.0 }, "seconds": 1.097537, "usage": { "completion_tokens": 130, "prompt_tokens": 624, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 754 }, "validation": { "assertions": [ { "passed": true, "type": "ordered_casefold", "values": [ "red", "green" ] } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "video-temporal-order-10s", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 11627, "kind": "video", "mime": "video/mp4", "path": "temporal-order-10s.mp4", "sha256": "4d82c50cd2e500fc58df6d4930efec12649a0eaf236bee53d0d7c7ca84e35980" } ], "modality": "video", "output": "The video shows two states in the following order:\n\n1. **FIRST RED**: The video begins with the text \"FIRST RED\" displayed in red font on a dark blue background. This state is visible from 00:00 to 00:03.\n2. **THEN GREEN**: The video then transitions to show the text \"THEN GREEN\" in green font on the same dark blue background. This state is visible from 00:04 to 00:08.", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 2, "sampling": { "fps": 1.0 }, "seconds": 0.830687, "usage": { "completion_tokens": 107, "prompt_tokens": 624, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 731 }, "validation": { "assertions": [ { "passed": true, "type": "ordered_casefold", "values": [ "red", "green" ] } ], "finish_reason_allowed": true, "visible_answer": true } } ], "configuration": { "allowed_finish_reasons": [ "length", "stop" ], "chat_template_kwargs": { "enable_thinking": false }, "concurrency": 1, "max_images_per_request": 4, "max_tokens": 512, "max_videos_per_request": 1, "reasoning_effort": null, "thinking_mode": "disabled", "timeout_seconds": 900.0 }, "corpus": { "cases": [ { "assertions": [ { "type": "contains_casefold", "value": "red car" }, { "type": "contains_casefold", "value": "green park" }, { "type": "contains_casefold", "value": "sunny day" } ], "id": "image-scene", "media": [ { "bytes": 13187, "kind": "image", "mime": "image/png", "path": "scene.png", "sha256": "30a1f11e40bf4e0a8595d39828e0aa362e8b28ac683c404a1068cc20bf7f7bda" } ], "modality": "image", "prompt": "Describe the scene and report every prominent label. Use the labels verbatim.", "repetitions": 2, "sampling": null }, { "assertions": [ { "type": "contains_casefold", "value": "ANVIL READY" }, { "type": "contains_casefold", "value": "42917" }, { "type": "contains_casefold", "value": "42" } ], "id": "image-ocr", "media": [ { "bytes": 15916, "kind": "image", "mime": "image/png", "path": "ocr.png", "sha256": "a8d5158019fb2b08ee9f6a153860ab2342384644f7c53ab58fdd45658124e83e" } ], "modality": "image", "prompt": "Transcribe all visible text exactly. Preserve every number.", "repetitions": 2, "sampling": null }, { "assertions": [ { "type": "ordered_casefold", "values": [ "Q1", "10", "Q2", "20", "Q3", "30" ] } ], "id": "image-chart", "media": [ { "bytes": 12635, "kind": "image", "mime": "image/png", "path": "chart.png", "sha256": "c905e453c4e16eb274e46992d1a901bbced8f2314b47b9d408971bb176ab2f3e" } ], "modality": "image", "prompt": "Read the chart and report each quarter with its value in ascending quarter order.", "repetitions": 2, "sampling": null }, { "assertions": [ { "type": "contains_casefold", "value": "ANVIL SERVING" }, { "type": "contains_casefold", "value": "READY" }, { "type": "contains_casefold", "value": "42" } ], "id": "image-ui", "media": [ { "bytes": 16940, "kind": "image", "mime": "image/png", "path": "ui.png", "sha256": "03fd14d8b68973c915de2848736d22eb6f67ba6bdb5da2a7a563085ac894e804" } ], "modality": "image", "prompt": "Report the application name, status, and GPU percentage.", "repetitions": 2, "sampling": null }, { "assertions": [ { "type": "contains_casefold", "value": "three" } ], "id": "image-spatial-count", "media": [ { "bytes": 8785, "kind": "image", "mime": "image/png", "path": "spatial.png", "sha256": "d48f47cfba9d65524d6a9ca37d593550e10400c1b5fa160bec8aea24e0164051" } ], "modality": "image", "prompt": "Count the blue boxes. Reply with the count written as an English word.", "repetitions": 2, "sampling": null }, { "assertions": [ { "type": "contains_casefold", "value": "meter b" }, { "type": "contains_casefold", "value": "75" } ], "id": "multi-image-comparison", "media": [ { "bytes": 8000, "kind": "image", "mime": "image/png", "path": "compare-a.png", "sha256": "7dac4f31d62c88d765150aa6051efea9ddaaa8c64312080be4c5b8f6ad96322a" }, { "bytes": 7679, "kind": "image", "mime": "image/png", "path": "compare-b.png", "sha256": "8f7cdfd733f1ee9731de6160e77af6aa50189edec6eaf9ddd7654a0c8300a43e" } ], "modality": "image", "prompt": "Compare the two meters. Identify which meter is higher and report its percentage.", "repetitions": 2, "sampling": null }, { "assertions": [ { "type": "ordered_casefold", "values": [ "red", "green" ] } ], "id": "video-temporal-order-10s", "media": [ { "bytes": 11627, "kind": "video", "mime": "video/mp4", "path": "temporal-order-10s.mp4", "sha256": "4d82c50cd2e500fc58df6d4930efec12649a0eaf236bee53d0d7c7ca84e35980" } ], "modality": "video", "prompt": "Report the two states in the order they appear.", "repetitions": 2, "sampling": { "fps": 1.0 } }, { "assertions": [ { "type": "ordered_casefold", "values": [ "closed", "open" ] } ], "id": "video-state-change-30s", "media": [ { "bytes": 14552, "kind": "video", "mime": "video/mp4", "path": "state-change-30s.mp4", "sha256": "4d720486425afff4a7450f0f7f3ce6e3eea591e063b66e48fc953415b9172e62" } ], "modality": "video", "prompt": "Describe the state change in temporal order.", "repetitions": 2, "sampling": { "fps": 1.0 } }, { "assertions": [ { "type": "contains_casefold", "value": "alert" }, { "type": "contains_casefold", "value": "42" } ], "id": "video-event-localization-60s", "media": [ { "bytes": 13829, "kind": "video", "mime": "video/mp4", "path": "event-localization-60s.mp4", "sha256": "5e0f40b3760e009f8ea0035b96bb4a8654fea58e0bb8a31a838eb78b224e7cbf" } ], "modality": "video", "prompt": "Identify the localized event and report its timestamp in seconds.", "repetitions": 2, "sampling": { "fps": 1.0 } }, { "assertions": [ { "type": "contains_casefold", "value": "ANVIL VIDEO OCR" }, { "type": "contains_casefold", "value": "7291" } ], "id": "video-ocr-30s", "media": [ { "bytes": 14731, "kind": "video", "mime": "video/mp4", "path": "video-ocr-30s.mp4", "sha256": "05c6cd1b1a9b4939f513944717f06d02dd5be54dd18b27870dccf2449b457e34" } ], "modality": "video", "prompt": "Transcribe every stable text label and preserve the numeric code.", "repetitions": 2, "sampling": { "num_frames": 16 } }, { "assertions": [ { "type": "ordered_casefold", "values": [ "alpha", "beta", "gamma" ] } ], "id": "video-continuity-120s", "media": [ { "bytes": 23622, "kind": "video", "mime": "video/mp4", "path": "continuity-120s.mp4", "sha256": "1ebc5459ed8c158ffb8294a0317005a13198748c529a49b66a2f76ff1c982b95" } ], "modality": "video", "prompt": "Report all phases in the order they appear across the full video.", "repetitions": 2, "sampling": { "num_frames": 32 } }, { "assertions": [ { "type": "ordered_casefold", "values": [ "red", "green" ] }, { "type": "contains_casefold", "value": "ready" } ], "id": "mixed-video-one-image", "media": [ { "bytes": 11627, "kind": "video", "mime": "video/mp4", "path": "temporal-order-10s.mp4", "sha256": "4d82c50cd2e500fc58df6d4930efec12649a0eaf236bee53d0d7c7ca84e35980" }, { "bytes": 16940, "kind": "image", "mime": "image/png", "path": "ui.png", "sha256": "03fd14d8b68973c915de2848736d22eb6f67ba6bdb5da2a7a563085ac894e804" } ], "modality": "mixed", "prompt": "Report the video's ordered colors and the dashboard's status.", "repetitions": 2, "sampling": { "fps": 1.0 } }, { "assertions": [ { "type": "contains_casefold", "value": "7291" }, { "type": "contains_casefold", "value": "42917" }, { "type": "contains_casefold", "value": "red car" }, { "type": "contains_casefold", "value": "Q3" }, { "type": "contains_casefold", "value": "ready" } ], "id": "mixed-video-four-images", "media": [ { "bytes": 14731, "kind": "video", "mime": "video/mp4", "path": "video-ocr-30s.mp4", "sha256": "05c6cd1b1a9b4939f513944717f06d02dd5be54dd18b27870dccf2449b457e34" }, { "bytes": 13187, "kind": "image", "mime": "image/png", "path": "scene.png", "sha256": "30a1f11e40bf4e0a8595d39828e0aa362e8b28ac683c404a1068cc20bf7f7bda" }, { "bytes": 15916, "kind": "image", "mime": "image/png", "path": "ocr.png", "sha256": "a8d5158019fb2b08ee9f6a153860ab2342384644f7c53ab58fdd45658124e83e" }, { "bytes": 12635, "kind": "image", "mime": "image/png", "path": "chart.png", "sha256": "c905e453c4e16eb274e46992d1a901bbced8f2314b47b9d408971bb176ab2f3e" }, { "bytes": 16940, "kind": "image", "mime": "image/png", "path": "ui.png", "sha256": "03fd14d8b68973c915de2848736d22eb6f67ba6bdb5da2a7a563085ac894e804" } ], "modality": "mixed", "prompt": "Inspect all five media inputs. Report both numeric codes, the red object, the highest chart quarter, and the dashboard status.", "repetitions": 2, "sampling": { "num_frames": 16 } }, { "assertions": [ { "type": "contains_casefold", "value": "color" } ], "id": "cc-video-chemical-traffic-light", "media": [ { "bytes": 459852, "kind": "video", "mime": "video/mp4", "path": "cc/chemical-traffic-light-normalized.mp4", "sha256": "1ab998a0339eaedb8494392be2c11f02bf9ce2f632fb01cf51b7089990d18f9c" } ], "modality": "video", "prompt": "Describe the experiment and the visible color changes.", "repetitions": 2, "sampling": { "num_frames": 24 } }, { "assertions": [ { "type": "contains_casefold", "value": "cat" }, { "type": "contains_casefold", "value": "door" } ], "id": "cc-video-cat-opening-door", "media": [ { "bytes": 490171, "kind": "video", "mime": "video/mp4", "path": "cc/cat-opening-door-normalized.mp4", "sha256": "b01ffe7800a3f74eead304a06bf35ece0e8917e107e1ce1e25837f3b5611cb61" } ], "modality": "video", "prompt": "Identify the animal and describe the main action it completes.", "repetitions": 2, "sampling": { "num_frames": 16 } } ], "path": "benchmarks/corpora/agents-a1-v1/corpus.json", "provenance": { "creative_commons": { "attribution": "cc/README.md", "role": "supplementary generalization evidence" }, "observed_date": "2026-07-28", "synthetic": { "generator": "generate.py", "license": "CC0-1.0", "runtime_image": "vllm/vllm-openai:nightly-f25953cc59f9b4ba9b04b16228d2b86dcfbcbdb1" } }, "schema": "multimodal-corpus/v1", "sha256": "ebff9dcc87a7fd13f801fc19eeea7271aec01a99fe560d721be99c1c9becad49" }, "failed_attempts": 2, "finished_at": "2026-08-27T04:00:38Z", "identity": { "base_url": "http://127.0.0.1:8000/v1", "engine_build_ref": null, "engine_revision": "d91c3682b0b429e4c70df63cd57f819588ce29b0", "hardware": "2x NVIDIA RTX PRO 6000 Blackwell Max-Q, TP=2, live router", "model": "vision.general", "model_revision": "7b719225242aacd3dbd3f9407468c2ee9a9d2594", "runtime_image": "sha256:59f06adce6f91401adf443bd168d45fdb2044d77671fd591c7c57a29d851cbae", "served_models": [ "llm.primary", "llm.auxiliary", "llm.secondary", "llm.voice", "vision.ocr", "vision.general", "vision.video" ] }, "passed": false, "passed_attempts": 28, "requests": 30, "run_id": "multimodal-20260827T035944Z", "schema": "multimodal-benchmark-evidence/v1", "started_at": "2026-08-27T03:59:44Z" }