{ "aggregates": { "case": { "cc-video-cat-opening-door": { "attempts": 2, "latency_p50_seconds": 1.5716035000000002, "latency_p95_seconds": 1.912714, "pass_rate": 1.0, "passed": 2 }, "cc-video-chemical-traffic-light": { "attempts": 2, "latency_p50_seconds": 2.7196555, "latency_p95_seconds": 3.052148, "pass_rate": 1.0, "passed": 2 }, "image-chart": { "attempts": 2, "latency_p50_seconds": 0.6246635, "latency_p95_seconds": 0.638804, "pass_rate": 1.0, "passed": 2 }, "image-ocr": { "attempts": 2, "latency_p50_seconds": 0.2759565, "latency_p95_seconds": 0.278392, "pass_rate": 1.0, "passed": 2 }, "image-scene": { "attempts": 2, "latency_p50_seconds": 0.8647085, "latency_p95_seconds": 0.880603, "pass_rate": 1.0, "passed": 2 }, "image-spatial-count": { "attempts": 2, "latency_p50_seconds": 0.705936, "latency_p95_seconds": 1.200565, "pass_rate": 1.0, "passed": 2 }, "image-ui": { "attempts": 2, "latency_p50_seconds": 0.6289685, "latency_p95_seconds": 0.660879, "pass_rate": 1.0, "passed": 2 }, "mixed-video-four-images": { "attempts": 2, "latency_p50_seconds": 2.1183475, "latency_p95_seconds": 2.794724, "pass_rate": 1.0, "passed": 2 }, "mixed-video-one-image": { "attempts": 2, "latency_p50_seconds": 1.0427724999999999, "latency_p95_seconds": 1.070708, "pass_rate": 1.0, "passed": 2 }, "multi-image-comparison": { "attempts": 2, "latency_p50_seconds": 0.9159014999999999, "latency_p95_seconds": 0.919298, "pass_rate": 1.0, "passed": 2 }, "video-continuity-120s": { "attempts": 2, "latency_p50_seconds": 0.8758665, "latency_p95_seconds": 1.034837, "pass_rate": 1.0, "passed": 2 }, "video-event-localization-60s": { "attempts": 2, "latency_p50_seconds": 1.050248, "latency_p95_seconds": 1.555101, "pass_rate": 1.0, "passed": 2 }, "video-ocr-30s": { "attempts": 2, "latency_p50_seconds": 0.36784300000000003, "latency_p95_seconds": 0.401279, "pass_rate": 1.0, "passed": 2 }, "video-state-change-30s": { "attempts": 2, "latency_p50_seconds": 1.3268295, "latency_p95_seconds": 1.44808, "pass_rate": 1.0, "passed": 2 }, "video-temporal-order-10s": { "attempts": 2, "latency_p50_seconds": 0.558175, "latency_p95_seconds": 0.57895, "pass_rate": 1.0, "passed": 2 } }, "modality": { "image": { "attempts": 12, "latency_p50_seconds": 0.6498415, "latency_p95_seconds": 1.200565, "pass_rate": 1.0, "passed": 12 }, "mixed": { "attempts": 4, "latency_p50_seconds": 1.2563395, "latency_p95_seconds": 2.794724, "pass_rate": 1.0, "passed": 4 }, "video": { "attempts": 14, "latency_p50_seconds": 1.1202079999999999, "latency_p95_seconds": 3.052148, "pass_rate": 1.0, "passed": 14 } } }, "attempts": [ { "case_id": "cc-video-cat-opening-door", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 490171, "kind": "video", "mime": "video/mp4", "path": "cc/cat-opening-door-normalized.mp4", "sha256": "b01ffe7800a3f74eead304a06bf35ece0e8917e107e1ce1e25837f3b5611cb61" } ], "modality": "video", "output": "The animal in the video is a cat, and its main action is opening a door.", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 1, "sampling": { "num_frames": 16 }, "seconds": 1.912714, "usage": { "completion_tokens": 19, "prompt_tokens": 11808, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 11827 }, "validation": { "assertions": [ { "passed": true, "type": "contains_casefold", "value": "cat" }, { "passed": true, "type": "contains_casefold", "value": "door" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "cc-video-cat-opening-door", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 490171, "kind": "video", "mime": "video/mp4", "path": "cc/cat-opening-door-normalized.mp4", "sha256": "b01ffe7800a3f74eead304a06bf35ece0e8917e107e1ce1e25837f3b5611cb61" } ], "modality": "video", "output": "The animal in the video is a cat, and its main action is opening a door.", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 2, "sampling": { "num_frames": 16 }, "seconds": 1.230493, "usage": { "completion_tokens": 19, "prompt_tokens": 11808, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 11827 }, "validation": { "assertions": [ { "passed": true, "type": "contains_casefold", "value": "cat" }, { "passed": true, "type": "contains_casefold", "value": "door" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "cc-video-chemical-traffic-light", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 459852, "kind": "video", "mime": "video/mp4", "path": "cc/chemical-traffic-light-normalized.mp4", "sha256": "1ab998a0339eaedb8494392be2c11f02bf9ce2f632fb01cf51b7089990d18f9c" } ], "modality": "video", "output": "In the video, a clear plastic bottle filled with a yellow liquid is placed on a table in a laboratory setting. The text \"Chemical Traffic Light\" appears on the screen, indicating that this is an experiment involving color changes. A hand enters the frame and shakes the bottle vigorously, causing the liquid inside to change colors. The liquid first turns green, then red, and finally yellow again. The text \"Green is the oxidized state\" appears on the screen, suggesting that the color changes are related to oxidation reactions. The video ends with the bottle still containing the yellow liquid.", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 1, "sampling": { "num_frames": 24 }, "seconds": 2.387163, "usage": { "completion_tokens": 118, "prompt_tokens": 12221, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 12339 }, "validation": { "assertions": [ { "passed": true, "type": "contains_casefold", "value": "color" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "cc-video-chemical-traffic-light", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 459852, "kind": "video", "mime": "video/mp4", "path": "cc/chemical-traffic-light-normalized.mp4", "sha256": "1ab998a0339eaedb8494392be2c11f02bf9ce2f632fb01cf51b7089990d18f9c" } ], "modality": "video", "output": "This video demonstrates a chemical reaction known as the \"Chemical Traffic Light.\" The experiment involves a clear plastic bottle filled with a yellow liquid, which is a mixture of sodium hydroxide (NaOH), dextrose (glucose), and 1% indigo carmine.\n\nInitially, the liquid in the bottle is yellow. When the bottle is shaken vigorously by a hand, the liquid changes color to green. This green color represents the oxidized state of the indigo carmine. After shaking, the bottle is left undisturbed on a black table. Over time, the green color gradually fades back to its original yellow hue. This process can be repeated by shaking the bottle again, causing the color to change from yellow to green and then back to yellow again.", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 2, "sampling": { "num_frames": 24 }, "seconds": 3.052148, "usage": { "completion_tokens": 158, "prompt_tokens": 12221, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 12379 }, "validation": { "assertions": [ { "passed": true, "type": "contains_casefold", "value": "color" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "image-chart", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 12635, "kind": "image", "mime": "image/png", "path": "chart.png", "sha256": "c905e453c4e16eb274e46992d1a901bbced8f2314b47b9d408971bb176ab2f3e" } ], "modality": "image", "output": "Based on the bar chart titled \"QUARTERLY UNITS\", here are the quarters and their corresponding values in ascending quarter order:\n\n- **Q1**: 10 units \n- **Q2**: 20 units \n- **Q3**: 30 units\n\nThe chart shows a clear upward trend in units sold or produced each quarter.", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 1, "sampling": null, "seconds": 0.610523, "usage": { "completion_tokens": 74, "prompt_tokens": 249, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 323 }, "validation": { "assertions": [ { "passed": true, "type": "ordered_casefold", "values": [ "Q1", "10", "Q2", "20", "Q3", "30" ] } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "image-chart", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 12635, "kind": "image", "mime": "image/png", "path": "chart.png", "sha256": "c905e453c4e16eb274e46992d1a901bbced8f2314b47b9d408971bb176ab2f3e" } ], "modality": "image", "output": "Based on the bar chart titled \"QUARTERLY UNITS\", here are the quarters and their corresponding values in ascending quarter order:\n\n- **Q1**: 10 units \n- **Q2**: 20 units \n- **Q3**: 30 units \n\nThe chart shows a clear upward trend in units sold from Q1 to Q3.", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 2, "sampling": null, "seconds": 0.638804, "usage": { "completion_tokens": 76, "prompt_tokens": 249, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 325 }, "validation": { "assertions": [ { "passed": true, "type": "ordered_casefold", "values": [ "Q1", "10", "Q2", "20", "Q3", "30" ] } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "image-ocr", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 15916, "kind": "image", "mime": "image/png", "path": "ocr.png", "sha256": "a8d5158019fb2b08ee9f6a153860ab2342384644f7c53ab58fdd45658124e83e" } ], "modality": "image", "output": "ANVIL READY\nCODE 42917\nTEMPERATURE 42 C", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 1, "sampling": null, "seconds": 0.273521, "usage": { "completion_tokens": 21, "prompt_tokens": 245, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 266 }, "validation": { "assertions": [ { "passed": true, "type": "contains_casefold", "value": "ANVIL READY" }, { "passed": true, "type": "contains_casefold", "value": "42917" }, { "passed": true, "type": "contains_casefold", "value": "42" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "image-ocr", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 15916, "kind": "image", "mime": "image/png", "path": "ocr.png", "sha256": "a8d5158019fb2b08ee9f6a153860ab2342384644f7c53ab58fdd45658124e83e" } ], "modality": "image", "output": "ANVIL READY\nCODE 42917\nTEMPERATURE 42 C", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 2, "sampling": null, "seconds": 0.278392, "usage": { "completion_tokens": 21, "prompt_tokens": 245, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 266 }, "validation": { "assertions": [ { "passed": true, "type": "contains_casefold", "value": "ANVIL READY" }, { "passed": true, "type": "contains_casefold", "value": "42917" }, { "passed": true, "type": "contains_casefold", "value": "42" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "image-scene", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 13187, "kind": "image", "mime": "image/png", "path": "scene.png", "sha256": "30a1f11e40bf4e0a8595d39828e0aa362e8b28ac683c404a1068cc20bf7f7bda" } ], "modality": "image", "output": "The image depicts a simple, stylized scene with a dark navy background. At the top, in bold yellow capital letters, is the label \u201cSUNNY DAY\u201d. Below that, on the left side, there\u2019s a red rectangle labeled \u201cRED CAR\u201d in white text above it. Beneath the red car and extending to the right is a long green rectangle labeled \u201cGREEN PARK\u201d in white text on its right end.\n\nProminent labels (verbatim):\n- SUNNY DAY\n- RED CAR\n- GREEN PARK", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 1, "sampling": null, "seconds": 0.880603, "usage": { "completion_tokens": 107, "prompt_tokens": 249, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 356 }, "validation": { "assertions": [ { "passed": true, "type": "contains_casefold", "value": "red car" }, { "passed": true, "type": "contains_casefold", "value": "green park" }, { "passed": true, "type": "contains_casefold", "value": "sunny day" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "image-scene", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 13187, "kind": "image", "mime": "image/png", "path": "scene.png", "sha256": "30a1f11e40bf4e0a8595d39828e0aa362e8b28ac683c404a1068cc20bf7f7bda" } ], "modality": "image", "output": "The image depicts a simple, stylized scene with a dark navy blue background. At the top, in bold yellow capital letters, is the label \u201cSUNNY DAY\u201d. Below that, on the left side, there\u2019s a red rectangle labeled \u201cRED CAR\u201d in white text above it. Beneath the red car and extending to the right is a larger green rectangle labeled \u201cGREEN PARK\u201d in white text on its right side.\n\nProminent labels (verbatim):\n- SUNNY DAY\n- RED CAR\n- GREEN PARK", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 2, "sampling": null, "seconds": 0.848814, "usage": { "completion_tokens": 108, "prompt_tokens": 249, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 357 }, "validation": { "assertions": [ { "passed": true, "type": "contains_casefold", "value": "red car" }, { "passed": true, "type": "contains_casefold", "value": "green park" }, { "passed": true, "type": "contains_casefold", "value": "sunny day" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "image-spatial-count", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 8785, "kind": "image", "mime": "image/png", "path": "spatial.png", "sha256": "d48f47cfba9d65524d6a9ca37d593550e10400c1b5fa160bec8aea24e0164051" } ], "modality": "image", "output": "three", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 1, "sampling": null, "seconds": 0.211307, "usage": { "completion_tokens": 2, "prompt_tokens": 249, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 251 }, "validation": { "assertions": [ { "passed": true, "type": "contains_casefold", "value": "three" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "image-spatial-count", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 8785, "kind": "image", "mime": "image/png", "path": "spatial.png", "sha256": "d48f47cfba9d65524d6a9ca37d593550e10400c1b5fa160bec8aea24e0164051" } ], "modality": "image", "output": "three", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 2, "sampling": null, "seconds": 1.200565, "usage": { "completion_tokens": 2, "prompt_tokens": 249, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 251 }, "validation": { "assertions": [ { "passed": true, "type": "contains_casefold", "value": "three" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "image-ui", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 16940, "kind": "image", "mime": "image/png", "path": "ui.png", "sha256": "03fd14d8b68973c915de2848736d22eb6f67ba6bdb5da2a7a563085ac894e804" } ], "modality": "image", "output": "Based on the image provided:\n\n- **Application Name**: ANVIL SERVING \n- **Status**: READY \n- **GPU Percentage**: 42%\n\nThis appears to be a status dashboard or monitoring display for an application called \u201cAnvil Serving,\u201d indicating it is currently ready to handle requests and is using 42% of available GPU resources.", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 1, "sampling": null, "seconds": 0.597058, "usage": { "completion_tokens": 74, "prompt_tokens": 245, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 319 }, "validation": { "assertions": [ { "passed": true, "type": "contains_casefold", "value": "ANVIL SERVING" }, { "passed": true, "type": "contains_casefold", "value": "READY" }, { "passed": true, "type": "contains_casefold", "value": "42" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "image-ui", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 16940, "kind": "image", "mime": "image/png", "path": "ui.png", "sha256": "03fd14d8b68973c915de2848736d22eb6f67ba6bdb5da2a7a563085ac894e804" } ], "modality": "image", "output": "Based on the image provided:\n\n- **Application Name**: ANVIL SERVING \n- **Status**: READY (displayed in green) \n- **GPU Usage**: 42 PERCENT (displayed in blue)\n\nThis appears to be a status dashboard or monitoring interface for an application called \u201cAnvil Serving,\u201d indicating it is currently ready to handle requests and is using 42% of GPU resources.", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 2, "sampling": null, "seconds": 0.660879, "usage": { "completion_tokens": 86, "prompt_tokens": 245, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 331 }, "validation": { "assertions": [ { "passed": true, "type": "contains_casefold", "value": "ANVIL SERVING" }, { "passed": true, "type": "contains_casefold", "value": "READY" }, { "passed": true, "type": "contains_casefold", "value": "42" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "mixed-video-four-images", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 14731, "kind": "video", "mime": "video/mp4", "path": "video-ocr-30s.mp4", "sha256": "05c6cd1b1a9b4939f513944717f06d02dd5be54dd18b27870dccf2449b457e34" }, { "bytes": 13187, "kind": "image", "mime": "image/png", "path": "scene.png", "sha256": "30a1f11e40bf4e0a8595d39828e0aa362e8b28ac683c404a1068cc20bf7f7bda" }, { "bytes": 15916, "kind": "image", "mime": "image/png", "path": "ocr.png", "sha256": "a8d5158019fb2b08ee9f6a153860ab2342384644f7c53ab58fdd45658124e83e" }, { "bytes": 12635, "kind": "image", "mime": "image/png", "path": "chart.png", "sha256": "c905e453c4e16eb274e46992d1a901bbced8f2314b47b9d408971bb176ab2f3e" }, { "bytes": 16940, "kind": "image", "mime": "image/png", "path": "ui.png", "sha256": "03fd14d8b68973c915de2848736d22eb6f67ba6bdb5da2a7a563085ac894e804" } ], "modality": "mixed", "output": "Based on the provided video frames, here is the analysis of the requested information:\n\n- **Numeric Codes:**\n - The first code, seen in the initial frames, is `7291`.\n - The second code, visible in the third frame, is `42917`.\n\n- **Red Object:**\n - In the second frame, there is a red rectangle labeled \"RED CAR\".\n\n- **Highest Chart Quarter:**\n - The fourth frame displays a bar chart titled \"QUARTERLY UNITS\".\n - The bars represent Q1 (10), Q2 (20), and Q3 (30).\n - The highest value is 30, which corresponds to **Q3**.\n\n- **Dashboard Status:**\n - The fifth frame shows a dashboard with the heading \"ANVIL SERVING\".\n - Within a box on the dashboard, the status is clearly indicated as \"STATUS READY\".\n\n---\n\nTo summarize the findings as requested:\n- **Numeric codes:** 7291, 42917\n- **Red object:** RED CAR\n- **Highest chart quarter:** Q3\n- **Dashboard status:** READY", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 1, "sampling": { "num_frames": 16 }, "seconds": 2.794724, "usage": { "completion_tokens": 254, "prompt_tokens": 2737, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 2991 }, "validation": { "assertions": [ { "passed": true, "type": "contains_casefold", "value": "7291" }, { "passed": true, "type": "contains_casefold", "value": "42917" }, { "passed": true, "type": "contains_casefold", "value": "red car" }, { "passed": true, "type": "contains_casefold", "value": "Q3" }, { "passed": true, "type": "contains_casefold", "value": "ready" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "mixed-video-four-images", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 14731, "kind": "video", "mime": "video/mp4", "path": "video-ocr-30s.mp4", "sha256": "05c6cd1b1a9b4939f513944717f06d02dd5be54dd18b27870dccf2449b457e34" }, { "bytes": 13187, "kind": "image", "mime": "image/png", "path": "scene.png", "sha256": "30a1f11e40bf4e0a8595d39828e0aa362e8b28ac683c404a1068cc20bf7f7bda" }, { "bytes": 15916, "kind": "image", "mime": "image/png", "path": "ocr.png", "sha256": "a8d5158019fb2b08ee9f6a153860ab2342384644f7c53ab58fdd45658124e83e" }, { "bytes": 12635, "kind": "image", "mime": "image/png", "path": "chart.png", "sha256": "c905e453c4e16eb274e46992d1a901bbced8f2314b47b9d408971bb176ab2f3e" }, { "bytes": 16940, "kind": "image", "mime": "image/png", "path": "ui.png", "sha256": "03fd14d8b68973c915de2848736d22eb6f67ba6bdb5da2a7a563085ac894e804" } ], "modality": "mixed", "output": "Based on the provided media inputs, here is the requested information:\n\n- **Numeric Codes:**\n - The first code, found in the initial set of frames, is `7291`.\n - The second code, found in the third image, is `42917`.\n\n- **Red Object:**\n - In the second image, the red object is explicitly labeled as a `RED CAR`.\n\n- **Highest Chart Quarter:**\n - The fourth image displays a bar chart titled \"QUARTERLY UNITS\". The values are Q1: 10, Q2: 20, and Q3: 30. Therefore, the highest quarter is `Q3`.\n\n- **Dashboard Status:**\n - The fifth image shows a dashboard with the heading \"ANVIL SERVING\". Within this dashboard, the status is clearly indicated as `STATUS READY`.", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 2, "sampling": { "num_frames": 16 }, "seconds": 1.441971, "usage": { "completion_tokens": 189, "prompt_tokens": 2737, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 2926 }, "validation": { "assertions": [ { "passed": true, "type": "contains_casefold", "value": "7291" }, { "passed": true, "type": "contains_casefold", "value": "42917" }, { "passed": true, "type": "contains_casefold", "value": "red car" }, { "passed": true, "type": "contains_casefold", "value": "Q3" }, { "passed": true, "type": "contains_casefold", "value": "ready" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "mixed-video-one-image", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 11627, "kind": "video", "mime": "video/mp4", "path": "temporal-order-10s.mp4", "sha256": "4d82c50cd2e500fc58df6d4930efec12649a0eaf236bee53d0d7c7ca84e35980" }, { "bytes": 16940, "kind": "image", "mime": "image/png", "path": "ui.png", "sha256": "03fd14d8b68973c915de2848736d22eb6f67ba6bdb5da2a7a563085ac894e804" } ], "modality": "mixed", "output": "Based on the video frames provided, here is a breakdown of the ordered colors and the dashboard's status:\n\n**Ordered Colors:**\nThe video displays text in a specific sequence of colors:\n1. **Red:** The video begins by showing the text \"FIRST RED\" in red.\n2. **Green:** The text then changes to \"THEN GREEN\" in green.\n\n**Dashboard Status:**\nThe final frame shows a dashboard with the following information:\n* **Title:** ANVIL SERVING\n* **Status:** STATUS READY (displayed in green)\n* **GPU Usage:** GPU 42 PERCENT (displayed in blue)", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 1, "sampling": { "fps": 1.0 }, "seconds": 1.014837, "usage": { "completion_tokens": 140, "prompt_tokens": 848, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 988 }, "validation": { "assertions": [ { "passed": true, "type": "ordered_casefold", "values": [ "red", "green" ] }, { "passed": true, "type": "contains_casefold", "value": "ready" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "mixed-video-one-image", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 11627, "kind": "video", "mime": "video/mp4", "path": "temporal-order-10s.mp4", "sha256": "4d82c50cd2e500fc58df6d4930efec12649a0eaf236bee53d0d7c7ca84e35980" }, { "bytes": 16940, "kind": "image", "mime": "image/png", "path": "ui.png", "sha256": "03fd14d8b68973c915de2848736d22eb6f67ba6bdb5da2a7a563085ac894e804" } ], "modality": "mixed", "output": "Based on the video frames provided, here is an analysis of the ordered colors and the dashboard's status:\n\n**Ordered Colors in the Video:**\nThe video displays text in a specific sequence of colors:\n1. **Red:** The video begins by showing the text \"FIRST RED\" in red letters.\n2. **Green:** The video then transitions to show the text \"THEN GREEN\" in green letters.\n\n**Dashboard Status:**\nThe final frame shows a dashboard with the following information:\n- **Title:** ANVIL SERVING\n- **Status:** STATUS READY (displayed in green text)\n- **GPU Usage:** GPU 42 PERCENT (displayed in blue text)", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 2, "sampling": { "fps": 1.0 }, "seconds": 1.070708, "usage": { "completion_tokens": 150, "prompt_tokens": 848, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 998 }, "validation": { "assertions": [ { "passed": true, "type": "ordered_casefold", "values": [ "red", "green" ] }, { "passed": true, "type": "contains_casefold", "value": "ready" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "multi-image-comparison", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 8000, "kind": "image", "mime": "image/png", "path": "compare-a.png", "sha256": "7dac4f31d62c88d765150aa6051efea9ddaaa8c64312080be4c5b8f6ad96322a" }, { "bytes": 7679, "kind": "image", "mime": "image/png", "path": "compare-b.png", "sha256": "8f7cdfd733f1ee9731de6160e77af6aa50189edec6eaf9ddd7654a0c8300a43e" } ], "modality": "image", "output": "Based on the visual information provided in the two images, we can compare the meters as follows:\n\n- **Meter A** shows a value of 25 percent. The orange bar fills one-quarter of the total track length.\n- **Meter B** shows a value of 75 percent. The green bar fills three-quarters of the total track length.\n\nBy comparing the two values, $75\\%$ is greater than $25\\%$.\n\n---\n\nThe meter that is higher is **Meter B**, and its percentage is **75 percent**.", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 1, "sampling": null, "seconds": 0.912505, "usage": { "completion_tokens": 118, "prompt_tokens": 471, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 589 }, "validation": { "assertions": [ { "passed": true, "type": "contains_casefold", "value": "meter b" }, { "passed": true, "type": "contains_casefold", "value": "75" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "multi-image-comparison", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 8000, "kind": "image", "mime": "image/png", "path": "compare-a.png", "sha256": "7dac4f31d62c88d765150aa6051efea9ddaaa8c64312080be4c5b8f6ad96322a" }, { "bytes": 7679, "kind": "image", "mime": "image/png", "path": "compare-b.png", "sha256": "8f7cdfd733f1ee9731de6160e77af6aa50189edec6eaf9ddd7654a0c8300a43e" } ], "modality": "image", "output": "Based on the visual information provided in the two images, we can compare the meters as follows:\n\n- **Meter A** shows a value of 25 percent. The orange bar fills one-quarter of the total track length.\n- **Meter B** shows a value of 75 percent. The green bar fills three-quarters of the total track length.\n\n---\n\nComparing the two values, $75\\%$ is greater than $25\\%$.\n\nThe meter that is higher is **Meter B**, and its percentage is **75 percent**.", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 2, "sampling": null, "seconds": 0.919298, "usage": { "completion_tokens": 118, "prompt_tokens": 471, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 589 }, "validation": { "assertions": [ { "passed": true, "type": "contains_casefold", "value": "meter b" }, { "passed": true, "type": "contains_casefold", "value": "75" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "video-continuity-120s", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 23622, "kind": "video", "mime": "video/mp4", "path": "continuity-120s.mp4", "sha256": "1ebc5459ed8c158ffb8294a0317005a13198748c529a49b66a2f76ff1c982b95" } ], "modality": "video", "output": "The phases in the video are: Phase Alpha, Phase Beta, Phase Gamma.", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 1, "sampling": { "num_frames": 32 }, "seconds": 1.034837, "usage": { "completion_tokens": 17, "prompt_tokens": 7292, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 7309 }, "validation": { "assertions": [ { "passed": true, "type": "ordered_casefold", "values": [ "alpha", "beta", "gamma" ] } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "video-continuity-120s", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 23622, "kind": "video", "mime": "video/mp4", "path": "continuity-120s.mp4", "sha256": "1ebc5459ed8c158ffb8294a0317005a13198748c529a49b66a2f76ff1c982b95" } ], "modality": "video", "output": "The phases appear in the following order: Phase Alpha, Phase Beta, and Phase Gamma.", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 2, "sampling": { "num_frames": 32 }, "seconds": 0.716896, "usage": { "completion_tokens": 19, "prompt_tokens": 7292, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 7311 }, "validation": { "assertions": [ { "passed": true, "type": "ordered_casefold", "values": [ "alpha", "beta", "gamma" ] } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "video-event-localization-60s", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 13829, "kind": "video", "mime": "video/mp4", "path": "event-localization-60s.mp4", "sha256": "5e0f40b3760e009f8ea0035b96bb4a8654fea58e0bb8a31a838eb78b224e7cbf" } ], "modality": "video", "output": "The localized event is 'alert', which occurs between 42 and 47 seconds.", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 1, "sampling": { "fps": 1.0 }, "seconds": 1.555101, "usage": { "completion_tokens": 20, "prompt_tokens": 3651, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 3671 }, "validation": { "assertions": [ { "passed": true, "type": "contains_casefold", "value": "alert" }, { "passed": true, "type": "contains_casefold", "value": "42" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "video-event-localization-60s", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 13829, "kind": "video", "mime": "video/mp4", "path": "event-localization-60s.mp4", "sha256": "5e0f40b3760e009f8ea0035b96bb4a8654fea58e0bb8a31a838eb78b224e7cbf" } ], "modality": "video", "output": "The localized event is 'an alert appears', and it occurs between 42 and 47 seconds.", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 2, "sampling": { "fps": 1.0 }, "seconds": 0.545395, "usage": { "completion_tokens": 23, "prompt_tokens": 3651, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 3674 }, "validation": { "assertions": [ { "passed": true, "type": "contains_casefold", "value": "alert" }, { "passed": true, "type": "contains_casefold", "value": "42" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "video-ocr-30s", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 14731, "kind": "video", "mime": "video/mp4", "path": "video-ocr-30s.mp4", "sha256": "05c6cd1b1a9b4939f513944717f06d02dd5be54dd18b27870dccf2449b457e34" } ], "modality": "video", "output": "ANVIL VIDEO OCR\nCODE 7291", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 1, "sampling": { "num_frames": 16 }, "seconds": 0.401279, "usage": { "completion_tokens": 13, "prompt_tokens": 1836, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 1849 }, "validation": { "assertions": [ { "passed": true, "type": "contains_casefold", "value": "ANVIL VIDEO OCR" }, { "passed": true, "type": "contains_casefold", "value": "7291" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "video-ocr-30s", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 14731, "kind": "video", "mime": "video/mp4", "path": "video-ocr-30s.mp4", "sha256": "05c6cd1b1a9b4939f513944717f06d02dd5be54dd18b27870dccf2449b457e34" } ], "modality": "video", "output": "ANVIL VIDEO OCR\nCODE 7291", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 2, "sampling": { "num_frames": 16 }, "seconds": 0.334407, "usage": { "completion_tokens": 13, "prompt_tokens": 1836, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 1849 }, "validation": { "assertions": [ { "passed": true, "type": "contains_casefold", "value": "ANVIL VIDEO OCR" }, { "passed": true, "type": "contains_casefold", "value": "7291" } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "video-state-change-30s", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 14552, "kind": "video", "mime": "video/mp4", "path": "state-change-30s.mp4", "sha256": "4d720486425afff4a7450f0f7f3ce6e3eea591e063b66e48fc953415b9172e62" } ], "modality": "video", "output": "Based on the video frames provided, here is the temporal description of the state change:\n\n* **00:00 - 00:14:** The video begins with a static display on a dark blue background. The text \"DOOR CLOSED\" is shown in white, capital letters. This state remains unchanged for the first 14 seconds.\n* **00:15 - 00:28:** At the 15-second mark, the state changes. The text on the screen switches to \"DOOR OPEN,\" displayed in bright green, capital letters. This new state remains static until the end of the video.", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 1, "sampling": { "fps": 1.0 }, "seconds": 1.205579, "usage": { "completion_tokens": 136, "prompt_tokens": 1832, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 1968 }, "validation": { "assertions": [ { "passed": true, "type": "ordered_casefold", "values": [ "closed", "open" ] } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "video-state-change-30s", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 14552, "kind": "video", "mime": "video/mp4", "path": "state-change-30s.mp4", "sha256": "4d720486425afff4a7450f0f7f3ce6e3eea591e063b66e48fc953415b9172e62" } ], "modality": "video", "output": "Based on the visual information provided in the video frames, here is the description of the state change in temporal order:\n\n- **Initial State (00:00 - 00:14):** The video begins with a static display. On a dark blue background, the text \"DOOR CLOSED\" is shown in white, capital letters. This state remains unchanged for the first 14 seconds.\n- **Transition (00:15):** At the 15-second mark, the text on the screen changes.\n- **Final State (00:15 - 00:28):** The text changes to \"DOOR OPEN\" and its color changes to bright green. This new state remains static until the end of the provided frames.\n\nIn summary, the temporal sequence shows a transition from a \"DOOR CLOSED\" state to a \"DOOR OPEN\" state.", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 2, "sampling": { "fps": 1.0 }, "seconds": 1.44808, "usage": { "completion_tokens": 186, "prompt_tokens": 1832, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 2018 }, "validation": { "assertions": [ { "passed": true, "type": "ordered_casefold", "values": [ "closed", "open" ] } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "video-temporal-order-10s", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 11627, "kind": "video", "mime": "video/mp4", "path": "temporal-order-10s.mp4", "sha256": "4d82c50cd2e500fc58df6d4930efec12649a0eaf236bee53d0d7c7ca84e35980" } ], "modality": "video", "output": "The video displays two states in the following order:\n\n1. **FIRST RED**: The video begins with the text \"FIRST RED\" in red letters on a dark blue background.\n2. **THEN GREEN**: The text then changes to \"THEN GREEN\" in green letters on the same dark blue background.", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 1, "sampling": { "fps": 1.0 }, "seconds": 0.57895, "usage": { "completion_tokens": 65, "prompt_tokens": 624, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 689 }, "validation": { "assertions": [ { "passed": true, "type": "ordered_casefold", "values": [ "red", "green" ] } ], "finish_reason_allowed": true, "visible_answer": true } }, { "case_id": "video-temporal-order-10s", "failure": null, "finish_reason": "stop", "media": [ { "bytes": 11627, "kind": "video", "mime": "video/mp4", "path": "temporal-order-10s.mp4", "sha256": "4d82c50cd2e500fc58df6d4930efec12649a0eaf236bee53d0d7c7ca84e35980" } ], "modality": "video", "output": "The video displays two states in the following order:\n\n1. **FIRST RED**: The video begins with the text \"FIRST RED\" in red letters on a dark blue background.\n2. **THEN GREEN**: The text then changes to \"THEN GREEN\" in green letters on the same dark blue background.", "passed": true, "reasoning": { "chars": 0, "field": null, "tokens": null }, "repetition": 2, "sampling": { "fps": 1.0 }, "seconds": 0.5374, "usage": { "completion_tokens": 65, "prompt_tokens": 624, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 689 }, "validation": { "assertions": [ { "passed": true, "type": "ordered_casefold", "values": [ "red", "green" ] } ], "finish_reason_allowed": true, "visible_answer": true } } ], "configuration": { "allowed_finish_reasons": [ "length", "stop" ], "chat_template_kwargs": { "enable_thinking": false }, "concurrency": 1, "max_images_per_request": 4, "max_tokens": 512, "max_videos_per_request": 1, "reasoning_effort": null, "thinking_mode": "disabled", "timeout_seconds": 900.0 }, "corpus": { "cases": [ { "assertions": [ { "type": "contains_casefold", "value": "red car" }, { "type": "contains_casefold", "value": "green park" }, { "type": "contains_casefold", "value": "sunny day" } ], "id": "image-scene", "media": [ { "bytes": 13187, "kind": "image", "mime": "image/png", "path": "scene.png", "sha256": "30a1f11e40bf4e0a8595d39828e0aa362e8b28ac683c404a1068cc20bf7f7bda" } ], "modality": "image", "prompt": "Describe the scene and report every prominent label. Use the labels verbatim.", "repetitions": 2, "sampling": null }, { "assertions": [ { "type": "contains_casefold", "value": "ANVIL READY" }, { "type": "contains_casefold", "value": "42917" }, { "type": "contains_casefold", "value": "42" } ], "id": "image-ocr", "media": [ { "bytes": 15916, "kind": "image", "mime": "image/png", "path": "ocr.png", "sha256": "a8d5158019fb2b08ee9f6a153860ab2342384644f7c53ab58fdd45658124e83e" } ], "modality": "image", "prompt": "Transcribe all visible text exactly. Preserve every number.", "repetitions": 2, "sampling": null }, { "assertions": [ { "type": "ordered_casefold", "values": [ "Q1", "10", "Q2", "20", "Q3", "30" ] } ], "id": "image-chart", "media": [ { "bytes": 12635, "kind": "image", "mime": "image/png", "path": "chart.png", "sha256": "c905e453c4e16eb274e46992d1a901bbced8f2314b47b9d408971bb176ab2f3e" } ], "modality": "image", "prompt": "Read the chart and report each quarter with its value in ascending quarter order.", "repetitions": 2, "sampling": null }, { "assertions": [ { "type": "contains_casefold", "value": "ANVIL SERVING" }, { "type": "contains_casefold", "value": "READY" }, { "type": "contains_casefold", "value": "42" } ], "id": "image-ui", "media": [ { "bytes": 16940, "kind": "image", "mime": "image/png", "path": "ui.png", "sha256": "03fd14d8b68973c915de2848736d22eb6f67ba6bdb5da2a7a563085ac894e804" } ], "modality": "image", "prompt": "Report the application name, status, and GPU percentage.", "repetitions": 2, "sampling": null }, { "assertions": [ { "type": "contains_casefold", "value": "three" } ], "id": "image-spatial-count", "media": [ { "bytes": 8785, "kind": "image", "mime": "image/png", "path": "spatial.png", "sha256": "d48f47cfba9d65524d6a9ca37d593550e10400c1b5fa160bec8aea24e0164051" } ], "modality": "image", "prompt": "Count the blue boxes. Reply with the count written as an English word.", "repetitions": 2, "sampling": null }, { "assertions": [ { "type": "contains_casefold", "value": "meter b" }, { "type": "contains_casefold", "value": "75" } ], "id": "multi-image-comparison", "media": [ { "bytes": 8000, "kind": "image", "mime": "image/png", "path": "compare-a.png", "sha256": "7dac4f31d62c88d765150aa6051efea9ddaaa8c64312080be4c5b8f6ad96322a" }, { "bytes": 7679, "kind": "image", "mime": "image/png", "path": "compare-b.png", "sha256": "8f7cdfd733f1ee9731de6160e77af6aa50189edec6eaf9ddd7654a0c8300a43e" } ], "modality": "image", "prompt": "Compare the two meters. Identify which meter is higher and report its percentage.", "repetitions": 2, "sampling": null }, { "assertions": [ { "type": "ordered_casefold", "values": [ "red", "green" ] } ], "id": "video-temporal-order-10s", "media": [ { "bytes": 11627, "kind": "video", "mime": "video/mp4", "path": "temporal-order-10s.mp4", "sha256": "4d82c50cd2e500fc58df6d4930efec12649a0eaf236bee53d0d7c7ca84e35980" } ], "modality": "video", "prompt": "Report the two states in the order they appear.", "repetitions": 2, "sampling": { "fps": 1.0 } }, { "assertions": [ { "type": "ordered_casefold", "values": [ "closed", "open" ] } ], "id": "video-state-change-30s", "media": [ { "bytes": 14552, "kind": "video", "mime": "video/mp4", "path": "state-change-30s.mp4", "sha256": "4d720486425afff4a7450f0f7f3ce6e3eea591e063b66e48fc953415b9172e62" } ], "modality": "video", "prompt": "Describe the state change in temporal order.", "repetitions": 2, "sampling": { "fps": 1.0 } }, { "assertions": [ { "type": "contains_casefold", "value": "alert" }, { "type": "contains_casefold", "value": "42" } ], "id": "video-event-localization-60s", "media": [ { "bytes": 13829, "kind": "video", "mime": "video/mp4", "path": "event-localization-60s.mp4", "sha256": "5e0f40b3760e009f8ea0035b96bb4a8654fea58e0bb8a31a838eb78b224e7cbf" } ], "modality": "video", "prompt": "Identify the localized event and report its timestamp in seconds.", "repetitions": 2, "sampling": { "fps": 1.0 } }, { "assertions": [ { "type": "contains_casefold", "value": "ANVIL VIDEO OCR" }, { "type": "contains_casefold", "value": "7291" } ], "id": "video-ocr-30s", "media": [ { "bytes": 14731, "kind": "video", "mime": "video/mp4", "path": "video-ocr-30s.mp4", "sha256": "05c6cd1b1a9b4939f513944717f06d02dd5be54dd18b27870dccf2449b457e34" } ], "modality": "video", "prompt": "Transcribe every stable text label and preserve the numeric code.", "repetitions": 2, "sampling": { "num_frames": 16 } }, { "assertions": [ { "type": "ordered_casefold", "values": [ "alpha", "beta", "gamma" ] } ], "id": "video-continuity-120s", "media": [ { "bytes": 23622, "kind": "video", "mime": "video/mp4", "path": "continuity-120s.mp4", "sha256": "1ebc5459ed8c158ffb8294a0317005a13198748c529a49b66a2f76ff1c982b95" } ], "modality": "video", "prompt": "Report all phases in the order they appear across the full video.", "repetitions": 2, "sampling": { "num_frames": 32 } }, { "assertions": [ { "type": "ordered_casefold", "values": [ "red", "green" ] }, { "type": "contains_casefold", "value": "ready" } ], "id": "mixed-video-one-image", "media": [ { "bytes": 11627, "kind": "video", "mime": "video/mp4", "path": "temporal-order-10s.mp4", "sha256": "4d82c50cd2e500fc58df6d4930efec12649a0eaf236bee53d0d7c7ca84e35980" }, { "bytes": 16940, "kind": "image", "mime": "image/png", "path": "ui.png", "sha256": "03fd14d8b68973c915de2848736d22eb6f67ba6bdb5da2a7a563085ac894e804" } ], "modality": "mixed", "prompt": "Report the video's ordered colors and the dashboard's status.", "repetitions": 2, "sampling": { "fps": 1.0 } }, { "assertions": [ { "type": "contains_casefold", "value": "7291" }, { "type": "contains_casefold", "value": "42917" }, { "type": "contains_casefold", "value": "red car" }, { "type": "contains_casefold", "value": "Q3" }, { "type": "contains_casefold", "value": "ready" } ], "id": "mixed-video-four-images", "media": [ { "bytes": 14731, "kind": "video", "mime": "video/mp4", "path": "video-ocr-30s.mp4", "sha256": "05c6cd1b1a9b4939f513944717f06d02dd5be54dd18b27870dccf2449b457e34" }, { "bytes": 13187, "kind": "image", "mime": "image/png", "path": "scene.png", "sha256": "30a1f11e40bf4e0a8595d39828e0aa362e8b28ac683c404a1068cc20bf7f7bda" }, { "bytes": 15916, "kind": "image", "mime": "image/png", "path": "ocr.png", "sha256": "a8d5158019fb2b08ee9f6a153860ab2342384644f7c53ab58fdd45658124e83e" }, { "bytes": 12635, "kind": "image", "mime": "image/png", "path": "chart.png", "sha256": "c905e453c4e16eb274e46992d1a901bbced8f2314b47b9d408971bb176ab2f3e" }, { "bytes": 16940, "kind": "image", "mime": "image/png", "path": "ui.png", "sha256": "03fd14d8b68973c915de2848736d22eb6f67ba6bdb5da2a7a563085ac894e804" } ], "modality": "mixed", "prompt": "Inspect all five media inputs. Report both numeric codes, the red object, the highest chart quarter, and the dashboard status.", "repetitions": 2, "sampling": { "num_frames": 16 } }, { "assertions": [ { "type": "contains_casefold", "value": "color" } ], "id": "cc-video-chemical-traffic-light", "media": [ { "bytes": 459852, "kind": "video", "mime": "video/mp4", "path": "cc/chemical-traffic-light-normalized.mp4", "sha256": "1ab998a0339eaedb8494392be2c11f02bf9ce2f632fb01cf51b7089990d18f9c" } ], "modality": "video", "prompt": "Describe the experiment and the visible color changes.", "repetitions": 2, "sampling": { "num_frames": 24 } }, { "assertions": [ { "type": "contains_casefold", "value": "cat" }, { "type": "contains_casefold", "value": "door" } ], "id": "cc-video-cat-opening-door", "media": [ { "bytes": 490171, "kind": "video", "mime": "video/mp4", "path": "cc/cat-opening-door-normalized.mp4", "sha256": "b01ffe7800a3f74eead304a06bf35ece0e8917e107e1ce1e25837f3b5611cb61" } ], "modality": "video", "prompt": "Identify the animal and describe the main action it completes.", "repetitions": 2, "sampling": { "num_frames": 16 } } ], "path": "benchmarks/corpora/agents-a1-v1/corpus.json", "provenance": { "creative_commons": { "attribution": "cc/README.md", "role": "supplementary generalization evidence" }, "observed_date": "2026-07-28", "synthetic": { "generator": "generate.py", "license": "CC0-1.0", "runtime_image": "vllm/vllm-openai:nightly-f25953cc59f9b4ba9b04b16228d2b86dcfbcbdb1" } }, "schema": "multimodal-corpus/v1", "sha256": "ebff9dcc87a7fd13f801fc19eeea7271aec01a99fe560d721be99c1c9becad49" }, "failed_attempts": 0, "finished_at": "2026-08-27T03:54:06Z", "identity": { "base_url": "http://127.0.0.1:18081/v1", "engine_build_ref": null, "engine_revision": "d91c3682b0b429e4c70df63cd57f819588ce29b0", "hardware": "2x NVIDIA RTX PRO 6000 Blackwell Max-Q, TP=2, isolated router", "model": "vision.general", "model_revision": "7b719225242aacd3dbd3f9407468c2ee9a9d2594", "runtime_image": "sha256:59f06adce6f91401adf443bd168d45fdb2044d77671fd591c7c57a29d851cbae", "served_models": [ "llm.primary", "vision.ocr", "vision.general", "vision.video" ] }, "passed": true, "passed_attempts": 30, "requests": 30, "run_id": "multimodal-20260827T035334Z", "schema": "multimodal-benchmark-evidence/v1", "started_at": "2026-08-27T03:53:34Z" }