{ "base_url": "http://127.0.0.1:8000/v1", "budget": { "max_completion_tokens": 256, "reasoning_headroom_tokens": 0, "visible_answer_tokens": 256 }, "checks": [ "smoke", "json", "tools", "streaming-tools", "tool-result", "responses", "image", "video" ], "evidence_policy": { "allowed_finish_reasons": [ "length", "stop", "tool_calls" ], "errors": [], "reasoning": "forbidden" }, "image_expect": [ "red car", "green park", "sunny day" ], "model": "llm.primary", "multimodal_input": { "bytes": 13187, "mime": "image/png", "sha256": "30a1f11e40bf4e0a8595d39828e0aa362e8b28ac683c404a1068cc20bf7f7bda" }, "observations": [ { "content": "```python\nsum(xs)\n```", "content_chars": 21, "content_excerpt": "```python\nsum(xs)\n```", "finish_reason": "stop", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "seconds": 1.206, "test": "smoke", "usage": { "completion_tokens": 9, "prompt_tokens": 28, "total_tokens": 37 }, "validation_detail": "contains sum(" }, { "content": "{\"language\":\"python\",\"ok\":true}", "content_chars": 31, "content_excerpt": "{\"language\":\"python\",\"ok\":true}", "finish_reason": "stop", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "seconds": 0.185, "test": "json", "usage": { "completion_tokens": 10, "prompt_tokens": 30, "total_tokens": 40 }, "validation_detail": "parsed keys=['language', 'ok']" }, { "content": "I'll check the weather in Oakland for you.\n\n", "content_chars": 44, "content_excerpt": "I'll check the weather in Oakland for you.\n\n", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 0, "seconds": 0.36, "test": "tools", "usage": { "completion_tokens": 38, "prompt_tokens": 5492, "total_tokens": 5530 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "...\n\n", "content_chars": 5, "content_excerpt": "...\n\n", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 1, "seconds": 0.65, "test": "tools", "usage": { "completion_tokens": 29, "prompt_tokens": 5492, "total_tokens": 5521 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 2, "seconds": 0.92, "test": "tools", "usage": { "completion_tokens": 27, "prompt_tokens": 5492, "total_tokens": 5519 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 3, "seconds": 1.184, "test": "tools", "usage": { "completion_tokens": 27, "prompt_tokens": 5492, "total_tokens": 5519 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 6, "seconds": 1.454, "test": "tools", "usage": { "completion_tokens": 27, "prompt_tokens": 5492, "total_tokens": 5519 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 4, "seconds": 1.742, "test": "tools", "usage": { "completion_tokens": 27, "prompt_tokens": 5492, "total_tokens": 5519 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 5, "seconds": 2.031, "test": "tools", "usage": { "completion_tokens": 27, "prompt_tokens": 5492, "total_tokens": 5519 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 7, "seconds": 2.306, "test": "tools", "usage": { "completion_tokens": 27, "prompt_tokens": 5492, "total_tokens": 5519 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 9, "seconds": 2.589, "test": "tools", "usage": { "completion_tokens": 27, "prompt_tokens": 5492, "total_tokens": 5519 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 8, "seconds": 2.879, "test": "tools", "usage": { "completion_tokens": 27, "prompt_tokens": 5492, "total_tokens": 5519 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 11, "seconds": 3.167, "test": "tools", "usage": { "completion_tokens": 27, "prompt_tokens": 5492, "total_tokens": 5519 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 13, "seconds": 3.447, "test": "tools", "usage": { "completion_tokens": 27, "prompt_tokens": 5492, "total_tokens": 5519 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "I'll check the current weather in Oakland for you.\n\n", "content_chars": 52, "content_excerpt": "I'll check the current weather in Oakland for you.\n\n", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 10, "seconds": 3.782, "test": "tools", "usage": { "completion_tokens": 39, "prompt_tokens": 5492, "total_tokens": 5531 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 12, "seconds": 4.068, "test": "tools", "usage": { "completion_tokens": 27, "prompt_tokens": 5492, "total_tokens": 5519 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 15, "seconds": 4.332, "test": "tools", "usage": { "completion_tokens": 27, "prompt_tokens": 5492, "total_tokens": 5519 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 14, "seconds": 4.611, "test": "tools", "usage": { "completion_tokens": 27, "prompt_tokens": 5492, "total_tokens": 5519 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "I'll check the weather in Oakland for you.\n\n", "content_chars": 44, "content_excerpt": "I'll check the weather in Oakland for you.\n\n", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 16, "seconds": 4.959, "test": "tools", "usage": { "completion_tokens": 38, "prompt_tokens": 5492, "total_tokens": 5530 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 17, "seconds": 5.237, "test": "tools", "usage": { "completion_tokens": 27, "prompt_tokens": 5492, "total_tokens": 5519 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 18, "seconds": 5.521, "test": "tools", "usage": { "completion_tokens": 27, "prompt_tokens": 5492, "total_tokens": 5519 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 19, "seconds": 5.792, "test": "tools", "usage": { "completion_tokens": 27, "prompt_tokens": 5492, "total_tokens": 5519 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content_chars": 0, "content_excerpt": "", "event_count": 5, "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "seconds": 1.262, "sse_done": true, "test": "streaming-tools", "usage": { "completion_tokens": 27, "prompt_tokens": 292, "total_tokens": 319 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "seconds": 0.262, "test": "tool-result-initial", "usage": { "completion_tokens": 27, "prompt_tokens": 292, "total_tokens": 319 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "OAKLAND 72F", "content_chars": 11, "content_excerpt": "OAKLAND 72F", "finish_reason": "stop", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "seconds": 0.168, "test": "tool-result-continuation", "usage": { "completion_tokens": 8, "prompt_tokens": 103, "total_tokens": 111 }, "validation_detail": "tool result retained" }, { "content_chars": 5, "content_excerpt": "READY", "finish_reason": "stop", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "response_status": "completed", "seconds": 0.142, "test": "responses", "usage": { "input_tokens": 16, "output_tokens": 2, "total_tokens": 18 }, "validation_detail": "completed exact READY" }, { "content": "This image is a simple, stylized graphic with a dark navy-blue background. It contains three labeled elements arranged vertically:\n\n1. **Top Label (Title):** \n - Text: **\u201cSUNNY DAY\u201d** \n - Color: Bright yellow, bold, uppercase \n - Position: Centered at the top \n - Status: Likely indicates weather or mood \u2014 not visually represented by an icon or color change, just text.\n\n2. **Middle Element (Object + Label):** \n - Visual: A solid red rectangle (representing a car) \n - Label: **\u201cRED CAR\u201d** in white, bold, uppercase text positioned above the red rectangle \n - Position: Left-aligned, below \u201cSUNNY DAY\u201d \n - Status: The red rectangle sits directly on top of the green rectangle below it \u2014 suggesting the car is parked or located within the park.\n\n3. **Bottom Element (Area + Label):** \n - Visual: A wide, solid green rectangle (representing a park) \n - Label: **\u201cGREEN PARK\u201d** in white, bold, uppercase text positioned on the right side of the green rectangle \n - Position: Spans most of the width at the bottom \n - Status: Acts as a", "content_chars": 1033, "content_excerpt": "This image is a simple, stylized graphic with a dark navy-blue background. It contains three labeled elements arranged vertically:\n\n1. **Top Label (Title):** \n - Text: **\u201cSUNNY DAY\u201d** \n - Color: Brigh", "expected": [ "red car", "green park", "sunny day" ], "finish_reason": "length", "image": { "bytes": 13187, "mime": "image/png", "sha256": "30a1f11e40bf4e0a8595d39828e0aa362e8b28ac683c404a1068cc20bf7f7bda" }, "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "seconds": 1.963, "test": "image", "usage": { "completion_tokens": 256, "prompt_tokens": 254, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 510 }, "validation_detail": "all expected text present" }, { "content": "The video begins with a dark blue background. The words \"FIRST RED\" appear in bold, red capital letters. This text remains on screen for a few seconds. Then, the text changes to \"THEN GREEN\" in bold, green capital letters, which also remains on screen for a few seconds before the video ends.", "content_chars": 292, "content_excerpt": "The video begins with a dark blue background. The words \"FIRST RED\" appear in bold, red capital letters. This text remains on screen for a few seconds. Then, the text changes to \"THEN GREEN\" in bold, ", "expected": [ "red", "green" ], "finish_reason": "stop", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "seconds": 0.674, "test": "video", "usage": { "completion_tokens": 65, "prompt_tokens": 635, "prompt_tokens_details": { "cached_tokens": 0 }, "total_tokens": 700 }, "validation_detail": "all expected text present", "video": { "bytes": 11627, "mime": "video/mp4", "sha256": "4d82c50cd2e500fc58df6d4930efec12649a0eaf236bee53d0d7c7ca84e35980" } } ], "ocr_expect": [], "passed": true, "results": [ { "detail": "1.2s got='```python\\nsum(xs)\\n```' finish='stop' visible=21 reasoning_chars=0 reasoning_tokens=None", "name": "smoke (short coding)", "passed": true }, { "detail": "parsed keys=['language', 'ok'] finish='stop' visible=31 reasoning_chars=0 reasoning_tokens=None", "name": "structured JSON", "passed": true }, { "detail": "20/20 clean (sample: 0.4s valid tool_call get_weather(city='Oakland') finish='tool_calls' visible=44 reasoning_chars=0 reasoning_tokens=None)", "name": "shared-prefix tool batch x20", "passed": true }, { "detail": "1.3s done=True events=5 valid tool_call get_weather(city='Oakland') finish='tool_calls'", "name": "streaming tool call", "passed": true }, { "detail": "0.4s tool result retained finish='stop' visible=11 reasoning_chars=0 reasoning_tokens=None", "name": "tool-result continuation", "passed": true }, { "detail": "0.1s status='completed' output='READY' reasoning_chars=0", "name": "Responses API subset", "passed": true }, { "detail": "2.0s all expected text present finish='length' visible=1033 reasoning_chars=0 reasoning_tokens=None", "name": "general image understanding", "passed": true }, { "detail": "0.7s all expected text present finish='stop' visible=292 reasoning_chars=0 reasoning_tokens=None", "name": "bounded video understanding", "passed": true } ], "schema_version": "preflight/v2", "thinking": { "chat_template_kwargs": { "enable_thinking": false }, "mode": "disabled", "reasoning_effort": null }, "video_expect": [ "red", "green" ], "video_input": { "bytes": 11627, "mime": "video/mp4", "sha256": "4d82c50cd2e500fc58df6d4930efec12649a0eaf236bee53d0d7c7ca84e35980" } }