{ "base_url": "http://127.0.0.1:8001/v1", "budget": { "max_completion_tokens": 1024, "reasoning_headroom_tokens": 0, "visible_answer_tokens": 1024 }, "checks": [ "smoke", "json", "needle", "tools", "long-tools", "streaming-tools", "tool-result", "responses", "image", "ocr" ], "evidence_policy": { "allowed_finish_reasons": [ "stop", "tool_calls" ], "errors": [], "reasoning": "forbidden" }, "image_expect": [ "ANVIL SERVING", "STATUS READY", "GPU 42 PERCENT" ], "model": "glm53-flash-ormandj-sglang-sm120-tp2-393k-c1-adaptive-mtp", "multimodal_input": { "bytes": 16940, "mime": "image/png", "sha256": "03fd14d8b68973c915de2848736d22eb6f67ba6bdb5da2a7a563085ac894e804" }, "observations": [ { "content": "```python\nsum(xs)\n```\n\nThat's it \u2014 Python's built-in `sum()` function takes an iterable and returns the total. If you need a starting value, you can pass one as the second argument: `sum(xs, 10)` starts counting from 10.", "content_chars": 220, "content_excerpt": "```python\nsum(xs)\n```\n\nThat's it \u2014 Python's built-in `sum()` function takes an iterable and returns the total. If you need a starting value, you can pass one as the second argument: `sum(xs, 10)` star", "finish_reason": "stop", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "seconds": 0.534, "test": "smoke", "usage": { "completion_tokens": 58, "prompt_tokens": 29, "prompt_tokens_details": null, "reasoning_tokens": 0, "total_tokens": 87 }, "validation_detail": "contains sum(" }, { "content": "{\"language\":\"python\",\"ok\":true}", "content_chars": 31, "content_excerpt": "{\"language\":\"python\",\"ok\":true}", "finish_reason": "stop", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "seconds": 0.168, "test": "json", "usage": { "completion_tokens": 10, "prompt_tokens": 31, "prompt_tokens_details": null, "reasoning_tokens": 0, "total_tokens": 41 }, "validation_detail": "parsed keys=['language', 'ok']" }, { "content": "ZEBRA-42917-QUARTZ", "content_chars": 18, "content_excerpt": "ZEBRA-42917-QUARTZ", "finish_reason": "stop", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "seconds": 54.31, "test": "needle", "usage": { "completion_tokens": 12, "prompt_tokens": 313547, "prompt_tokens_details": null, "reasoning_tokens": 0, "total_tokens": 313559 }, "validation_detail": "needle present" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 0, "seconds": 0.915, "test": "tools", "usage": { "completion_tokens": 12, "prompt_tokens": 4968, "prompt_tokens_details": null, "reasoning_tokens": 0, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 1, "seconds": 1.08, "test": "tools", "usage": { "completion_tokens": 12, "prompt_tokens": 4968, "prompt_tokens_details": { "cached_tokens": 4928 }, "reasoning_tokens": 0, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 2, "seconds": 1.249, "test": "tools", "usage": { "completion_tokens": 12, "prompt_tokens": 4968, "prompt_tokens_details": { "cached_tokens": 4928 }, "reasoning_tokens": 0, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 3, "seconds": 1.413, "test": "tools", "usage": { "completion_tokens": 12, "prompt_tokens": 4968, "prompt_tokens_details": { "cached_tokens": 4928 }, "reasoning_tokens": 0, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 4, "seconds": 1.574, "test": "tools", "usage": { "completion_tokens": 12, "prompt_tokens": 4968, "prompt_tokens_details": { "cached_tokens": 4928 }, "reasoning_tokens": 0, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 5, "seconds": 1.739, "test": "tools", "usage": { "completion_tokens": 12, "prompt_tokens": 4968, "prompt_tokens_details": { "cached_tokens": 4928 }, "reasoning_tokens": 0, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 6, "seconds": 1.903, "test": "tools", "usage": { "completion_tokens": 12, "prompt_tokens": 4968, "prompt_tokens_details": { "cached_tokens": 4928 }, "reasoning_tokens": 0, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 7, "seconds": 2.068, "test": "tools", "usage": { "completion_tokens": 12, "prompt_tokens": 4968, "prompt_tokens_details": { "cached_tokens": 4928 }, "reasoning_tokens": 0, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 8, "seconds": 2.236, "test": "tools", "usage": { "completion_tokens": 12, "prompt_tokens": 4968, "prompt_tokens_details": { "cached_tokens": 4928 }, "reasoning_tokens": 0, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 9, "seconds": 2.397, "test": "tools", "usage": { "completion_tokens": 12, "prompt_tokens": 4968, "prompt_tokens_details": { "cached_tokens": 4928 }, "reasoning_tokens": 0, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 10, "seconds": 2.562, "test": "tools", "usage": { "completion_tokens": 12, "prompt_tokens": 4968, "prompt_tokens_details": { "cached_tokens": 4928 }, "reasoning_tokens": 0, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 11, "seconds": 2.729, "test": "tools", "usage": { "completion_tokens": 12, "prompt_tokens": 4968, "prompt_tokens_details": { "cached_tokens": 4928 }, "reasoning_tokens": 0, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 12, "seconds": 2.89, "test": "tools", "usage": { "completion_tokens": 12, "prompt_tokens": 4968, "prompt_tokens_details": { "cached_tokens": 4928 }, "reasoning_tokens": 0, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 13, "seconds": 3.058, "test": "tools", "usage": { "completion_tokens": 12, "prompt_tokens": 4968, "prompt_tokens_details": { "cached_tokens": 4928 }, "reasoning_tokens": 0, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 14, "seconds": 3.219, "test": "tools", "usage": { "completion_tokens": 12, "prompt_tokens": 4968, "prompt_tokens_details": { "cached_tokens": 4928 }, "reasoning_tokens": 0, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 15, "seconds": 3.384, "test": "tools", "usage": { "completion_tokens": 12, "prompt_tokens": 4968, "prompt_tokens_details": { "cached_tokens": 4928 }, "reasoning_tokens": 0, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 16, "seconds": 3.548, "test": "tools", "usage": { "completion_tokens": 12, "prompt_tokens": 4968, "prompt_tokens_details": { "cached_tokens": 4928 }, "reasoning_tokens": 0, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 17, "seconds": 3.716, "test": "tools", "usage": { "completion_tokens": 12, "prompt_tokens": 4968, "prompt_tokens_details": { "cached_tokens": 4928 }, "reasoning_tokens": 0, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 18, "seconds": 3.881, "test": "tools", "usage": { "completion_tokens": 12, "prompt_tokens": 4968, "prompt_tokens_details": { "cached_tokens": 4928 }, "reasoning_tokens": 0, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 19, "seconds": 4.046, "test": "tools", "usage": { "completion_tokens": 12, "prompt_tokens": 4968, "prompt_tokens_details": { "cached_tokens": 4928 }, "reasoning_tokens": 0, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "measured_prompt_tokens": 135169, "minimum_prompt_tokens": 100000, "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "requested_context_tokens": 160000, "seconds": 22.899, "test": "long-tools", "usage": { "completion_tokens": 12, "prompt_tokens": 135169, "prompt_tokens_details": null, "reasoning_tokens": 0, "total_tokens": 135181 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content_chars": 0, "content_excerpt": "", "event_count": 7, "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "seconds": 0.496, "sse_done": true, "test": "streaming-tools", "usage": { "completion_tokens": 12, "prompt_tokens": 167, "prompt_tokens_details": null, "reasoning_tokens": 0, "total_tokens": 179 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "seconds": 0.174, "test": "tool-result-initial", "usage": { "completion_tokens": 12, "prompt_tokens": 167, "prompt_tokens_details": { "cached_tokens": 128 }, "reasoning_tokens": 0, "total_tokens": 179 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "OAKLAND 72F", "content_chars": 11, "content_excerpt": "OAKLAND 72F", "finish_reason": "stop", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "seconds": 0.163, "test": "tool-result-continuation", "usage": { "completion_tokens": 7, "prompt_tokens": 71, "prompt_tokens_details": null, "reasoning_tokens": 0, "total_tokens": 78 }, "validation_detail": "tool result retained" }, { "content_chars": 5, "content_excerpt": "READY", "finish_reason": "stop", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "response_status": "completed", "seconds": 0.149, "test": "responses", "usage": { "input_tokens": 17, "input_tokens_details": { "cache_write_tokens": 0, "cached_tokens": 0 }, "output_tokens": 2, "output_tokens_details": { "reasoning_tokens": 0 }, "total_tokens": 19 }, "validation_detail": "completed exact READY" }, { "content": "# Image Description\n\nThis is a **system status dashboard** with a dark navy blue background, displaying monitoring information for a service called \"ANVIL SERVING.\"\n\n## Content Breakdown\n\n**Title (top):**\n- **ANVIL SERVING** \u2014 displayed in large, bold white text\n\n**Status Panel (centered box, slightly lighter navy):**\n\n| Label | Value | Color |\n|-------|-------|-------|\n| STATUS | READY | Bright green |\n| GPU | 42 PERCENT | Light blue |\n\n## Key Takeaways\n\n- \u2705 **Status: READY** \u2014 The service is operational and available (indicated by the green text)\n- \ud83d\udcca **GPU Utilization: 42%** \u2014 The GPU is under moderate load, with roughly 58% of capacity still available\n\nThe color coding follows common dashboard conventions: green for healthy/ready status, and blue for informational metrics. Overall, this appears to be a healthy system snapshot showing a GPU inference or serving workload running at moderate utilization.", "content_chars": 917, "content_excerpt": "# Image Description\n\nThis is a **system status dashboard** with a dark navy blue background, displaying monitoring information for a service called \"ANVIL SERVING.\"\n\n## Content Breakdown\n\n**Title (top", "expected": [ "ANVIL SERVING", "STATUS READY", "GPU 42 PERCENT" ], "finish_reason": "stop", "image": { "bytes": 16940, "mime": "image/png", "sha256": "03fd14d8b68973c915de2848736d22eb6f67ba6bdb5da2a7a563085ac894e804" }, "passed": false, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "seconds": 1.453, "test": "image", "usage": { "completion_tokens": 212, "prompt_tokens": 334, "prompt_tokens_details": { "cached_tokens": 0, "image_tokens": 299 }, "reasoning_tokens": 0, "total_tokens": 546 }, "validation_detail": "missing expected text: ['STATUS READY', 'GPU 42 PERCENT']" }, { "content": "ANVIL SERVING\n\nSTATUS READY\n\nGPU 42 PERCENT", "content_chars": 43, "content_excerpt": "ANVIL SERVING\n\nSTATUS READY\n\nGPU 42 PERCENT", "expected": [ "ANVIL SERVING", "STATUS READY", "GPU 42 PERCENT" ], "finish_reason": "stop", "image": { "bytes": 16940, "mime": "image/png", "sha256": "03fd14d8b68973c915de2848736d22eb6f67ba6bdb5da2a7a563085ac894e804" }, "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "seconds": 0.245, "test": "ocr", "usage": { "completion_tokens": 16, "prompt_tokens": 335, "prompt_tokens_details": { "cached_tokens": 0, "image_tokens": 299 }, "reasoning_tokens": 0, "total_tokens": 351 }, "validation_detail": "all expected text present" } ], "ocr_expect": [ "ANVIL SERVING", "STATUS READY", "GPU 42 PERCENT" ], "passed": false, "results": [ { "detail": "0.5s got=\"```python\\nsum(xs)\\n```\\n\\nThat's it \u2014 Python's built-\" finish='stop' visible=220 reasoning_chars=0 reasoning_tokens=None", "name": "smoke (short coding)", "passed": true }, { "detail": "parsed keys=['language', 'ok'] finish='stop' visible=31 reasoning_chars=0 reasoning_tokens=None", "name": "structured JSON", "passed": true }, { "detail": "54.3s ctx~380000 got='ZEBRA-42917-QUARTZ' finish='stop' visible=18 reasoning_chars=0 reasoning_tokens=None", "name": "needle @ ~380000 ctx", "passed": true }, { "detail": "20/20 clean (sample: 0.9s valid tool_call get_weather(city='Oakland') finish='tool_calls' visible=0 reasoning_chars=0 reasoning_tokens=None)", "name": "shared-prefix tool batch x20", "passed": true }, { "detail": "22.9s measured_prompt=135169 valid tool_call get_weather(city='Oakland') finish='tool_calls' visible=0 reasoning_chars=0 reasoning_tokens=None", "name": "tool call after >=100K prompt (target 160000)", "passed": true }, { "detail": "0.5s done=True events=7 valid tool_call get_weather(city='Oakland') finish='tool_calls'", "name": "streaming tool call", "passed": true }, { "detail": "0.3s tool result retained finish='stop' visible=11 reasoning_chars=0 reasoning_tokens=None", "name": "tool-result continuation", "passed": true }, { "detail": "0.1s status='completed' output='READY' reasoning_chars=0", "name": "Responses API subset", "passed": true }, { "detail": "1.5s missing=['STATUS READY', 'GPU 42 PERCENT'] finish='stop' visible=917 reasoning_chars=0 reasoning_tokens=None", "name": "general image understanding", "passed": false }, { "detail": "0.2s all expected text present finish='stop' visible=43 reasoning_chars=0 reasoning_tokens=None", "name": "verbatim image OCR", "passed": true } ], "schema_version": "preflight/v2", "thinking": { "chat_template_kwargs": { "enable_thinking": false }, "mode": "disabled", "reasoning_effort": null }, "video_expect": [], "video_input": null }