{ "base_url": "http://127.0.0.1:8001/v1", "budget": { "max_completion_tokens": 4096, "reasoning_headroom_tokens": 3584, "visible_answer_tokens": 512 }, "checks": [ "smoke", "json", "needle", "tools", "long-tools", "streaming-tools", "tool-result", "responses", "image", "ocr" ], "evidence_policy": { "allowed_finish_reasons": [ "stop", "tool_calls" ], "errors": [], "reasoning": "any" }, "image_expect": [ "ANVIL READY", "42917" ], "model": "glm53-flash-tr3-4bpw-tp2-262k-fixed-mtp5-vision", "multimodal_input": { "bytes": 15916, "mime": "image/png", "sha256": "a8d5158019fb2b08ee9f6a153860ab2342384644f7c53ab58fdd45658124e83e" }, "observations": [ { "content": "```python\nsum(xs)\n```\n\nIf you want it as a lambda:\n\n```python\ntotal = lambda xs: sum(xs)\n```\n\nOr, if you want to avoid the built-in entirely (e.g., as an exercise):\n\n```python\ntotal = lambda xs: 0 if not xs else xs[0] + total(xs[1:])\n```", "content_chars": 237, "content_excerpt": "```python\nsum(xs)\n```\n\nIf you want it as a lambda:\n\n```python\ntotal = lambda xs: sum(xs)\n```\n\nOr, if you want to avoid the built-in entirely (e.g., as an exercise):\n\n```python\ntotal = lambda xs: 0 if ", "finish_reason": "stop", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "seconds": 1.918, "test": "smoke", "usage": { "completion_tokens": 75, "prompt_tokens": 28, "prompt_tokens_details": null, "total_tokens": 103 }, "validation_detail": "contains sum(" }, { "content": "{\"language\":\"python\",\"ok\":true}", "content_chars": 31, "content_excerpt": "{\"language\":\"python\",\"ok\":true}", "finish_reason": "stop", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "seconds": 0.279, "test": "json", "usage": { "completion_tokens": 11, "prompt_tokens": 30, "prompt_tokens_details": null, "total_tokens": 41 }, "validation_detail": "parsed keys=['language', 'ok']" }, { "content": "ZEBRA-42917-QUARTZ", "content_chars": 18, "content_excerpt": "ZEBRA-42917-QUARTZ", "finish_reason": "stop", "passed": true, "reasoning_chars": 18, "reasoning_excerpt": "ZEBRA-42917-QUARTZ", "reasoning_field": "reasoning", "reasoning_tokens": null, "seconds": 62.593, "test": "needle", "usage": { "completion_tokens": 24, "prompt_tokens": 206296, "prompt_tokens_details": null, "total_tokens": 206320 }, "validation_detail": "needle present" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 0, "seconds": 3.173, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "prompt_tokens_details": null, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 1, "seconds": 4.399, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "prompt_tokens_details": null, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 2, "seconds": 6.316, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "prompt_tokens_details": null, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 3, "seconds": 7.53, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "prompt_tokens_details": null, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 4, "seconds": 9.421, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "prompt_tokens_details": null, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 5, "seconds": 10.628, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "prompt_tokens_details": null, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 6, "seconds": 12.538, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "prompt_tokens_details": null, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 8, "seconds": 13.802, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "prompt_tokens_details": null, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 9, "seconds": 15.035, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "prompt_tokens_details": null, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 10, "seconds": 16.947, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "prompt_tokens_details": null, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 11, "seconds": 18.163, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "prompt_tokens_details": null, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 12, "seconds": 20.055, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "prompt_tokens_details": null, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 7, "seconds": 21.27, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "prompt_tokens_details": null, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 13, "seconds": 23.157, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "prompt_tokens_details": null, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 15, "seconds": 24.421, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "prompt_tokens_details": null, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 14, "seconds": 25.66, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "prompt_tokens_details": null, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 16, "seconds": 27.578, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "prompt_tokens_details": null, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 17, "seconds": 28.797, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "prompt_tokens_details": null, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 18, "seconds": 30.494, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "prompt_tokens_details": null, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 19, "seconds": 30.659, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "prompt_tokens_details": null, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "measured_prompt_tokens": 211101, "minimum_prompt_tokens": 100000, "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "requested_context_tokens": 250000, "seconds": 63.488, "test": "long-tools", "usage": { "completion_tokens": 13, "prompt_tokens": 211101, "prompt_tokens_details": null, "total_tokens": 211114 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content_chars": 0, "content_excerpt": "", "event_count": 13, "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 123, "reasoning_excerpt": "The user wants to know the weather in Oakland. I have a get_weather tool available. Let me call it with the city \"Oakland\".", "reasoning_field": "stream_delta", "reasoning_tokens": null, "seconds": 0.91, "sse_done": true, "test": "streaming-tools", "usage": { "completion_tokens": 42, "prompt_tokens": 166, "total_tokens": 208 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "seconds": 0.36, "test": "tool-result-initial", "usage": { "completion_tokens": 13, "prompt_tokens": 166, "prompt_tokens_details": null, "total_tokens": 179 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "OAKLAND 72F", "content_chars": 11, "content_excerpt": "OAKLAND 72F", "finish_reason": "stop", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "seconds": 0.356, "test": "tool-result-continuation", "usage": { "completion_tokens": 8, "prompt_tokens": 70, "prompt_tokens_details": null, "total_tokens": 78 }, "validation_detail": "tool result retained" }, { "content_chars": 5, "content_excerpt": "READY", "finish_reason": "stop", "passed": true, "reasoning_chars": 136, "reasoning_excerpt": "The user has asked me to reply with exactly \"READY\". This is a simple instruction. I should comply with exactly that word, nothing more.", "reasoning_field": "reasoning_text", "reasoning_tokens": null, "response_status": "completed", "seconds": 0.765, "test": "responses", "usage": { "input_tokens": 16, "input_tokens_details": { "cached_tokens": 0, "cached_tokens_per_turn": [], "input_tokens_per_turn": [] }, "output_tokens": 32, "output_tokens_details": { "output_tokens_per_turn": [], "reasoning_tokens": 0, "tool_output_tokens": 0, "tool_output_tokens_per_turn": [] }, "total_tokens": 48 }, "validation_detail": "completed exact READY" }, { "content": "This image shows a dark navy blue status screen with three lines of text:\n\n1. **\"ANVIL READY\"** \u2014 displayed in bright green, indicating a positive/ready status\n2. **\"CODE 42917\"** \u2014 displayed in white, likely an identification or access code\n3. **\"TEMPERATURE 42 C\"** \u2014 displayed in light blue, showing a temperature reading of 42 degrees Celsius\n\n**Summary:** The screen appears to be a machine or system status display indicating that a device called \"Anvil\" is operational and ready, with an associated code of 42917 and a current temperature of 42\u00b0C. The green \"READY\" text suggests normal operation, though 42\u00b0C is quite warm \u2014 possibly a preheating temperature for a forge, industrial equipment, or similar apparatus.", "content_chars": 723, "content_excerpt": "This image shows a dark navy blue status screen with three lines of text:\n\n1. **\"ANVIL READY\"** \u2014 displayed in bright green, indicating a positive/ready status\n2. **\"CODE 42917\"** \u2014 displayed in white", "expected": [ "ANVIL READY", "42917" ], "finish_reason": "stop", "image": { "bytes": 15916, "mime": "image/png", "sha256": "a8d5158019fb2b08ee9f6a153860ab2342384644f7c53ab58fdd45658124e83e" }, "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "seconds": 3.823, "test": "image", "usage": { "completion_tokens": 172, "prompt_tokens": 333, "prompt_tokens_details": null, "total_tokens": 505 }, "validation_detail": "all expected text present" }, { "content": "ANVIL READY\nCODE 42917\nTEMPERATURE 42 C", "content_chars": 39, "content_excerpt": "ANVIL READY\nCODE 42917\nTEMPERATURE 42 C", "expected": [ "ANVIL READY", "42917", "42" ], "finish_reason": "stop", "image": { "bytes": 15916, "mime": "image/png", "sha256": "a8d5158019fb2b08ee9f6a153860ab2342384644f7c53ab58fdd45658124e83e" }, "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "seconds": 0.606, "test": "ocr", "usage": { "completion_tokens": 19, "prompt_tokens": 334, "prompt_tokens_details": null, "total_tokens": 353 }, "validation_detail": "all expected text present" } ], "ocr_expect": [ "ANVIL READY", "42917", "42" ], "passed": true, "results": [ { "detail": "1.9s got='```python\\nsum(xs)\\n```\\n\\nIf you want it as a lambda:' finish='stop' visible=237 reasoning_chars=0 reasoning_tokens=None", "name": "smoke (short coding)", "passed": true }, { "detail": "parsed keys=['language', 'ok'] finish='stop' visible=31 reasoning_chars=0 reasoning_tokens=None", "name": "structured JSON", "passed": true }, { "detail": "62.6s ctx~250000 got='ZEBRA-42917-QUARTZ' finish='stop' visible=18 reasoning_chars=18 reasoning_tokens=None", "name": "needle @ ~250000 ctx", "passed": true }, { "detail": "20/20 clean (sample: 3.2s valid tool_call get_weather(city='Oakland') finish='tool_calls' visible=0 reasoning_chars=0 reasoning_tokens=None)", "name": "shared-prefix tool batch x20", "passed": true }, { "detail": "63.5s measured_prompt=211101 valid tool_call get_weather(city='Oakland') finish='tool_calls' visible=0 reasoning_chars=0 reasoning_tokens=None", "name": "tool call after >=100K prompt (target 250000)", "passed": true }, { "detail": "0.9s done=True events=13 valid tool_call get_weather(city='Oakland') finish='tool_calls'", "name": "streaming tool call", "passed": true }, { "detail": "0.7s tool result retained finish='stop' visible=11 reasoning_chars=0 reasoning_tokens=None", "name": "tool-result continuation", "passed": true }, { "detail": "0.8s status='completed' output='READY' reasoning_chars=136", "name": "Responses API subset", "passed": true }, { "detail": "3.8s all expected text present finish='stop' visible=723 reasoning_chars=0 reasoning_tokens=None", "name": "general image understanding", "passed": true }, { "detail": "0.6s all expected text present finish='stop' visible=39 reasoning_chars=0 reasoning_tokens=None", "name": "verbatim image OCR", "passed": true } ], "schema_version": "preflight/v2", "thinking": { "chat_template_kwargs": null, "mode": "default", "reasoning_effort": "low" }, "video_expect": [], "video_input": null }