{ "base_url": "http://127.0.0.1:8001/v1", "budget": { "max_completion_tokens": 8192, "reasoning_headroom_tokens": 7168, "visible_answer_tokens": 1024 }, "checks": [ "smoke", "json", "needle", "tools", "long-tools", "streaming-tools", "tool-result", "responses" ], "evidence_policy": { "allowed_finish_reasons": [ "stop", "tool_calls" ], "errors": [], "reasoning": "any" }, "image_expect": [], "model": "glm53-flash-exl3-k3-dflash2-k5-fp8-tp2-524k-vision-xgfix", "multimodal_input": null, "observations": [ { "content": "```python\nsum(xs)\n```\n\nThat's it \u2014 Python's built-in `sum()` does exactly this. If you want it as a lambda:\n\n```python\ntotal = lambda xs: sum(xs)\n```", "content_chars": 149, "content_excerpt": "```python\nsum(xs)\n```\n\nThat's it \u2014 Python's built-in `sum()` does exactly this. If you want it as a lambda:\n\n```python\ntotal = lambda xs: sum(xs)\n```", "finish_reason": "stop", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "seconds": 1.426, "test": "smoke", "usage": { "completion_tokens": 45, "prompt_tokens": 28, "prompt_tokens_details": null, "total_tokens": 73 }, "validation_detail": "contains sum(" }, { "content": "{\"language\":\"python\",\"ok\":true}", "content_chars": 31, "content_excerpt": "{\"language\":\"python\",\"ok\":true}", "finish_reason": "stop", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "seconds": 0.174, "test": "json", "usage": { "completion_tokens": 11, "prompt_tokens": 30, "prompt_tokens_details": null, "total_tokens": 41 }, "validation_detail": "parsed keys=['language', 'ok']" }, { "content": "ZEBRA-42917-QUARTZ", "content_chars": 18, "content_excerpt": "ZEBRA-42917-QUARTZ", "finish_reason": "stop", "passed": true, "reasoning_chars": 18, "reasoning_excerpt": "ZEBRA-42917-QUARTZ", "reasoning_field": "reasoning", "reasoning_tokens": null, "seconds": 67.025, "test": "needle", "usage": { "completion_tokens": 24, "prompt_tokens": 206296, "prompt_tokens_details": null, "total_tokens": 206320 }, "validation_detail": "needle present" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 2, "seconds": 3.715, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "prompt_tokens_details": null, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 0, "seconds": 5.445, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "prompt_tokens_details": null, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 1, "seconds": 6.614, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "prompt_tokens_details": null, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 3, "seconds": 8.327, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "prompt_tokens_details": null, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 5, "seconds": 9.49, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "prompt_tokens_details": null, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 6, "seconds": 11.276, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "prompt_tokens_details": null, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 4, "seconds": 12.479, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "prompt_tokens_details": null, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 7, "seconds": 13.667, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "prompt_tokens_details": null, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 8, "seconds": 15.417, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "prompt_tokens_details": null, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 9, "seconds": 16.597, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "prompt_tokens_details": null, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 10, "seconds": 18.335, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "prompt_tokens_details": null, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 11, "seconds": 19.506, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "prompt_tokens_details": null, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 13, "seconds": 21.231, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "prompt_tokens_details": null, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 12, "seconds": 22.406, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "prompt_tokens_details": null, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 14, "seconds": 24.133, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "prompt_tokens_details": null, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 15, "seconds": 25.303, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "prompt_tokens_details": null, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 16, "seconds": 27.639, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "prompt_tokens_details": null, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 17, "seconds": 28.212, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "prompt_tokens_details": null, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 19, "seconds": 33.863, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "prompt_tokens_details": null, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 18, "seconds": 33.973, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "prompt_tokens_details": null, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "measured_prompt_tokens": 211101, "minimum_prompt_tokens": 100000, "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "requested_context_tokens": 250000, "seconds": 62.556, "test": "long-tools", "usage": { "completion_tokens": 13, "prompt_tokens": 211101, "prompt_tokens_details": null, "total_tokens": 211114 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content_chars": 0, "content_excerpt": "", "event_count": 13, "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 123, "reasoning_excerpt": "The user wants to know the weather in Oakland. I have a get_weather tool available. Let me call it with the city \"Oakland\".", "reasoning_field": "stream_delta", "reasoning_tokens": null, "seconds": 5.537, "sse_done": true, "test": "streaming-tools", "usage": { "completion_tokens": 42, "prompt_tokens": 166, "total_tokens": 208 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "seconds": 0.255, "test": "tool-result-initial", "usage": { "completion_tokens": 13, "prompt_tokens": 166, "prompt_tokens_details": null, "total_tokens": 179 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "OAKLAND 72F", "content_chars": 11, "content_excerpt": "OAKLAND 72F", "finish_reason": "stop", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "seconds": 0.256, "test": "tool-result-continuation", "usage": { "completion_tokens": 8, "prompt_tokens": 70, "prompt_tokens_details": null, "total_tokens": 78 }, "validation_detail": "tool result retained" }, { "content_chars": 5, "content_excerpt": "READY", "finish_reason": "stop", "passed": true, "reasoning_chars": 135, "reasoning_excerpt": "The user wants me to reply with exactly \"READY\". This is a simple instruction. I should just output the word \"READY\" with nothing else.", "reasoning_field": "reasoning_text", "reasoning_tokens": null, "response_status": "completed", "seconds": 0.478, "test": "responses", "usage": { "input_tokens": 16, "input_tokens_details": { "cached_tokens": 0, "cached_tokens_per_turn": [], "input_tokens_per_turn": [] }, "output_tokens": 33, "output_tokens_details": { "output_tokens_per_turn": [], "reasoning_tokens": 0, "tool_output_tokens": 0, "tool_output_tokens_per_turn": [] }, "total_tokens": 49 }, "validation_detail": "completed exact READY" } ], "ocr_expect": [], "passed": true, "results": [ { "detail": "1.4s got=\"```python\\nsum(xs)\\n```\\n\\nThat's it \u2014 Python's built-\" finish='stop' visible=149 reasoning_chars=0 reasoning_tokens=None", "name": "smoke (short coding)", "passed": true }, { "detail": "parsed keys=['language', 'ok'] finish='stop' visible=31 reasoning_chars=0 reasoning_tokens=None", "name": "structured JSON", "passed": true }, { "detail": "67.0s ctx~250000 got='ZEBRA-42917-QUARTZ' finish='stop' visible=18 reasoning_chars=18 reasoning_tokens=None", "name": "needle @ ~250000 ctx", "passed": true }, { "detail": "20/20 clean (sample: 3.7s valid tool_call get_weather(city='Oakland') finish='tool_calls' visible=0 reasoning_chars=0 reasoning_tokens=None)", "name": "shared-prefix tool batch x20", "passed": true }, { "detail": "62.6s measured_prompt=211101 valid tool_call get_weather(city='Oakland') finish='tool_calls' visible=0 reasoning_chars=0 reasoning_tokens=None", "name": "tool call after >=100K prompt (target 250000)", "passed": true }, { "detail": "5.5s done=True events=13 valid tool_call get_weather(city='Oakland') finish='tool_calls'", "name": "streaming tool call", "passed": true }, { "detail": "0.5s tool result retained finish='stop' visible=11 reasoning_chars=0 reasoning_tokens=None", "name": "tool-result continuation", "passed": true }, { "detail": "0.5s status='completed' output='READY' reasoning_chars=135", "name": "Responses API subset", "passed": true } ], "schema_version": "preflight/v2", "thinking": { "chat_template_kwargs": null, "mode": "default", "reasoning_effort": "low" }, "video_expect": [], "video_input": null }