{ "base_url": "http://127.0.0.1:8080/v1", "budget": { "max_completion_tokens": 256, "reasoning_headroom_tokens": 0, "visible_answer_tokens": 256 }, "checks": [ "smoke", "json", "needle", "tools", "streaming-tools", "tool-result", "responses" ], "evidence_policy": { "allowed_finish_reasons": [ "stop", "tool_calls" ], "errors": [], "reasoning": "forbidden" }, "image_expect": [], "model": "qwen38-huihui-ninfer-schemafix-mtp3-64k", "multimodal_input": null, "observations": [ { "content": "```python\nsum(xs)\n```", "content_chars": 21, "content_excerpt": "```python\nsum(xs)\n```", "finish_reason": "stop", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "seconds": 0.356, "test": "smoke", "usage": { "completion_tokens": 9, "prompt_tokens": 28, "total_tokens": 37 }, "validation_detail": "contains sum(" }, { "content": "{\"language\":\"python\",\"ok\":true}", "content_chars": 31, "content_excerpt": "{\"language\":\"python\",\"ok\":true}", "finish_reason": "stop", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "seconds": 0.123, "test": "json", "usage": { "completion_tokens": 10, "prompt_tokens": 30, "total_tokens": 40 }, "validation_detail": "parsed keys=['language', 'ok']" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 1, "seconds": 1.592, "test": "tools", "usage": { "completion_tokens": 27, "prompt_tokens": 5485, "total_tokens": 5512 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 0, "seconds": 2.486, "test": "tools", "usage": { "completion_tokens": 27, "prompt_tokens": 5485, "total_tokens": 5512 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 2, "seconds": 3.382, "test": "tools", "usage": { "completion_tokens": 27, "prompt_tokens": 5485, "total_tokens": 5512 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 3, "seconds": 4.285, "test": "tools", "usage": { "completion_tokens": 27, "prompt_tokens": 5485, "total_tokens": 5512 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 4, "seconds": 5.186, "test": "tools", "usage": { "completion_tokens": 27, "prompt_tokens": 5485, "total_tokens": 5512 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 17, "seconds": 6.081, "test": "tools", "usage": { "completion_tokens": 27, "prompt_tokens": 5485, "total_tokens": 5512 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 6, "seconds": 6.986, "test": "tools", "usage": { "completion_tokens": 27, "prompt_tokens": 5485, "total_tokens": 5512 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 5, "seconds": 7.889, "test": "tools", "usage": { "completion_tokens": 27, "prompt_tokens": 5485, "total_tokens": 5512 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 8, "seconds": 8.79, "test": "tools", "usage": { "completion_tokens": 27, "prompt_tokens": 5485, "total_tokens": 5512 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 13, "seconds": 9.689, "test": "tools", "usage": { "completion_tokens": 27, "prompt_tokens": 5485, "total_tokens": 5512 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 16, "seconds": 10.59, "test": "tools", "usage": { "completion_tokens": 27, "prompt_tokens": 5485, "total_tokens": 5512 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 7, "seconds": 11.496, "test": "tools", "usage": { "completion_tokens": 27, "prompt_tokens": 5485, "total_tokens": 5512 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 15, "seconds": 12.397, "test": "tools", "usage": { "completion_tokens": 27, "prompt_tokens": 5485, "total_tokens": 5512 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 11, "seconds": 13.301, "test": "tools", "usage": { "completion_tokens": 27, "prompt_tokens": 5485, "total_tokens": 5512 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 9, "seconds": 14.203, "test": "tools", "usage": { "completion_tokens": 27, "prompt_tokens": 5485, "total_tokens": 5512 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 10, "seconds": 15.099, "test": "tools", "usage": { "completion_tokens": 27, "prompt_tokens": 5485, "total_tokens": 5512 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 12, "seconds": 15.996, "test": "tools", "usage": { "completion_tokens": 27, "prompt_tokens": 5485, "total_tokens": 5512 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content_chars": 0, "content_excerpt": "", "event_count": 4, "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "seconds": 0.252, "sse_done": true, "test": "streaming-tools", "usage": { "completion_tokens": 27, "prompt_tokens": 285, "total_tokens": 312 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "seconds": 0.177, "test": "tool-result-initial", "usage": { "completion_tokens": 27, "prompt_tokens": 285, "total_tokens": 312 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "OAKLAND 72F", "content_chars": 11, "content_excerpt": "OAKLAND 72F", "finish_reason": "stop", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "seconds": 0.13, "test": "tool-result-continuation", "usage": { "completion_tokens": 8, "prompt_tokens": 99, "total_tokens": 107 }, "validation_detail": "tool result retained" }, { "content_chars": 5, "content_excerpt": "READY", "finish_reason": "stop", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "response_status": "completed", "seconds": 0.081, "test": "responses", "usage": { "input_tokens": 16, "input_tokens_details": { "cached_tokens": 0 }, "output_tokens": 2, "output_tokens_details": { "reasoning_tokens": 0 }, "total_tokens": 18 }, "validation_detail": "completed exact READY" } ], "ocr_expect": [], "passed": false, "results": [ { "detail": "0.4s got='```python\\nsum(xs)\\n```' finish='stop' visible=21 reasoning_chars=0 reasoning_tokens=None", "name": "smoke (short coding)", "passed": true }, { "detail": "parsed keys=['language', 'ok'] finish='stop' visible=31 reasoning_chars=0 reasoning_tokens=None", "name": "structured JSON", "passed": true }, { "detail": "error: HTTP 400: {\"error\":{\"code\":\"context_length_exceeded\",\"message\":\"prepared prompt exceeds Engine max_context 65536\",\"param\":\"messages\",\"type\":\"invalid_request_error\"}}", "name": "needle @ ~128000 ctx", "passed": false }, { "detail": "17/20 clean (sample: error: HTTP 429: {\"error\":{\"code\":\"server_overloaded\",\"message\":\"inference request queue is full\",\"param\":null,\"type\":\"rate_limit_error\"}})", "name": "shared-prefix tool batch x20", "passed": false }, { "detail": "0.3s done=True events=4 valid tool_call get_weather(city='Oakland') finish='tool_calls'", "name": "streaming tool call", "passed": true }, { "detail": "0.3s tool result retained finish='stop' visible=11 reasoning_chars=0 reasoning_tokens=None", "name": "tool-result continuation", "passed": true }, { "detail": "0.1s status='completed' output='READY' reasoning_chars=0", "name": "Responses API subset", "passed": true } ], "schema_version": "preflight/v2", "thinking": { "chat_template_kwargs": null, "mode": "unsupported", "reasoning_effort": null }, "video_expect": [], "video_input": null }