{ "base_url": "http://127.0.0.1:8000/v1", "budget": { "max_completion_tokens": 8192, "reasoning_headroom_tokens": 7168, "visible_answer_tokens": 1024 }, "checks": [ "smoke", "json", "tools", "streaming-tools", "tool-result", "responses", "image", "ocr" ], "evidence_policy": { "allowed_finish_reasons": [ "stop", "tool_calls" ], "errors": [], "reasoning": "any" }, "image_expect": [ "ANVIL READY", "42917", "42" ], "model": "llm.primary", "multimodal_input": { "bytes": 15916, "mime": "image/png", "sha256": "a8d5158019fb2b08ee9f6a153860ab2342384644f7c53ab58fdd45658124e83e" }, "observations": [ { "content": "```python\nsum(xs)\n```\n\nThat's it \u2014 Python's built-in `sum()` does exactly this. If you want it as a lambda:\n\n```python\ntotal = lambda xs: sum(xs)\n```", "content_chars": 149, "content_excerpt": "```python\nsum(xs)\n```\n\nThat's it \u2014 Python's built-in `sum()` does exactly this. If you want it as a lambda:\n\n```python\ntotal = lambda xs: sum(xs)\n```", "finish_reason": "stop", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "seconds": 0.848, "test": "smoke", "usage": { "completion_tokens": 45, "prompt_tokens": 28, "total_tokens": 73 }, "validation_detail": "contains sum(" }, { "content": "{\"language\":\"python\",\"ok\":true}", "content_chars": 31, "content_excerpt": "{\"language\":\"python\",\"ok\":true}", "finish_reason": "stop", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "seconds": 0.172, "test": "json", "usage": { "completion_tokens": 11, "prompt_tokens": 30, "total_tokens": 41 }, "validation_detail": "parsed keys=['language', 'ok']" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 0, "seconds": 8.619, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 1, "seconds": 10.985, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 2, "seconds": 11.566, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 4, "seconds": 13.309, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 3, "seconds": 14.488, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 5, "seconds": 16.22, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 6, "seconds": 17.389, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 8, "seconds": 19.121, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 9, "seconds": 20.328, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 7, "seconds": 22.071, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 10, "seconds": 23.246, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 11, "seconds": 25.034, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 12, "seconds": 26.203, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 15, "seconds": 27.39, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 13, "seconds": 29.138, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 16, "seconds": 30.312, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 17, "seconds": 32.065, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 18, "seconds": 33.262, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 14, "seconds": 34.112, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "request_index": 19, "seconds": 34.183, "test": "tools", "usage": { "completion_tokens": 13, "prompt_tokens": 4967, "total_tokens": 4980 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content_chars": 0, "content_excerpt": "", "event_count": 5, "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "seconds": 0.518, "sse_done": true, "test": "streaming-tools", "usage": { "completion_tokens": 42, "prompt_tokens": 166, "total_tokens": 208 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "", "content_chars": 0, "content_excerpt": "", "finish_reason": "tool_calls", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "seconds": 0.242, "test": "tool-result-initial", "usage": { "completion_tokens": 13, "prompt_tokens": 166, "total_tokens": 179 }, "validation_detail": "valid tool_call get_weather(city='Oakland')" }, { "content": "OAKLAND 72F", "content_chars": 11, "content_excerpt": "OAKLAND 72F", "finish_reason": "stop", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "seconds": 0.201, "test": "tool-result-continuation", "usage": { "completion_tokens": 8, "prompt_tokens": 70, "total_tokens": 78 }, "validation_detail": "tool result retained" }, { "content_chars": 5, "content_excerpt": "READY", "finish_reason": "stop", "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "response_status": "completed", "seconds": 0.431, "test": "responses", "usage": { "input_tokens": 16, "output_tokens": 33, "total_tokens": 49 }, "validation_detail": "completed exact READY" }, { "content": "This image shows a dark navy blue status display with three lines of text:\n\n1. **ANVIL READY** \u2014 in bright green, indicating the system/anvil is operational and ready\n2. **CODE 42917** \u2014 in white, likely an identification or reference code\n3. **TEMPERATURE 42 C** \u2014 in light blue, showing the current temperature reading of 42\u00b0C\n\nOverall, it appears to be a machine or process status screen confirming that the \"anvil\" is ready for use, with an associated code and a temperature reading of 42 degrees Celsius.", "content_chars": 509, "content_excerpt": "This image shows a dark navy blue status display with three lines of text:\n\n1. **ANVIL READY** \u2014 in bright green, indicating the system/anvil is operational and ready\n2. **CODE 42917** \u2014 in white, lik", "expected": [ "ANVIL READY", "42917", "42" ], "finish_reason": "stop", "image": { "bytes": 15916, "mime": "image/png", "sha256": "a8d5158019fb2b08ee9f6a153860ab2342384644f7c53ab58fdd45658124e83e" }, "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "seconds": 1.45, "test": "image", "usage": { "completion_tokens": 123, "prompt_tokens": 333, "total_tokens": 456 }, "validation_detail": "all expected text present" }, { "content": "ANVIL READY\nCODE 42917\nTEMPERATURE 42 C", "content_chars": 39, "content_excerpt": "ANVIL READY\nCODE 42917\nTEMPERATURE 42 C", "expected": [ "ANVIL READY", "42917", "42" ], "finish_reason": "stop", "image": { "bytes": 15916, "mime": "image/png", "sha256": "a8d5158019fb2b08ee9f6a153860ab2342384644f7c53ab58fdd45658124e83e" }, "passed": true, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "seconds": 0.341, "test": "ocr", "usage": { "completion_tokens": 19, "prompt_tokens": 334, "total_tokens": 353 }, "validation_detail": "all expected text present" } ], "ocr_expect": [ "ANVIL READY", "42917", "42" ], "passed": true, "results": [ { "detail": "0.8s got=\"```python\\nsum(xs)\\n```\\n\\nThat's it \u2014 Python's built-\" finish='stop' visible=149 reasoning_chars=0 reasoning_tokens=None", "name": "smoke (short coding)", "passed": true }, { "detail": "parsed keys=['language', 'ok'] finish='stop' visible=31 reasoning_chars=0 reasoning_tokens=None", "name": "structured JSON", "passed": true }, { "detail": "20/20 clean (sample: 8.6s valid tool_call get_weather(city='Oakland') finish='tool_calls' visible=0 reasoning_chars=0 reasoning_tokens=None)", "name": "shared-prefix tool batch x20", "passed": true }, { "detail": "0.5s done=True events=5 valid tool_call get_weather(city='Oakland') finish='tool_calls'", "name": "streaming tool call", "passed": true }, { "detail": "0.4s tool result retained finish='stop' visible=11 reasoning_chars=0 reasoning_tokens=None", "name": "tool-result continuation", "passed": true }, { "detail": "0.4s status='completed' output='READY' reasoning_chars=0", "name": "Responses API subset", "passed": true }, { "detail": "1.4s all expected text present finish='stop' visible=509 reasoning_chars=0 reasoning_tokens=None", "name": "general image understanding", "passed": true }, { "detail": "0.3s all expected text present finish='stop' visible=39 reasoning_chars=0 reasoning_tokens=None", "name": "verbatim image OCR", "passed": true } ], "schema_version": "preflight/v2", "thinking": { "chat_template_kwargs": null, "mode": "default", "reasoning_effort": "low" }, "video_expect": [], "video_input": null }