{ "context": { "cap_tokens": 260864, "max_model_len": 262144, "targets": [ { "attempted_context_tokens": 32768, "chars_per_token": 3.0, "clamped_tokens": 32768, "e2e_ms": 6962.919473648071, "estimated_prompt_tokens": 32768, "output_tokens": 0, "status": "passed", "target_tokens": 32768, "ttft_ms": 6962.918519973755, "usage": { "completion_tokens": 256, "prompt_tokens": 27774, "total_tokens": 28030 } }, { "attempted_context_tokens": 131072, "chars_per_token": 3.539, "clamped_tokens": 131072, "e2e_ms": 44608.45160484314, "estimated_prompt_tokens": 131071, "output_tokens": 0, "status": "passed", "target_tokens": 131072, "ttft_ms": 44608.4508895874, "usage": { "completion_tokens": 256, "prompt_tokens": 152736, "total_tokens": 152992 } }, { "attempted_context_tokens": 240000, "chars_per_token": 3.037, "clamped_tokens": 240000, "e2e_ms": 97333.83226394653, "estimated_prompt_tokens": 239999, "output_tokens": 0, "status": "passed", "target_tokens": 240000, "ttft_ms": 97333.83178710938, "usage": { "completion_tokens": 256, "prompt_tokens": 243409, "total_tokens": 243665 } } ] }, "evaluation_protocol": { "budget_semantics": "visible_answer_tokens plus reasoning_headroom_tokens are sent as one max_tokens cap; the endpoint does not hard-partition the channels", "minimum_comparable_repetitions": 3, "minimum_pass_rate": 1.0, "reasoning_headroom_tokens": 2048, "records_finish_reason": true, "records_full_visible_answer": true, "records_reasoning_channel_metadata": true, "repetitions": 3, "version": 3, "visible_answer_tokens": 512 }, "failures": [], "identity": { "base_url": "http://127.0.0.1:39038/v1", "candidate_id": "google/gemma-4-12B-it-qat-w4a16-ct@5d8bb23cdbff01e89d2a1a47f3b3d29b877bca76", "config_id": "heavy-12b-w4a16-256k-thinking", "engine": "vllm-0.25.1", "gpu": "RTX-PRO-6000-REDACTED_GPU_UUID_COMPUTE_A", "model": "gemma4-heavy-lab", "started_at": "2026-07-16T15:52:42Z" }, "intelligence": { "checks": [ { "attempt_count": 3, "attempts": [ { "attempt": 1, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 2560, "reasoning_headroom_tokens": 2048, "visible_answer_tokens": 512 }, "content": "```diff\n--- app.py\n+++ app.py\n@@ -1,2 +1,2 @@\n-timeout = 30\n+timeout = 45\n retries = 2\n```", "content_excerpt": "```diff\n--- app.py\n+++ app.py\n@@ -1,2 +1,2 @@\n-timeout = 30\n+timeout = 45\n retries = 2\n```", "failure_class": null, "finish_reason": "stop", "latency_ms": 2919.1534519195557, "reasoning_chars": 776, "reasoning_excerpt": "* Input file content:\n ```python\n timeout = 30\n retries = 2\n ```\n * Task: Change `timeout` to 45, leave `retries` unchanged.\n * Output format: Unified diff only", "reasoning_field": "reasoning", "reasoning_tokens": null, "status": "passed", "text_checks": [ { "name": "diff_shape", "passed": true }, { "name": "removes_old_timeout", "passed": true }, { "name": "adds_new_timeout", "passed": true } ], "usage": { "completion_tokens": 322, "prompt_tokens": 57, "prompt_tokens_details": null, "total_tokens": 379 } }, { "attempt": 2, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 2560, "reasoning_headroom_tokens": 2048, "visible_answer_tokens": 512 }, "content": "```diff\n--- app.py\n+++ app.py\n@@ -1,2 +1,2 @@\n-timeout = 30\n+timeout = 45\n retries = 2\n```", "content_excerpt": "```diff\n--- app.py\n+++ app.py\n@@ -1,2 +1,2 @@\n-timeout = 30\n+timeout = 45\n retries = 2\n```", "failure_class": null, "finish_reason": "stop", "latency_ms": 2980.93843460083, "reasoning_chars": 776, "reasoning_excerpt": "* Input file content:\n ```python\n timeout = 30\n retries = 2\n ```\n * Task: Change `timeout` to 45, leave `retries` unchanged.\n * Output format: Unified diff only", "reasoning_field": "reasoning", "reasoning_tokens": null, "status": "passed", "text_checks": [ { "name": "diff_shape", "passed": true }, { "name": "removes_old_timeout", "passed": true }, { "name": "adds_new_timeout", "passed": true } ], "usage": { "completion_tokens": 322, "prompt_tokens": 57, "prompt_tokens_details": null, "total_tokens": 379 } }, { "attempt": 3, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 2560, "reasoning_headroom_tokens": 2048, "visible_answer_tokens": 512 }, "content": "```diff\n--- app.py\n+++ app.py\n@@ -1,2 +1,2 @@\n-timeout = 30\n+timeout = 45\n retries = 2\n```", "content_excerpt": "```diff\n--- app.py\n+++ app.py\n@@ -1,2 +1,2 @@\n-timeout = 30\n+timeout = 45\n retries = 2\n```", "failure_class": null, "finish_reason": "stop", "latency_ms": 2979.736804962158, "reasoning_chars": 776, "reasoning_excerpt": "* Input file content:\n ```python\n timeout = 30\n retries = 2\n ```\n * Task: Change `timeout` to 45, leave `retries` unchanged.\n * Output format: Unified diff only", "reasoning_field": "reasoning", "reasoning_tokens": null, "status": "passed", "text_checks": [ { "name": "diff_shape", "passed": true }, { "name": "removes_old_timeout", "passed": true }, { "name": "adds_new_timeout", "passed": true } ], "usage": { "completion_tokens": 322, "prompt_tokens": 57, "prompt_tokens_details": null, "total_tokens": 379 } } ], "id": "unified_diff_timeout_edit", "pass_count": 3, "pass_rate": 1.0, "required_pass_rate": 1.0, "status": "passed", "validator": "deterministic_text_checks" }, { "attempt_count": 3, "attempts": [ { "attempt": 1, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 2560, "reasoning_headroom_tokens": 2048, "visible_answer_tokens": 512 }, "content": "The total latency (3000ms) exceeds the 2500ms timeout; implement LLM streaming or use a faster model to reduce response time.", "content_excerpt": "The total latency (3000ms) exceeds the 2500ms timeout; implement LLM streaming or use a faster model to reduce response time.", "failure_class": null, "finish_reason": "stop", "latency_ms": 3959.9449634552, "reasoning_chars": 1235, "reasoning_excerpt": "* Components: STT (Speech-to-Text), LLM (Large Language Model), TTS (Text-to-Speech).\n * Total Turn Timeout: 2500 ms.\n * Actual Latency:\n * STT: 550 ms\n * LLM: 1800 ms\n ", "reasoning_field": "reasoning", "reasoning_tokens": null, "status": "passed", "text_checks": [ { "name": "identifies_budget_overrun", "passed": true }, { "name": "offers_latency_fix", "passed": true } ], "usage": { "completion_tokens": 431, "prompt_tokens": 81, "prompt_tokens_details": null, "total_tokens": 512 } }, { "attempt": 2, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 2560, "reasoning_headroom_tokens": 2048, "visible_answer_tokens": 512 }, "content": "The total latency (3000ms) exceeds the 2500ms timeout; implement LLM streaming or use a faster model to reduce response time.", "content_excerpt": "The total latency (3000ms) exceeds the 2500ms timeout; implement LLM streaming or use a faster model to reduce response time.", "failure_class": null, "finish_reason": "stop", "latency_ms": 3924.7443675994873, "reasoning_chars": 1235, "reasoning_excerpt": "* Components: STT (Speech-to-Text), LLM (Large Language Model), TTS (Text-to-Speech).\n * Total Turn Timeout: 2500 ms.\n * Actual Latency:\n * STT: 550 ms\n * LLM: 1800 ms\n ", "reasoning_field": "reasoning", "reasoning_tokens": null, "status": "passed", "text_checks": [ { "name": "identifies_budget_overrun", "passed": true }, { "name": "offers_latency_fix", "passed": true } ], "usage": { "completion_tokens": 431, "prompt_tokens": 81, "prompt_tokens_details": null, "total_tokens": 512 } }, { "attempt": 3, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 2560, "reasoning_headroom_tokens": 2048, "visible_answer_tokens": 512 }, "content": "The total latency (3000ms) exceeds the 2500ms timeout; implement LLM streaming or use a faster model to reduce response time.", "content_excerpt": "The total latency (3000ms) exceeds the 2500ms timeout; implement LLM streaming or use a faster model to reduce response time.", "failure_class": null, "finish_reason": "stop", "latency_ms": 3906.348943710327, "reasoning_chars": 1235, "reasoning_excerpt": "* Components: STT (Speech-to-Text), LLM (Large Language Model), TTS (Text-to-Speech).\n * Total Turn Timeout: 2500 ms.\n * Actual Latency:\n * STT: 550 ms\n * LLM: 1800 ms\n ", "reasoning_field": "reasoning", "reasoning_tokens": null, "status": "passed", "text_checks": [ { "name": "identifies_budget_overrun", "passed": true }, { "name": "offers_latency_fix", "passed": true } ], "usage": { "completion_tokens": 431, "prompt_tokens": 81, "prompt_tokens_details": null, "total_tokens": 512 } } ], "id": "parallel_timeout_triage", "pass_count": 3, "pass_rate": 1.0, "required_pass_rate": 1.0, "status": "passed", "validator": "deterministic_text_checks" } ], "status": "passed" }, "run_id": "fast-bakeoff-20260716T155242Z", "schema": "anvil-serving.fast-tier-bakeoff/v1", "score_inputs": { "e2e_p50_ms": 44608.45160484314, "intelligence_pass_rate": 1.0, "operational_fit_notes": [ "endpoint was already loaded; benchmark did not start or stop serves" ], "session_recall_passed": true, "thinking_mode": "enabled", "tool_call_passed": true, "ttft_p50_ms": 44608.4508895874, "usable_context_tokens": 240000, "voice_latency_ms": null }, "selection": { "context_targets": [ 32768, 131072, 240000 ], "endpoint_already_loaded": true, "requests_per_context": 1, "suites": [ "chat", "context", "tool", "session", "intelligence" ] }, "session": { "checks": [ { "attempt_count": 3, "attempts": [ { "attempt": 1, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 2560, "reasoning_headroom_tokens": 2048, "visible_answer_tokens": 512 }, "content": "RIVER-918", "content_excerpt": "RIVER-918", "expected": "RIVER-918", "failure_class": null, "finish_reason": "stop", "latency_ms": 926.8085956573486, "reasoning_chars": 320, "reasoning_excerpt": "The user is asking for the session code that was provided in the previous turn.\nThe previous turn was: \"Remember this session code: RIVER-918. Reply with ok.\"\nThe session code is \"RIVER-918\".\nThe user", "reasoning_field": "reasoning", "reasoning_tokens": null, "status": "passed", "usage": { "completion_tokens": 97, "prompt_tokens": 55, "prompt_tokens_details": null, "total_tokens": 152 } }, { "attempt": 2, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 2560, "reasoning_headroom_tokens": 2048, "visible_answer_tokens": 512 }, "content": "RIVER-918", "content_excerpt": "RIVER-918", "expected": "RIVER-918", "failure_class": null, "finish_reason": "stop", "latency_ms": 914.2584800720215, "reasoning_chars": 320, "reasoning_excerpt": "The user is asking for the session code that was provided in the previous turn.\nThe previous turn was: \"Remember this session code: RIVER-918. Reply with ok.\"\nThe session code is \"RIVER-918\".\nThe user", "reasoning_field": "reasoning", "reasoning_tokens": null, "status": "passed", "usage": { "completion_tokens": 97, "prompt_tokens": 55, "prompt_tokens_details": null, "total_tokens": 152 } }, { "attempt": 3, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 2560, "reasoning_headroom_tokens": 2048, "visible_answer_tokens": 512 }, "content": "RIVER-918", "content_excerpt": "RIVER-918", "expected": "RIVER-918", "failure_class": null, "finish_reason": "stop", "latency_ms": 907.5801372528076, "reasoning_chars": 320, "reasoning_excerpt": "The user is asking for the session code that was provided in the previous turn.\nThe previous turn was: \"Remember this session code: RIVER-918. Reply with ok.\"\nThe session code is \"RIVER-918\".\nThe user", "reasoning_field": "reasoning", "reasoning_tokens": null, "status": "passed", "usage": { "completion_tokens": 97, "prompt_tokens": 55, "prompt_tokens_details": null, "total_tokens": 152 } } ], "name": "single_request_multiturn_recall", "pass_count": 3, "pass_rate": 1.0, "required_pass_rate": 1.0, "status": "passed" } ], "status": "passed" }, "source_recipe": { "ref": null, "serve_command": "vllm serve google/gemma-4-12B-it-qat-w4a16-ct --revision 5d8bb23cdbff01e89d2a1a47f3b3d29b877bca76 --tokenizer google/gemma-4-12B-it --tokenizer-revision 12ace6d648d72bd41519e140f1185f34d38c7e3d --max-model-len 262144 --kv-cache-dtype fp8" }, "suites": {}, "thinking": { "chat_template_kwargs": { "enable_thinking": true }, "control_evidence": "C:\\Users\\operator\\ai-code\\anvil-serving\\docs\\findings\\2026-07-16-gemma4-chat-template-bakeoff-evidence\\heavy-12b-thinking-control.json", "control_evidence_sha256": "4d27b5bbe7b8cdac9a53dd48b9a49478a0fcd768dd0527e24fa073ec8a0eb9de", "control_mechanism": "chat_template_kwargs", "control_requested": { "enable_thinking": true }, "control_status": "verified", "mode": "enabled", "reasoning_effort": null, "unsupported": false }, "timing": { "chat": { "e2e_p50_ms": 44608.45160484314, "e2e_p95_ms": 97333.83226394653, "output_tokens": 0, "ttft_p50_ms": 44608.4508895874, "ttft_p95_ms": 97333.83178710938 }, "wall_ms": 175943.5019493103 }, "tool": { "checks": [ { "attempt_count": 3, "attempts": [ { "arguments": { "zip": "98101" }, "attempt": 1, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 2560, "reasoning_headroom_tokens": 2048, "visible_answer_tokens": 512 }, "content": "", "content_excerpt": "", "failure_class": null, "finish_reason": "tool_calls", "latency_ms": 1214.5638465881348, "reasoning_chars": 328, "reasoning_excerpt": "The user wants to call the `record_weather_zip` tool with the argument `zip=\"98101\"`. I should check the tool definition to ensure the parameter name and type are correct.\n\nTool: `record_weather_zip`\n", "reasoning_field": "reasoning", "reasoning_tokens": null, "status": "passed", "tool_call_count": 1, "usage": { "completion_tokens": 128, "prompt_tokens": 81, "prompt_tokens_details": null, "total_tokens": 209 }, "valid_tool_call_count": 1, "validation_errors": [] }, { "arguments": { "zip": "98101" }, "attempt": 2, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 2560, "reasoning_headroom_tokens": 2048, "visible_answer_tokens": 512 }, "content": "", "content_excerpt": "", "failure_class": null, "finish_reason": "tool_calls", "latency_ms": 1199.047565460205, "reasoning_chars": 328, "reasoning_excerpt": "The user wants to call the `record_weather_zip` tool with the argument `zip=\"98101\"`. I should check the tool definition to ensure the parameter name and type are correct.\n\nTool: `record_weather_zip`\n", "reasoning_field": "reasoning", "reasoning_tokens": null, "status": "passed", "tool_call_count": 1, "usage": { "completion_tokens": 128, "prompt_tokens": 81, "prompt_tokens_details": null, "total_tokens": 209 }, "valid_tool_call_count": 1, "validation_errors": [] }, { "arguments": { "zip": "98101" }, "attempt": 3, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 2560, "reasoning_headroom_tokens": 2048, "visible_answer_tokens": 512 }, "content": "", "content_excerpt": "", "failure_class": null, "finish_reason": "tool_calls", "latency_ms": 1196.2833404541016, "reasoning_chars": 328, "reasoning_excerpt": "The user wants to call the `record_weather_zip` tool with the argument `zip=\"98101\"`. I should check the tool definition to ensure the parameter name and type are correct.\n\nTool: `record_weather_zip`\n", "reasoning_field": "reasoning", "reasoning_tokens": null, "status": "passed", "tool_call_count": 1, "usage": { "completion_tokens": 128, "prompt_tokens": 81, "prompt_tokens_details": null, "total_tokens": 209 }, "valid_tool_call_count": 1, "validation_errors": [] } ], "expected_arguments": { "zip": "98101" }, "expected_function": "record_weather_zip", "name": "openai_tool_call_smoke", "pass_count": 3, "pass_rate": 1.0, "required_pass_rate": 1.0, "status": "passed" } ], "status": "passed" }, "voice": { "llm_latency_ms": null, "status": "not_run", "stt_latency_ms": null, "total_turn_latency_ms": null, "tts_latency_ms": null } }