{ "context": { "cap_tokens": 64256, "max_model_len": 65536, "targets": [ { "attempted_context_tokens": 64000, "chars_per_token": 3.0, "clamped_tokens": 64000, "content_chunks": 19, "decode_tok_s": 143.59680789826064, "e2e_ms": 12924.405799945816, "effective_prefill_tok_s": 4988.220475702129, "estimated_prompt_tokens": 64000, "generation_ms": 403.9086999837309, "mean_inter_token_latency_ms": 6.963943103167774, "output_token_source": "usage", "output_tokens": 59, "prompt_tokens": 62455, "reasoning_chunks": 0, "status": "passed", "target_tokens": 64000, "time_to_first_output_ms": 12520.497099962085, "ttft_ms": 12520.498499972746, "usage": { "completion_tokens": 59, "prompt_tokens": 62455, "total_tokens": 62514 }, "visible_generation_ms": 403.9072999730706 } ] }, "evaluation_protocol": { "budget_semantics": "visible_answer_tokens plus reasoning_headroom_tokens are sent as one max_tokens cap; the endpoint does not hard-partition the channels", "minimum_comparable_repetitions": 3, "minimum_pass_rate": 1.0, "reasoning_headroom_tokens": 0, "records_finish_reason": true, "records_full_visible_answer": true, "records_reasoning_channel_metadata": true, "repetitions": 3, "version": 3, "visible_answer_tokens": 256 }, "failures": [], "identity": { "base_url": "http://127.0.0.1:8080/v1", "candidate_id": "qwen38-huihui", "config_id": "baked-mtp3-64k", "engine": "ninfer", "gpu": "RTX5090", "model": "qwen38-huihui-ninfer-schemafix-mtp3-64k", "started_at": "2026-09-19T23:28:34Z" }, "intelligence": { "checks": [ { "attempt_count": 3, "attempts": [ { "attempt": 1, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 256, "reasoning_headroom_tokens": 0, "visible_answer_tokens": 256 }, "content": "```diff\n--- a/app.py\n+++ b/app.py\n@@ -1,2 +1,2 @@\n-timeout = 30\n+timeout = 45\n retries = 2\n```", "content_chars": 94, "content_excerpt": "```diff\n--- a/app.py\n+++ b/app.py\n@@ -1,2 +1,2 @@\n-timeout = 30\n+timeout = 45\n retries = 2\n```", "failure_class": null, "finish_reason": "stop", "latency_ms": 317.93450005352497, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "status": "passed", "text_checks": [ { "name": "diff_shape", "passed": true }, { "name": "removes_old_timeout", "passed": true }, { "name": "adds_new_timeout", "passed": true } ], "usage": { "completion_tokens": 45, "prompt_tokens": 51, "total_tokens": 96 } }, { "attempt": 2, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 256, "reasoning_headroom_tokens": 0, "visible_answer_tokens": 256 }, "content": "```diff\n--- a/app.py\n+++ b/app.py\n@@ -1,2 +1,2 @@\n-timeout = 30\n+timeout = 45\n retries = 2\n```", "content_chars": 94, "content_excerpt": "```diff\n--- a/app.py\n+++ b/app.py\n@@ -1,2 +1,2 @@\n-timeout = 30\n+timeout = 45\n retries = 2\n```", "failure_class": null, "finish_reason": "stop", "latency_ms": 293.4961999999359, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "status": "passed", "text_checks": [ { "name": "diff_shape", "passed": true }, { "name": "removes_old_timeout", "passed": true }, { "name": "adds_new_timeout", "passed": true } ], "usage": { "completion_tokens": 45, "prompt_tokens": 51, "total_tokens": 96 } }, { "attempt": 3, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 256, "reasoning_headroom_tokens": 0, "visible_answer_tokens": 256 }, "content": "```diff\n--- a/app.py\n+++ b/app.py\n@@ -1,2 +1,2 @@\n-timeout = 30\n+timeout = 45\n retries = 2\n```", "content_chars": 94, "content_excerpt": "```diff\n--- a/app.py\n+++ b/app.py\n@@ -1,2 +1,2 @@\n-timeout = 30\n+timeout = 45\n retries = 2\n```", "failure_class": null, "finish_reason": "stop", "latency_ms": 293.8708000583574, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "status": "passed", "text_checks": [ { "name": "diff_shape", "passed": true }, { "name": "removes_old_timeout", "passed": true }, { "name": "adds_new_timeout", "passed": true } ], "usage": { "completion_tokens": 45, "prompt_tokens": 51, "total_tokens": 96 } } ], "id": "unified_diff_timeout_edit", "pass_count": 3, "pass_rate": 1.0, "required_pass_rate": 1.0, "status": "passed", "validator": "deterministic_text_checks" }, { "attempt_count": 3, "attempts": [ { "attempt": 1, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 256, "reasoning_headroom_tokens": 0, "visible_answer_tokens": 256 }, "content": "The total latency of 3000 ms exceeds the 2500 ms timeout because the LLM takes too long, so you should reduce the LLM's response time by using a faster model or limiting its output length.", "content_chars": 188, "content_excerpt": "The total latency of 3000 ms exceeds the 2500 ms timeout because the LLM takes too long, so you should reduce the LLM's response time by using a faster model or limiting its output length.", "failure_class": null, "finish_reason": "stop", "latency_ms": 442.8418999304995, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "status": "passed", "text_checks": [ { "name": "identifies_budget_overrun", "passed": true }, { "name": "offers_latency_fix", "passed": true } ], "usage": { "completion_tokens": 49, "prompt_tokens": 79, "total_tokens": 128 } }, { "attempt": 2, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 256, "reasoning_headroom_tokens": 0, "visible_answer_tokens": 256 }, "content": "The total latency of 3000 ms exceeds the 2500 ms timeout because the LLM takes too long, so you should reduce the LLM's response time by using a faster model or limiting its output length.", "content_chars": 188, "content_excerpt": "The total latency of 3000 ms exceeds the 2500 ms timeout because the LLM takes too long, so you should reduce the LLM's response time by using a faster model or limiting its output length.", "failure_class": null, "finish_reason": "stop", "latency_ms": 410.55040003266186, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "status": "passed", "text_checks": [ { "name": "identifies_budget_overrun", "passed": true }, { "name": "offers_latency_fix", "passed": true } ], "usage": { "completion_tokens": 49, "prompt_tokens": 79, "total_tokens": 128 } }, { "attempt": 3, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 256, "reasoning_headroom_tokens": 0, "visible_answer_tokens": 256 }, "content": "The total latency of 3000 ms exceeds the 2500 ms timeout because the LLM takes too long, so you should reduce the LLM's response time by using a faster model or limiting its output length.", "content_chars": 188, "content_excerpt": "The total latency of 3000 ms exceeds the 2500 ms timeout because the LLM takes too long, so you should reduce the LLM's response time by using a faster model or limiting its output length.", "failure_class": null, "finish_reason": "stop", "latency_ms": 410.32839997205883, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "status": "passed", "text_checks": [ { "name": "identifies_budget_overrun", "passed": true }, { "name": "offers_latency_fix", "passed": true } ], "usage": { "completion_tokens": 49, "prompt_tokens": 79, "total_tokens": 128 } } ], "id": "parallel_timeout_triage", "pass_count": 3, "pass_rate": 1.0, "required_pass_rate": 1.0, "status": "passed", "validator": "deterministic_text_checks" } ], "status": "passed" }, "run_id": "fast-bakeoff-20260919T232834Z", "sampling": { "temperature": { "effective_request": 0.0, "requested": null, "sent": true }, "top_p": { "effective_request": null, "requested": null, "sent": false } }, "schema": "anvil-serving.fast-tier-bakeoff/v1", "score_inputs": { "e2e_p50_ms": 12924.405799945816, "intelligence_pass_rate": 1.0, "operational_fit_notes": [ "endpoint was already loaded; benchmark did not start or stop serves" ], "session_recall_passed": true, "thinking_mode": "unsupported", "tool_call_passed": true, "ttft_p50_ms": 12520.498499972746, "usable_context_tokens": 64000, "voice_latency_ms": null }, "selection": { "context_targets": [ 64000 ], "endpoint_already_loaded": true, "requests_per_context": 1, "suites": [ "tool", "intelligence", "session", "context" ] }, "session": { "checks": [ { "attempt_count": 3, "attempts": [ { "attempt": 1, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 256, "reasoning_headroom_tokens": 0, "visible_answer_tokens": 256 }, "content": "RIVER-918", "content_chars": 9, "content_excerpt": "RIVER-918", "expected": "RIVER-918", "failure_class": null, "finish_reason": "stop", "latency_ms": 143.41099990997463, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "status": "passed", "usage": { "completion_tokens": 7, "prompt_tokens": 52, "total_tokens": 59 } }, { "attempt": 2, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 256, "reasoning_headroom_tokens": 0, "visible_answer_tokens": 256 }, "content": "RIVER-918", "content_chars": 9, "content_excerpt": "RIVER-918", "expected": "RIVER-918", "failure_class": null, "finish_reason": "stop", "latency_ms": 79.1169999865815, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "status": "passed", "usage": { "completion_tokens": 7, "prompt_tokens": 52, "total_tokens": 59 } }, { "attempt": 3, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 256, "reasoning_headroom_tokens": 0, "visible_answer_tokens": 256 }, "content": "RIVER-918", "content_chars": 9, "content_excerpt": "RIVER-918", "expected": "RIVER-918", "failure_class": null, "finish_reason": "stop", "latency_ms": 78.69350002147257, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "status": "passed", "usage": { "completion_tokens": 7, "prompt_tokens": 52, "total_tokens": 59 } } ], "name": "single_request_multiturn_recall", "pass_count": 3, "pass_rate": 1.0, "required_pass_rate": 1.0, "status": "passed" } ], "status": "passed" }, "source_recipe": { "ref": null, "serve_command": null }, "suites": {}, "thinking": { "chat_template_kwargs": null, "control_evidence": null, "control_evidence_sha256": null, "control_mechanism": "unsupported", "control_requested": null, "control_status": "unsupported", "mode": "unsupported", "reasoning_effort": null, "unsupported": true }, "timing": { "chat": { "completion_tokens_max": 59.0, "completion_tokens_mean": 59.0, "completion_tokens_mean_ci95_high": 59.0, "completion_tokens_mean_ci95_low": 59.0, "completion_tokens_min": 59.0, "completion_tokens_p25": 59.0, "completion_tokens_p50": 59.0, "completion_tokens_p75": 59.0, "completion_tokens_p90": 59.0, "completion_tokens_p95": 59.0, "completion_tokens_p99": 59.0, "completion_tokens_samples": 1, "completion_tokens_stddev": 0.0, "content_chunks": 19, "controlled_output_exact_adherence": 0, "controlled_output_exact_adherence_rate": null, "controlled_output_samples": 0, "controlled_output_unobservable": 0, "decode_tok_s_max": 143.59680789826064, "decode_tok_s_mean": 143.59680789826064, "decode_tok_s_mean_ci95_high": 143.59680789826064, "decode_tok_s_mean_ci95_low": 143.59680789826064, "decode_tok_s_min": 143.59680789826064, "decode_tok_s_p25": 143.59680789826064, "decode_tok_s_p50": 143.59680789826064, "decode_tok_s_p75": 143.59680789826064, "decode_tok_s_p90": 143.59680789826064, "decode_tok_s_p95": 143.59680789826064, "decode_tok_s_p99": 143.59680789826064, "decode_tok_s_samples": 1, "decode_tok_s_stddev": 0.0, "e2e_max_ms": 12924.405799945816, "e2e_mean_ci95_high_ms": 12924.405799945816, "e2e_mean_ci95_low_ms": 12924.405799945816, "e2e_mean_ms": 12924.405799945816, "e2e_min_ms": 12924.405799945816, "e2e_p25_ms": 12924.405799945816, "e2e_p50_ms": 12924.405799945816, "e2e_p75_ms": 12924.405799945816, "e2e_p90_ms": 12924.405799945816, "e2e_p95_ms": 12924.405799945816, "e2e_p99_ms": 12924.405799945816, "e2e_samples": 1, "e2e_stddev_ms": 0.0, "effective_prefill_tok_s_max": 4988.220475702129, "effective_prefill_tok_s_mean": 4988.220475702129, "effective_prefill_tok_s_mean_ci95_high": 4988.220475702129, "effective_prefill_tok_s_mean_ci95_low": 4988.220475702129, "effective_prefill_tok_s_min": 4988.220475702129, "effective_prefill_tok_s_p25": 4988.220475702129, "effective_prefill_tok_s_p50": 4988.220475702129, "effective_prefill_tok_s_p75": 4988.220475702129, "effective_prefill_tok_s_p90": 4988.220475702129, "effective_prefill_tok_s_p95": 4988.220475702129, "effective_prefill_tok_s_p99": 4988.220475702129, "effective_prefill_tok_s_samples": 1, "effective_prefill_tok_s_stddev": 0.0, "generation_max_ms": 403.9086999837309, "generation_mean_ci95_high_ms": 403.9086999837309, "generation_mean_ci95_low_ms": 403.9086999837309, "generation_mean_ms": 403.9086999837309, "generation_min_ms": 403.9086999837309, "generation_p25_ms": 403.9086999837309, "generation_p50_ms": 403.9086999837309, "generation_p75_ms": 403.9086999837309, "generation_p90_ms": 403.9086999837309, "generation_p95_ms": 403.9086999837309, "generation_p99_ms": 403.9086999837309, "generation_samples": 1, "generation_stddev_ms": 0.0, "mean_inter_token_latency_ms_max": 6.963943103167774, "mean_inter_token_latency_ms_mean": 6.963943103167774, "mean_inter_token_latency_ms_mean_ci95_high": 6.963943103167774, "mean_inter_token_latency_ms_mean_ci95_low": 6.963943103167774, "mean_inter_token_latency_ms_min": 6.963943103167774, "mean_inter_token_latency_ms_p25": 6.963943103167774, "mean_inter_token_latency_ms_p50": 6.963943103167774, "mean_inter_token_latency_ms_p75": 6.963943103167774, "mean_inter_token_latency_ms_p90": 6.963943103167774, "mean_inter_token_latency_ms_p95": 6.963943103167774, "mean_inter_token_latency_ms_p99": 6.963943103167774, "mean_inter_token_latency_ms_samples": 1, "mean_inter_token_latency_ms_stddev": 0.0, "mean_time_per_output_token_max_ms": 6.963943103167774, "mean_time_per_output_token_mean_ci95_high_ms": 6.963943103167774, "mean_time_per_output_token_mean_ci95_low_ms": 6.963943103167774, "mean_time_per_output_token_mean_ms": 6.963943103167774, "mean_time_per_output_token_min_ms": 6.963943103167774, "mean_time_per_output_token_p25_ms": 6.963943103167774, "mean_time_per_output_token_p50_ms": 6.963943103167774, "mean_time_per_output_token_p75_ms": 6.963943103167774, "mean_time_per_output_token_p90_ms": 6.963943103167774, "mean_time_per_output_token_p95_ms": 6.963943103167774, "mean_time_per_output_token_p99_ms": 6.963943103167774, "mean_time_per_output_token_samples": 1, "mean_time_per_output_token_stddev_ms": 0.0, "output_tokens": 59, "prompt_token_samples": 1, "prompt_tokens": 62455, "prompt_tokens_max": 62455.0, "prompt_tokens_mean": 62455.0, "prompt_tokens_mean_ci95_high": 62455.0, "prompt_tokens_mean_ci95_low": 62455.0, "prompt_tokens_min": 62455.0, "prompt_tokens_p25": 62455.0, "prompt_tokens_p50": 62455.0, "prompt_tokens_p75": 62455.0, "prompt_tokens_p90": 62455.0, "prompt_tokens_p95": 62455.0, "prompt_tokens_p99": 62455.0, "prompt_tokens_samples": 1, "prompt_tokens_stddev": 0.0, "reasoning_chunks": 0, "request_canary_passed": 0, "request_canary_samples": 0, "time_to_first_output_max_ms": 12520.497099962085, "time_to_first_output_mean_ci95_high_ms": 12520.497099962085, "time_to_first_output_mean_ci95_low_ms": 12520.497099962085, "time_to_first_output_mean_ms": 12520.497099962085, "time_to_first_output_min_ms": 12520.497099962085, "time_to_first_output_p25_ms": 12520.497099962085, "time_to_first_output_p50_ms": 12520.497099962085, "time_to_first_output_p75_ms": 12520.497099962085, "time_to_first_output_p90_ms": 12520.497099962085, "time_to_first_output_p95_ms": 12520.497099962085, "time_to_first_output_p99_ms": 12520.497099962085, "time_to_first_output_samples": 1, "time_to_first_output_stddev_ms": 0.0, "tpot_max_ms": 6.963943103167774, "tpot_mean_ci95_high_ms": 6.963943103167774, "tpot_mean_ci95_low_ms": 6.963943103167774, "tpot_mean_ms": 6.963943103167774, "tpot_min_ms": 6.963943103167774, "tpot_p25_ms": 6.963943103167774, "tpot_p50_ms": 6.963943103167774, "tpot_p75_ms": 6.963943103167774, "tpot_p90_ms": 6.963943103167774, "tpot_p95_ms": 6.963943103167774, "tpot_p99_ms": 6.963943103167774, "tpot_samples": 1, "tpot_stddev_ms": 0.0, "ttft_max_ms": 12520.498499972746, "ttft_mean_ci95_high_ms": 12520.498499972746, "ttft_mean_ci95_low_ms": 12520.498499972746, "ttft_mean_ms": 12520.498499972746, "ttft_min_ms": 12520.498499972746, "ttft_p25_ms": 12520.498499972746, "ttft_p50_ms": 12520.498499972746, "ttft_p75_ms": 12520.498499972746, "ttft_p90_ms": 12520.498499972746, "ttft_p95_ms": 12520.498499972746, "ttft_p99_ms": 12520.498499972746, "ttft_samples": 1, "ttft_stddev_ms": 0.0 }, "wall_ms": 16065.857500070706 }, "tool": { "checks": [ { "attempt_count": 3, "attempts": [ { "arguments": { "zip": "98101" }, "attempt": 1, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 256, "reasoning_headroom_tokens": 0, "visible_answer_tokens": 256 }, "content": "", "content_chars": 0, "content_excerpt": "", "failure_class": null, "finish_reason": "tool_calls", "latency_ms": 274.9343999894336, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "status": "passed", "tool_argument_observations": [ { "arguments_chars": 15, "arguments_excerpt": "{\"zip\":\"98101\"}", "arguments_truncated": false, "arguments_wire_type": "str", "name": "record_weather_zip" } ], "tool_call_count": 1, "usage": { "completion_tokens": 32, "prompt_tokens": 292, "total_tokens": 324 }, "valid_tool_call_count": 1, "validation_errors": [] }, { "arguments": { "zip": "98101" }, "attempt": 2, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 256, "reasoning_headroom_tokens": 0, "visible_answer_tokens": 256 }, "content": "", "content_chars": 0, "content_excerpt": "", "failure_class": null, "finish_reason": "tool_calls", "latency_ms": 196.92719995509833, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "status": "passed", "tool_argument_observations": [ { "arguments_chars": 15, "arguments_excerpt": "{\"zip\":\"98101\"}", "arguments_truncated": false, "arguments_wire_type": "str", "name": "record_weather_zip" } ], "tool_call_count": 1, "usage": { "completion_tokens": 32, "prompt_tokens": 292, "total_tokens": 324 }, "valid_tool_call_count": 1, "validation_errors": [] }, { "arguments": { "zip": "98101" }, "attempt": 3, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 256, "reasoning_headroom_tokens": 0, "visible_answer_tokens": 256 }, "content": "", "content_chars": 0, "content_excerpt": "", "failure_class": null, "finish_reason": "tool_calls", "latency_ms": 196.900499984622, "reasoning_chars": 0, "reasoning_excerpt": "", "reasoning_field": null, "reasoning_tokens": null, "status": "passed", "tool_argument_observations": [ { "arguments_chars": 15, "arguments_excerpt": "{\"zip\":\"98101\"}", "arguments_truncated": false, "arguments_wire_type": "str", "name": "record_weather_zip" } ], "tool_call_count": 1, "usage": { "completion_tokens": 32, "prompt_tokens": 292, "total_tokens": 324 }, "valid_tool_call_count": 1, "validation_errors": [] } ], "expected_arguments": { "zip": "98101" }, "expected_function": "record_weather_zip", "name": "openai_tool_call_smoke", "pass_count": 3, "pass_rate": 1.0, "required_pass_rate": 1.0, "status": "passed" } ], "status": "passed" }, "voice": { "status": "not_run", "stt_latency_ms": null, "total_turn_latency_ms": null, "tts_latency_ms": null } }