{
  "context": {
    "cap_tokens": 251648,
    "max_model_len": 252928,
    "targets": [
      {
        "attempted_context_tokens": 32768,
        "chars_per_token": 3.0,
        "clamped_tokens": 32768,
        "content_chunks": 18,
        "decode_tok_s": 189.91822612425395,
        "e2e_ms": 4856.060600024648,
        "effective_prefill_tok_s": 6830.021831778826,
        "estimated_prompt_tokens": 32768,
        "generation_ms": 284.33290001703426,
        "mean_inter_token_latency_ms": 5.265424074389523,
        "output_token_source": "usage",
        "output_tokens": 55,
        "prompt_tokens": 31225,
        "reasoning_chunks": 0,
        "status": "passed",
        "target_tokens": 32768,
        "time_to_first_output_ms": 4571.727700007614,
        "ttft_ms": 4571.729000017513,
        "usage": {
          "completion_tokens": 55,
          "completion_tokens_details": {
            "reasoning_tokens": 0
          },
          "prompt_tokens": 31225,
          "prompt_tokens_details": {
            "cached_tokens": 0
          },
          "total_tokens": 31280
        },
        "visible_generation_ms": 284.3316000071354
      }
    ]
  },
  "evaluation_protocol": {
    "budget_semantics": "visible_answer_tokens plus reasoning_headroom_tokens are sent as one max_tokens cap; the endpoint does not hard-partition the channels",
    "minimum_comparable_repetitions": 3,
    "minimum_pass_rate": 1.0,
    "reasoning_headroom_tokens": 0,
    "records_finish_reason": true,
    "records_full_visible_answer": true,
    "records_reasoning_channel_metadata": true,
    "repetitions": 3,
    "version": 3,
    "visible_answer_tokens": 256
  },
  "failures": [],
  "identity": {
    "base_url": "http://127.0.0.1:8081/v1",
    "candidate_id": "neroued-qwen38-27b-nvfp4",
    "config_id": "ninfer-mtp3-int8kv-252928-c1",
    "engine": "ninfer-e3aeaf8c",
    "gpu": "rtx-5090",
    "model": "qwen38-27b-ninfer-nvfp4-mtp3-252k",
    "started_at": "2026-09-03T12:30:35Z"
  },
  "intelligence": {
    "checks": [
      {
        "attempt_count": 3,
        "attempts": [
          {
            "attempt": 1,
            "budget": {
              "legacy_max_tokens_as_visible": false,
              "max_completion_tokens": 256,
              "reasoning_headroom_tokens": 0,
              "visible_answer_tokens": 256
            },
            "content": "```diff\n--- a/app.py\n+++ b/app.py\n@@ -1,2 +1,2 @@\n-timeout = 30\n+timeout = 45\n retries = 2\n```",
            "content_chars": 94,
            "content_excerpt": "```diff\n--- a/app.py\n+++ b/app.py\n@@ -1,2 +1,2 @@\n-timeout = 30\n+timeout = 45\n retries = 2\n```",
            "failure_class": null,
            "finish_reason": "stop",
            "latency_ms": 284.47569999843836,
            "reasoning_chars": 0,
            "reasoning_excerpt": "",
            "reasoning_field": null,
            "reasoning_tokens": 0,
            "status": "passed",
            "text_checks": [
              {
                "name": "diff_shape",
                "passed": true
              },
              {
                "name": "removes_old_timeout",
                "passed": true
              },
              {
                "name": "adds_new_timeout",
                "passed": true
              }
            ],
            "usage": {
              "completion_tokens": 45,
              "completion_tokens_details": {
                "reasoning_tokens": 0
              },
              "prompt_tokens": 51,
              "prompt_tokens_details": {
                "cached_tokens": 0
              },
              "total_tokens": 96
            }
          },
          {
            "attempt": 2,
            "budget": {
              "legacy_max_tokens_as_visible": false,
              "max_completion_tokens": 256,
              "reasoning_headroom_tokens": 0,
              "visible_answer_tokens": 256
            },
            "content": "```diff\n--- a/app.py\n+++ b/app.py\n@@ -1,2 +1,2 @@\n-timeout = 30\n+timeout = 45\n retries = 2\n```",
            "content_chars": 94,
            "content_excerpt": "```diff\n--- a/app.py\n+++ b/app.py\n@@ -1,2 +1,2 @@\n-timeout = 30\n+timeout = 45\n retries = 2\n```",
            "failure_class": null,
            "finish_reason": "stop",
            "latency_ms": 260.8934000018053,
            "reasoning_chars": 0,
            "reasoning_excerpt": "",
            "reasoning_field": null,
            "reasoning_tokens": 0,
            "status": "passed",
            "text_checks": [
              {
                "name": "diff_shape",
                "passed": true
              },
              {
                "name": "removes_old_timeout",
                "passed": true
              },
              {
                "name": "adds_new_timeout",
                "passed": true
              }
            ],
            "usage": {
              "completion_tokens": 45,
              "completion_tokens_details": {
                "reasoning_tokens": 0
              },
              "prompt_tokens": 51,
              "prompt_tokens_details": {
                "cached_tokens": 44
              },
              "total_tokens": 96
            }
          },
          {
            "attempt": 3,
            "budget": {
              "legacy_max_tokens_as_visible": false,
              "max_completion_tokens": 256,
              "reasoning_headroom_tokens": 0,
              "visible_answer_tokens": 256
            },
            "content": "```diff\n--- a/app.py\n+++ b/app.py\n@@ -1,2 +1,2 @@\n-timeout = 30\n+timeout = 45\n retries = 2\n```",
            "content_chars": 94,
            "content_excerpt": "```diff\n--- a/app.py\n+++ b/app.py\n@@ -1,2 +1,2 @@\n-timeout = 30\n+timeout = 45\n retries = 2\n```",
            "failure_class": null,
            "finish_reason": "stop",
            "latency_ms": 260.95320004969835,
            "reasoning_chars": 0,
            "reasoning_excerpt": "",
            "reasoning_field": null,
            "reasoning_tokens": 0,
            "status": "passed",
            "text_checks": [
              {
                "name": "diff_shape",
                "passed": true
              },
              {
                "name": "removes_old_timeout",
                "passed": true
              },
              {
                "name": "adds_new_timeout",
                "passed": true
              }
            ],
            "usage": {
              "completion_tokens": 45,
              "completion_tokens_details": {
                "reasoning_tokens": 0
              },
              "prompt_tokens": 51,
              "prompt_tokens_details": {
                "cached_tokens": 44
              },
              "total_tokens": 96
            }
          }
        ],
        "id": "unified_diff_timeout_edit",
        "pass_count": 3,
        "pass_rate": 1.0,
        "required_pass_rate": 1.0,
        "status": "passed",
        "validator": "deterministic_text_checks"
      },
      {
        "attempt_count": 3,
        "attempts": [
          {
            "attempt": 1,
            "budget": {
              "legacy_max_tokens_as_visible": false,
              "max_completion_tokens": 256,
              "reasoning_headroom_tokens": 0,
              "visible_answer_tokens": 256
            },
            "content": "The total latency of 3000 ms exceeds the 2500 ms timeout because the LLM's 1800 ms response time is too slow, so you should optimize the LLM by using a smaller/faster model or implementing streaming to reduce end-to-end latency.",
            "content_chars": 228,
            "content_excerpt": "The total latency of 3000 ms exceeds the 2500 ms timeout because the LLM's 1800 ms response time is too slow, so you should optimize the LLM by using a smaller/faster model or implementing streaming t",
            "failure_class": null,
            "finish_reason": "stop",
            "latency_ms": 385.4888000059873,
            "reasoning_chars": 0,
            "reasoning_excerpt": "",
            "reasoning_field": null,
            "reasoning_tokens": 0,
            "status": "passed",
            "text_checks": [
              {
                "name": "identifies_budget_overrun",
                "passed": true
              },
              {
                "name": "offers_latency_fix",
                "passed": true
              }
            ],
            "usage": {
              "completion_tokens": 61,
              "completion_tokens_details": {
                "reasoning_tokens": 0
              },
              "prompt_tokens": 79,
              "prompt_tokens_details": {
                "cached_tokens": 0
              },
              "total_tokens": 140
            }
          },
          {
            "attempt": 2,
            "budget": {
              "legacy_max_tokens_as_visible": false,
              "max_completion_tokens": 256,
              "reasoning_headroom_tokens": 0,
              "visible_answer_tokens": 256
            },
            "content": "The total latency of 3000 ms exceeds the 2500 ms timeout because the LLM's 1800 ms response time is too slow, so you should optimize the LLM by using a smaller/faster model or implementing streaming to reduce end-to-end latency.",
            "content_chars": 228,
            "content_excerpt": "The total latency of 3000 ms exceeds the 2500 ms timeout because the LLM's 1800 ms response time is too slow, so you should optimize the LLM by using a smaller/faster model or implementing streaming t",
            "failure_class": null,
            "finish_reason": "stop",
            "latency_ms": 356.47429997334257,
            "reasoning_chars": 0,
            "reasoning_excerpt": "",
            "reasoning_field": null,
            "reasoning_tokens": 0,
            "status": "passed",
            "text_checks": [
              {
                "name": "identifies_budget_overrun",
                "passed": true
              },
              {
                "name": "offers_latency_fix",
                "passed": true
              }
            ],
            "usage": {
              "completion_tokens": 61,
              "completion_tokens_details": {
                "reasoning_tokens": 0
              },
              "prompt_tokens": 79,
              "prompt_tokens_details": {
                "cached_tokens": 72
              },
              "total_tokens": 140
            }
          },
          {
            "attempt": 3,
            "budget": {
              "legacy_max_tokens_as_visible": false,
              "max_completion_tokens": 256,
              "reasoning_headroom_tokens": 0,
              "visible_answer_tokens": 256
            },
            "content": "The total latency of 3000 ms exceeds the 2500 ms timeout because the LLM's 1800 ms response time is too slow, so you should optimize the LLM by using a smaller/faster model or implementing streaming to reduce end-to-end latency.",
            "content_chars": 228,
            "content_excerpt": "The total latency of 3000 ms exceeds the 2500 ms timeout because the LLM's 1800 ms response time is too slow, so you should optimize the LLM by using a smaller/faster model or implementing streaming t",
            "failure_class": null,
            "finish_reason": "stop",
            "latency_ms": 356.6403000149876,
            "reasoning_chars": 0,
            "reasoning_excerpt": "",
            "reasoning_field": null,
            "reasoning_tokens": 0,
            "status": "passed",
            "text_checks": [
              {
                "name": "identifies_budget_overrun",
                "passed": true
              },
              {
                "name": "offers_latency_fix",
                "passed": true
              }
            ],
            "usage": {
              "completion_tokens": 61,
              "completion_tokens_details": {
                "reasoning_tokens": 0
              },
              "prompt_tokens": 79,
              "prompt_tokens_details": {
                "cached_tokens": 72
              },
              "total_tokens": 140
            }
          }
        ],
        "id": "parallel_timeout_triage",
        "pass_count": 3,
        "pass_rate": 1.0,
        "required_pass_rate": 1.0,
        "status": "passed",
        "validator": "deterministic_text_checks"
      }
    ],
    "status": "passed"
  },
  "run_id": "fast-bakeoff-20260903T123035Z",
  "schema": "anvil-serving.fast-tier-bakeoff/v1",
  "score_inputs": {
    "e2e_p50_ms": 4856.060600024648,
    "intelligence_pass_rate": 1.0,
    "operational_fit_notes": [
      "endpoint was already loaded; benchmark did not start or stop serves"
    ],
    "session_recall_passed": false,
    "thinking_mode": "disabled",
    "tool_call_passed": true,
    "ttft_p50_ms": 4571.729000017513,
    "usable_context_tokens": 32768,
    "voice_latency_ms": null
  },
  "selection": {
    "context_targets": [
      32768
    ],
    "endpoint_already_loaded": true,
    "requests_per_context": 1,
    "suites": [
      "chat",
      "tool",
      "intelligence"
    ]
  },
  "session": {
    "checks": [],
    "status": "not_run"
  },
  "source_recipe": {
    "ref": "neroued/Qwen3.8-27B-NInfer-NVFP4-MTP3-252K",
    "serve_command": null
  },
  "suites": {},
  "thinking": {
    "chat_template_kwargs": {
      "enable_thinking": false
    },
    "control_evidence": null,
    "control_evidence_sha256": null,
    "control_mechanism": "chat_template_kwargs",
    "control_requested": {
      "enable_thinking": false
    },
    "control_status": "requested_unverified",
    "mode": "disabled",
    "reasoning_effort": null,
    "unsupported": false
  },
  "timing": {
    "chat": {
      "content_chunks": 18,
      "decode_tok_s_p50": 189.91822612425395,
      "decode_tok_s_p95": 189.91822612425395,
      "e2e_p50_ms": 4856.060600024648,
      "e2e_p95_ms": 4856.060600024648,
      "effective_prefill_tok_s_p50": 6830.021831778826,
      "effective_prefill_tok_s_p95": 6830.021831778826,
      "generation_p50_ms": 284.33290001703426,
      "generation_p95_ms": 284.33290001703426,
      "mean_inter_token_latency_ms_p50": 5.265424074389523,
      "mean_inter_token_latency_ms_p95": 5.265424074389523,
      "output_tokens": 55,
      "prompt_token_samples": 1,
      "prompt_tokens": 31225,
      "prompt_tokens_p50": 31225.0,
      "prompt_tokens_p95": 31225.0,
      "reasoning_chunks": 0,
      "time_to_first_output_p50_ms": 4571.727700007614,
      "time_to_first_output_p95_ms": 4571.727700007614,
      "ttft_p50_ms": 4571.729000017513,
      "ttft_p95_ms": 4571.729000017513,
      "ttft_samples": 1
    },
    "wall_ms": 7358.341699989978
  },
  "tool": {
    "checks": [
      {
        "attempt_count": 3,
        "attempts": [
          {
            "arguments": {
              "zip": "98101"
            },
            "attempt": 1,
            "budget": {
              "legacy_max_tokens_as_visible": false,
              "max_completion_tokens": 256,
              "reasoning_headroom_tokens": 0,
              "visible_answer_tokens": 256
            },
            "content": "",
            "content_chars": 0,
            "content_excerpt": "",
            "failure_class": null,
            "finish_reason": "tool_calls",
            "latency_ms": 231.30680003669113,
            "reasoning_chars": 0,
            "reasoning_excerpt": "",
            "reasoning_field": null,
            "reasoning_tokens": 0,
            "status": "passed",
            "tool_call_count": 1,
            "usage": {
              "completion_tokens": 32,
              "completion_tokens_details": {
                "reasoning_tokens": 0
              },
              "prompt_tokens": 292,
              "prompt_tokens_details": {
                "cached_tokens": 0
              },
              "total_tokens": 324
            },
            "valid_tool_call_count": 1,
            "validation_errors": []
          },
          {
            "arguments": {
              "zip": "98101"
            },
            "attempt": 2,
            "budget": {
              "legacy_max_tokens_as_visible": false,
              "max_completion_tokens": 256,
              "reasoning_headroom_tokens": 0,
              "visible_answer_tokens": 256
            },
            "content": "",
            "content_chars": 0,
            "content_excerpt": "",
            "failure_class": null,
            "finish_reason": "tool_calls",
            "latency_ms": 182.25539999548346,
            "reasoning_chars": 0,
            "reasoning_excerpt": "",
            "reasoning_field": null,
            "reasoning_tokens": 0,
            "status": "passed",
            "tool_call_count": 1,
            "usage": {
              "completion_tokens": 32,
              "completion_tokens_details": {
                "reasoning_tokens": 0
              },
              "prompt_tokens": 292,
              "prompt_tokens_details": {
                "cached_tokens": 285
              },
              "total_tokens": 324
            },
            "valid_tool_call_count": 1,
            "validation_errors": []
          },
          {
            "arguments": {
              "zip": "98101"
            },
            "attempt": 3,
            "budget": {
              "legacy_max_tokens_as_visible": false,
              "max_completion_tokens": 256,
              "reasoning_headroom_tokens": 0,
              "visible_answer_tokens": 256
            },
            "content": "",
            "content_chars": 0,
            "content_excerpt": "",
            "failure_class": null,
            "finish_reason": "tool_calls",
            "latency_ms": 182.1695999824442,
            "reasoning_chars": 0,
            "reasoning_excerpt": "",
            "reasoning_field": null,
            "reasoning_tokens": 0,
            "status": "passed",
            "tool_call_count": 1,
            "usage": {
              "completion_tokens": 32,
              "completion_tokens_details": {
                "reasoning_tokens": 0
              },
              "prompt_tokens": 292,
              "prompt_tokens_details": {
                "cached_tokens": 285
              },
              "total_tokens": 324
            },
            "valid_tool_call_count": 1,
            "validation_errors": []
          }
        ],
        "expected_arguments": {
          "zip": "98101"
        },
        "expected_function": "record_weather_zip",
        "name": "openai_tool_call_smoke",
        "pass_count": 3,
        "pass_rate": 1.0,
        "required_pass_rate": 1.0,
        "status": "passed"
      }
    ],
    "status": "passed"
  },
  "voice": {
    "status": "not_run",
    "stt_latency_ms": null,
    "total_turn_latency_ms": null,
    "tts_latency_ms": null
  }
}