{ "base_url": "http://127.0.0.1:39272/v1", "completed": 50, "concurrency": 8, "context_distribution": { "4096": 50 }, "context_seed": 0, "context_tokens": 4096, "engine": "sglang", "failed": 0, "failures": [], "finished_at": "2026-09-05T04:36:54Z", "gpu": "NVIDIA RTX PRO 6000 Max-Q replica A", "manifest": null, "max_context_tokens": 262144, "max_tokens": 512, "measurement_protocol": "capacity-v3", "metrics": { "completion_tokens_max": 512.0, "completion_tokens_mean": 512.0, "completion_tokens_mean_ci95_high": 512.0, "completion_tokens_mean_ci95_low": 512.0, "completion_tokens_min": 512.0, "completion_tokens_p25": 512.0, "completion_tokens_p50": 512.0, "completion_tokens_p75": 512.0, "completion_tokens_p90": 512.0, "completion_tokens_p95": 512.0, "completion_tokens_p99": 512.0, "completion_tokens_samples": 50, "completion_tokens_stddev": 0.0, "content_chunks": 2224, "content_chunks_s": null, "decode_tok_s_max": 352.4208415286776, "decode_tok_s_mean": 196.5928036261601, "decode_tok_s_mean_ci95_high": 214.48987883063313, "decode_tok_s_mean_ci95_low": 178.69572842168708, "decode_tok_s_min": 120.97445085714787, "decode_tok_s_p25": 139.27643792042562, "decode_tok_s_p50": 183.06261023867216, "decode_tok_s_p75": 247.5045870703598, "decode_tok_s_p90": 297.4413409001829, "decode_tok_s_p95": 308.84655601745976, "decode_tok_s_p99": 352.4208415286776, "decode_tok_s_samples": 50, "decode_tok_s_stddev": 64.56705734943104, "e2e_max_ms": 9831.872400012799, "e2e_mean_ci95_high_ms": 5925.712315443215, "e2e_mean_ci95_low_ms": 5059.856680577378, "e2e_mean_ms": 5492.784498010296, "e2e_min_ms": 4667.5514000235125, "e2e_p25_ms": 4841.308300034143, "e2e_p50_ms": 4907.281200052239, "e2e_p75_ms": 4989.190499996766, "e2e_p90_ms": 9427.985499962233, "e2e_p95_ms": 9690.99350005854, "e2e_p99_ms": 9831.872400012799, "e2e_samples": 50, "e2e_stddev_ms": 1561.8683442403471, "effective_prefill_tok_s_max": 4261.552457870067, "effective_prefill_tok_s_mean": 1758.4182998660672, "effective_prefill_tok_s_mean_ci95_high": 2003.9871375188245, "effective_prefill_tok_s_mean_ci95_low": 1512.8494622133098, "effective_prefill_tok_s_min": 639.535257399504, "effective_prefill_tok_s_p25": 1174.820532366914, "effective_prefill_tok_s_p50": 1504.4086089086334, "effective_prefill_tok_s_p75": 2317.4596709091807, "effective_prefill_tok_s_p90": 3076.2875138258687, "effective_prefill_tok_s_p95": 3631.3894431372128, "effective_prefill_tok_s_p99": 4261.552457870067, "effective_prefill_tok_s_samples": 50, "effective_prefill_tok_s_stddev": 885.9356650630771, "generation_max_ms": 4224.032400059514, "generation_mean_ci95_high_ms": 3109.446475666196, "generation_mean_ci95_low_ms": 2628.1159283433176, "generation_mean_ms": 2868.781202004757, "generation_min_ms": 1449.9709999654442, "generation_p25_ms": 2064.6081999875605, "generation_p50_ms": 2780.4737999103963, "generation_p75_ms": 3668.9622999401763, "generation_p90_ms": 4146.187899983488, "generation_p95_ms": 4175.16749992501, "generation_p99_ms": 4224.032400059514, "generation_samples": 50, "generation_stddev_ms": 868.2451377659173, "mean_inter_token_latency_ms_max": 8.266208219294548, "mean_inter_token_latency_ms_mean": 5.614053232886021, "mean_inter_token_latency_ms_mean_ci95_high": 6.085022457272399, "mean_inter_token_latency_ms_mean_ci95_low": 5.143084008499643, "mean_inter_token_latency_ms_min": 2.837516633983257, "mean_inter_token_latency_ms_p25": 4.040329158488377, "mean_inter_token_latency_ms_p50": 5.441240312936197, "mean_inter_token_latency_ms_p75": 7.179965361918153, "mean_inter_token_latency_ms_p90": 8.113870645760251, "mean_inter_token_latency_ms_p95": 8.17058219163407, "mean_inter_token_latency_ms_p99": 8.266208219294548, "mean_inter_token_latency_ms_samples": 50, "mean_inter_token_latency_ms_stddev": 1.6991098586417168, "output_token_sources": { "usage": 50 }, "output_tokens": 25600, "prefix_cache_hit_avg": null, "prompt_token_samples": 50, "prompt_tokens": 181847, "prompt_tokens_max": 3655.0, "prompt_tokens_mean": 3636.94, "prompt_tokens_mean_ci95_high": 3640.4171490850986, "prompt_tokens_mean_ci95_low": 3633.4628509149015, "prompt_tokens_min": 3613.0, "prompt_tokens_p25": 3627.0, "prompt_tokens_p50": 3636.0, "prompt_tokens_p75": 3649.0, "prompt_tokens_p90": 3653.0, "prompt_tokens_p95": 3655.0, "prompt_tokens_p99": 3655.0, "prompt_tokens_samples": 50, "prompt_tokens_stddev": 12.544467843212312, "reasoning_chunks": 0, "request_canary_passed": 50, "request_canary_samples": 50, "throughput_tok_s": 711.7062458047608, "time_to_first_output_max_ms": 5663.487599929795, "time_to_first_output_mean_ci95_high_ms": 2991.8289701412305, "time_to_first_output_mean_ci95_low_ms": 2256.1776218698487, "time_to_first_output_mean_ms": 2624.0032960055396, "time_to_first_output_min_ms": 847.8130999719724, "time_to_first_output_p25_ms": 1568.7413000268862, "time_to_first_output_p50_ms": 2412.769000045955, "time_to_first_output_p75_ms": 3081.3217000104487, "time_to_first_output_p90_ms": 5281.797599978745, "time_to_first_output_p95_ms": 5486.428799922578, "time_to_first_output_p99_ms": 5663.487599929795, "time_to_first_output_samples": 50, "time_to_first_output_stddev_ms": 1327.0001452850008, "tpot_max_ms": 8.266208219294548, "tpot_mean_ci95_high_ms": 6.085022457272399, "tpot_mean_ci95_low_ms": 5.143084008499643, "tpot_mean_ms": 5.614053232886021, "tpot_min_ms": 2.837516633983257, "tpot_p25_ms": 4.040329158488377, "tpot_p50_ms": 5.441240312936197, "tpot_p75_ms": 7.179965361918153, "tpot_p90_ms": 8.113870645760251, "tpot_p95_ms": 8.17058219163407, "tpot_p99_ms": 8.266208219294548, "tpot_samples": 50, "tpot_stddev_ms": 1.6991098586417168, "ttft_max_ms": 5663.4879999328405, "ttft_mean_ci95_high_ms": 2991.829406596181, "ttft_mean_ci95_low_ms": 2256.1780734123467, "ttft_mean_ms": 2624.003740004264, "ttft_min_ms": 847.813899978064, "ttft_p25_ms": 1568.7419000314549, "ttft_p50_ms": 2412.7695000497624, "ttft_p75_ms": 3081.3221000134945, "ttft_p90_ms": 5281.797999981791, "ttft_p95_ms": 5486.429099924862, "ttft_p99_ms": 5663.4879999328405, "ttft_samples": 50, "ttft_stddev_ms": 1327.0001180694217 }, "model": "qwen38-27b-inferact-nvfp4-pro6000-dflash2-k12-chunk1k", "prompt_cache_mode": "unique", "request_canaries": true, "request_timings": [ { "content_chunks": 44, "decode_tok_s": 123.35016141634507, "e2e_ms": 4990.491100004874, "effective_prefill_tok_s": 4261.552457870067, "generation_ms": 4142.678000032902, "mean_inter_token_latency_ms": 8.10700195701155, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3613, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00000", "passed": true }, "request_index": 0, "time_to_first_output_ms": 847.8130999719724, "tpot_ms": 8.10700195701155, "ttft_ms": 847.813899978064, "visible_content_truncated": false, "visible_generation_ms": 4142.67720002681 }, { "content_chunks": 44, "decode_tok_s": 139.93167500617767, "e2e_ms": 4989.190499996766, "effective_prefill_tok_s": 2702.9890572209024, "generation_ms": 3651.7821999732405, "mean_inter_token_latency_ms": 7.146344814037652, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3615, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00001", "passed": true }, "request_index": 1, "time_to_first_output_ms": 1337.408300023526, "tpot_ms": 7.146344814037652, "ttft_ms": 1337.409000028856, "visible_content_truncated": false, "visible_generation_ms": 3651.7814999679103 }, { "content_chunks": 44, "decode_tok_s": 155.77202413874278, "e2e_ms": 4990.5137999448925, "effective_prefill_tok_s": 2115.107210288109, "generation_ms": 3280.4349999642, "mean_inter_token_latency_ms": 6.419637964704893, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3617, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00002", "passed": true }, "request_index": 2, "time_to_first_output_ms": 1710.0787999806926, "tpot_ms": 6.419637964704893, "ttft_ms": 1710.0792999844998, "visible_content_truncated": false, "visible_generation_ms": 3280.4344999603927 }, { "content_chunks": 44, "decode_tok_s": 183.78162743934774, "e2e_ms": 4989.201599964872, "effective_prefill_tok_s": 1637.594274817744, "generation_ms": 2780.4737999103963, "mean_inter_token_latency_ms": 5.441240312936197, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3617, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00003", "passed": true }, "request_index": 3, "time_to_first_output_ms": 2208.7278000544757, "tpot_ms": 5.441240312936197, "ttft_ms": 2208.728300058283, "visible_content_truncated": false, "visible_generation_ms": 2780.473299906589 }, { "content_chunks": 44, "decode_tok_s": 212.35977086561113, "e2e_ms": 4989.784999983385, "effective_prefill_tok_s": 1400.817622293909, "generation_ms": 2406.293799984269, "mean_inter_token_latency_ms": 4.708989823843971, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3619, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00004", "passed": true }, "request_index": 4, "time_to_first_output_ms": 2583.4911999991164, "tpot_ms": 4.708989823843971, "ttft_ms": 2583.491600002162, "visible_content_truncated": false, "visible_generation_ms": 2406.293399981223 }, { "content_chunks": 45, "decode_tok_s": 262.3394922616727, "e2e_ms": 5029.1793999494985, "effective_prefill_tok_s": 1174.820532366914, "generation_ms": 1947.8576999390498, "mean_inter_token_latency_ms": 3.8118545987065553, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3620, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00005", "passed": true }, "request_index": 5, "time_to_first_output_ms": 3081.3217000104487, "tpot_ms": 3.8118545987065553, "ttft_ms": 3081.3221000134945, "visible_content_truncated": false, "visible_generation_ms": 1947.857299936004 }, { "content_chunks": 44, "decode_tok_s": 308.84655601745976, "e2e_ms": 4987.936200108379, "effective_prefill_tok_s": 1086.5805827836282, "generation_ms": 1654.5433000428602, "mean_inter_token_latency_ms": 3.237853816130842, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3622, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00006", "passed": true }, "request_index": 6, "time_to_first_output_ms": 3333.392900065519, "tpot_ms": 3.237853816130842, "ttft_ms": 3333.3935000700876, "visible_content_truncated": false, "visible_generation_ms": 1654.5427000382915 }, { "content_chunks": 44, "decode_tok_s": 122.58944999267452, "e2e_ms": 9831.872400012799, "effective_prefill_tok_s": 639.535257399504, "generation_ms": 4168.384800083004, "mean_inter_token_latency_ms": 8.157308806424664, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3622, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00007", "passed": true }, "request_index": 7, "time_to_first_output_ms": 5663.487599929795, "tpot_ms": 8.157308806424664, "ttft_ms": 5663.4879999328405, "visible_content_truncated": false, "visible_generation_ms": 4168.384400079958 }, { "content_chunks": 44, "decode_tok_s": 139.27643792042562, "e2e_ms": 4846.355499932542, "effective_prefill_tok_s": 3076.2875138258687, "generation_ms": 3668.9622999401763, "mean_inter_token_latency_ms": 7.179965361918153, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3622, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00008", "passed": true }, "request_index": 8, "time_to_first_output_ms": 1177.3931999923661, "tpot_ms": 7.179965361918153, "ttft_ms": 1177.3934999946505, "visible_content_truncated": false, "visible_generation_ms": 3668.961999937892 }, { "content_chunks": 44, "decode_tok_s": 308.14149013025633, "e2e_ms": 4844.772899989039, "effective_prefill_tok_s": 1137.3180346099662, "generation_ms": 1658.3290999988094, "mean_inter_token_latency_ms": 3.2452624266121513, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3624, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00009", "passed": true }, "request_index": 9, "time_to_first_output_ms": 3186.4437999902293, "tpot_ms": 3.2452624266121513, "ttft_ms": 3186.444199993275, "visible_content_truncated": false, "visible_generation_ms": 1658.3286999957636 }, { "content_chunks": 44, "decode_tok_s": 183.06261023867216, "e2e_ms": 4845.168200088665, "effective_prefill_tok_s": 1765.5306195913915, "generation_ms": 2791.3947000633925, "mean_inter_token_latency_ms": 5.462611937501746, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3626, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00010", "passed": true }, "request_index": 10, "time_to_first_output_ms": 2053.7735000252724, "tpot_ms": 5.462611937501746, "ttft_ms": 2053.7738000275567, "visible_content_truncated": false, "visible_generation_ms": 2791.394400061108 }, { "content_chunks": 44, "decode_tok_s": 266.8009686704603, "e2e_ms": 4845.822300063446, "effective_prefill_tok_s": 1237.6571678451367, "generation_ms": 1915.2853999985382, "mean_inter_token_latency_ms": 3.748112328764263, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3627, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00011", "passed": true }, "request_index": 11, "time_to_first_output_ms": 2930.536900064908, "tpot_ms": 3.748112328764263, "ttft_ms": 2930.5375000694767, "visible_content_truncated": false, "visible_generation_ms": 1915.2847999939695 }, { "content_chunks": 44, "decode_tok_s": 211.4968442989546, "e2e_ms": 4844.939699978568, "effective_prefill_tok_s": 1493.312824071562, "generation_ms": 2416.1116998875514, "mean_inter_token_latency_ms": 4.728202935200688, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3627, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00012", "passed": true }, "request_index": 12, "time_to_first_output_ms": 2428.8280000910163, "tpot_ms": 4.728202935200688, "ttft_ms": 2428.8283999776468, "visible_content_truncated": false, "visible_generation_ms": 2416.111300000921 }, { "content_chunks": 45, "decode_tok_s": 153.3680923663602, "e2e_ms": 4885.863999952562, "effective_prefill_tok_s": 2334.6044100823074, "generation_ms": 3331.8533999845386, "mean_inter_token_latency_ms": 6.5202610567212105, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3628, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00013", "passed": true }, "request_index": 13, "time_to_first_output_ms": 1554.0105999680236, "tpot_ms": 6.5202610567212105, "ttft_ms": 1554.0110999718308, "visible_content_truncated": false, "visible_generation_ms": 3331.8528999807313 }, { "content_chunks": 44, "decode_tok_s": 120.97445085714787, "e2e_ms": 9710.461199982092, "effective_prefill_tok_s": 661.4503044405153, "generation_ms": 4224.032400059514, "mean_inter_token_latency_ms": 8.266208219294548, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3629, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00014", "passed": true }, "request_index": 14, "time_to_first_output_ms": 5486.428799922578, "tpot_ms": 8.266208219294548, "ttft_ms": 5486.429099924862, "visible_content_truncated": false, "visible_generation_ms": 4224.032100057229 }, { "content_chunks": 45, "decode_tok_s": 151.29575338328988, "e2e_ms": 4943.861000007018, "effective_prefill_tok_s": 2317.4596709091807, "generation_ms": 3377.490699989721, "mean_inter_token_latency_ms": 6.609570841467165, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3630, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00015", "passed": true }, "request_index": 15, "time_to_first_output_ms": 1566.3703000172973, "tpot_ms": 6.609570841467165, "ttft_ms": 1566.370700020343, "visible_content_truncated": false, "visible_generation_ms": 3377.4902999866754 }, { "content_chunks": 44, "decode_tok_s": 180.19694785793624, "e2e_ms": 4907.281200052239, "effective_prefill_tok_s": 1751.8747690633268, "generation_ms": 2835.786100011319, "mean_inter_token_latency_ms": 5.549483561665986, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3629, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00016", "passed": true }, "request_index": 16, "time_to_first_output_ms": 2071.49510004092, "tpot_ms": 5.549483561665986, "ttft_ms": 2071.4956000447273, "visible_content_truncated": false, "visible_generation_ms": 2835.7856000075117 }, { "content_chunks": 44, "decode_tok_s": 137.42868668096443, "e2e_ms": 4905.269299983047, "effective_prefill_tok_s": 3058.1884808482687, "generation_ms": 3718.292100005783, "mean_inter_token_latency_ms": 7.276501174179614, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3630, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00017", "passed": true }, "request_index": 17, "time_to_first_output_ms": 1186.9771999772638, "tpot_ms": 7.276501174179614, "ttft_ms": 1186.9775999803096, "visible_content_truncated": false, "visible_generation_ms": 3718.2917000027373 }, { "content_chunks": 45, "decode_tok_s": 205.53475739359766, "e2e_ms": 4942.376299994066, "effective_prefill_tok_s": 1478.3125723763424, "generation_ms": 2486.197500023991, "mean_inter_token_latency_ms": 4.865357142904092, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3631, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00018", "passed": true }, "request_index": 18, "time_to_first_output_ms": 2456.1787999700755, "tpot_ms": 4.865357142904092, "ttft_ms": 2456.1791999731213, "visible_content_truncated": false, "visible_generation_ms": 2486.197100020945 }, { "content_chunks": 44, "decode_tok_s": 263.01831271976215, "e2e_ms": 4905.711199971847, "effective_prefill_tok_s": 1225.8340337447983, "generation_ms": 1942.8304999601096, "mean_inter_token_latency_ms": 3.8020166339728174, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3632, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00019", "passed": true }, "request_index": 19, "time_to_first_output_ms": 2962.8807000117376, "tpot_ms": 3.8020166339728174, "ttft_ms": 2962.881200015545, "visible_content_truncated": false, "visible_generation_ms": 1942.8299999563023 }, { "content_chunks": 45, "decode_tok_s": 297.4413409001829, "e2e_ms": 4941.338300006464, "effective_prefill_tok_s": 1127.0874035634358, "generation_ms": 1717.9858000017703, "mean_inter_token_latency_ms": 3.3620074364026817, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3633, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00020", "passed": true }, "request_index": 20, "time_to_first_output_ms": 3223.352500004694, "tpot_ms": 3.3620074364026817, "ttft_ms": 3223.3529000077397, "visible_content_truncated": false, "visible_generation_ms": 1717.9853999987245 }, { "content_chunks": 45, "decode_tok_s": 122.49612878527971, "e2e_ms": 9690.99350005854, "effective_prefill_tok_s": 658.4009506327475, "generation_ms": 4171.560400049202, "mean_inter_token_latency_ms": 8.16352328776752, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3634, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00021", "passed": true }, "request_index": 21, "time_to_first_output_ms": 5519.433100009337, "tpot_ms": 8.16352328776752, "ttft_ms": 5519.433700013906, "visible_content_truncated": false, "visible_generation_ms": 4171.559800044633 }, { "content_chunks": 44, "decode_tok_s": 181.82046548440087, "e2e_ms": 4785.388300078921, "effective_prefill_tok_s": 1840.5775291977109, "generation_ms": 2810.4647001018748, "mean_inter_token_latency_ms": 5.49993091996453, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3635, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00022", "passed": true }, "request_index": 22, "time_to_first_output_ms": 1974.9235999770463, "tpot_ms": 5.49993091996453, "ttft_ms": 1974.9240999808535, "visible_content_truncated": false, "visible_generation_ms": 2810.4642000980675 }, { "content_chunks": 45, "decode_tok_s": 138.38136268015887, "e2e_ms": 4825.8705999469385, "effective_prefill_tok_s": 3208.6784692946403, "generation_ms": 3692.6937999669462, "mean_inter_token_latency_ms": 7.226406653555668, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3636, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00023", "passed": true }, "request_index": 23, "time_to_first_output_ms": 1133.1767999799922, "tpot_ms": 7.226406653555668, "ttft_ms": 1133.177399984561, "visible_content_truncated": false, "visible_generation_ms": 3692.6931999623775 }, { "content_chunks": 45, "decode_tok_s": 153.29142027990494, "e2e_ms": 4825.090800062753, "effective_prefill_tok_s": 2437.6984022750958, "generation_ms": 3333.5198999848217, "mean_inter_token_latency_ms": 6.523522309167949, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3636, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00024", "passed": true }, "request_index": 24, "time_to_first_output_ms": 1491.5709000779316, "tpot_ms": 6.523522309167949, "ttft_ms": 1491.5713000809774, "visible_content_truncated": false, "visible_generation_ms": 3333.519499981776 }, { "content_chunks": 45, "decode_tok_s": 196.07341310766557, "e2e_ms": 4825.432399986312, "effective_prefill_tok_s": 1638.8303572895031, "generation_ms": 2606.1667000176385, "mean_inter_token_latency_ms": 5.100130528410252, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3637, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00025", "passed": true }, "request_index": 25, "time_to_first_output_ms": 2219.2656999686733, "tpot_ms": 5.100130528410252, "ttft_ms": 2219.266299973242, "visible_content_truncated": false, "visible_generation_ms": 2606.16610001307 }, { "content_chunks": 45, "decode_tok_s": 122.33008016444926, "e2e_ms": 9590.31410003081, "effective_prefill_tok_s": 672.2591211496891, "generation_ms": 4177.22280009184, "mean_inter_token_latency_ms": 8.174604305463484, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3639, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00026", "passed": true }, "request_index": 26, "time_to_first_output_ms": 5413.091299938969, "tpot_ms": 8.174604305463484, "ttft_ms": 5413.091699942015, "visible_content_truncated": false, "visible_generation_ms": 4177.2224000887945 }, { "content_chunks": 44, "decode_tok_s": 308.9219874057994, "e2e_ms": 4790.056300000288, "effective_prefill_tok_s": 1160.4261209694043, "generation_ms": 1654.1392999934033, "mean_inter_token_latency_ms": 3.237063209380437, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3639, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00027", "passed": true }, "request_index": 27, "time_to_first_output_ms": 3135.9170000068843, "tpot_ms": 3.237063209380437, "ttft_ms": 3135.9173000091687, "visible_content_truncated": false, "visible_generation_ms": 1654.138999991119 }, { "content_chunks": 45, "decode_tok_s": 239.94940682015044, "e2e_ms": 4931.358700036071, "effective_prefill_tok_s": 1299.1912071056154, "generation_ms": 2129.6156001044437, "mean_inter_token_latency_ms": 4.167545205683843, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3640, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00028", "passed": true }, "request_index": 28, "time_to_first_output_ms": 2801.7430999316275, "tpot_ms": 4.167545205683843, "ttft_ms": 2801.7435999354348, "visible_content_truncated": false, "visible_generation_ms": 2129.6151001006365 }, { "content_chunks": 44, "decode_tok_s": 134.5285718678837, "e2e_ms": 4801.371500012465, "effective_prefill_tok_s": 3631.3894431372128, "generation_ms": 3798.449599999003, "mean_inter_token_latency_ms": 7.433365166338558, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3642, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00029", "passed": true }, "request_index": 29, "time_to_first_output_ms": 1002.9219000134617, "tpot_ms": 7.433365166338558, "ttft_ms": 1002.9223000165075, "visible_content_truncated": false, "visible_generation_ms": 3798.4491999959573 }, { "content_chunks": 45, "decode_tok_s": 151.2246026225863, "e2e_ms": 4944.482600083575, "effective_prefill_tok_s": 2326.5577396668773, "generation_ms": 3379.0798000991344, "mean_inter_token_latency_ms": 6.6126806264170925, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3642, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00030", "passed": true }, "request_index": 30, "time_to_first_output_ms": 1565.4027999844402, "tpot_ms": 6.6126806264170925, "ttft_ms": 1565.403199987486, "visible_content_truncated": false, "visible_generation_ms": 3379.0794000960886 }, { "content_chunks": 44, "decode_tok_s": 213.48449985904713, "e2e_ms": 4800.700600026175, "effective_prefill_tok_s": 1513.4493425285013, "generation_ms": 2393.6163999605924, "mean_inter_token_latency_ms": 4.684180821840689, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3643, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00031", "passed": true }, "request_index": 31, "time_to_first_output_ms": 2407.084200065583, "tpot_ms": 4.684180821840689, "ttft_ms": 2407.084600068629, "visible_content_truncated": false, "visible_generation_ms": 2393.6159999575466 }, { "content_chunks": 45, "decode_tok_s": 176.2448256510053, "e2e_ms": 4944.206500076689, "effective_prefill_tok_s": 1782.054709051173, "generation_ms": 2899.375900044106, "mean_inter_token_latency_ms": 5.673925440399424, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3644, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00032", "passed": true }, "request_index": 32, "time_to_first_output_ms": 2044.8306000325829, "tpot_ms": 5.673925440399424, "ttft_ms": 2044.8310000356287, "visible_content_truncated": false, "visible_generation_ms": 2899.37550004106 }, { "content_chunks": 44, "decode_tok_s": 267.43611507827814, "e2e_ms": 4799.739300040528, "effective_prefill_tok_s": 1261.6811075273254, "generation_ms": 1910.736700054258, "mean_inter_token_latency_ms": 3.739210763315574, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3645, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00033", "passed": true }, "request_index": 33, "time_to_first_output_ms": 2889.00259998627, "tpot_ms": 3.739210763315574, "ttft_ms": 2889.002999989316, "visible_content_truncated": false, "visible_generation_ms": 1910.7363000512123 }, { "content_chunks": 45, "decode_tok_s": 282.5693371755133, "e2e_ms": 4944.080600049347, "effective_prefill_tok_s": 1162.748089723746, "generation_ms": 1808.405700023286, "mean_inter_token_latency_ms": 3.538954403176685, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3646, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00034", "passed": true }, "request_index": 34, "time_to_first_output_ms": 3135.6749000260606, "tpot_ms": 3.538954403176685, "ttft_ms": 3135.675200028345, "visible_content_truncated": false, "visible_generation_ms": 1808.4054000210017 }, { "content_chunks": 44, "decode_tok_s": 123.24574098584269, "e2e_ms": 9427.985499962233, "effective_prefill_tok_s": 690.2952888642822, "generation_ms": 4146.187899983488, "mean_inter_token_latency_ms": 8.113870645760251, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3646, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00035", "passed": true }, "request_index": 35, "time_to_first_output_ms": 5281.797599978745, "tpot_ms": 8.113870645760251, "ttft_ms": 5281.797999981791, "visible_content_truncated": false, "visible_generation_ms": 4146.187499980442 }, { "content_chunks": 45, "decode_tok_s": 132.63302404190927, "e2e_ms": 4850.177300046198, "effective_prefill_tok_s": 3656.3547838619443, "generation_ms": 3852.7358000865206, "mean_inter_token_latency_ms": 7.539600391558749, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3647, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00036", "passed": true }, "request_index": 36, "time_to_first_output_ms": 997.4414999596775, "tpot_ms": 7.539600391558749, "ttft_ms": 997.4418999627233, "visible_content_truncated": false, "visible_generation_ms": 3852.735400083475 }, { "content_chunks": 45, "decode_tok_s": 200.98411419060534, "e2e_ms": 4955.258500063792, "effective_prefill_tok_s": 1512.370226876464, "generation_ms": 2542.4895000178367, "mean_inter_token_latency_ms": 4.975517612559368, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3649, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00037", "passed": true }, "request_index": 37, "time_to_first_output_ms": 2412.769000045955, "tpot_ms": 4.975517612559368, "ttft_ms": 2412.7695000497624, "visible_content_truncated": false, "visible_generation_ms": 2542.4890000140294 }, { "content_chunks": 45, "decode_tok_s": 150.8929540893852, "e2e_ms": 4955.24799998384, "effective_prefill_tok_s": 2326.0686768031546, "generation_ms": 3386.5066999569535, "mean_inter_token_latency_ms": 6.627214677019478, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3649, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00038", "passed": true }, "request_index": 38, "time_to_first_output_ms": 1568.7413000268862, "tpot_ms": 6.627214677019478, "ttft_ms": 1568.7419000314549, "visible_content_truncated": false, "visible_generation_ms": 3386.506099952385 }, { "content_chunks": 44, "decode_tok_s": 185.09157958189417, "e2e_ms": 4811.317799962126, "effective_prefill_tok_s": 1780.0342869183328, "generation_ms": 2760.795500013046, "mean_inter_token_latency_ms": 5.402730919790697, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3650, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00039", "passed": true }, "request_index": 39, "time_to_first_output_ms": 2050.5222999490798, "tpot_ms": 5.402730919790697, "ttft_ms": 2050.5226999521255, "visible_content_truncated": false, "visible_generation_ms": 2760.79510001 }, { "content_chunks": 44, "decode_tok_s": 305.76801745619946, "e2e_ms": 4667.5514000235125, "effective_prefill_tok_s": 1218.4825683452784, "generation_ms": 1671.2015999946743, "mean_inter_token_latency_ms": 3.270453228952396, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3651, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00040", "passed": true }, "request_index": 40, "time_to_first_output_ms": 2996.349800028838, "tpot_ms": 3.270453228952396, "ttft_ms": 2996.3501000311226, "visible_content_truncated": false, "visible_generation_ms": 1671.20129999239 }, { "content_chunks": 44, "decode_tok_s": 122.39029931354324, "e2e_ms": 9468.606199952774, "effective_prefill_tok_s": 689.910700199635, "generation_ms": 4175.16749992501, "mean_inter_token_latency_ms": 8.17058219163407, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3652, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00041", "passed": true }, "request_index": 41, "time_to_first_output_ms": 5293.438700027764, "tpot_ms": 8.17058219163407, "ttft_ms": 5293.439200031571, "visible_content_truncated": false, "visible_generation_ms": 4175.166999921203 }, { "content_chunks": 45, "decode_tok_s": 247.5045870703598, "e2e_ms": 4811.647100024857, "effective_prefill_tok_s": 1329.7955117965034, "generation_ms": 2064.6081999875605, "mean_inter_token_latency_ms": 4.040329158488377, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3653, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00042", "passed": true }, "request_index": 42, "time_to_first_output_ms": 2747.038900037296, "tpot_ms": 4.040329158488377, "ttft_ms": 2747.0392000395805, "visible_content_truncated": false, "visible_generation_ms": 2064.607899985276 }, { "content_chunks": 45, "decode_tok_s": 131.92505190607275, "e2e_ms": 4882.280400022864, "effective_prefill_tok_s": 3620.8863588398, "generation_ms": 3873.4114000108093, "mean_inter_token_latency_ms": 7.580061448162054, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3653, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00043", "passed": true }, "request_index": 43, "time_to_first_output_ms": 1008.869000012055, "tpot_ms": 7.580061448162054, "ttft_ms": 1008.8693000143394, "visible_content_truncated": false, "visible_generation_ms": 3873.411100008525 }, { "content_chunks": 45, "decode_tok_s": 150.00480250147925, "e2e_ms": 4987.091599963605, "effective_prefill_tok_s": 2311.87687206748, "generation_ms": 3406.5576000139117, "mean_inter_token_latency_ms": 6.666453228990043, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3654, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00044", "passed": true }, "request_index": 44, "time_to_first_output_ms": 1580.5339999496937, "tpot_ms": 6.666453228990043, "ttft_ms": 1580.5345999542624, "visible_content_truncated": false, "visible_generation_ms": 3406.557000009343 }, { "content_chunks": 44, "decode_tok_s": 184.0729361137324, "e2e_ms": 4840.783899999224, "effective_prefill_tok_s": 1769.2553880103392, "generation_ms": 2776.073499931954, "mean_inter_token_latency_ms": 5.432629158379558, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3653, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00045", "passed": true }, "request_index": 45, "time_to_first_output_ms": 2064.71040006727, "tpot_ms": 5.432629158379558, "ttft_ms": 2064.7108000703156, "visible_content_truncated": false, "visible_generation_ms": 2776.073099928908 }, { "content_chunks": 45, "decode_tok_s": 199.81589565179908, "e2e_ms": 4986.215500044636, "effective_prefill_tok_s": 1504.4086089086334, "generation_ms": 2557.354100048542, "mean_inter_token_latency_ms": 5.004606849410063, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3654, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00046", "passed": true }, "request_index": 46, "time_to_first_output_ms": 2428.8613999960944, "tpot_ms": 5.004606849410063, "ttft_ms": 2428.8618999999017, "visible_content_truncated": false, "visible_generation_ms": 2557.353600044735 }, { "content_chunks": 45, "decode_tok_s": 246.4837358572896, "e2e_ms": 4842.89940004237, "effective_prefill_tok_s": 1319.6183049989704, "generation_ms": 2073.159100022167, "mean_inter_token_latency_ms": 4.057062818047293, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3655, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00047", "passed": true }, "request_index": 47, "time_to_first_output_ms": 2769.740300020203, "tpot_ms": 4.057062818047293, "ttft_ms": 2769.7408000240102, "visible_content_truncated": false, "visible_generation_ms": 2073.1586000183597 }, { "content_chunks": 45, "decode_tok_s": 352.4208415286776, "e2e_ms": 6644.377099932171, "effective_prefill_tok_s": 703.6415577949156, "generation_ms": 1449.9709999654442, "mean_inter_token_latency_ms": 2.837516633983257, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3655, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00048", "passed": true }, "request_index": 48, "time_to_first_output_ms": 5194.406099966727, "tpot_ms": 2.837516633983257, "ttft_ms": 5194.40629996825, "visible_content_truncated": false, "visible_generation_ms": 1449.9707999639213 }, { "content_chunks": 45, "decode_tok_s": 280.646933481051, "e2e_ms": 4841.308300034143, "effective_prefill_tok_s": 1210.0584694555107, "generation_ms": 1820.7930999342352, "mean_inter_token_latency_ms": 3.5631958902822607, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3655, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00049", "passed": true }, "request_index": 49, "time_to_first_output_ms": 3020.515200099908, "tpot_ms": 3.5631958902822607, "ttft_ms": 3020.5155999865383, "visible_content_truncated": false, "visible_generation_ms": 1820.7927000476047 } ], "requests": 50, "response_words": 256, "run_id": "benchmark-20260905T043618Z", "schema": "anvil-serving.benchmark/v1", "serve_flags": { "no_thinking": false, "reasoning_effort": null, "shared_prefix_burst": false, "thinking_mode": "disabled" }, "source_recipe": null, "started_at": "2026-09-05T04:36:18Z", "tier": null, "timing_methodology": { "clock": "client time.perf_counter", "decode": "usage completion tokens after the first token divided by client-observed generation time; completion tokens may include reasoning tokens", "distributions": "request-level mean, sample standard deviation, normal-approximation 95% confidence interval for the mean, min, p25, p50, p75, p90, p95, p99, and max using the nearest-rank percentile method", "effective_prefill": "usage.prompt_tokens divided by client-observed time to first output; includes queueing, scheduling, prefill, and first-token work", "generation": "client-observed E2E minus time to first output", "mean_inter_token_latency": "client-observed generation time divided by completion tokens after the first token; not a raw per-token timestamp trace", "request_canaries": "when enabled, each response must contain its own synthetic marker and no marker assigned to another request; failures are excluded from performance metrics", "time_to_first_output": "request start through the first non-empty streamed reasoning or content delta", "tpot": "alias of mean_inter_token_latency for capacity-v3: client-observed generation time divided by completion tokens after the first token", "ttft": "request start through the first non-empty streamed content delta" }, "wall_clock_ms": 35969.89649999887 }