{ "base_url": "http://127.0.0.1:39278/v1", "completed": 50, "concurrency": 8, "context_distribution": { "4096": 50 }, "context_seed": 0, "context_tokens": 4096, "engine": "sglang", "failed": 0, "failures": [], "finished_at": "2026-09-05T04:36:54Z", "gpu": "NVIDIA RTX PRO 6000 Max-Q replica B", "manifest": null, "max_context_tokens": 262144, "max_tokens": 512, "measurement_protocol": "capacity-v3", "metrics": { "completion_tokens_max": 512.0, "completion_tokens_mean": 512.0, "completion_tokens_mean_ci95_high": 512.0, "completion_tokens_mean_ci95_low": 512.0, "completion_tokens_min": 512.0, "completion_tokens_p25": 512.0, "completion_tokens_p50": 512.0, "completion_tokens_p75": 512.0, "completion_tokens_p90": 512.0, "completion_tokens_p95": 512.0, "completion_tokens_p99": 512.0, "completion_tokens_samples": 50, "completion_tokens_stddev": 0.0, "content_chunks": 2221, "content_chunks_s": null, "decode_tok_s_max": 363.50358555595983, "decode_tok_s_mean": 199.31879028124098, "decode_tok_s_mean_ci95_high": 217.4467414010066, "decode_tok_s_mean_ci95_low": 181.19083916147537, "decode_tok_s_min": 121.64626195210138, "decode_tok_s_p25": 143.34357730559648, "decode_tok_s_p50": 180.76293277919467, "decode_tok_s_p75": 249.68498646707354, "decode_tok_s_p90": 306.86842394828494, "decode_tok_s_p95": 310.0791187157258, "decode_tok_s_p99": 363.50358555595983, "decode_tok_s_samples": 50, "decode_tok_s_stddev": 65.39998553981903, "e2e_max_ms": 9651.010400033556, "e2e_mean_ci95_high_ms": 5858.474999295532, "e2e_mean_ci95_low_ms": 4994.657708707281, "e2e_mean_ms": 5426.566354001407, "e2e_min_ms": 4686.001800000668, "e2e_p25_ms": 4763.01690004766, "e2e_p50_ms": 4847.066399990581, "e2e_p75_ms": 4884.6237999387085, "e2e_p90_ms": 9367.989799939096, "e2e_p95_ms": 9553.644199972041, "e2e_p99_ms": 9651.010400033556, "e2e_samples": 50, "e2e_stddev_ms": 1558.1914894927124, "effective_prefill_tok_s_max": 4291.720727923479, "effective_prefill_tok_s_mean": 1761.8002979608684, "effective_prefill_tok_s_mean_ci95_high": 1996.6851134211045, "effective_prefill_tok_s_mean_ci95_low": 1526.9154825006324, "effective_prefill_tok_s_min": 657.1104805646363, "effective_prefill_tok_s_p25": 1193.0225281771359, "effective_prefill_tok_s_p50": 1529.3129274817397, "effective_prefill_tok_s_p75": 2388.9922255440547, "effective_prefill_tok_s_p90": 3151.9206765148583, "effective_prefill_tok_s_p95": 3203.8460985928587, "effective_prefill_tok_s_p99": 4291.720727923479, "effective_prefill_tok_s_samples": 50, "effective_prefill_tok_s_stddev": 847.3910500494071, "generation_max_ms": 4200.704500079155, "generation_mean_ci95_high_ms": 3061.2593927199637, "generation_mean_ci95_low_ms": 2591.1984592710937, "generation_mean_ms": 2826.2289259955287, "generation_min_ms": 1405.7633000193164, "generation_p25_ms": 2046.5788000728935, "generation_p50_ms": 2749.8188000172377, "generation_p75_ms": 3564.861499937251, "generation_p90_ms": 4041.633499902673, "generation_p95_ms": 4173.28769993037, "generation_p99_ms": 4200.704500079155, "generation_samples": 50, "generation_stddev_ms": 847.9165143177916, "mean_inter_token_latency_ms_max": 8.220556751622613, "mean_inter_token_latency_ms_mean": 5.530780677094969, "mean_inter_token_latency_ms_mean_ci95_high": 5.990722882035154, "mean_inter_token_latency_ms_mean_ci95_low": 5.070838472154783, "mean_inter_token_latency_ms_min": 2.7510045010162747, "mean_inter_token_latency_ms_p25": 4.005046575485115, "mean_inter_token_latency_ms_p50": 5.381250097881091, "mean_inter_token_latency_ms_p75": 6.976245596746089, "mean_inter_token_latency_ms_p90": 7.909263209202883, "mean_inter_token_latency_ms_p95": 8.16690352236863, "mean_inter_token_latency_ms_p99": 8.220556751622613, "mean_inter_token_latency_ms_samples": 50, "mean_inter_token_latency_ms_stddev": 1.659327816668868, "output_token_sources": { "usage": 50 }, "output_tokens": 25600, "prefix_cache_hit_avg": null, "prompt_token_samples": 50, "prompt_tokens": 181847, "prompt_tokens_max": 3655.0, "prompt_tokens_mean": 3636.94, "prompt_tokens_mean_ci95_high": 3640.4171490850986, "prompt_tokens_mean_ci95_low": 3633.4628509149015, "prompt_tokens_min": 3613.0, "prompt_tokens_p25": 3627.0, "prompt_tokens_p50": 3636.0, "prompt_tokens_p75": 3649.0, "prompt_tokens_p90": 3653.0, "prompt_tokens_p95": 3655.0, "prompt_tokens_p99": 3655.0, "prompt_tokens_samples": 50, "prompt_tokens_stddev": 12.544467843212312, "reasoning_chunks": 0, "request_canary_passed": 50, "request_canary_samples": 50, "throughput_tok_s": 720.0036947443282, "time_to_first_output_max_ms": 5512.010700069368, "time_to_first_output_mean_ci95_high_ms": 2966.4661266235603, "time_to_first_output_mean_ci95_low_ms": 2234.2087293881955, "time_to_first_output_mean_ms": 2600.337428005878, "time_to_first_output_min_ms": 842.7855001064017, "time_to_first_output_p25_ms": 1527.4223000742495, "time_to_first_output_p50_ms": 2337.9850999917835, "time_to_first_output_p75_ms": 3041.0154999699444, "time_to_first_output_p90_ms": 5332.8445999650285, "time_to_first_output_p95_ms": 5424.043099978007, "time_to_first_output_p99_ms": 5512.010700069368, "time_to_first_output_samples": 50, "time_to_first_output_stddev_ms": 1320.877987650862, "tpot_max_ms": 8.220556751622613, "tpot_mean_ci95_high_ms": 5.990722882035154, "tpot_mean_ci95_low_ms": 5.070838472154783, "tpot_mean_ms": 5.530780677094969, "tpot_min_ms": 2.7510045010162747, "tpot_p25_ms": 4.005046575485115, "tpot_p50_ms": 5.381250097881091, "tpot_p75_ms": 6.976245596746089, "tpot_p90_ms": 7.909263209202883, "tpot_p95_ms": 8.16690352236863, "tpot_p99_ms": 8.220556751622613, "tpot_samples": 50, "tpot_stddev_ms": 1.659327816668868, "ttft_max_ms": 5512.011000071652, "ttft_mean_ci95_high_ms": 2966.4665624536756, "ttft_mean_ci95_low_ms": 2234.2091695647505, "ttft_mean_ms": 2600.337866009213, "ttft_min_ms": 842.7861001109704, "ttft_p25_ms": 1527.4226000765339, "ttft_p50_ms": 2337.9854999948293, "ttft_p75_ms": 3041.0157999722287, "ttft_p90_ms": 5332.844899967313, "ttft_p95_ms": 5424.043399980292, "ttft_p99_ms": 5512.011000071652, "ttft_samples": 50, "ttft_stddev_ms": 1320.8779798105636 }, "model": "qwen38-27b-inferact-nvfp4-pro6000-dp2-replica-b-dflash2-k12-chunk1k", "prompt_cache_mode": "unique", "request_canaries": true, "request_timings": [ { "content_chunks": 44, "decode_tok_s": 143.86943344854205, "e2e_ms": 4867.13360005524, "effective_prefill_tok_s": 2746.8976705825967, "generation_ms": 3551.8316000234336, "mean_inter_token_latency_ms": 6.950746771083041, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3613, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00000", "passed": true }, "request_index": 0, "time_to_first_output_ms": 1315.3020000318065, "tpot_ms": 6.950746771083041, "ttft_ms": 1315.3028000378981, "visible_content_truncated": false, "visible_generation_ms": 3551.830800017342 }, { "content_chunks": 44, "decode_tok_s": 159.93568018701708, "e2e_ms": 4866.181100020185, "effective_prefill_tok_s": 2163.1853147065576, "generation_ms": 3195.0344000943005, "mean_inter_token_latency_ms": 6.252513503119962, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3615, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00001", "passed": true }, "request_index": 1, "time_to_first_output_ms": 1671.1466999258846, "tpot_ms": 6.252513503119962, "ttft_ms": 1671.1470999289304, "visible_content_truncated": false, "visible_generation_ms": 3195.0340000912547 }, { "content_chunks": 44, "decode_tok_s": 126.98277325937143, "e2e_ms": 4866.953400080092, "effective_prefill_tok_s": 4291.720727923479, "generation_ms": 4024.1678999736905, "mean_inter_token_latency_ms": 7.8750839529817815, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3617, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00002", "passed": true }, "request_index": 2, "time_to_first_output_ms": 842.7855001064017, "tpot_ms": 7.8750839529817815, "ttft_ms": 842.7861001109704, "visible_content_truncated": false, "visible_generation_ms": 4024.167299969122 }, { "content_chunks": 44, "decode_tok_s": 216.4696277685605, "e2e_ms": 4866.788599989377, "effective_prefill_tok_s": 1443.2321569040735, "generation_ms": 2360.6082999613136, "mean_inter_token_latency_ms": 4.619585714210007, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3617, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00003", "passed": true }, "request_index": 3, "time_to_first_output_ms": 2506.1803000280634, "tpot_ms": 4.619585714210007, "ttft_ms": 2506.180700031109, "visible_content_truncated": false, "visible_generation_ms": 2360.607899958268 }, { "content_chunks": 44, "decode_tok_s": 188.12642463860206, "e2e_ms": 4865.434300038032, "effective_prefill_tok_s": 1683.9013355459374, "generation_ms": 2716.258500004187, "mean_inter_token_latency_ms": 5.3155743640003665, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3619, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00004", "passed": true }, "request_index": 4, "time_to_first_output_ms": 2149.175800033845, "tpot_ms": 5.3155743640003665, "ttft_ms": 2149.1761000361294, "visible_content_truncated": false, "visible_generation_ms": 2716.258200001903 }, { "content_chunks": 45, "decode_tok_s": 264.88756222932693, "e2e_ms": 4911.149900057353, "effective_prefill_tok_s": 1213.9383597369683, "generation_ms": 1929.1203999891877, "mean_inter_token_latency_ms": 3.7751866927381363, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3620, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00005", "passed": true }, "request_index": 5, "time_to_first_output_ms": 2982.0295000681654, "tpot_ms": 3.7751866927381363, "ttft_ms": 2982.029900071211, "visible_content_truncated": false, "visible_generation_ms": 1929.119999986142 }, { "content_chunks": 44, "decode_tok_s": 310.92020514535756, "e2e_ms": 4865.238600061275, "effective_prefill_tok_s": 1124.2406680839463, "generation_ms": 1643.5085000703111, "mean_inter_token_latency_ms": 3.2162592956366165, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3622, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00006", "passed": true }, "request_index": 6, "time_to_first_output_ms": 3221.730099990964, "tpot_ms": 3.2162592956366165, "ttft_ms": 3221.730599994771, "visible_content_truncated": false, "visible_generation_ms": 1643.5080000665039 }, { "content_chunks": 44, "decode_tok_s": 126.43402723485576, "e2e_ms": 9553.644199972041, "effective_prefill_tok_s": 657.1104805646363, "generation_ms": 4041.633499902673, "mean_inter_token_latency_ms": 7.909263209202883, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3622, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00007", "passed": true }, "request_index": 7, "time_to_first_output_ms": 5512.010700069368, "tpot_ms": 7.909263209202883, "ttft_ms": 5512.011000071652, "visible_content_truncated": false, "visible_generation_ms": 4041.6331999003887 }, { "content_chunks": 44, "decode_tok_s": 143.34357730559648, "e2e_ms": 4689.2649999354035, "effective_prefill_tok_s": 3221.263541073958, "generation_ms": 3564.861499937251, "mean_inter_token_latency_ms": 6.976245596746089, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3622, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00008", "passed": true }, "request_index": 8, "time_to_first_output_ms": 1124.4034999981523, "tpot_ms": 6.976245596746089, "ttft_ms": 1124.4038000004366, "visible_content_truncated": false, "visible_generation_ms": 3564.861199934967 }, { "content_chunks": 44, "decode_tok_s": 159.34767614199234, "e2e_ms": 4689.417900051922, "effective_prefill_tok_s": 2444.365063941182, "generation_ms": 3206.8242999957874, "mean_inter_token_latency_ms": 6.2755857142774705, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3624, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00009", "passed": true }, "request_index": 9, "time_to_first_output_ms": 1482.5936000561342, "tpot_ms": 6.2755857142774705, "ttft_ms": 1482.5941000599414, "visible_content_truncated": false, "visible_generation_ms": 3206.82379999198 }, { "content_chunks": 44, "decode_tok_s": 187.2510678474566, "e2e_ms": 4689.3436999525875, "effective_prefill_tok_s": 1849.6345084449918, "generation_ms": 2728.95639995113, "mean_inter_token_latency_ms": 5.340423483270313, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3626, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00010", "passed": true }, "request_index": 10, "time_to_first_output_ms": 1960.3873000014573, "tpot_ms": 5.340423483270313, "ttft_ms": 1960.388100007549, "visible_content_truncated": false, "visible_generation_ms": 2728.9555999450386 }, { "content_chunks": 44, "decode_tok_s": 270.1003414888494, "e2e_ms": 4687.821100000292, "effective_prefill_tok_s": 1297.24203551889, "generation_ms": 1891.8894999660552, "mean_inter_token_latency_ms": 3.702327788583278, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3627, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00011", "passed": true }, "request_index": 11, "time_to_first_output_ms": 2795.931600034237, "tpot_ms": 3.702327788583278, "ttft_ms": 2795.9320000372827, "visible_content_truncated": false, "visible_generation_ms": 1891.8890999630094 }, { "content_chunks": 45, "decode_tok_s": 211.9029055060782, "e2e_ms": 4729.705000063404, "effective_prefill_tok_s": 1564.5603063318124, "generation_ms": 2411.4818000234663, "mean_inter_token_latency_ms": 4.719142465799347, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3627, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00012", "passed": true }, "request_index": 12, "time_to_first_output_ms": 2318.223200039938, "tpot_ms": 4.719142465799347, "ttft_ms": 2318.223600042984, "visible_content_truncated": false, "visible_generation_ms": 2411.4814000204206 }, { "content_chunks": 44, "decode_tok_s": 310.0791187157258, "e2e_ms": 4688.981999992393, "effective_prefill_tok_s": 1193.0225281771359, "generation_ms": 1647.9665000224486, "mean_inter_token_latency_ms": 3.22498336599305, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3628, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00013", "passed": true }, "request_index": 13, "time_to_first_output_ms": 3041.0154999699444, "tpot_ms": 3.22498336599305, "ttft_ms": 3041.0157999722287, "visible_content_truncated": false, "visible_generation_ms": 1647.9662000201643 }, { "content_chunks": 44, "decode_tok_s": 125.47270491247981, "e2e_ms": 9367.989799939096, "effective_prefill_tok_s": 685.3129577281528, "generation_ms": 4072.5988999474794, "mean_inter_token_latency_ms": 7.969860860953971, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3629, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00014", "passed": true }, "request_index": 14, "time_to_first_output_ms": 5295.390899991617, "tpot_ms": 7.969860860953971, "ttft_ms": 5295.391399995424, "visible_content_truncated": false, "visible_generation_ms": 4072.598399943672 }, { "content_chunks": 45, "decode_tok_s": 210.71888623820348, "e2e_ms": 4763.01690004766, "effective_prefill_tok_s": 1552.6189623760893, "generation_ms": 2425.0318000558764, "mean_inter_token_latency_ms": 4.745659099913652, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3630, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00015", "passed": true }, "request_index": 15, "time_to_first_output_ms": 2337.9850999917835, "tpot_ms": 4.745659099913652, "ttft_ms": 2337.9854999948293, "visible_content_truncated": false, "visible_generation_ms": 2425.0314000528306 }, { "content_chunks": 44, "decode_tok_s": 142.26062658830037, "e2e_ms": 4724.699900019914, "effective_prefill_tok_s": 3203.8460985928587, "generation_ms": 3591.998800053261, "mean_inter_token_latency_ms": 7.029351859204033, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3629, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00016", "passed": true }, "request_index": 16, "time_to_first_output_ms": 1132.7010999666527, "tpot_ms": 7.029351859204033, "ttft_ms": 1132.701399968937, "visible_content_truncated": false, "visible_generation_ms": 3591.998500050977 }, { "content_chunks": 44, "decode_tok_s": 185.83042635274612, "e2e_ms": 4724.349600030109, "effective_prefill_tok_s": 1838.4114342386235, "generation_ms": 2749.8188000172377, "mean_inter_token_latency_ms": 5.381250097881091, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3630, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00017", "passed": true }, "request_index": 17, "time_to_first_output_ms": 1974.5308000128716, "tpot_ms": 5.381250097881091, "ttft_ms": 1974.5313000166789, "visible_content_truncated": false, "visible_generation_ms": 2749.8183000134304 }, { "content_chunks": 45, "decode_tok_s": 263.0269230882869, "e2e_ms": 4762.518199975602, "effective_prefill_tok_s": 1287.7022168962137, "generation_ms": 1942.7669000579044, "mean_inter_token_latency_ms": 3.801892172324666, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3631, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00018", "passed": true }, "request_index": 18, "time_to_first_output_ms": 2819.751299917698, "tpot_ms": 3.801892172324666, "ttft_ms": 2819.7516999207437, "visible_content_truncated": false, "visible_generation_ms": 1942.7665000548586 }, { "content_chunks": 44, "decode_tok_s": 158.14741108470895, "e2e_ms": 4723.655400099233, "effective_prefill_tok_s": 2433.5125770849713, "generation_ms": 3231.1626001028344, "mean_inter_token_latency_ms": 6.323214481610243, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3632, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00019", "passed": true }, "request_index": 19, "time_to_first_output_ms": 1492.4927999963984, "tpot_ms": 6.323214481610243, "ttft_ms": 1492.4931999994442, "visible_content_truncated": false, "visible_generation_ms": 3231.1622000997886 }, { "content_chunks": 45, "decode_tok_s": 301.8345634620259, "e2e_ms": 4762.059799977578, "effective_prefill_tok_s": 1183.7425907228755, "generation_ms": 1692.9804000537843, "mean_inter_token_latency_ms": 3.3130731899291277, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3633, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00020", "passed": true }, "request_index": 20, "time_to_first_output_ms": 3069.079399923794, "tpot_ms": 3.3130731899291277, "ttft_ms": 3069.0799999283627, "visible_content_truncated": false, "visible_generation_ms": 1692.9798000492156 }, { "content_chunks": 45, "decode_tok_s": 121.64626195210138, "e2e_ms": 9544.722200022079, "effective_prefill_tok_s": 680.0127177795111, "generation_ms": 4200.704500079155, "mean_inter_token_latency_ms": 8.220556751622613, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3634, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00021", "passed": true }, "request_index": 21, "time_to_first_output_ms": 5344.017699942924, "tpot_ms": 8.220556751622613, "ttft_ms": 5344.018299947493, "visible_content_truncated": false, "visible_generation_ms": 4200.703900074586 }, { "content_chunks": 44, "decode_tok_s": 139.06792107065527, "e2e_ms": 4821.85200008098, "effective_prefill_tok_s": 3168.0638248137116, "generation_ms": 3674.4635000359267, "mean_inter_token_latency_ms": 7.190730919835473, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3635, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00022", "passed": true }, "request_index": 22, "time_to_first_output_ms": 1147.3885000450537, "tpot_ms": 7.190730919835473, "ttft_ms": 1147.3891000496224, "visible_content_truncated": false, "visible_generation_ms": 3674.462900031358 }, { "content_chunks": 45, "decode_tok_s": 195.32077921443303, "e2e_ms": 4861.053599976003, "effective_prefill_tok_s": 1619.7112985063222, "generation_ms": 2616.2091000005603, "mean_inter_token_latency_ms": 5.119782974560783, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3636, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00023", "passed": true }, "request_index": 23, "time_to_first_output_ms": 2244.844499975443, "tpot_ms": 5.119782974560783, "ttft_ms": 2244.8450999800116, "visible_content_truncated": false, "visible_generation_ms": 2616.2084999959916 }, { "content_chunks": 45, "decode_tok_s": 178.28340642084905, "e2e_ms": 4860.603699926287, "effective_prefill_tok_s": 1823.122335728646, "generation_ms": 2866.2229999899864, "mean_inter_token_latency_ms": 5.609046966712302, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3636, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00024", "passed": true }, "request_index": 24, "time_to_first_output_ms": 1994.3806999363005, "tpot_ms": 5.609046966712302, "ttft_ms": 1994.3812999408692, "visible_content_truncated": false, "visible_generation_ms": 2866.2223999854177 }, { "content_chunks": 44, "decode_tok_s": 154.31314915218596, "e2e_ms": 4821.546299965121, "effective_prefill_tok_s": 2408.4528019346326, "generation_ms": 3311.448200023733, "mean_inter_token_latency_ms": 6.480329158559164, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3637, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00025", "passed": true }, "request_index": 25, "time_to_first_output_ms": 1510.0980999413878, "tpot_ms": 6.480329158559164, "ttft_ms": 1510.098599945195, "visible_content_truncated": false, "visible_generation_ms": 3311.447700019926 }, { "content_chunks": 45, "decode_tok_s": 121.66384522425943, "e2e_ms": 9651.010400033556, "effective_prefill_tok_s": 667.5945846090965, "generation_ms": 4200.097399996594, "mean_inter_token_latency_ms": 8.219368688838735, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3639, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00026", "passed": true }, "request_index": 26, "time_to_first_output_ms": 5450.913000036962, "tpot_ms": 8.219368688838735, "ttft_ms": 5450.91350004077, "visible_content_truncated": false, "visible_generation_ms": 4200.096899992786 }, { "content_chunks": 45, "decode_tok_s": 239.9563814729042, "e2e_ms": 4964.605799992569, "effective_prefill_tok_s": 1283.574294802867, "generation_ms": 2129.5536999823526, "mean_inter_token_latency_ms": 4.167424070415563, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3639, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00027", "passed": true }, "request_index": 27, "time_to_first_output_ms": 2835.0521000102162, "tpot_ms": 4.167424070415563, "ttft_ms": 2835.052500013262, "visible_content_truncated": false, "visible_generation_ms": 2129.5532999793068 }, { "content_chunks": 45, "decode_tok_s": 285.32752613129435, "e2e_ms": 4964.550400036387, "effective_prefill_tok_s": 1146.9530074536626, "generation_ms": 1790.9242999739945, "mean_inter_token_latency_ms": 3.5047442269549793, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3640, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00028", "passed": true }, "request_index": 28, "time_to_first_output_ms": 3173.6261000623927, "tpot_ms": 3.5047442269549793, "ttft_ms": 3173.6265000654384, "visible_content_truncated": false, "visible_generation_ms": 1790.9238999709487 }, { "content_chunks": 45, "decode_tok_s": 147.15785905879181, "e2e_ms": 4866.885400027968, "effective_prefill_tok_s": 2611.8313088685036, "generation_ms": 3472.461499972269, "mean_inter_token_latency_ms": 6.795423679006397, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3642, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00029", "passed": true }, "request_index": 29, "time_to_first_output_ms": 1394.423900055699, "tpot_ms": 6.795423679006397, "ttft_ms": 1394.4242000579834, "visible_content_truncated": false, "visible_generation_ms": 3472.4611999699846 }, { "content_chunks": 45, "decode_tok_s": 137.70271503643377, "e2e_ms": 4866.043400019407, "effective_prefill_tok_s": 3152.835383265926, "generation_ms": 3710.8927000081167, "mean_inter_token_latency_ms": 7.2620209393505215, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3642, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00030", "passed": true }, "request_index": 30, "time_to_first_output_ms": 1155.1507000112906, "tpot_ms": 7.2620209393505215, "ttft_ms": 1155.1511000143364, "visible_content_truncated": false, "visible_generation_ms": 3710.892300005071 }, { "content_chunks": 44, "decode_tok_s": 179.52975080678306, "e2e_ms": 4827.374899992719, "effective_prefill_tok_s": 1838.9238029777248, "generation_ms": 2846.3248999323696, "mean_inter_token_latency_ms": 5.570107436266868, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3643, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00031", "passed": true }, "request_index": 31, "time_to_first_output_ms": 1981.0500000603497, "tpot_ms": 5.570107436266868, "ttft_ms": 1981.050500064157, "visible_content_truncated": false, "visible_generation_ms": 2846.3243999285623 }, { "content_chunks": 44, "decode_tok_s": 216.65495769307768, "e2e_ms": 4827.337899943814, "effective_prefill_tok_s": 1476.0512906000133, "generation_ms": 2358.5889999521896, "mean_inter_token_latency_ms": 4.6156340507870635, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3644, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00032", "passed": true }, "request_index": 32, "time_to_first_output_ms": 2468.748899991624, "tpot_ms": 4.6156340507870635, "ttft_ms": 2468.7494999961928, "visible_content_truncated": false, "visible_generation_ms": 2358.588399947621 }, { "content_chunks": 45, "decode_tok_s": 249.5480421234679, "e2e_ms": 4882.831999915652, "effective_prefill_tok_s": 1285.6552861970233, "generation_ms": 2047.7019000099972, "mean_inter_token_latency_ms": 4.007244422720151, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3645, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00033", "passed": true }, "request_index": 33, "time_to_first_output_ms": 2835.130099905655, "tpot_ms": 4.007244422720151, "ttft_ms": 2835.1304999087006, "visible_content_truncated": false, "visible_generation_ms": 2047.7015000069514 }, { "content_chunks": 44, "decode_tok_s": 123.40786645356695, "e2e_ms": 9491.260999930091, "effective_prefill_tok_s": 681.429082009127, "generation_ms": 4140.740899951197, "mean_inter_token_latency_ms": 8.103211154503322, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3646, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00034", "passed": true }, "request_index": 34, "time_to_first_output_ms": 5350.520099978894, "tpot_ms": 8.103211154503322, "ttft_ms": 5350.520699983463, "visible_content_truncated": false, "visible_generation_ms": 4140.7402999466285 }, { "content_chunks": 44, "decode_tok_s": 309.7468295540272, "e2e_ms": 4686.001800000668, "effective_prefill_tok_s": 1200.8165025172364, "generation_ms": 1649.734399979934, "mean_inter_token_latency_ms": 3.2284430527983057, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3646, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00035", "passed": true }, "request_index": 35, "time_to_first_output_ms": 3036.2674000207335, "tpot_ms": 3.2284430527983057, "ttft_ms": 3036.2678000237793, "visible_content_truncated": false, "visible_generation_ms": 1649.7339999768883 }, { "content_chunks": 45, "decode_tok_s": 180.76293277919467, "e2e_ms": 4846.019799937494, "effective_prefill_tok_s": 1806.238858954616, "generation_ms": 2826.9069999223575, "mean_inter_token_latency_ms": 5.532107631941991, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3647, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00036", "passed": true }, "request_index": 36, "time_to_first_output_ms": 2019.1128000151366, "tpot_ms": 5.532107631941991, "ttft_ms": 2019.113100017421, "visible_content_truncated": false, "visible_generation_ms": 2826.906699920073 }, { "content_chunks": 45, "decode_tok_s": 138.5708561173621, "e2e_ms": 4845.350899966434, "effective_prefill_tok_s": 3151.9206765148583, "generation_ms": 3687.6441000495106, "mean_inter_token_latency_ms": 7.216524657631136, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3649, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00037", "passed": true }, "request_index": 37, "time_to_first_output_ms": 1157.706799916923, "tpot_ms": 7.216524657631136, "ttft_ms": 1157.7070999192074, "visible_content_truncated": false, "visible_generation_ms": 3687.6438000472263 }, { "content_chunks": 44, "decode_tok_s": 155.84767149954632, "e2e_ms": 4806.265000021085, "effective_prefill_tok_s": 2388.9922255440547, "generation_ms": 3278.8426999468356, "mean_inter_token_latency_ms": 6.41652191770418, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3649, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00038", "passed": true }, "request_index": 38, "time_to_first_output_ms": 1527.4223000742495, "tpot_ms": 6.41652191770418, "ttft_ms": 1527.4226000765339, "visible_content_truncated": false, "visible_generation_ms": 3278.8423999445513 }, { "content_chunks": 45, "decode_tok_s": 207.84228390819982, "e2e_ms": 4845.287699950859, "effective_prefill_tok_s": 1529.3129274817397, "generation_ms": 2458.5949999745935, "mean_inter_token_latency_ms": 4.811340508756543, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3650, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00039", "passed": true }, "request_index": 39, "time_to_first_output_ms": 2386.6926999762654, "tpot_ms": 4.811340508756543, "ttft_ms": 2386.69299997855, "visible_content_truncated": false, "visible_generation_ms": 2458.594699972309 }, { "content_chunks": 44, "decode_tok_s": 306.86842394828494, "e2e_ms": 4805.47900008969, "effective_prefill_tok_s": 1162.6388072888235, "generation_ms": 1665.2087999973446, "mean_inter_token_latency_ms": 3.2587256360026315, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3651, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00040", "passed": true }, "request_index": 40, "time_to_first_output_ms": 3140.2702000923455, "tpot_ms": 3.2587256360026315, "ttft_ms": 3140.27050009463, "visible_content_truncated": false, "visible_generation_ms": 1665.2084999950603 }, { "content_chunks": 45, "decode_tok_s": 259.7673145713514, "e2e_ms": 4844.988099997863, "effective_prefill_tok_s": 1269.0059881677548, "generation_ms": 1967.1451000031084, "mean_inter_token_latency_ms": 3.8495990215325016, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3652, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00041", "passed": true }, "request_index": 41, "time_to_first_output_ms": 2877.842999994755, "tpot_ms": 3.8495990215325016, "ttft_ms": 2877.8433999978006, "visible_content_truncated": false, "visible_generation_ms": 1967.1447000000626 }, { "content_chunks": 44, "decode_tok_s": 122.44542833903493, "e2e_ms": 9597.330799908377, "effective_prefill_tok_s": 673.4828489867294, "generation_ms": 4173.28769993037, "mean_inter_token_latency_ms": 8.16690352236863, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3653, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00042", "passed": true }, "request_index": 42, "time_to_first_output_ms": 5424.043099978007, "tpot_ms": 8.16690352236863, "ttft_ms": 5424.043399980292, "visible_content_truncated": false, "visible_generation_ms": 4173.2873999280855 }, { "content_chunks": 44, "decode_tok_s": 172.72249420640162, "e2e_ms": 4847.636600024998, "effective_prefill_tok_s": 1933.690558745055, "generation_ms": 2958.5028999717906, "mean_inter_token_latency_ms": 5.789633855130706, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3653, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00043", "passed": true }, "request_index": 43, "time_to_first_output_ms": 1889.1337000532076, "tpot_ms": 5.789633855130706, "ttft_ms": 1889.1342000570148, "visible_content_truncated": false, "visible_generation_ms": 2958.5023999679834 }, { "content_chunks": 45, "decode_tok_s": 137.57970536582502, "e2e_ms": 4884.6237999387085, "effective_prefill_tok_s": 3121.974359020812, "generation_ms": 3714.210599893704, "mean_inter_token_latency_ms": 7.268513894116838, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3654, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00044", "passed": true }, "request_index": 44, "time_to_first_output_ms": 1170.4132000450045, "tpot_ms": 7.268513894116838, "ttft_ms": 1170.4136000480503, "visible_content_truncated": false, "visible_generation_ms": 3714.2101998906583 }, { "content_chunks": 44, "decode_tok_s": 154.72153817341547, "e2e_ms": 4847.066399990581, "effective_prefill_tok_s": 2365.382966629412, "generation_ms": 3302.707599941641, "mean_inter_token_latency_ms": 6.4632242660306085, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3653, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00045", "passed": true }, "request_index": 45, "time_to_first_output_ms": 1544.3588000489399, "tpot_ms": 6.4632242660306085, "ttft_ms": 1544.3593000527471, "visible_content_truncated": false, "visible_generation_ms": 3302.707099937834 }, { "content_chunks": 45, "decode_tok_s": 202.39587603386298, "e2e_ms": 4985.871599987149, "effective_prefill_tok_s": 1484.6919483820204, "generation_ms": 2524.7550000203773, "mean_inter_token_latency_ms": 4.940812133112284, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3654, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00046", "passed": true }, "request_index": 46, "time_to_first_output_ms": 2461.116599966772, "tpot_ms": 4.940812133112284, "ttft_ms": 2461.1168999690562, "visible_content_truncated": false, "visible_generation_ms": 2524.754700018093 }, { "content_chunks": 45, "decode_tok_s": 249.68498646707354, "e2e_ms": 4985.716600087471, "effective_prefill_tok_s": 1243.5619724879425, "generation_ms": 2046.5788000728935, "mean_inter_token_latency_ms": 4.005046575485115, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3655, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00047", "passed": true }, "request_index": 47, "time_to_first_output_ms": 2939.137800014578, "tpot_ms": 4.005046575485115, "ttft_ms": 2939.138300018385, "visible_content_truncated": false, "visible_generation_ms": 2046.5783000690863 }, { "content_chunks": 44, "decode_tok_s": 306.9571630876217, "e2e_ms": 4845.044699963182, "effective_prefill_tok_s": 1149.256396515546, "generation_ms": 1664.7273999406025, "mean_inter_token_latency_ms": 3.2577835615275976, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3655, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00048", "passed": true }, "request_index": 48, "time_to_first_output_ms": 3180.3173000225797, "tpot_ms": 3.2577835615275976, "ttft_ms": 3180.317600024864, "visible_content_truncated": false, "visible_generation_ms": 1664.7270999383181 }, { "content_chunks": 44, "decode_tok_s": 363.50358555595983, "e2e_ms": 6738.607899984345, "effective_prefill_tok_s": 685.3753060841054, "generation_ms": 1405.7633000193164, "mean_inter_token_latency_ms": 2.7510045010162747, "output_token_source": "usage", "output_tokens": 512, "planned_context_tokens": 4096, "prompt_tokens": 3655, "reasoning_chunks": 0, "request_canary": { "foreign_marker_count": 0, "marker": "ANVIL_REQ_0_00049", "passed": true }, "request_index": 49, "time_to_first_output_ms": 5332.8445999650285, "tpot_ms": 2.7510045010162747, "ttft_ms": 5332.844899967313, "visible_content_truncated": false, "visible_generation_ms": 1405.763000017032 } ], "requests": 50, "response_words": 256, "run_id": "benchmark-20260905T043618Z", "schema": "anvil-serving.benchmark/v1", "serve_flags": { "no_thinking": false, "reasoning_effort": null, "shared_prefix_burst": false, "thinking_mode": "disabled" }, "source_recipe": null, "started_at": "2026-09-05T04:36:18Z", "tier": null, "timing_methodology": { "clock": "client time.perf_counter", "decode": "usage completion tokens after the first token divided by client-observed generation time; completion tokens may include reasoning tokens", "distributions": "request-level mean, sample standard deviation, normal-approximation 95% confidence interval for the mean, min, p25, p50, p75, p90, p95, p99, and max using the nearest-rank percentile method", "effective_prefill": "usage.prompt_tokens divided by client-observed time to first output; includes queueing, scheduling, prefill, and first-token work", "generation": "client-observed E2E minus time to first output", "mean_inter_token_latency": "client-observed generation time divided by completion tokens after the first token; not a raw per-token timestamp trace", "request_canaries": "when enabled, each response must contain its own synthetic marker and no marker assigned to another request; failures are excluded from performance metrics", "time_to_first_output": "request start through the first non-empty streamed reasoning or content delta", "tpot": "alias of mean_inter_token_latency for capacity-v3: client-observed generation time divided by completion tokens after the first token", "ttft": "request start through the first non-empty streamed content delta" }, "wall_clock_ms": 35555.37309998181 }