{ "base_url": "http://127.0.0.1:39272/v1", "completed": 100, "concurrency": 8, "context_distribution": { "4096": 100 }, "context_seed": 0, "context_tokens": 4096, "engine": "sglang", "failed": 0, "failures": [], "finished_at": "2026-09-05T03:53:33Z", "gpu": "Primary Node / RTX PRO 6000 Blackwell Max-Q / TP=1", "manifest": null, "max_context_tokens": 262144, "max_tokens": 512, "measurement_protocol": "capacity-v3", "metrics": { "content_chunks": 1339, "content_chunks_s": null, "decode_tok_s_max": 117.43932363886229, "decode_tok_s_mean": 74.14468895892715, "decode_tok_s_min": 45.52403061578605, "decode_tok_s_p25": 63.599496171468495, "decode_tok_s_p50": 70.20638963172141, "decode_tok_s_p75": 82.35716225861665, "decode_tok_s_p90": 100.54413629903459, "decode_tok_s_p95": 106.23385440932458, "decode_tok_s_p99": 113.07493346863872, "decode_tok_s_samples": 100, "e2e_max_ms": 1609.8959000082687, "e2e_mean_ms": 858.2939599978272, "e2e_min_ms": 524.1460999241099, "e2e_p25_ms": 758.1216000253335, "e2e_p50_ms": 816.8178000487387, "e2e_p75_ms": 934.7544999327511, "e2e_p90_ms": 1066.6811999399215, "e2e_p95_ms": 1137.546099955216, "e2e_p99_ms": 1496.908999979496, "e2e_samples": 100, "effective_prefill_tok_s_max": 30860.05976314626, "effective_prefill_tok_s_mean": 18938.14744072151, "effective_prefill_tok_s_min": 3759.4841648543584, "effective_prefill_tok_s_p25": 14750.063159166495, "effective_prefill_tok_s_p50": 19737.438552646418, "effective_prefill_tok_s_p75": 24086.444610062634, "effective_prefill_tok_s_p90": 24734.111720501933, "effective_prefill_tok_s_p95": 25069.784734374127, "effective_prefill_tok_s_p99": 28329.47796554701, "effective_prefill_tok_s_samples": 100, "generation_max_ms": 1017.723900033161, "generation_mean_ms": 638.1497169996146, "generation_min_ms": 380.27879991568625, "generation_p25_ms": 537.692700047046, "generation_p50_ms": 633.3753999788314, "generation_p75_ms": 715.1647000573575, "generation_p90_ms": 788.958300021477, "generation_p95_ms": 874.6128999628127, "generation_p99_ms": 987.5217999797314, "generation_samples": 100, "mean_inter_token_latency_ms_max": 21.96642051403148, "mean_inter_token_latency_ms_mean": 14.092985208661242, "mean_inter_token_latency_ms_min": 8.515035415863773, "mean_inter_token_latency_ms_p25": 12.137270734145693, "mean_inter_token_latency_ms_p50": 14.170325000304729, "mean_inter_token_latency_ms_p75": 15.615378721973839, "mean_inter_token_latency_ms_p90": 17.531182978579654, "mean_inter_token_latency_ms_p95": 18.60878510559176, "mean_inter_token_latency_ms_p99": 21.01110212722833, "mean_inter_token_latency_ms_samples": 100, "output_token_sources": { "usage": 100 }, "output_tokens": 4636, "prefix_cache_hit_avg": null, "prompt_token_samples": 100, "prompt_tokens": 361390, "prompt_tokens_max": 3614.0, "prompt_tokens_mean": 3613.9, "prompt_tokens_min": 3613.0, "prompt_tokens_p25": 3614.0, "prompt_tokens_p50": 3614.0, "prompt_tokens_p75": 3614.0, "prompt_tokens_p90": 3614.0, "prompt_tokens_p95": 3614.0, "prompt_tokens_p99": 3614.0, "prompt_tokens_samples": 100, "reasoning_chunks": 0, "throughput_tok_s": 420.4162930219674, "time_to_first_output_max_ms": 961.0361000522971, "time_to_first_output_mean_ms": 220.14424299821258, "time_to_first_output_min_ms": 117.10930010303855, "time_to_first_output_p25_ms": 150.0242999754846, "time_to_first_output_p50_ms": 182.9254999756813, "time_to_first_output_p75_ms": 242.94529994949698, "time_to_first_output_p90_ms": 325.65200002864003, "time_to_first_output_p95_ms": 341.93789993878454, "time_to_first_output_p99_ms": 651.77319990471, "time_to_first_output_samples": 100, "tpot_max_ms": 21.96642051403148, "tpot_mean_ms": 14.092985208661242, "tpot_min_ms": 8.515035415863773, "tpot_p25_ms": 12.137270734145693, "tpot_p50_ms": 14.170325000304729, "tpot_p75_ms": 15.615378721973839, "tpot_p90_ms": 17.531182978579654, "tpot_p95_ms": 18.60878510559176, "tpot_p99_ms": 21.01110212722833, "tpot_samples": 100, "ttft_max_ms": 961.03630005382, "ttft_mean_ms": 220.14449199778028, "ttft_min_ms": 117.1096001053229, "ttft_p25_ms": 150.02459997776896, "ttft_p50_ms": 182.92579997796565, "ttft_p75_ms": 242.94549995101988, "ttft_p90_ms": 325.6523000309244, "ttft_p95_ms": 341.9381999410689, "ttft_p99_ms": 651.773399906233, "ttft_samples": 100 }, "model": "qwen38-27b-inferact-nvfp4-pro6000-dflash2-k12-chunk1k", "request_timings": [ { "content_chunks": 16, "decode_tok_s": 54.518757587664886, "e2e_ms": 1075.5434000166133, "effective_prefill_tok_s": 15586.90769261766, "generation_ms": 843.7463000882417, "mean_inter_token_latency_ms": 18.342310871483516, "output_token_source": "usage", "output_tokens": 47, "planned_context_tokens": 4096, "prompt_tokens": 3613, "reasoning_chunks": 0, "request_index": 0, "time_to_first_output_ms": 231.7970999283716, "tpot_ms": 18.342310871483516, "ttft_ms": 231.79779993370175, "visible_generation_ms": 843.7456000829116 }, { "content_chunks": 13, "decode_tok_s": 98.26943345674891, "e2e_ms": 795.373700093478, "effective_prefill_tok_s": 11393.99703533625, "generation_ms": 478.2769000157714, "mean_inter_token_latency_ms": 10.17610425565471, "output_token_source": "usage", "output_tokens": 48, "planned_context_tokens": 4096, "prompt_tokens": 3613, "reasoning_chunks": 0, "request_index": 1, "time_to_first_output_ms": 317.0968000777066, "tpot_ms": 10.17610425565471, "ttft_ms": 317.0975000830367, "visible_generation_ms": 478.27620001044124 }, { "content_chunks": 13, "decode_tok_s": 100.54413629903459, "e2e_ms": 794.8768000351265, "effective_prefill_tok_s": 11034.74310125786, "generation_ms": 467.45640004519373, "mean_inter_token_latency_ms": 9.945880852025399, "output_token_source": "usage", "output_tokens": 48, "planned_context_tokens": 4096, "prompt_tokens": 3613, "reasoning_chunks": 0, "request_index": 2, "time_to_first_output_ms": 327.4203999899328, "tpot_ms": 9.945880852025399, "ttft_ms": 327.4204999906942, "visible_generation_ms": 467.4563000444323 }, { "content_chunks": 14, "decode_tok_s": 92.65368634075392, "e2e_ms": 834.4147000461817, "effective_prefill_tok_s": 11043.883925076281, "generation_ms": 507.2653000243008, "mean_inter_token_latency_ms": 10.792878723921294, "output_token_source": "usage", "output_tokens": 48, "planned_context_tokens": 4096, "prompt_tokens": 3613, "reasoning_chunks": 0, "request_index": 3, "time_to_first_output_ms": 327.14940002188087, "tpot_ms": 10.792878723921294, "ttft_ms": 327.14960002340376, "visible_generation_ms": 507.2651000227779 }, { "content_chunks": 13, "decode_tok_s": 106.80960355133062, "e2e_ms": 794.7770000901073, "effective_prefill_tok_s": 11060.622802039232, "generation_ms": 468.1226999964565, "mean_inter_token_latency_ms": 9.36245399992913, "output_token_source": "usage", "output_tokens": 51, "planned_context_tokens": 4096, "prompt_tokens": 3613, "reasoning_chunks": 0, "request_index": 4, "time_to_first_output_ms": 326.6543000936508, "tpot_ms": 9.36245399992913, "ttft_ms": 326.65440009441227, "visible_generation_ms": 468.12259999569505 }, { "content_chunks": 14, "decode_tok_s": 86.66439763500826, "e2e_ms": 833.844400011003, "effective_prefill_tok_s": 11078.10539527427, "generation_ms": 507.7056000009179, "mean_inter_token_latency_ms": 11.538763636384498, "output_token_source": "usage", "output_tokens": 45, "planned_context_tokens": 4096, "prompt_tokens": 3613, "reasoning_chunks": 0, "request_index": 5, "time_to_first_output_ms": 326.1388000100851, "tpot_ms": 11.538763636384498, "ttft_ms": 326.139000011608, "visible_generation_ms": 507.705399999395 }, { "content_chunks": 14, "decode_tok_s": 98.435699248269, "e2e_ms": 833.5977999959141, "effective_prefill_tok_s": 11094.665470140666, "generation_ms": 507.94579996727407, "mean_inter_token_latency_ms": 10.158915999345481, "output_token_source": "usage", "output_tokens": 51, "planned_context_tokens": 4096, "prompt_tokens": 3613, "reasoning_chunks": 0, "request_index": 6, "time_to_first_output_ms": 325.65200002864003, "tpot_ms": 10.158915999345481, "ttft_ms": 325.6523000309244, "visible_generation_ms": 507.9454999649897 }, { "content_chunks": 14, "decode_tok_s": 72.43475401494929, "e2e_ms": 1609.8959000082687, "effective_prefill_tok_s": 3759.4841648543584, "generation_ms": 648.8597999559715, "mean_inter_token_latency_ms": 13.805527658637692, "output_token_source": "usage", "output_tokens": 48, "planned_context_tokens": 4096, "prompt_tokens": 3613, "reasoning_chunks": 0, "request_index": 7, "time_to_first_output_ms": 961.0361000522971, "tpot_ms": 13.805527658637692, "ttft_ms": 961.03630005382, "visible_generation_ms": 648.8595999544486 }, { "content_chunks": 15, "decode_tok_s": 54.660863508527925, "e2e_ms": 1045.6099999137223, "effective_prefill_tok_s": 21574.27090928159, "generation_ms": 878.1419999431819, "mean_inter_token_latency_ms": 18.29462499881629, "output_token_source": "usage", "output_tokens": 49, "planned_context_tokens": 4096, "prompt_tokens": 3613, "reasoning_chunks": 0, "request_index": 8, "time_to_first_output_ms": 167.4679999705404, "tpot_ms": 18.29462499881629, "ttft_ms": 167.4681999720633, "visible_generation_ms": 878.141799941659 }, { "content_chunks": 14, "decode_tok_s": 72.39035094301059, "e2e_ms": 816.8178000487387, "effective_prefill_tok_s": 21562.425403385067, "generation_ms": 649.2578000761569, "mean_inter_token_latency_ms": 13.81399574630121, "output_token_source": "usage", "output_tokens": 48, "planned_context_tokens": 4096, "prompt_tokens": 3613, "reasoning_chunks": 0, "request_index": 9, "time_to_first_output_ms": 167.55999997258186, "tpot_ms": 13.81399574630121, "ttft_ms": 167.5600999733433, "visible_generation_ms": 649.2577000753954 }, { "content_chunks": 18, "decode_tok_s": 56.007331652664085, "e2e_ms": 1184.928100090474, "effective_prefill_tok_s": 21614.289585795235, "generation_ms": 1017.723900033161, "mean_inter_token_latency_ms": 17.854805263739667, "output_token_source": "usage", "output_tokens": 58, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 10, "time_to_first_output_ms": 167.20420005731285, "tpot_ms": 17.854805263739667, "ttft_ms": 167.20440005883574, "visible_generation_ms": 1017.7237000316381 }, { "content_chunks": 11, "decode_tok_s": 106.23385440932458, "e2e_ms": 635.1666999980807, "effective_prefill_tok_s": 14500.912222942554, "generation_ms": 385.94099995680153, "mean_inter_token_latency_ms": 9.413195120897598, "output_token_source": "usage", "output_tokens": 42, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 11, "time_to_first_output_ms": 249.2257000412792, "tpot_ms": 9.413195120897598, "ttft_ms": 249.2259000428021, "visible_generation_ms": 385.94079995527864 }, { "content_chunks": 12, "decode_tok_s": 100.9878490668117, "e2e_ms": 674.9050999060273, "effective_prefill_tok_s": 14507.571516090724, "generation_ms": 425.7937999209389, "mean_inter_token_latency_ms": 9.902181393510206, "output_token_source": "usage", "output_tokens": 44, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 12, "time_to_first_output_ms": 249.1112999850884, "tpot_ms": 9.902181393510206, "ttft_ms": 249.11139998584986, "visible_generation_ms": 425.79369992017746 }, { "content_chunks": 14, "decode_tok_s": 74.20559750437234, "e2e_ms": 882.4055000441149, "effective_prefill_tok_s": 14512.301922749846, "generation_ms": 633.3753999788314, "mean_inter_token_latency_ms": 13.476072339975136, "output_token_source": "usage", "output_tokens": 48, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 13, "time_to_first_output_ms": 249.03010006528348, "tpot_ms": 13.476072339975136, "ttft_ms": 249.03020006604493, "visible_generation_ms": 633.37529997807 }, { "content_chunks": 14, "decode_tok_s": 65.72854780181008, "e2e_ms": 1259.644800098613, "effective_prefill_tok_s": 6636.27519490092, "generation_ms": 715.0622000917792, "mean_inter_token_latency_ms": 15.214089363654876, "output_token_source": "usage", "output_tokens": 48, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 14, "time_to_first_output_ms": 544.5826000068337, "tpot_ms": 15.214089363654876, "ttft_ms": 544.5828000083566, "visible_generation_ms": 715.0620000902563 }, { "content_chunks": 17, "decode_tok_s": 47.593886029619455, "e2e_ms": 1137.546099955216, "effective_prefill_tok_s": 24089.430849472796, "generation_ms": 987.5217999797314, "mean_inter_token_latency_ms": 21.01110212722833, "output_token_source": "usage", "output_tokens": 48, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 15, "time_to_first_output_ms": 150.0242999754846, "tpot_ms": 21.01110212722833, "ttft_ms": 150.02459997776896, "visible_generation_ms": 987.5214999774471 }, { "content_chunks": 15, "decode_tok_s": 67.32943663337696, "e2e_ms": 981.2076999805868, "effective_prefill_tok_s": 12763.665580074188, "generation_ms": 698.0601999675855, "mean_inter_token_latency_ms": 14.852344680161394, "output_token_source": "usage", "output_tokens": 48, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 16, "time_to_first_output_ms": 283.14750001300126, "tpot_ms": 14.852344680161394, "ttft_ms": 283.1478000152856, "visible_generation_ms": 698.0598999653012 }, { "content_chunks": 13, "decode_tok_s": 76.95257989595154, "e2e_ms": 762.9873999394476, "effective_prefill_tok_s": 18901.0113202883, "generation_ms": 571.7807000037283, "mean_inter_token_latency_ms": 12.995015909175642, "output_token_source": "usage", "output_tokens": 45, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 17, "time_to_first_output_ms": 191.20669993571937, "tpot_ms": 12.995015909175642, "ttft_ms": 191.20689993724227, "visible_generation_ms": 571.7805000022054 }, { "content_chunks": 15, "decode_tok_s": 69.75200112346509, "e2e_ms": 878.814399940893, "effective_prefill_tok_s": 18954.99944867151, "generation_ms": 688.1522999610752, "mean_inter_token_latency_ms": 14.336506249189066, "output_token_source": "usage", "output_tokens": 49, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 18, "time_to_first_output_ms": 190.66209997981787, "tpot_ms": 14.336506249189066, "ttft_ms": 190.66239998210222, "visible_generation_ms": 688.1519999587908 }, { "content_chunks": 13, "decode_tok_s": 69.50257652831982, "e2e_ms": 890.3495999984443, "effective_prefill_tok_s": 13303.026676533445, "generation_ms": 618.6820999719203, "mean_inter_token_latency_ms": 14.38795581330047, "output_token_source": "usage", "output_tokens": 44, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 19, "time_to_first_output_ms": 271.66750002652407, "tpot_ms": 14.38795581330047, "ttft_ms": 271.6678000288084, "visible_generation_ms": 618.6817999696359 }, { "content_chunks": 13, "decode_tok_s": 69.47651225401734, "e2e_ms": 767.221900052391, "effective_prefill_tok_s": 24368.25598774099, "generation_ms": 618.9141999930143, "mean_inter_token_latency_ms": 14.393353488209636, "output_token_source": "usage", "output_tokens": 44, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 20, "time_to_first_output_ms": 148.30770005937666, "tpot_ms": 14.393353488209636, "ttft_ms": 148.308000061661, "visible_generation_ms": 618.91389999073 }, { "content_chunks": 16, "decode_tok_s": 49.08233920494639, "e2e_ms": 1496.908999979496, "effective_prefill_tok_s": 6963.921145951358, "generation_ms": 977.9484999598935, "mean_inter_token_latency_ms": 20.373927082497783, "output_token_source": "usage", "output_tokens": 49, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 21, "time_to_first_output_ms": 518.9605000196025, "tpot_ms": 20.373927082497783, "ttft_ms": 518.9607000211254, "visible_generation_ms": 977.9482999583706 }, { "content_chunks": 14, "decode_tok_s": 66.45025598905804, "e2e_ms": 931.7288000602275, "effective_prefill_tok_s": 22004.732167418737, "generation_ms": 767.4914000090212, "mean_inter_token_latency_ms": 15.048850980569043, "output_token_source": "usage", "output_tokens": 52, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 22, "time_to_first_output_ms": 164.23740005120635, "tpot_ms": 15.048850980569043, "ttft_ms": 164.23760005272925, "visible_generation_ms": 767.4912000074983 }, { "content_chunks": 14, "decode_tok_s": 59.747909590815865, "e2e_ms": 1102.1723999874666, "effective_prefill_tok_s": 11453.599293270516, "generation_ms": 786.6383999353275, "mean_inter_token_latency_ms": 16.736987232666543, "output_token_source": "usage", "output_tokens": 48, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 23, "time_to_first_output_ms": 315.53400005213916, "tpot_ms": 16.736987232666543, "ttft_ms": 315.5343000544235, "visible_generation_ms": 786.6380999330431 }, { "content_chunks": 13, "decode_tok_s": 75.83342254906319, "e2e_ms": 775.619599968195, "effective_prefill_tok_s": 17326.078814189692, "generation_ms": 567.0322999358177, "mean_inter_token_latency_ms": 13.186797672925993, "output_token_source": "usage", "output_tokens": 44, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 24, "time_to_first_output_ms": 208.58730003237724, "tpot_ms": 13.186797672925993, "ttft_ms": 208.5876000346616, "visible_generation_ms": 567.0319999335334 }, { "content_chunks": 12, "decode_tok_s": 91.24061576134224, "e2e_ms": 734.6343999961391, "effective_prefill_tok_s": 17328.936686775134, "generation_ms": 526.0814999928698, "mean_inter_token_latency_ms": 10.960031249851454, "output_token_source": "usage", "output_tokens": 49, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 25, "time_to_first_output_ms": 208.55290000326931, "tpot_ms": 10.960031249851454, "ttft_ms": 208.5531000047922, "visible_generation_ms": 526.0812999913469 }, { "content_chunks": 11, "decode_tok_s": 117.43932363886229, "e2e_ms": 617.9770999588072, "effective_prefill_tok_s": 17270.76099372267, "generation_ms": 408.7216999614611, "mean_inter_token_latency_ms": 8.515035415863773, "output_token_source": "usage", "output_tokens": 49, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 26, "time_to_first_output_ms": 209.2553999973461, "tpot_ms": 8.515035415863773, "ttft_ms": 209.255599998869, "visible_generation_ms": 408.7214999599382 }, { "content_chunks": 11, "decode_tok_s": 95.27604279267499, "e2e_ms": 618.2010000338778, "effective_prefill_tok_s": 17303.11719069193, "generation_ms": 409.3368999892846, "mean_inter_token_latency_ms": 10.495817948443195, "output_token_source": "usage", "output_tokens": 40, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 27, "time_to_first_output_ms": 208.86410004459321, "tpot_ms": 10.495817948443195, "ttft_ms": 208.8643000461161, "visible_generation_ms": 409.3366999877617 }, { "content_chunks": 13, "decode_tok_s": 76.3514201338954, "e2e_ms": 759.0049999998882, "effective_prefill_tok_s": 17298.73586312219, "generation_ms": 550.0880000181496, "mean_inter_token_latency_ms": 13.097333333765468, "output_token_source": "usage", "output_tokens": 43, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 28, "time_to_first_output_ms": 208.91699998173863, "tpot_ms": 13.097333333765468, "ttft_ms": 208.91719998326153, "visible_generation_ms": 550.0878000166267 }, { "content_chunks": 14, "decode_tok_s": 64.11826135418853, "e2e_ms": 883.0634000478312, "effective_prefill_tok_s": 24086.444610062634, "generation_ms": 733.0205000471324, "mean_inter_token_latency_ms": 15.596180852066647, "output_token_source": "usage", "output_tokens": 48, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 29, "time_to_first_output_ms": 150.04290000069886, "tpot_ms": 15.596180852066647, "ttft_ms": 150.04310000222176, "visible_generation_ms": 733.0203000456095 }, { "content_chunks": 14, "decode_tok_s": 64.03943303615223, "e2e_ms": 883.5636998992413, "effective_prefill_tok_s": 24151.15119469187, "generation_ms": 733.9227999327704, "mean_inter_token_latency_ms": 15.615378721973839, "output_token_source": "usage", "output_tokens": 48, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 30, "time_to_first_output_ms": 149.6408999664709, "tpot_ms": 15.615378721973839, "ttft_ms": 149.6410999679938, "visible_generation_ms": 733.9225999312475 }, { "content_chunks": 16, "decode_tok_s": 50.93405414895459, "e2e_ms": 1131.1138999881223, "effective_prefill_tok_s": 24181.61235107705, "generation_ms": 981.6615000599995, "mean_inter_token_latency_ms": 19.63323000119999, "output_token_source": "usage", "output_tokens": 51, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 31, "time_to_first_output_ms": 149.45239992812276, "tpot_ms": 19.63323000119999, "ttft_ms": 149.4526999304071, "visible_generation_ms": 981.6612000577152 }, { "content_chunks": 13, "decode_tok_s": 59.39637746836182, "e2e_ms": 882.0309999864548, "effective_prefill_tok_s": 16031.918328805805, "generation_ms": 656.6056999145076, "mean_inter_token_latency_ms": 16.836043587551476, "output_token_source": "usage", "output_tokens": 40, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 32, "time_to_first_output_ms": 225.42530007194728, "tpot_ms": 16.836043587551476, "ttft_ms": 225.42550007347018, "visible_generation_ms": 656.6054999129847 }, { "content_chunks": 13, "decode_tok_s": 69.68103588012652, "e2e_ms": 842.3259999835864, "effective_prefill_tok_s": 16045.933810386274, "generation_ms": 617.0975998975337, "mean_inter_token_latency_ms": 14.351106974361247, "output_token_source": "usage", "output_tokens": 44, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 33, "time_to_first_output_ms": 225.22840008605272, "tpot_ms": 14.351106974361247, "ttft_ms": 225.22870008833706, "visible_generation_ms": 617.0972998952493 }, { "content_chunks": 12, "decode_tok_s": 93.97241274807347, "e2e_ms": 741.3629000075161, "effective_prefill_tok_s": 12735.136645806968, "generation_ms": 457.58110005408525, "mean_inter_token_latency_ms": 10.641420931490355, "output_token_source": "usage", "output_tokens": 44, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 34, "time_to_first_output_ms": 283.7817999534309, "tpot_ms": 10.641420931490355, "ttft_ms": 283.78209995571524, "visible_generation_ms": 457.5808000518009 }, { "content_chunks": 14, "decode_tok_s": 77.51578752391653, "e2e_ms": 776.352999964729, "effective_prefill_tok_s": 19756.67690114531, "generation_ms": 593.4274999890476, "mean_inter_token_latency_ms": 12.900597825848862, "output_token_source": "usage", "output_tokens": 47, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 35, "time_to_first_output_ms": 182.9254999756813, "tpot_ms": 12.900597825848862, "ttft_ms": 182.92579997796565, "visible_generation_ms": 593.4271999867633 }, { "content_chunks": 15, "decode_tok_s": 67.38087527334622, "e2e_ms": 880.134699982591, "effective_prefill_tok_s": 19791.092803659525, "generation_ms": 697.5272999843583, "mean_inter_token_latency_ms": 14.84100638264592, "output_token_source": "usage", "output_tokens": 48, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 36, "time_to_first_output_ms": 182.60739999823272, "tpot_ms": 14.84100638264592, "ttft_ms": 182.60770000051707, "visible_generation_ms": 697.5269999820739 }, { "content_chunks": 13, "decode_tok_s": 66.07027234244244, "e2e_ms": 835.9824999934062, "effective_prefill_tok_s": 23332.76303407445, "generation_ms": 681.0929999919608, "mean_inter_token_latency_ms": 15.135399999821352, "output_token_source": "usage", "output_tokens": 46, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 37, "time_to_first_output_ms": 154.8895000014454, "tpot_ms": 15.135399999821352, "ttft_ms": 154.88960000220686, "visible_generation_ms": 681.0928999911994 }, { "content_chunks": 14, "decode_tok_s": 55.49186322976924, "e2e_ms": 875.3458999563009, "effective_prefill_tok_s": 23388.633800351196, "generation_ms": 720.8263999782503, "mean_inter_token_latency_ms": 18.020659999456257, "output_token_source": "usage", "output_tokens": 41, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 38, "time_to_first_output_ms": 154.5194999780506, "tpot_ms": 18.020659999456257, "ttft_ms": 154.51979998033494, "visible_generation_ms": 720.8260999759659 }, { "content_chunks": 13, "decode_tok_s": 68.97773384479433, "e2e_ms": 835.5017000576481, "effective_prefill_tok_s": 23448.895149002252, "generation_ms": 681.3792999600992, "mean_inter_token_latency_ms": 14.497431914044663, "output_token_source": "usage", "output_tokens": 48, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 39, "time_to_first_output_ms": 154.12240009754896, "tpot_ms": 14.497431914044663, "ttft_ms": 154.12260009907186, "visible_generation_ms": 681.3790999585763 }, { "content_chunks": 14, "decode_tok_s": 65.24647758884404, "e2e_ms": 874.3991999654099, "effective_prefill_tok_s": 23459.336944447794, "generation_ms": 720.3454000409693, "mean_inter_token_latency_ms": 15.326497873212112, "output_token_source": "usage", "output_tokens": 48, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 40, "time_to_first_output_ms": 154.05379992444068, "tpot_ms": 15.326497873212112, "ttft_ms": 154.05389992520213, "visible_generation_ms": 720.3453000402078 }, { "content_chunks": 11, "decode_tok_s": 89.2703211254313, "e2e_ms": 753.6947999615222, "effective_prefill_tok_s": 16731.31882250647, "generation_ms": 537.692700047046, "mean_inter_token_latency_ms": 11.201931250980124, "output_token_source": "usage", "output_tokens": 49, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 41, "time_to_first_output_ms": 216.00209991447628, "tpot_ms": 11.201931250980124, "ttft_ms": 216.00239991676062, "visible_generation_ms": 537.6924000447616 }, { "content_chunks": 13, "decode_tok_s": 82.35716225861665, "e2e_ms": 835.1149999070913, "effective_prefill_tok_s": 13067.11231261704, "generation_ms": 558.5427999030799, "mean_inter_token_latency_ms": 12.142234780501736, "output_token_source": "usage", "output_tokens": 47, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 42, "time_to_first_output_ms": 276.5722000040114, "tpot_ms": 12.142234780501736, "ttft_ms": 276.57259989064187, "visible_generation_ms": 558.5424000164494 }, { "content_chunks": 12, "decode_tok_s": 107.83868140049641, "e2e_ms": 626.2985999928787, "effective_prefill_tok_s": 19945.99019803361, "generation_ms": 445.10930008254945, "mean_inter_token_latency_ms": 9.273110418386446, "output_token_source": "usage", "output_tokens": 49, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 43, "time_to_first_output_ms": 181.18929991032928, "tpot_ms": 9.273110418386446, "ttft_ms": 181.18949991185218, "visible_generation_ms": 445.10910008102655 }, { "content_chunks": 12, "decode_tok_s": 100.99647611558359, "e2e_ms": 626.397900050506, "effective_prefill_tok_s": 19984.75982037147, "generation_ms": 445.5601000227034, "mean_inter_token_latency_ms": 9.901335556060076, "output_token_source": "usage", "output_tokens": 46, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 44, "time_to_first_output_ms": 180.8378000278026, "tpot_ms": 9.901335556060076, "ttft_ms": 180.83830003160983, "visible_generation_ms": 445.55960001889616 }, { "content_chunks": 12, "decode_tok_s": 100.99373797938452, "e2e_ms": 691.3920999504626, "effective_prefill_tok_s": 18409.48002313079, "generation_ms": 495.08020002394915, "mean_inter_token_latency_ms": 9.901604000478983, "output_token_source": "usage", "output_tokens": 51, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 45, "time_to_first_output_ms": 196.31189992651343, "tpot_ms": 9.901604000478983, "ttft_ms": 196.31209992803633, "visible_generation_ms": 495.08000002242625 }, { "content_chunks": 12, "decode_tok_s": 84.94801990164028, "e2e_ms": 648.7923000240698, "effective_prefill_tok_s": 23410.935766974824, "generation_ms": 494.42000000271946, "mean_inter_token_latency_ms": 11.77190476196951, "output_token_source": "usage", "output_tokens": 43, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 46, "time_to_first_output_ms": 154.37230002135038, "tpot_ms": 11.77190476196951, "ttft_ms": 154.37250002287328, "visible_generation_ms": 494.41980000119656 }, { "content_chunks": 14, "decode_tok_s": 75.91370445998402, "e2e_ms": 786.5795000689104, "effective_prefill_tok_s": 23424.546897300508, "generation_ms": 632.2969000320882, "mean_inter_token_latency_ms": 13.172852084001837, "output_token_source": "usage", "output_tokens": 49, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 47, "time_to_first_output_ms": 154.28260003682226, "tpot_ms": 13.172852084001837, "ttft_ms": 154.2827000375837, "visible_generation_ms": 632.2968000313267 }, { "content_chunks": 16, "decode_tok_s": 59.16279572455619, "e2e_ms": 934.5719000557438, "effective_prefill_tok_s": 16086.079665930787, "generation_ms": 709.9056000588462, "mean_inter_token_latency_ms": 16.902514287115384, "output_token_source": "usage", "output_tokens": 43, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 48, "time_to_first_output_ms": 224.66629999689758, "tpot_ms": 16.902514287115384, "ttft_ms": 224.66659999918193, "visible_generation_ms": 709.9053000565618 }, { "content_chunks": 13, "decode_tok_s": 89.8147751811587, "e2e_ms": 747.6713999640197, "effective_prefill_tok_s": 16107.164793559201, "generation_ms": 523.2991999946535, "mean_inter_token_latency_ms": 11.134025531801138, "output_token_source": "usage", "output_tokens": 48, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 49, "time_to_first_output_ms": 224.3721999693662, "tpot_ms": 11.134025531801138, "ttft_ms": 224.372599972412, "visible_generation_ms": 523.2987999916077 }, { "content_chunks": 14, "decode_tok_s": 70.57001162489183, "e2e_ms": 791.2850000429899, "effective_prefill_tok_s": 16100.00356170974, "generation_ms": 566.8130000121891, "mean_inter_token_latency_ms": 14.170325000304729, "output_token_source": "usage", "output_tokens": 41, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 50, "time_to_first_output_ms": 224.4720000308007, "tpot_ms": 14.170325000304729, "ttft_ms": 224.47210003156215, "visible_generation_ms": 566.8129000114277 }, { "content_chunks": 11, "decode_tok_s": 111.5050379373762, "e2e_ms": 609.5794000430033, "effective_prefill_tok_s": 16137.775702731577, "generation_ms": 385.63280005473644, "mean_inter_token_latency_ms": 8.96820465243573, "output_token_source": "usage", "output_tokens": 44, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 51, "time_to_first_output_ms": 223.94659998826683, "tpot_ms": 8.96820465243573, "ttft_ms": 223.94699999131262, "visible_generation_ms": 385.63240005169064 }, { "content_chunks": 16, "decode_tok_s": 65.90969383120053, "e2e_ms": 1440.731499926187, "effective_prefill_tok_s": 5544.873585671167, "generation_ms": 788.958300021477, "mean_inter_token_latency_ms": 15.17227500041302, "output_token_source": "usage", "output_tokens": 53, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 52, "time_to_first_output_ms": 651.77319990471, "tpot_ms": 15.17227500041302, "ttft_ms": 651.773399906233, "visible_generation_ms": 788.9581000199541 }, { "content_chunks": 13, "decode_tok_s": 65.95272837402058, "e2e_ms": 753.5317000001669, "effective_prefill_tok_s": 24578.897661496303, "generation_ms": 606.4950000727549, "mean_inter_token_latency_ms": 15.162375001818873, "output_token_source": "usage", "output_tokens": 41, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 53, "time_to_first_output_ms": 147.03669992741197, "tpot_ms": 15.162375001818873, "ttft_ms": 147.03679992817342, "visible_generation_ms": 606.4949000719935 }, { "content_chunks": 14, "decode_tok_s": 77.12434773666982, "e2e_ms": 794.8393999831751, "effective_prefill_tok_s": 24662.931975813204, "generation_ms": 648.3037000289187, "mean_inter_token_latency_ms": 12.966074000578374, "output_token_source": "usage", "output_tokens": 51, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 54, "time_to_first_output_ms": 146.53569995425642, "tpot_ms": 12.966074000578374, "ttft_ms": 146.5358999557793, "visible_generation_ms": 648.3035000273958 }, { "content_chunks": 14, "decode_tok_s": 72.43765660892603, "e2e_ms": 795.0602000346407, "effective_prefill_tok_s": 24715.09931802396, "generation_ms": 648.8337999908254, "mean_inter_token_latency_ms": 13.8049744678899, "output_token_source": "usage", "output_tokens": 48, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 55, "time_to_first_output_ms": 146.22640004381537, "tpot_ms": 13.8049744678899, "ttft_ms": 146.22690004762262, "visible_generation_ms": 648.8332999870181 }, { "content_chunks": 14, "decode_tok_s": 74.17271921185791, "e2e_ms": 797.5062999175861, "effective_prefill_tok_s": 24034.35304445432, "generation_ms": 647.1382000017911, "mean_inter_token_latency_ms": 13.482045833370648, "output_token_source": "usage", "output_tokens": 49, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 56, "time_to_first_output_ms": 150.36809991579503, "tpot_ms": 13.482045833370648, "ttft_ms": 150.36829991731793, "visible_generation_ms": 647.1380000002682 }, { "content_chunks": 12, "decode_tok_s": 92.60943232844323, "e2e_ms": 657.6222999719903, "effective_prefill_tok_s": 24074.94008479322, "generation_ms": 507.50770000740886, "mean_inter_token_latency_ms": 10.798036170370402, "output_token_source": "usage", "output_tokens": 48, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 57, "time_to_first_output_ms": 150.11459996458143, "tpot_ms": 10.798036170370402, "ttft_ms": 150.11489996686578, "visible_generation_ms": 507.5074000051245 }, { "content_chunks": 14, "decode_tok_s": 60.468869565379485, "e2e_ms": 949.9056999338791, "effective_prefill_tok_s": 12530.846030271363, "generation_ms": 661.4974000258371, "mean_inter_token_latency_ms": 16.537435000645928, "output_token_source": "usage", "output_tokens": 41, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 58, "time_to_first_output_ms": 288.40829990804195, "tpot_ms": 16.537435000645928, "ttft_ms": 288.40849990956485, "visible_generation_ms": 661.4972000243142 }, { "content_chunks": 14, "decode_tok_s": 60.521083504040156, "e2e_ms": 806.4339000266045, "effective_prefill_tok_s": 24837.258905618066, "generation_ms": 660.9266999876127, "mean_inter_token_latency_ms": 16.523167499690317, "output_token_source": "usage", "output_tokens": 41, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 59, "time_to_first_output_ms": 145.50720003899187, "tpot_ms": 16.523167499690317, "ttft_ms": 145.50740004051477, "visible_generation_ms": 660.9264999860898 }, { "content_chunks": 16, "decode_tok_s": 53.7380594340632, "e2e_ms": 1066.6811999399215, "effective_prefill_tok_s": 18816.223189515007, "generation_ms": 874.6128999628127, "mean_inter_token_latency_ms": 18.60878510559176, "output_token_source": "usage", "output_tokens": 48, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 60, "time_to_first_output_ms": 192.06829997710884, "tpot_ms": 18.60878510559176, "ttft_ms": 192.06859997939318, "visible_generation_ms": 874.6125999605283 }, { "content_chunks": 12, "decode_tok_s": 72.66032077171572, "e2e_ms": 742.5381999928504, "effective_prefill_tok_s": 23974.5156371049, "generation_ms": 591.7948000133038, "mean_inter_token_latency_ms": 13.76266976775125, "output_token_source": "usage", "output_tokens": 44, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 61, "time_to_first_output_ms": 150.74339997954667, "tpot_ms": 13.76266976775125, "ttft_ms": 150.74359998106956, "visible_generation_ms": 591.7946000117809 }, { "content_chunks": 14, "decode_tok_s": 65.41264270413008, "e2e_ms": 884.0052000014111, "effective_prefill_tok_s": 24060.915271008438, "generation_ms": 733.8031000690535, "mean_inter_token_latency_ms": 15.287564584771948, "output_token_source": "usage", "output_tokens": 49, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 62, "time_to_first_output_ms": 150.20209993235767, "tpot_ms": 15.287564584771948, "ttft_ms": 150.20239993464202, "visible_generation_ms": 733.8028000667691 }, { "content_chunks": 12, "decode_tok_s": 72.61666205385387, "e2e_ms": 742.2954000066966, "effective_prefill_tok_s": 24070.097653195797, "generation_ms": 592.1505999285728, "mean_inter_token_latency_ms": 13.770944184385414, "output_token_source": "usage", "output_tokens": 44, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 63, "time_to_first_output_ms": 150.1448000781238, "tpot_ms": 13.770944184385414, "ttft_ms": 150.14490007888526, "visible_generation_ms": 592.1504999278113 }, { "content_chunks": 12, "decode_tok_s": 73.22950639291717, "e2e_ms": 743.8804999692366, "effective_prefill_tok_s": 23065.312353071633, "generation_ms": 587.1949999127537, "mean_inter_token_latency_ms": 13.655697672389621, "output_token_source": "usage", "output_tokens": 44, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 64, "time_to_first_output_ms": 156.68550005648285, "tpot_ms": 13.655697672389621, "ttft_ms": 156.68590005952865, "visible_generation_ms": 587.1945999097079 }, { "content_chunks": 13, "decode_tok_s": 80.84435749034793, "e2e_ms": 787.2383999638259, "effective_prefill_tok_s": 23107.91923293819, "generation_ms": 630.8418000116944, "mean_inter_token_latency_ms": 12.369447059052831, "output_token_source": "usage", "output_tokens": 52, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 65, "time_to_first_output_ms": 156.3965999521315, "tpot_ms": 12.369447059052831, "ttft_ms": 156.39689995441586, "visible_generation_ms": 630.8415000094101 }, { "content_chunks": 14, "decode_tok_s": 57.04121628425432, "e2e_ms": 988.413600018248, "effective_prefill_tok_s": 21976.551854996687, "generation_ms": 823.9655999932438, "mean_inter_token_latency_ms": 17.531182978579654, "output_token_source": "usage", "output_tokens": 48, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 66, "time_to_first_output_ms": 164.44800002500415, "tpot_ms": 17.531182978579654, "ttft_ms": 164.44840002804995, "visible_generation_ms": 823.965199990198 }, { "content_chunks": 12, "decode_tok_s": 68.30785556815422, "e2e_ms": 793.3500999351963, "effective_prefill_tok_s": 22057.14962442566, "generation_ms": 629.5029999455437, "mean_inter_token_latency_ms": 14.639604649896365, "output_token_source": "usage", "output_tokens": 44, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 67, "time_to_first_output_ms": 163.84709998965263, "tpot_ms": 14.639604649896365, "ttft_ms": 163.84759999345988, "visible_generation_ms": 629.5024999417365 }, { "content_chunks": 12, "decode_tok_s": 63.599496171468495, "e2e_ms": 778.4139999421313, "effective_prefill_tok_s": 24177.43858169739, "generation_ms": 628.93579993397, "mean_inter_token_latency_ms": 15.72339499834925, "output_token_source": "usage", "output_tokens": 41, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 68, "time_to_first_output_ms": 149.47820000816137, "tpot_ms": 15.72339499834925, "ttft_ms": 149.47840000968426, "visible_generation_ms": 628.9355999324471 }, { "content_chunks": 12, "decode_tok_s": 68.4340366756812, "e2e_ms": 777.7039000065997, "effective_prefill_tok_s": 24196.31283241909, "generation_ms": 628.3422999549657, "mean_inter_token_latency_ms": 14.612611626859668, "output_token_source": "usage", "output_tokens": 44, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 69, "time_to_first_output_ms": 149.361600051634, "tpot_ms": 14.612611626859668, "ttft_ms": 149.3618000531569, "visible_generation_ms": 628.3420999534428 }, { "content_chunks": 13, "decode_tok_s": 59.52952329546024, "e2e_ms": 821.2020000210032, "effective_prefill_tok_s": 24211.728682395544, "generation_ms": 671.9354999950156, "mean_inter_token_latency_ms": 16.79838749987539, "output_token_source": "usage", "output_tokens": 41, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 70, "time_to_first_output_ms": 149.26650002598763, "tpot_ms": 16.79838749987539, "ttft_ms": 149.26670002751052, "visible_generation_ms": 671.9352999934927 }, { "content_chunks": 15, "decode_tok_s": 64.09257531435631, "e2e_ms": 898.1519000371918, "effective_prefill_tok_s": 24216.806755403068, "generation_ms": 748.9167000167072, "mean_inter_token_latency_ms": 15.602431250348067, "output_token_source": "usage", "output_tokens": 49, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 71, "time_to_first_output_ms": 149.23520002048463, "tpot_ms": 15.602431250348067, "ttft_ms": 149.23540002200752, "visible_generation_ms": 748.9165000151843 }, { "content_chunks": 16, "decode_tok_s": 66.03181311414623, "e2e_ms": 997.8449000045657, "effective_prefill_tok_s": 17181.24021273961, "generation_ms": 787.4991999706253, "mean_inter_token_latency_ms": 15.144215384050487, "output_token_source": "usage", "output_tokens": 53, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 72, "time_to_first_output_ms": 210.34570003394037, "tpot_ms": 15.144215384050487, "ttft_ms": 210.34600003622472, "visible_generation_ms": 787.498899968341 }, { "content_chunks": 11, "decode_tok_s": 82.39084567724996, "e2e_ms": 719.6698000188917, "effective_prefill_tok_s": 16276.221994876203, "generation_ms": 497.6281000999734, "mean_inter_token_latency_ms": 12.137270734145693, "output_token_source": "usage", "output_tokens": 42, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 73, "time_to_first_output_ms": 222.0416999189183, "tpot_ms": 12.137270734145693, "ttft_ms": 222.04199992120266, "visible_generation_ms": 497.62780009768903 }, { "content_chunks": 12, "decode_tok_s": 86.34356103194379, "e2e_ms": 615.1198000879958, "effective_prefill_tok_s": 30860.05976314626, "generation_ms": 498.0104999849573, "mean_inter_token_latency_ms": 11.58163953453389, "output_token_source": "usage", "output_tokens": 44, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 74, "time_to_first_output_ms": 117.10930010303855, "tpot_ms": 11.58163953453389, "ttft_ms": 117.1096001053229, "visible_generation_ms": 498.01019998267293 }, { "content_chunks": 13, "decode_tok_s": 74.97500833227828, "e2e_ms": 942.137899925001, "effective_prefill_tok_s": 10569.170602752713, "generation_ms": 600.1999999862164, "mean_inter_token_latency_ms": 13.337777777471477, "output_token_source": "usage", "output_tokens": 46, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 75, "time_to_first_output_ms": 341.93789993878454, "tpot_ms": 13.337777777471477, "ttft_ms": 341.9381999410689, "visible_generation_ms": 600.1996999839321 }, { "content_chunks": 15, "decode_tok_s": 64.68283433392578, "e2e_ms": 826.3563000364229, "effective_prefill_tok_s": 24734.111720501933, "generation_ms": 680.2423000335693, "mean_inter_token_latency_ms": 15.460052273490213, "output_token_source": "usage", "output_tokens": 45, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 76, "time_to_first_output_ms": 146.11400000285357, "tpot_ms": 15.460052273490213, "ttft_ms": 146.11420000437647, "visible_generation_ms": 680.2421000320464 }, { "content_chunks": 14, "decode_tok_s": 70.20638963172141, "e2e_ms": 786.5364999743178, "effective_prefill_tok_s": 24826.68036145108, "generation_ms": 640.9672999288887, "mean_inter_token_latency_ms": 14.243717776197526, "output_token_source": "usage", "output_tokens": 46, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 77, "time_to_first_output_ms": 145.56920004542917, "tpot_ms": 14.243717776197526, "ttft_ms": 145.56940004695207, "visible_generation_ms": 640.9670999273658 }, { "content_chunks": 12, "decode_tok_s": 81.82362932843283, "e2e_ms": 758.1216000253335, "effective_prefill_tok_s": 19671.731391292113, "generation_ms": 574.4062000885606, "mean_inter_token_latency_ms": 12.221408512522565, "output_token_source": "usage", "output_tokens": 48, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 78, "time_to_first_output_ms": 183.71539993677288, "tpot_ms": 12.221408512522565, "ttft_ms": 183.71569993905723, "visible_generation_ms": 574.4059000862762 }, { "content_chunks": 13, "decode_tok_s": 70.83144991366706, "e2e_ms": 836.3864000421017, "effective_prefill_tok_s": 24992.513997822338, "generation_ms": 691.7831000173464, "mean_inter_token_latency_ms": 14.118022449333601, "output_token_source": "usage", "output_tokens": 50, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 79, "time_to_first_output_ms": 144.6033000247553, "tpot_ms": 14.118022449333601, "ttft_ms": 144.60340002551675, "visible_generation_ms": 691.783000016585 }, { "content_chunks": 14, "decode_tok_s": 45.52403061578605, "e2e_ms": 1000.8480000542477, "effective_prefill_tok_s": 25069.784734374127, "generation_ms": 856.6904000472277, "mean_inter_token_latency_ms": 21.96642051403148, "output_token_source": "usage", "output_tokens": 40, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 80, "time_to_first_output_ms": 144.15760000701994, "tpot_ms": 21.96642051403148, "ttft_ms": 144.15790000930429, "visible_generation_ms": 856.6901000449434 }, { "content_chunks": 13, "decode_tok_s": 62.922568740306836, "e2e_ms": 999.9287000391632, "effective_prefill_tok_s": 12691.210968489373, "generation_ms": 715.1647000573575, "mean_inter_token_latency_ms": 15.892548890163502, "output_token_source": "usage", "output_tokens": 46, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 81, "time_to_first_output_ms": 284.7639999818057, "tpot_ms": 15.892548890163502, "ttft_ms": 284.76429998409003, "visible_generation_ms": 715.1644000550732 }, { "content_chunks": 13, "decode_tok_s": 47.91504441323494, "e2e_ms": 955.4925999836996, "effective_prefill_tok_s": 25531.25353997235, "generation_ms": 813.9406000263989, "mean_inter_token_latency_ms": 20.87027179554869, "output_token_source": "usage", "output_tokens": 40, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 82, "time_to_first_output_ms": 141.55199995730072, "tpot_ms": 20.87027179554869, "ttft_ms": 141.55219995882362, "visible_generation_ms": 813.940400024876 }, { "content_chunks": 12, "decode_tok_s": 58.88877430428208, "e2e_ms": 936.4927000133321, "effective_prefill_tok_s": 17517.95663421284, "generation_ms": 730.1901000319049, "mean_inter_token_latency_ms": 16.981165117021042, "output_token_source": "usage", "output_tokens": 44, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 83, "time_to_first_output_ms": 206.3025999814272, "tpot_ms": 16.981165117021042, "ttft_ms": 206.3027999829501, "visible_generation_ms": 730.189900030382 }, { "content_chunks": 12, "decode_tok_s": 58.50316150972183, "e2e_ms": 934.7544999327511, "effective_prefill_tok_s": 21828.245422145483, "generation_ms": 769.1892000148073, "mean_inter_token_latency_ms": 17.093093333662384, "output_token_source": "usage", "output_tokens": 46, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 84, "time_to_first_output_ms": 165.56529991794378, "tpot_ms": 17.093093333662384, "ttft_ms": 165.56549991946667, "visible_generation_ms": 769.1890000132844 }, { "content_chunks": 13, "decode_tok_s": 59.87314078985344, "e2e_ms": 994.5343999424949, "effective_prefill_tok_s": 14875.776566787963, "generation_ms": 751.589099992998, "mean_inter_token_latency_ms": 16.701979999844397, "output_token_source": "usage", "output_tokens": 46, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 85, "time_to_first_output_ms": 242.94529994949698, "tpot_ms": 16.701979999844397, "ttft_ms": 242.94549995101988, "visible_generation_ms": 751.588899991475 }, { "content_chunks": 12, "decode_tok_s": 60.57786436331132, "e2e_ms": 987.8615000052378, "effective_prefill_tok_s": 14750.063159166495, "generation_ms": 742.8456000052392, "mean_inter_token_latency_ms": 16.507680000116427, "output_token_source": "usage", "output_tokens": 46, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 86, "time_to_first_output_ms": 245.0158999999985, "tpot_ms": 16.507680000116427, "ttft_ms": 245.01620000228286, "visible_generation_ms": 742.8453000029549 }, { "content_chunks": 12, "decode_tok_s": 61.18040621758625, "e2e_ms": 1010.470800101757, "effective_prefill_tok_s": 11747.825479588555, "generation_ms": 702.8394000371918, "mean_inter_token_latency_ms": 16.34510232644632, "output_token_source": "usage", "output_tokens": 44, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 87, "time_to_first_output_ms": 307.63140006456524, "tpot_ms": 16.34510232644632, "ttft_ms": 307.6317000668496, "visible_generation_ms": 702.8391000349075 }, { "content_chunks": 10, "decode_tok_s": 72.8474209000991, "e2e_ms": 705.9198999777436, "effective_prefill_tok_s": 25255.119335654716, "generation_ms": 562.820199993439, "mean_inter_token_latency_ms": 13.727321951059489, "output_token_source": "usage", "output_tokens": 42, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 88, "time_to_first_output_ms": 143.0996999843046, "tpot_ms": 13.727321951059489, "ttft_ms": 143.0998999858275, "visible_generation_ms": 562.8199999919161 }, { "content_chunks": 15, "decode_tok_s": 59.89236797886449, "e2e_ms": 966.6906999191269, "effective_prefill_tok_s": 15574.934438169095, "generation_ms": 734.651199891232, "mean_inter_token_latency_ms": 16.69661817934618, "output_token_source": "usage", "output_tokens": 45, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 89, "time_to_first_output_ms": 232.03950002789497, "tpot_ms": 16.69661817934618, "ttft_ms": 232.039799913764, "visible_generation_ms": 734.6509000053629 }, { "content_chunks": 12, "decode_tok_s": 69.12049793192251, "e2e_ms": 1091.3792999926955, "effective_prefill_tok_s": 7701.203530537319, "generation_ms": 622.101999935694, "mean_inter_token_latency_ms": 14.467488370597536, "output_token_source": "usage", "output_tokens": 44, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 90, "time_to_first_output_ms": 469.2773000570014, "tpot_ms": 14.467488370597536, "ttft_ms": 469.2775000585243, "visible_generation_ms": 622.1017999341711 }, { "content_chunks": 13, "decode_tok_s": 65.00239374734713, "e2e_ms": 808.7317000608891, "effective_prefill_tok_s": 24548.69526064368, "generation_ms": 661.5141000365838, "mean_inter_token_latency_ms": 15.384048838060089, "output_token_source": "usage", "output_tokens": 44, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 91, "time_to_first_output_ms": 147.21760002430528, "tpot_ms": 15.384048838060089, "ttft_ms": 147.21790002658963, "visible_generation_ms": 661.5138000342995 }, { "content_chunks": 12, "decode_tok_s": 79.11756318039356, "e2e_ms": 770.1238000299782, "effective_prefill_tok_s": 15946.781694891431, "generation_ms": 543.4949999907985, "mean_inter_token_latency_ms": 12.639418604437175, "output_token_source": "usage", "output_tokens": 44, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 92, "time_to_first_output_ms": 226.62880003917962, "tpot_ms": 12.639418604437175, "ttft_ms": 226.62900004070252, "visible_generation_ms": 543.4947999892756 }, { "content_chunks": 14, "decode_tok_s": 69.17644703128961, "e2e_ms": 806.9922999711707, "effective_prefill_tok_s": 28329.47796554701, "generation_ms": 679.4220000738278, "mean_inter_token_latency_ms": 14.455787235613357, "output_token_source": "usage", "output_tokens": 48, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 93, "time_to_first_output_ms": 127.57029989734292, "tpot_ms": 14.455787235613357, "ttft_ms": 127.57049989886582, "visible_generation_ms": 679.4218000723049 }, { "content_chunks": 15, "decode_tok_s": 70.89029004549586, "e2e_ms": 811.3450999371707, "effective_prefill_tok_s": 24361.504774137582, "generation_ms": 662.9962999140844, "mean_inter_token_latency_ms": 14.106304253491157, "output_token_source": "usage", "output_tokens": 48, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 94, "time_to_first_output_ms": 148.34880002308637, "tpot_ms": 14.106304253491157, "ttft_ms": 148.34900002460927, "visible_generation_ms": 662.9960999125615 }, { "content_chunks": 15, "decode_tok_s": 70.88385378674865, "e2e_ms": 811.2962000304833, "effective_prefill_tok_s": 24379.434118007463, "generation_ms": 663.0565000232309, "mean_inter_token_latency_ms": 14.107585106877254, "output_token_source": "usage", "output_tokens": 48, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 95, "time_to_first_output_ms": 148.2397000072524, "tpot_ms": 14.107585106877254, "ttft_ms": 148.2399000087753, "visible_generation_ms": 663.056300021708 }, { "content_chunks": 14, "decode_tok_s": 82.03212063225885, "e2e_ms": 876.3096999609843, "effective_prefill_tok_s": 11913.104879631024, "generation_ms": 572.9463000316173, "mean_inter_token_latency_ms": 12.190346809183346, "output_token_source": "usage", "output_tokens": 48, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 96, "time_to_first_output_ms": 303.363399929367, "tpot_ms": 12.190346809183346, "ttft_ms": 303.3635999308899, "visible_generation_ms": 572.9461000300944 }, { "content_chunks": 13, "decode_tok_s": 100.7854944751177, "e2e_ms": 651.7088999971747, "effective_prefill_tok_s": 24807.167859878227, "generation_ms": 506.02520001120865, "mean_inter_token_latency_ms": 9.922062745317817, "output_token_source": "usage", "output_tokens": 52, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 97, "time_to_first_output_ms": 145.6836999859661, "tpot_ms": 9.922062745317817, "ttft_ms": 145.68399998825043, "visible_generation_ms": 506.0249000089243 }, { "content_chunks": 15, "decode_tok_s": 91.50371378637084, "e2e_ms": 663.9586000237614, "effective_prefill_tok_s": 19737.438552646418, "generation_ms": 480.8548000873998, "mean_inter_token_latency_ms": 10.92851818380454, "output_token_source": "usage", "output_tokens": 45, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 98, "time_to_first_output_ms": 183.1037999363616, "tpot_ms": 10.92851818380454, "ttft_ms": 183.1039999378845, "visible_generation_ms": 480.8546000858769 }, { "content_chunks": 12, "decode_tok_s": 113.07493346863872, "e2e_ms": 524.1460999241099, "effective_prefill_tok_s": 25120.371340731323, "generation_ms": 380.27879991568625, "mean_inter_token_latency_ms": 8.84369302129503, "output_token_source": "usage", "output_tokens": 44, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 99, "time_to_first_output_ms": 143.86730000842363, "tpot_ms": 8.84369302129503, "ttft_ms": 143.86750000994653, "visible_generation_ms": 380.27859991416335 } ], "requests": 100, "run_id": "benchmark-20260905T035322Z", "schema": "anvil-serving.benchmark/v1", "serve_flags": { "no_thinking": false, "reasoning_effort": null, "shared_prefix_burst": false, "thinking_mode": "disabled" }, "source_recipe": null, "started_at": "2026-09-05T03:53:22Z", "tier": null, "timing_methodology": { "clock": "client time.perf_counter", "decode": "usage completion tokens after the first token divided by client-observed generation time; completion tokens may include reasoning tokens", "distributions": "request-level mean, min, p25, p50, p75, p90, p95, p99, and max using the nearest-rank percentile method", "effective_prefill": "usage.prompt_tokens divided by client-observed time to first output; includes queueing, scheduling, prefill, and first-token work", "generation": "client-observed E2E minus time to first output", "mean_inter_token_latency": "client-observed generation time divided by completion tokens after the first token; not a raw per-token timestamp trace", "time_to_first_output": "request start through the first non-empty streamed reasoning or content delta", "tpot": "alias of mean_inter_token_latency for capacity-v3: client-observed generation time divided by completion tokens after the first token", "ttft": "request start through the first non-empty streamed content delta" }, "wall_clock_ms": 11027.165399980731 }