{ "base_url": "http://127.0.0.1:39275/v1", "completed": 16, "concurrency": 8, "context_distribution": { "4096": 16 }, "context_seed": 0, "context_tokens": 4096, "engine": "sglang", "failed": 0, "failures": [], "finished_at": "2026-09-05T03:44:12Z", "gpu": "Primary Node / RTX PRO 6000 Blackwell Max-Q / TP=1", "manifest": null, "max_context_tokens": 262144, "max_tokens": 512, "measurement_protocol": "capacity-v3", "metrics": { "content_chunks": 220, "content_chunks_s": null, "decode_tok_s_max": 107.63447332700808, "decode_tok_s_mean": 79.49894582790925, "decode_tok_s_min": 54.01449582491984, "decode_tok_s_p25": 66.13108424879205, "decode_tok_s_p50": 79.68477442975667, "decode_tok_s_p75": 87.47009498954598, "decode_tok_s_p90": 101.80436320765526, "decode_tok_s_p95": 107.63447332700808, "decode_tok_s_p99": 107.63447332700808, "decode_tok_s_samples": 16, "e2e_max_ms": 981.3783999998122, "e2e_mean_ms": 795.8254187687999, "e2e_min_ms": 646.6454999754205, "e2e_p25_ms": 686.2048000330105, "e2e_p50_ms": 724.6608999557793, "e2e_p75_ms": 907.5150999706239, "e2e_p90_ms": 978.3892000559717, "e2e_p95_ms": 981.3783999998122, "e2e_p99_ms": 981.3783999998122, "e2e_samples": 16, "effective_prefill_tok_s_max": 29151.885039176177, "effective_prefill_tok_s_mean": 20719.655067276486, "effective_prefill_tok_s_min": 7431.942919190169, "effective_prefill_tok_s_p25": 19532.295324460418, "effective_prefill_tok_s_p50": 19656.637225328206, "effective_prefill_tok_s_p75": 22051.625084428797, "effective_prefill_tok_s_p90": 29089.521335148136, "effective_prefill_tok_s_p95": 29151.885039176177, "effective_prefill_tok_s_p99": 29151.885039176177, "effective_prefill_tok_s_samples": 16, "generation_max_ms": 794.9251000536606, "generation_mean_ms": 605.3390000088257, "generation_min_ms": 461.66979998815805, "generation_p25_ms": 501.698000007309, "generation_p50_ms": 551.4057000400499, "generation_p75_ms": 722.0284000504762, "generation_p90_ms": 756.0740999178961, "generation_p95_ms": 794.9251000536606, "generation_p99_ms": 794.9251000536606, "generation_samples": 16, "mean_inter_token_latency_ms_max": 18.51354871924298, "mean_inter_token_latency_ms_mean": 12.964825771454898, "mean_inter_token_latency_ms_min": 9.290703703839055, "mean_inter_token_latency_ms_p25": 11.400875000452453, "mean_inter_token_latency_ms_p50": 12.468253487049667, "mean_inter_token_latency_ms_p75": 13.925123075471046, "mean_inter_token_latency_ms_p90": 15.843626830113552, "mean_inter_token_latency_ms_p95": 18.51354871924298, "mean_inter_token_latency_ms_p99": 18.51354871924298, "mean_inter_token_latency_ms_samples": 16, "output_token_sources": { "usage": 16 }, "output_tokens": 768, "prefix_cache_hit_avg": null, "prompt_token_samples": 16, "prompt_tokens": 57814, "prompt_tokens_max": 3614.0, "prompt_tokens_mean": 3613.375, "prompt_tokens_min": 3613.0, "prompt_tokens_p25": 3613.0, "prompt_tokens_p50": 3613.0, "prompt_tokens_p75": 3614.0, "prompt_tokens_p90": 3614.0, "prompt_tokens_p95": 3614.0, "prompt_tokens_p99": 3614.0, "prompt_tokens_samples": 16, "reasoning_chunks": 0, "throughput_tok_s": 391.35223662242504, "time_to_first_output_max_ms": 486.2792999483645, "time_to_first_output_mean_ms": 190.48641875997419, "time_to_first_output_min_ms": 123.97139996755868, "time_to_first_output_p25_ms": 149.9644999857992, "time_to_first_output_p50_ms": 183.65590006578714, "time_to_first_output_p75_ms": 184.7303999820724, "time_to_first_output_p90_ms": 194.9657000368461, "time_to_first_output_p95_ms": 486.2792999483645, "time_to_first_output_p99_ms": 486.2792999483645, "time_to_first_output_samples": 16, "tpot_max_ms": 18.51354871924298, "tpot_mean_ms": 12.964825771454898, "tpot_min_ms": 9.290703703839055, "tpot_p25_ms": 11.400875000452453, "tpot_p50_ms": 12.468253487049667, "tpot_p75_ms": 13.925123075471046, "tpot_p90_ms": 15.843626830113552, "tpot_p95_ms": 18.51354871924298, "tpot_p99_ms": 18.51354871924298, "tpot_samples": 16, "ttft_max_ms": 486.27959995064884, "ttft_mean_ms": 190.486681261973, "ttft_min_ms": 123.97169996984303, "ttft_p25_ms": 149.96479998808354, "ttft_p50_ms": 183.65660007111728, "ttft_p75_ms": 184.7305999835953, "ttft_p90_ms": 194.965900038369, "ttft_p95_ms": 486.27959995064884, "ttft_p99_ms": 486.27959995064884, "ttft_samples": 16 }, "model": "qwen38-27b-inferact-nvfp4-pro6000-dflash2-k12-chunk1k-lazy96", "request_timings": [ { "content_chunks": 14, "decode_tok_s": 85.23669595106885, "e2e_ms": 722.3520000698045, "effective_prefill_tok_s": 21135.292190419605, "generation_ms": 551.4057000400499, "mean_inter_token_latency_ms": 11.732036171064891, "output_token_source": "usage", "output_tokens": 48, "planned_context_tokens": 4096, "prompt_tokens": 3613, "reasoning_chunks": 0, "request_index": 0, "time_to_first_output_ms": 170.94630002975464, "tpot_ms": 11.732036171064891, "ttft_ms": 170.94690003432333, "visible_generation_ms": 551.4051000354812 }, { "content_chunks": 12, "decode_tok_s": 101.80436320765526, "e2e_ms": 646.6454999754205, "effective_prefill_tok_s": 19532.295324460418, "generation_ms": 461.66979998815805, "mean_inter_token_latency_ms": 9.822761701875702, "output_token_source": "usage", "output_tokens": 48, "planned_context_tokens": 4096, "prompt_tokens": 3613, "reasoning_chunks": 0, "request_index": 1, "time_to_first_output_ms": 184.9756999872625, "tpot_ms": 9.822761701875702, "ttft_ms": 184.9758999887854, "visible_generation_ms": 461.66959998663515 }, { "content_chunks": 13, "decode_tok_s": 107.63447332700808, "e2e_ms": 686.2048000330105, "effective_prefill_tok_s": 19581.93410484987, "generation_ms": 501.698000007309, "mean_inter_token_latency_ms": 9.290703703839055, "output_token_source": "usage", "output_tokens": 55, "planned_context_tokens": 4096, "prompt_tokens": 3613, "reasoning_chunks": 0, "request_index": 2, "time_to_first_output_ms": 184.50680002570152, "tpot_ms": 9.290703703839055, "ttft_ms": 184.50700002722442, "visible_generation_ms": 501.6978000057861 }, { "content_chunks": 14, "decode_tok_s": 87.47009498954598, "e2e_ms": 722.0569000346586, "effective_prefill_tok_s": 19558.231890098392, "generation_ms": 537.3265000525862, "mean_inter_token_latency_ms": 11.432478724523111, "output_token_source": "usage", "output_tokens": 48, "planned_context_tokens": 4096, "prompt_tokens": 3613, "reasoning_chunks": 0, "request_index": 3, "time_to_first_output_ms": 184.7303999820724, "tpot_ms": 11.432478724523111, "ttft_ms": 184.7305999835953, "visible_generation_ms": 537.3263000510633 }, { "content_chunks": 15, "decode_tok_s": 71.81265074500413, "e2e_ms": 908.2690000068396, "effective_prefill_tok_s": 19618.53274434932, "generation_ms": 724.1063999244943, "mean_inter_token_latency_ms": 13.925123075471046, "output_token_source": "usage", "output_tokens": 53, "planned_context_tokens": 4096, "prompt_tokens": 3613, "reasoning_chunks": 0, "request_index": 4, "time_to_first_output_ms": 184.1626000823453, "tpot_ms": 13.925123075471046, "ttft_ms": 184.1628000838682, "visible_generation_ms": 724.1061999229714 }, { "content_chunks": 13, "decode_tok_s": 87.71256591799438, "e2e_ms": 685.2944000856951, "effective_prefill_tok_s": 19672.65956991195, "generation_ms": 501.63850001990795, "mean_inter_token_latency_ms": 11.400875000452453, "output_token_source": "usage", "output_tokens": 45, "planned_context_tokens": 4096, "prompt_tokens": 3613, "reasoning_chunks": 0, "request_index": 5, "time_to_first_output_ms": 183.65590006578714, "tpot_ms": 11.400875000452453, "ttft_ms": 183.65660007111728, "visible_generation_ms": 501.6378000145778 }, { "content_chunks": 15, "decode_tok_s": 64.94318508269473, "e2e_ms": 907.5150999706239, "effective_prefill_tok_s": 19656.637225328206, "generation_ms": 723.7094999291003, "mean_inter_token_latency_ms": 15.3980744665766, "output_token_source": "usage", "output_tokens": 48, "planned_context_tokens": 4096, "prompt_tokens": 3613, "reasoning_chunks": 0, "request_index": 6, "time_to_first_output_ms": 183.80560004152358, "tpot_ms": 15.3980744665766, "ttft_ms": 183.80570004228503, "visible_generation_ms": 723.7093999283388 }, { "content_chunks": 17, "decode_tok_s": 76.7367894105775, "e2e_ms": 978.3892000559717, "effective_prefill_tok_s": 19693.226085945345, "generation_ms": 794.9251000536606, "mean_inter_token_latency_ms": 13.031559017273125, "output_token_source": "usage", "output_tokens": 62, "planned_context_tokens": 4096, "prompt_tokens": 3613, "reasoning_chunks": 0, "request_index": 7, "time_to_first_output_ms": 183.46410000231117, "tpot_ms": 13.031559017273125, "ttft_ms": 183.46430000383407, "visible_generation_ms": 794.9249000521377 }, { "content_chunks": 15, "decode_tok_s": 66.13108424879205, "e2e_ms": 919.9168999912217, "effective_prefill_tok_s": 22051.625084428797, "generation_ms": 756.0740999178961, "mean_inter_token_latency_ms": 15.121481998357922, "output_token_source": "usage", "output_tokens": 51, "planned_context_tokens": 4096, "prompt_tokens": 3613, "reasoning_chunks": 0, "request_index": 8, "time_to_first_output_ms": 163.84280007332563, "tpot_ms": 15.121481998357922, "ttft_ms": 163.84290007408708, "visible_generation_ms": 756.0739999171346 }, { "content_chunks": 11, "decode_tok_s": 80.20369501138748, "e2e_ms": 660.3376999264583, "effective_prefill_tok_s": 29089.521335148136, "generation_ms": 536.1348999431357, "mean_inter_token_latency_ms": 12.468253487049667, "output_token_source": "usage", "output_tokens": 44, "planned_context_tokens": 4096, "prompt_tokens": 3613, "reasoning_chunks": 0, "request_index": 9, "time_to_first_output_ms": 124.20279998332262, "tpot_ms": 12.468253487049667, "ttft_ms": 124.20299998484552, "visible_generation_ms": 536.1346999416128 }, { "content_chunks": 13, "decode_tok_s": 54.01449582491984, "e2e_ms": 845.9998000180349, "effective_prefill_tok_s": 29151.885039176177, "generation_ms": 722.0284000504762, "mean_inter_token_latency_ms": 18.51354871924298, "output_token_source": "usage", "output_tokens": 40, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 10, "time_to_first_output_ms": 123.97139996755868, "tpot_ms": 18.51354871924298, "ttft_ms": 123.97169996984303, "visible_generation_ms": 722.0281000481918 }, { "content_chunks": 13, "decode_tok_s": 63.1168614814461, "e2e_ms": 844.5544000715017, "effective_prefill_tok_s": 18536.59386916262, "generation_ms": 649.5887000346556, "mean_inter_token_latency_ms": 15.843626830113552, "output_token_source": "usage", "output_tokens": 42, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 11, "time_to_first_output_ms": 194.9657000368461, "tpot_ms": 15.843626830113552, "ttft_ms": 194.965900038369, "visible_generation_ms": 649.5885000331327 }, { "content_chunks": 13, "decode_tok_s": 73.08794623177805, "e2e_ms": 810.196700040251, "effective_prefill_tok_s": 18580.967312085388, "generation_ms": 615.6966000562534, "mean_inter_token_latency_ms": 13.682146667916742, "output_token_source": "usage", "output_tokens": 46, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 12, "time_to_first_output_ms": 194.50009998399764, "tpot_ms": 13.682146667916742, "ttft_ms": 194.50029998552054, "visible_generation_ms": 615.6964000547305 }, { "content_chunks": 14, "decode_tok_s": 83.52236068036503, "e2e_ms": 724.6608999557793, "effective_prefill_tok_s": 24099.036774318094, "generation_ms": 574.6963999699801, "mean_inter_token_latency_ms": 11.972841666041253, "output_token_source": "usage", "output_tokens": 49, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 13, "time_to_first_output_ms": 149.9644999857992, "tpot_ms": 11.972841666041253, "ttft_ms": 149.96479998808354, "visible_generation_ms": 574.6960999676958 }, { "content_chunks": 13, "decode_tok_s": 79.68477442975667, "e2e_ms": 689.4350000657141, "effective_prefill_tok_s": 24124.099607551307, "generation_ms": 539.6263001020998, "mean_inter_token_latency_ms": 12.549448839583716, "output_token_source": "usage", "output_tokens": 44, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 14, "time_to_first_output_ms": 149.80869996361434, "tpot_ms": 12.549448839583716, "ttft_ms": 149.80889996513724, "visible_generation_ms": 539.6261001005769 }, { "content_chunks": 15, "decode_tok_s": 88.87109670655386, "e2e_ms": 981.3783999998122, "effective_prefill_tok_s": 7431.942919190169, "generation_ms": 495.09910005144775, "mean_inter_token_latency_ms": 11.25225227389654, "output_token_source": "usage", "output_tokens": 45, "planned_context_tokens": 4096, "prompt_tokens": 3614, "reasoning_chunks": 0, "request_index": 15, "time_to_first_output_ms": 486.2792999483645, "tpot_ms": 11.25225227389654, "ttft_ms": 486.27959995064884, "visible_generation_ms": 495.0988000491634 } ], "requests": 16, "run_id": "benchmark-20260905T034410Z", "schema": "anvil-serving.benchmark/v1", "serve_flags": { "no_thinking": false, "reasoning_effort": null, "shared_prefix_burst": false, "thinking_mode": "disabled" }, "source_recipe": null, "started_at": "2026-09-05T03:44:10Z", "tier": null, "timing_methodology": { "clock": "client time.perf_counter", "decode": "usage completion tokens after the first token divided by client-observed generation time; completion tokens may include reasoning tokens", "distributions": "request-level mean, min, p25, p50, p75, p90, p95, p99, and max using the nearest-rank percentile method", "effective_prefill": "usage.prompt_tokens divided by client-observed time to first output; includes queueing, scheduling, prefill, and first-token work", "generation": "client-observed E2E minus time to first output", "mean_inter_token_latency": "client-observed generation time divided by completion tokens after the first token; not a raw per-token timestamp trace", "time_to_first_output": "request start through the first non-empty streamed reasoning or content delta", "tpot": "alias of mean_inter_token_latency for capacity-v3: client-observed generation time divided by completion tokens after the first token", "ttft": "request start through the first non-empty streamed content delta" }, "wall_clock_ms": 1962.4265000456944 }