{ "base_url": "http://127.0.0.1:39272/v1", "completed": 8, "concurrency": 4, "context_distribution": { "4096": 8 }, "context_seed": 0, "context_tokens": 4096, "engine": "sglang", "failed": 0, "failures": [], "finished_at": "2026-09-05T03:25:37Z", "gpu": "Primary Node / RTX PRO 6000 Blackwell Max-Q / TP=1", "manifest": null, "max_context_tokens": 262144, "max_tokens": 512, "measurement_protocol": "capacity-v3", "metrics": { "content_chunks": 113, "content_chunks_s": null, "decode_tok_s_max": 120.41127368521478, "decode_tok_s_mean": 104.72819689698355, "decode_tok_s_min": 79.57711583944473, "decode_tok_s_p25": 95.31084821520938, "decode_tok_s_p50": 107.94639336955407, "decode_tok_s_p75": 111.44267281385079, "decode_tok_s_p90": 120.41127368521478, "decode_tok_s_p95": 120.41127368521478, "decode_tok_s_p99": 120.41127368521478, "decode_tok_s_samples": 8, "e2e_max_ms": 878.1014999840409, "e2e_mean_ms": 655.3639125049813, "e2e_min_ms": 557.8785999678075, "e2e_p25_ms": 564.9980999296531, "e2e_p50_ms": 598.5700000310317, "e2e_p75_ms": 670.7387000788003, "e2e_p90_ms": 878.1014999840409, "e2e_p95_ms": 878.1014999840409, "e2e_p99_ms": 878.1014999840409, "e2e_samples": 8, "effective_prefill_tok_s_max": 26539.42240346298, "effective_prefill_tok_s_mean": 20866.554063042837, "effective_prefill_tok_s_min": 17152.69123136191, "effective_prefill_tok_s_p25": 19517.238694210573, "effective_prefill_tok_s_p50": 20655.711652078397, "effective_prefill_tok_s_p75": 20718.929061991155, "effective_prefill_tok_s_p90": 26539.42240346298, "effective_prefill_tok_s_p95": 26539.42240346298, "effective_prefill_tok_s_p99": 26539.42240346298, "effective_prefill_tok_s_samples": 8, "generation_max_ms": 703.7198999896646, "generation_mean_ms": 479.8499750031624, "generation_min_ms": 390.3288999572396, "generation_p25_ms": 421.74150003120303, "generation_p50_ms": 435.40129996836185, "generation_p75_ms": 485.6203000526875, "generation_p90_ms": 703.7198999896646, "generation_p95_ms": 703.7198999896646, "generation_p99_ms": 703.7198999896646, "generation_samples": 8, "mean_inter_token_latency_ms_max": 12.566426785529725, "mean_inter_token_latency_ms_mean": 9.694142428224653, "mean_inter_token_latency_ms_min": 8.304870211856162, "mean_inter_token_latency_ms_p25": 8.823414544828914, "mean_inter_token_latency_ms_p50": 9.012272341196683, "mean_inter_token_latency_ms_p75": 10.11708958443099, "mean_inter_token_latency_ms_p90": 12.566426785529725, "mean_inter_token_latency_ms_p95": 12.566426785529725, "mean_inter_token_latency_ms_p99": 12.566426785529725, "mean_inter_token_latency_ms_samples": 8, "output_token_sources": { "usage": 8 }, "output_tokens": 402, "prefix_cache_hit_avg": null, "prompt_token_samples": 8, "prompt_tokens": 28904, "prompt_tokens_max": 3613.0, "prompt_tokens_mean": 3613.0, "prompt_tokens_min": 3613.0, "prompt_tokens_p25": 3613.0, "prompt_tokens_p50": 3613.0, "prompt_tokens_p75": 3613.0, "prompt_tokens_p90": 3613.0, "prompt_tokens_p95": 3613.0, "prompt_tokens_p99": 3613.0, "prompt_tokens_samples": 8, "reasoning_chunks": 0, "throughput_tok_s": 279.6469019661643, "time_to_first_output_max_ms": 210.63750004395843, "time_to_first_output_mean_ms": 175.51393750181887, "time_to_first_output_min_ms": 136.1370999366045, "time_to_first_output_p25_ms": 163.1687000626698, "time_to_first_output_p50_ms": 174.66919997241348, "time_to_first_output_p75_ms": 185.08369999472052, "time_to_first_output_p90_ms": 210.63750004395843, "time_to_first_output_p95_ms": 210.63750004395843, "time_to_first_output_p99_ms": 210.63750004395843, "time_to_first_output_samples": 8, "tpot_max_ms": 12.566426785529725, "tpot_mean_ms": 9.694142428224653, "tpot_min_ms": 8.304870211856162, "tpot_p25_ms": 8.823414544828914, "tpot_p50_ms": 9.012272341196683, "tpot_p75_ms": 10.11708958443099, "tpot_p90_ms": 12.566426785529725, "tpot_p95_ms": 12.566426785529725, "tpot_p99_ms": 12.566426785529725, "tpot_samples": 8, "ttft_max_ms": 210.63780004624277, "ttft_mean_ms": 175.5142374895513, "ttft_min_ms": 136.1372999381274, "ttft_p25_ms": 163.1694999523461, "ttft_p50_ms": 174.66939997393638, "ttft_p75_ms": 185.08389999624342, "ttft_p90_ms": 210.63780004624277, "ttft_p95_ms": 210.63780004624277, "ttft_p99_ms": 210.63780004624277, "ttft_samples": 8 }, "model": "qwen38-27b-inferact-nvfp4-pro6000-dflash2-k12-chunk1k", "request_timings": [ { "content_chunks": 14, "decode_tok_s": 107.94639336955407, "e2e_ms": 598.5700000310317, "effective_prefill_tok_s": 22142.727119921405, "generation_ms": 435.40129996836185, "mean_inter_token_latency_ms": 9.263857446135358, "output_token_source": "usage", "output_tokens": 48, "planned_context_tokens": 4096, "prompt_tokens": 3613, "reasoning_chunks": 0, "request_index": 0, "time_to_first_output_ms": 163.1687000626698, "tpot_ms": 9.263857446135358, "ttft_ms": 163.1694999523461, "visible_generation_ms": 435.4005000786856 }, { "content_chunks": 13, "decode_tok_s": 110.9598070432053, "e2e_ms": 598.4921000199392, "effective_prefill_tok_s": 20655.711652078397, "generation_ms": 423.5768000362441, "mean_inter_token_latency_ms": 9.012272341196683, "output_token_source": "usage", "output_tokens": 48, "planned_context_tokens": 4096, "prompt_tokens": 3613, "reasoning_chunks": 0, "request_index": 1, "time_to_first_output_ms": 174.91529998369515, "tpot_ms": 9.012272341196683, "ttft_ms": 174.91549998521805, "visible_generation_ms": 423.5766000347212 }, { "content_chunks": 13, "decode_tok_s": 120.41127368521478, "e2e_ms": 564.9980999296531, "effective_prefill_tok_s": 20684.814498323816, "generation_ms": 390.3288999572396, "mean_inter_token_latency_ms": 8.304870211856162, "output_token_source": "usage", "output_tokens": 48, "planned_context_tokens": 4096, "prompt_tokens": 3613, "reasoning_chunks": 0, "request_index": 2, "time_to_first_output_ms": 174.66919997241348, "tpot_ms": 8.304870211856162, "ttft_ms": 174.66939997393638, "visible_generation_ms": 390.32869995571673 }, { "content_chunks": 18, "decode_tok_s": 79.57711583944473, "e2e_ms": 878.1014999840409, "effective_prefill_tok_s": 20718.929061991155, "generation_ms": 703.7198999896646, "mean_inter_token_latency_ms": 12.566426785529725, "output_token_source": "usage", "output_tokens": 57, "planned_context_tokens": 4096, "prompt_tokens": 3613, "reasoning_chunks": 0, "request_index": 3, "time_to_first_output_ms": 174.3815999943763, "tpot_ms": 12.566426785529725, "ttft_ms": 174.38189999666065, "visible_generation_ms": 703.7195999873802 }, { "content_chunks": 13, "decode_tok_s": 95.31084821520938, "e2e_ms": 703.7608000682667, "effective_prefill_tok_s": 17152.69123136191, "generation_ms": 493.12330002430826, "mean_inter_token_latency_ms": 10.491985106900175, "output_token_source": "usage", "output_tokens": 48, "planned_context_tokens": 4096, "prompt_tokens": 3613, "reasoning_chunks": 0, "request_index": 4, "time_to_first_output_ms": 210.63750004395843, "tpot_ms": 10.491985106900175, "ttft_ms": 210.63780004624277, "visible_generation_ms": 493.1230000220239 }, { "content_chunks": 14, "decode_tok_s": 98.84265545487334, "e2e_ms": 670.7387000788003, "effective_prefill_tok_s": 19517.238694210573, "generation_ms": 485.6203000526875, "mean_inter_token_latency_ms": 10.11708958443099, "output_token_source": "usage", "output_tokens": 49, "planned_context_tokens": 4096, "prompt_tokens": 3613, "reasoning_chunks": 0, "request_index": 5, "time_to_first_output_ms": 185.1184000261128, "tpot_ms": 10.11708958443099, "ttft_ms": 185.1186000276357, "visible_generation_ms": 485.6201000511646 }, { "content_chunks": 14, "decode_tok_s": 113.33480875451603, "e2e_ms": 670.3714999603108, "effective_prefill_tok_s": 19520.897842992443, "generation_ms": 485.28779996559024, "mean_inter_token_latency_ms": 8.823414544828914, "output_token_source": "usage", "output_tokens": 56, "planned_context_tokens": 4096, "prompt_tokens": 3613, "reasoning_chunks": 0, "request_index": 6, "time_to_first_output_ms": 185.08369999472052, "tpot_ms": 8.823414544828914, "ttft_ms": 185.08389999624342, "visible_generation_ms": 485.28759996406734 }, { "content_chunks": 14, "decode_tok_s": 111.44267281385079, "e2e_ms": 557.8785999678075, "effective_prefill_tok_s": 26539.42240346298, "generation_ms": 421.74150003120303, "mean_inter_token_latency_ms": 8.973223404919214, "output_token_source": "usage", "output_tokens": 48, "planned_context_tokens": 4096, "prompt_tokens": 3613, "reasoning_chunks": 0, "request_index": 7, "time_to_first_output_ms": 136.1370999366045, "tpot_ms": 8.973223404919214, "ttft_ms": 136.1372999381274, "visible_generation_ms": 421.74130002968013 } ], "requests": 8, "run_id": "benchmark-20260905T032535Z", "schema": "anvil-serving.benchmark/v1", "serve_flags": { "no_thinking": false, "reasoning_effort": null, "shared_prefix_burst": false, "thinking_mode": "disabled" }, "source_recipe": null, "started_at": "2026-09-05T03:25:35Z", "tier": null, "timing_methodology": { "clock": "client time.perf_counter", "decode": "usage completion tokens after the first token divided by client-observed generation time; completion tokens may include reasoning tokens", "distributions": "request-level mean, min, p25, p50, p75, p90, p95, p99, and max using the nearest-rank percentile method", "effective_prefill": "usage.prompt_tokens divided by client-observed time to first output; includes queueing, scheduling, prefill, and first-token work", "generation": "client-observed E2E minus time to first output", "mean_inter_token_latency": "client-observed generation time divided by completion tokens after the first token; not a raw per-token timestamp trace", "time_to_first_output": "request start through the first non-empty streamed reasoning or content delta", "tpot": "alias of mean_inter_token_latency for capacity-v3: client-observed generation time divided by completion tokens after the first token", "ttft": "request start through the first non-empty streamed content delta" }, "wall_clock_ms": 1437.5270999735221 }