{ "base_url": "http://127.0.0.1:39272/v1", "completed": 8, "concurrency": 8, "context_distribution": { "81920": 8 }, "context_seed": 0, "context_tokens": 81920, "engine": "sglang", "failed": 0, "failures": [], "finished_at": "2026-09-05T03:53:21Z", "gpu": "Primary Node / RTX PRO 6000 Blackwell Max-Q / TP=1", "manifest": null, "max_context_tokens": 262144, "max_tokens": 512, "measurement_protocol": "capacity-v3", "metrics": { "content_chunks": 152, "content_chunks_s": null, "decode_tok_s_max": 89.82609799452156, "decode_tok_s_mean": 12.66319509810194, "decode_tok_s_min": 0.5665850659670995, "decode_tok_s_p25": 0.5698906840233378, "decode_tok_s_p50": 0.9888948003924265, "decode_tok_s_p75": 3.1700245100103914, "decode_tok_s_p90": 89.82609799452156, "decode_tok_s_p95": 89.82609799452156, "decode_tok_s_p99": 89.82609799452156, "decode_tok_s_samples": 8, "e2e_max_ms": 140194.93150000926, "e2e_mean_ms": 130495.52130000666, "e2e_min_ms": 121163.5329000419, "e2e_p25_ms": 121232.16200002935, "e2e_p50_ms": 121419.27489999216, "e2e_p75_ms": 139496.2018999504, "e2e_p90_ms": 140194.93150000926, "e2e_p95_ms": 140194.93150000926, "e2e_p99_ms": 140194.93150000926, "e2e_samples": 8, "effective_prefill_tok_s_max": 4338.300099568515, "effective_prefill_tok_s_mean": 1509.2234511571353, "effective_prefill_tok_s_min": 556.742411145022, "effective_prefill_tok_s_p25": 646.9409709219631, "effective_prefill_tok_s_p50": 905.7014115624244, "effective_prefill_tok_s_p75": 1503.9433338007639, "effective_prefill_tok_s_p90": 4338.300099568515, "effective_prefill_tok_s_p95": 4338.300099568515, "effective_prefill_tok_s_p99": 4338.300099568515, "effective_prefill_tok_s_samples": 8, "generation_max_ms": 103528.62689993344, "generation_mean_ms": 52880.63993748801, "generation_min_ms": 757.0183000061661, "generation_p25_ms": 18337.413499946706, "generation_p50_ms": 35710.286199930124, "generation_p75_ms": 86483.04190009367, "generation_p90_ms": 103528.62689993344, "generation_p95_ms": 103528.62689993344, "generation_p99_ms": 103528.62689993344, "generation_samples": 8, "mean_inter_token_latency_ms_max": 1764.9600387774217, "mean_inter_token_latency_ms_mean": 881.9010501983731, "mean_inter_token_latency_ms_min": 11.132622058914208, "mean_inter_token_latency_ms_p25": 269.66784558745155, "mean_inter_token_latency_ms_p50": 575.9723580633892, "mean_inter_token_latency_ms_p75": 1352.0681630766305, "mean_inter_token_latency_ms_p90": 1764.9600387774217, "mean_inter_token_latency_ms_p95": 1764.9600387774217, "mean_inter_token_latency_ms_p99": 1764.9600387774217, "mean_inter_token_latency_ms_samples": 8, "output_token_sources": { "usage": 8 }, "output_tokens": 511, "prefix_cache_hit_avg": null, "prompt_token_samples": 8, "prompt_tokens": 620979, "prompt_tokens_max": 77631.0, "prompt_tokens_mean": 77622.375, "prompt_tokens_min": 77615.0, "prompt_tokens_p25": 77617.0, "prompt_tokens_p50": 77620.0, "prompt_tokens_p75": 77626.0, "prompt_tokens_p90": 77631.0, "prompt_tokens_p95": 77631.0, "prompt_tokens_p99": 77631.0, "prompt_tokens_samples": 8, "reasoning_chunks": 0, "throughput_tok_s": 3.6446794604748183, "time_to_first_output_max_ms": 139437.9132000031, "time_to_first_output_mean_ms": 77614.88136251864, "time_to_first_output_min_ms": 17890.648000058718, "time_to_first_output_p25_ms": 34680.49099994823, "time_to_first_output_p50_ms": 68710.10819997173, "time_to_first_output_p75_ms": 102894.74850008264, "time_to_first_output_p90_ms": 139437.9132000031, "time_to_first_output_p95_ms": 139437.9132000031, "time_to_first_output_p99_ms": 139437.9132000031, "time_to_first_output_samples": 8, "tpot_max_ms": 1764.9600387774217, "tpot_mean_ms": 881.9010501983731, "tpot_min_ms": 11.132622058914208, "tpot_p25_ms": 269.66784558745155, "tpot_p50_ms": 575.9723580633892, "tpot_p75_ms": 1352.0681630766305, "tpot_p90_ms": 1764.9600387774217, "tpot_p95_ms": 1764.9600387774217, "tpot_p99_ms": 1764.9600387774217, "tpot_samples": 8, "ttft_max_ms": 139437.91340000462, "ttft_mean_ms": 77614.88181252207, "ttft_min_ms": 17890.64880006481, "ttft_p25_ms": 34680.4915999528, "ttft_p50_ms": 68710.10869997554, "ttft_p75_ms": 102894.74870008416, "ttft_p90_ms": 139437.91340000462, "ttft_p95_ms": 139437.91340000462, "ttft_p99_ms": 139437.91340000462, "ttft_samples": 8 }, "model": "qwen38-27b-inferact-nvfp4-pro6000-dflash2-k12-chunk1k", "request_timings": [ { "content_chunks": 19, "decode_tok_s": 0.5698906840233378, "e2e_ms": 121419.27489999216, "effective_prefill_tok_s": 4338.300099568515, "generation_ms": 103528.62689993344, "mean_inter_token_latency_ms": 1754.7224898293805, "output_token_source": "usage", "output_tokens": 60, "planned_context_tokens": 81920, "prompt_tokens": 77615, "reasoning_chunks": 0, "request_index": 0, "time_to_first_output_ms": 17890.648000058718, "tpot_ms": 1754.7224898293805, "ttft_ms": 17890.64880006481, "visible_generation_ms": 103528.62609992735 }, { "content_chunks": 17, "decode_tok_s": 0.5665850659670995, "e2e_ms": 121163.5329000419, "effective_prefill_tok_s": 2238.05943232222, "generation_ms": 86483.04190009367, "mean_inter_token_latency_ms": 1764.9600387774217, "output_token_source": "usage", "output_tokens": 50, "planned_context_tokens": 81920, "prompt_tokens": 77617, "reasoning_chunks": 0, "request_index": 1, "time_to_first_output_ms": 34680.49099994823, "tpot_ms": 1764.9600387774217, "ttft_ms": 34680.4915999528, "visible_generation_ms": 86483.0413000891 }, { "content_chunks": 20, "decode_tok_s": 0.9888948003924265, "e2e_ms": 139496.2018999504, "effective_prefill_tok_s": 1129.659114698235, "generation_ms": 70786.09369997866, "mean_inter_token_latency_ms": 1011.2299099996952, "output_token_source": "usage", "output_tokens": 71, "planned_context_tokens": 81920, "prompt_tokens": 77619, "reasoning_chunks": 0, "request_index": 2, "time_to_first_output_ms": 68710.10819997173, "tpot_ms": 1011.2299099996952, "ttft_ms": 68710.10869997554, "visible_generation_ms": 70786.09319997486 }, { "content_chunks": 20, "decode_tok_s": 0.7396076820006622, "e2e_ms": 139495.4177000327, "effective_prefill_tok_s": 1503.9433338007639, "generation_ms": 87884.43059998099, "mean_inter_token_latency_ms": 1352.0681630766305, "output_token_source": "usage", "output_tokens": 66, "planned_context_tokens": 81920, "prompt_tokens": 77620, "reasoning_chunks": 0, "request_index": 3, "time_to_first_output_ms": 51610.987100051716, "tpot_ms": 1352.0681630766305, "ttft_ms": 51610.98750005476, "visible_generation_ms": 87884.43019997794 }, { "content_chunks": 19, "decode_tok_s": 1.7361944301673313, "e2e_ms": 121415.13219999615, "effective_prefill_tok_s": 905.7014115624244, "generation_ms": 35710.286199930124, "mean_inter_token_latency_ms": 575.9723580633892, "output_token_source": "usage", "output_tokens": 63, "planned_context_tokens": 81920, "prompt_tokens": 77623, "reasoning_chunks": 0, "request_index": 4, "time_to_first_output_ms": 85704.84600006603, "tpot_ms": 575.9723580633892, "ttft_ms": 85704.84650006983, "visible_generation_ms": 35710.28569992632 }, { "content_chunks": 20, "decode_tok_s": 3.1700245100103914, "e2e_ms": 139547.51730000135, "effective_prefill_tok_s": 646.9409709219631, "generation_ms": 19558.208400034346, "mean_inter_token_latency_ms": 315.45497419410236, "output_token_source": "usage", "output_tokens": 63, "planned_context_tokens": 81920, "prompt_tokens": 77626, "reasoning_chunks": 0, "request_index": 5, "time_to_first_output_ms": 119989.308899967, "tpot_ms": 315.45497419410236, "ttft_ms": 119989.30929997005, "visible_generation_ms": 19558.2080000313 }, { "content_chunks": 18, "decode_tok_s": 3.70826561773271, "e2e_ms": 121232.16200002935, "effective_prefill_tok_s": 754.4408352379388, "generation_ms": 18337.413499946706, "mean_inter_token_latency_ms": 269.66784558745155, "output_token_source": "usage", "output_tokens": 69, "planned_context_tokens": 81920, "prompt_tokens": 77628, "reasoning_chunks": 0, "request_index": 6, "time_to_first_output_ms": 102894.74850008264, "tpot_ms": 269.66784558745155, "ttft_ms": 102894.74870008416, "visible_generation_ms": 18337.413299945183 }, { "content_chunks": 19, "decode_tok_s": 89.82609799452156, "e2e_ms": 140194.93150000926, "effective_prefill_tok_s": 556.742411145022, "generation_ms": 757.0183000061661, "mean_inter_token_latency_ms": 11.132622058914208, "output_token_source": "usage", "output_tokens": 69, "planned_context_tokens": 81920, "prompt_tokens": 77631, "reasoning_chunks": 0, "request_index": 7, "time_to_first_output_ms": 139437.9132000031, "tpot_ms": 11.132622058914208, "ttft_ms": 139437.91340000462, "visible_generation_ms": 757.0181000046432 } ], "requests": 8, "run_id": "benchmark-20260905T035101Z", "schema": "anvil-serving.benchmark/v1", "serve_flags": { "no_thinking": false, "reasoning_effort": null, "shared_prefix_burst": false, "thinking_mode": "disabled" }, "source_recipe": null, "started_at": "2026-09-05T03:51:01Z", "tier": null, "timing_methodology": { "clock": "client time.perf_counter", "decode": "usage completion tokens after the first token divided by client-observed generation time; completion tokens may include reasoning tokens", "distributions": "request-level mean, min, p25, p50, p75, p90, p95, p99, and max using the nearest-rank percentile method", "effective_prefill": "usage.prompt_tokens divided by client-observed time to first output; includes queueing, scheduling, prefill, and first-token work", "generation": "client-observed E2E minus time to first output", "mean_inter_token_latency": "client-observed generation time divided by completion tokens after the first token; not a raw per-token timestamp trace", "time_to_first_output": "request start through the first non-empty streamed reasoning or content delta", "tpot": "alias of mean_inter_token_latency for capacity-v3: client-observed generation time divided by completion tokens after the first token", "ttft": "request start through the first non-empty streamed content delta" }, "wall_clock_ms": 140204.3734000763 }