{ "context": { "cap_tokens": 128000, "max_model_len": 131072, "targets": [] }, "evaluation_protocol": { "budget_semantics": "visible_answer_tokens plus reasoning_headroom_tokens are sent as one max_tokens cap; the endpoint does not hard-partition the channels", "minimum_comparable_repetitions": 3, "minimum_pass_rate": 1.0, "reasoning_headroom_tokens": 1792, "records_finish_reason": true, "records_full_visible_answer": true, "records_reasoning_channel_metadata": true, "repetitions": 3, "version": 3, "visible_answer_tokens": 256 }, "failures": [ { "error": "answer", "eval_id": "mmlu-pro-30-computer-science", "failure_classes": [ "reasoning_budget_exhausted" ], "pass_rate": 0.0, "suite": "hf-mmlu-pro-10-repeated" }, { "error": "answer", "eval_id": "mmlu-pro-40-engineering", "failure_classes": [ "reasoning_budget_exhausted" ], "pass_rate": 0.0, "suite": "hf-mmlu-pro-10-repeated" } ], "identity": { "base_url": "http://127.0.0.1:39038/v1", "candidate_id": "RadixArk-Qwen3.8-27B-NVFP4", "config_id": "stock-template-128k", "engine": "sglang", "gpu": "isolated-rtx5090-host", "model": "qwen38-27b-radixark-nvfp4-sglang-rtx5090-128k-mm", "started_at": "2026-08-20T10:26:33Z" }, "intelligence": { "checks": [], "status": "not_run" }, "run_id": "fast-bakeoff-20260820T102633Z", "schema": "anvil-serving.fast-tier-bakeoff/v1", "score_inputs": { "e2e_p50_ms": null, "intelligence_pass_rate": null, "operational_fit_notes": [ "endpoint was already loaded; benchmark did not start or stop serves" ], "session_recall_passed": false, "thinking_mode": "enabled", "tool_call_passed": false, "ttft_p50_ms": null, "usable_context_tokens": null, "voice_latency_ms": null }, "selection": { "context_targets": [ 32768 ], "endpoint_already_loaded": true, "requests_per_context": 1, "suites": [] }, "session": { "checks": [], "status": "not_run" }, "source_recipe": { "ref": "RadixArk/Qwen3.8-27B-NVFP4@554ebba9b5f1b79dc11246341960360e6ef05ef4", "serve_command": null }, "suites": { "hf-mmlu-pro-10-repeated": { "checks": [ { "attempt_count": 3, "attempts": [ { "attempt": 1, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 2048, "reasoning_headroom_tokens": 1792, "visible_answer_tokens": 256 }, "content": "\n\nThe pituitary gland is located in the head, within the cranial cavity (specifically in the sella turcica of the sphenoid bone).\n\nFINAL=J", "content_chars": 138, "content_excerpt": "\n\nThe pituitary gland is located in the head, within the cranial cavity (specifically in the sella turcica of the sphenoid bone).\n\nFINAL=J", "failure_class": null, "finish_reason": "stop", "latency_ms": 2193.851399992127, "reasoning_chars": 444, "reasoning_excerpt": "We need answer user's multiple choice. Need reason through ten-option question, finish with FINAL=. Question: Which body cavity contains pituitary gland? Pituitary gland is in cranial cavity, ", "reasoning_field": "reasoning_content", "reasoning_tokens": null, "status": "passed", "text_checks": [ { "name": "answer", "passed": true } ], "usage": { "completion_tokens": 150, "prompt_tokens": 137, "prompt_tokens_details": null, "reasoning_tokens": 112, "total_tokens": 287 } }, { "attempt": 2, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 2048, "reasoning_headroom_tokens": 1792, "visible_answer_tokens": 256 }, "content": "\n\nThe pituitary gland is located in the head, within the cranial cavity (specifically in the sella turcica of the sphenoid bone).\n\nFINAL=J", "content_chars": 138, "content_excerpt": "\n\nThe pituitary gland is located in the head, within the cranial cavity (specifically in the sella turcica of the sphenoid bone).\n\nFINAL=J", "failure_class": null, "finish_reason": "stop", "latency_ms": 2052.835899987258, "reasoning_chars": 444, "reasoning_excerpt": "We need answer user's multiple choice. Need reason through ten-option question, finish with FINAL=. Question: Which body cavity contains pituitary gland? Pituitary gland is in cranial cavity, ", "reasoning_field": "reasoning_content", "reasoning_tokens": null, "status": "passed", "text_checks": [ { "name": "answer", "passed": true } ], "usage": { "completion_tokens": 150, "prompt_tokens": 137, "prompt_tokens_details": null, "reasoning_tokens": 112, "total_tokens": 287 } }, { "attempt": 3, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 2048, "reasoning_headroom_tokens": 1792, "visible_answer_tokens": 256 }, "content": "\n\nThe pituitary gland is located in the head, within the cranial cavity (specifically in the sella turcica of the sphenoid bone).\n\nFINAL=J", "content_chars": 138, "content_excerpt": "\n\nThe pituitary gland is located in the head, within the cranial cavity (specifically in the sella turcica of the sphenoid bone).\n\nFINAL=J", "failure_class": null, "finish_reason": "stop", "latency_ms": 2052.929400000721, "reasoning_chars": 444, "reasoning_excerpt": "We need answer user's multiple choice. Need reason through ten-option question, finish with FINAL=. Question: Which body cavity contains pituitary gland? Pituitary gland is in cranial cavity, ", "reasoning_field": "reasoning_content", "reasoning_tokens": null, "status": "passed", "text_checks": [ { "name": "answer", "passed": true } ], "usage": { "completion_tokens": 150, "prompt_tokens": 137, "prompt_tokens_details": null, "reasoning_tokens": 112, "total_tokens": 287 } } ], "id": "mmlu-pro-5-health", "pass_count": 3, "pass_rate": 1.0, "required_pass_rate": 1.0, "status": "passed", "validator": "deterministic_text_checks" }, { "attempt_count": 3, "attempts": [ { "attempt": 1, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 2048, "reasoning_headroom_tokens": 1792, "visible_answer_tokens": 256 }, "content": "\n\nThe light-gathering power is proportional to the area of the aperture, which scales with the square of the diameter.\n\n- Pupil diameter: \\(5\\text{ mm}\\)\n- Telescope aperture: \\(50\\text{ cm} = 500\\text{ mm}\\)\n\nDiameter ratio:\n\n\\[\n\\frac{500}{5} = 100\n\\]\n\nArea ratio:\n\n\\[\n100^2 = 10000\n\\]\n\nSo the telescope gathers about **10,000 times** more light than the eye.\n\nFINAL=E", "content_chars": 369, "content_excerpt": "\n\nThe light-gathering power is proportional to the area of the aperture, which scales with the square of the diameter.\n\n- Pupil diameter: \\(5\\text{ mm}\\)\n- Telescope aperture: \\(50\\text{ cm} = 500\\tex", "failure_class": null, "finish_reason": "stop", "latency_ms": 4095.0208000140265, "reasoning_chars": 514, "reasoning_excerpt": "We need answer multiple choice. Need reason. Pupil diameter 5 mm, telescope aperture 50 cm. Light gathering proportional to area = pi (d/2)^2, ratio = (50 cm / 5 mm)^2. Convert: 50 cm = 500 mm. Ratio ", "reasoning_field": "reasoning_content", "reasoning_tokens": null, "status": "passed", "text_checks": [ { "name": "answer", "passed": true } ], "usage": { "completion_tokens": 306, "prompt_tokens": 197, "prompt_tokens_details": null, "reasoning_tokens": 172, "total_tokens": 503 } }, { "attempt": 2, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 2048, "reasoning_headroom_tokens": 1792, "visible_answer_tokens": 256 }, "content": "\n\nThe light-gathering power is proportional to the area of the aperture, which scales with the square of the diameter.\n\n- Pupil diameter: \\(5\\text{ mm}\\)\n- Telescope aperture: \\(50\\text{ cm} = 500\\text{ mm}\\)\n\nDiameter ratio:\n\n\\[\n\\frac{500}{5} = 100\n\\]\n\nArea ratio:\n\n\\[\n100^2 = 10000\n\\]\n\nSo the telescope gathers about **10,000 times** more light than the eye.\n\nFINAL=E", "content_chars": 369, "content_excerpt": "\n\nThe light-gathering power is proportional to the area of the aperture, which scales with the square of the diameter.\n\n- Pupil diameter: \\(5\\text{ mm}\\)\n- Telescope aperture: \\(50\\text{ cm} = 500\\tex", "failure_class": null, "finish_reason": "stop", "latency_ms": 4099.5335999759845, "reasoning_chars": 514, "reasoning_excerpt": "We need answer multiple choice. Need reason. Pupil diameter 5 mm, telescope aperture 50 cm. Light gathering proportional to area = pi (d/2)^2, ratio = (50 cm / 5 mm)^2. Convert: 50 cm = 500 mm. Ratio ", "reasoning_field": "reasoning_content", "reasoning_tokens": null, "status": "passed", "text_checks": [ { "name": "answer", "passed": true } ], "usage": { "completion_tokens": 306, "prompt_tokens": 197, "prompt_tokens_details": null, "reasoning_tokens": 172, "total_tokens": 503 } }, { "attempt": 3, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 2048, "reasoning_headroom_tokens": 1792, "visible_answer_tokens": 256 }, "content": "\n\nThe light-gathering power is proportional to the area of the aperture, which scales with the square of the diameter.\n\n- Pupil diameter: \\(5\\text{ mm}\\)\n- Telescope aperture: \\(50\\text{ cm} = 500\\text{ mm}\\)\n\nDiameter ratio:\n\n\\[\n\\frac{500}{5} = 100\n\\]\n\nArea ratio:\n\n\\[\n100^2 = 10000\n\\]\n\nSo the telescope gathers about **10,000 times** more light than the eye.\n\nFINAL=E", "content_chars": 369, "content_excerpt": "\n\nThe light-gathering power is proportional to the area of the aperture, which scales with the square of the diameter.\n\n- Pupil diameter: \\(5\\text{ mm}\\)\n- Telescope aperture: \\(50\\text{ cm} = 500\\tex", "failure_class": null, "finish_reason": "stop", "latency_ms": 4086.3985999603756, "reasoning_chars": 514, "reasoning_excerpt": "We need answer multiple choice. Need reason. Pupil diameter 5 mm, telescope aperture 50 cm. Light gathering proportional to area = pi (d/2)^2, ratio = (50 cm / 5 mm)^2. Convert: 50 cm = 500 mm. Ratio ", "reasoning_field": "reasoning_content", "reasoning_tokens": null, "status": "passed", "text_checks": [ { "name": "answer", "passed": true } ], "usage": { "completion_tokens": 306, "prompt_tokens": 197, "prompt_tokens_details": null, "reasoning_tokens": 172, "total_tokens": 503 } } ], "id": "mmlu-pro-10-physics", "pass_count": 3, "pass_rate": 1.0, "required_pass_rate": 1.0, "status": "passed", "validator": "deterministic_text_checks" }, { "attempt_count": 3, "attempts": [ { "attempt": 1, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 2048, "reasoning_headroom_tokens": 1792, "visible_answer_tokens": 256 }, "content": "\n\nThe sentence is describing **buycotts**, which are the positive counterpart to **boycotts**:\n\n- **Boycotts** punish or pressure companies by withholding purchases.\n- **Buycotts** reward companies for favourable behaviour by encouraging purchases.\n\nSo the first two blanks must be:\n\n> In contrast to **boycotts**, **buycotts** aim to reward favourable behaviour by companies.\n\nThat narrows the options to **C, F, H, J**.\n\nThe next part says the success of such campaigns has been heightened through the use of some technology, which helps the company achieve something. Since buycotts are meant to reward companies, the likely outcome is **increased sales**, not decreased sales or charitable donations. Also, **digital technology** is the most plausible modern enabler of consumer campaigns, more than \u201cphysical technology\u201d or a specific \u201cblockchain technology\u201d in this general context.\n\nThus the best completion is:\n\n> In contrast to **boycotts**, **buycotts** aim to reward favourable behaviour by companies. The success of such campaigns has been heightened through the use of **digital technology**, which allow campaigns to facilitate the company in achieving **increased sales**.\n\nThat matches option **F**.\n\nFINAL=F", "content_chars": 1224, "content_excerpt": "\n\nThe sentence is describing **buycotts**, which are the positive counterpart to **boycotts**:\n\n- **Boycotts** punish or pressure companies by withholding purchases.\n- **Buycotts** reward companies fo", "failure_class": null, "finish_reason": "stop", "latency_ms": 15445.872100011911, "reasoning_chars": 4394, "reasoning_excerpt": "We need answer a multiple-choice fill-in-the-blank. Need reason through ten options, finish with FINAL=. Need determine correct option.\n\nQuestion: \"In contrast to _______, _______ aim to rewar", "reasoning_field": "reasoning_content", "reasoning_tokens": null, "status": "passed", "text_checks": [ { "name": "answer", "passed": true } ], "usage": { "completion_tokens": 1178, "prompt_tokens": 270, "prompt_tokens_details": null, "reasoning_tokens": 918, "total_tokens": 1448 } }, { "attempt": 2, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 2048, "reasoning_headroom_tokens": 1792, "visible_answer_tokens": 256 }, "content": "\n\nThe sentence is describing **buycotts**, which are the positive counterpart to **boycotts**:\n\n- **Boycotts** punish or pressure companies by withholding purchases.\n- **Buycotts** reward companies for favourable behaviour by encouraging purchases.\n\nSo the first two blanks must be:\n\n> In contrast to **boycotts**, **buycotts** aim to reward favourable behaviour by companies.\n\nThat narrows the options to **C, F, H, J**.\n\nThe next part says the success of such campaigns has been heightened through the use of some technology, which helps the company achieve something. Since buycotts are meant to reward companies, the likely outcome is **increased sales**, not decreased sales or charitable donations. Also, **digital technology** is the most plausible modern enabler of consumer campaigns, more than \u201cphysical technology\u201d or a specific \u201cblockchain technology\u201d in this general context.\n\nThus the best completion is:\n\n> In contrast to **boycotts**, **buycotts** aim to reward favourable behaviour by companies. The success of such campaigns has been heightened through the use of **digital technology**, which allow campaigns to facilitate the company in achieving **increased sales**.\n\nThat matches option **F**.\n\nFINAL=F", "content_chars": 1224, "content_excerpt": "\n\nThe sentence is describing **buycotts**, which are the positive counterpart to **boycotts**:\n\n- **Boycotts** punish or pressure companies by withholding purchases.\n- **Buycotts** reward companies fo", "failure_class": null, "finish_reason": "stop", "latency_ms": 15435.674599953927, "reasoning_chars": 4394, "reasoning_excerpt": "We need answer a multiple-choice fill-in-the-blank. Need reason through ten options, finish with FINAL=. Need determine correct option.\n\nQuestion: \"In contrast to _______, _______ aim to rewar", "reasoning_field": "reasoning_content", "reasoning_tokens": null, "status": "passed", "text_checks": [ { "name": "answer", "passed": true } ], "usage": { "completion_tokens": 1178, "prompt_tokens": 270, "prompt_tokens_details": null, "reasoning_tokens": 918, "total_tokens": 1448 } }, { "attempt": 3, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 2048, "reasoning_headroom_tokens": 1792, "visible_answer_tokens": 256 }, "content": "\n\nThe sentence is describing **buycotts**, which are the positive counterpart to **boycotts**:\n\n- **Boycotts** punish or pressure companies by withholding purchases.\n- **Buycotts** reward companies for favourable behaviour by encouraging purchases.\n\nSo the first two blanks must be:\n\n> In contrast to **boycotts**, **buycotts** aim to reward favourable behaviour by companies.\n\nThat narrows the options to **C, F, H, J**.\n\nThe next part says the success of such campaigns has been heightened through the use of some technology, which helps the company achieve something. Since buycotts are meant to reward companies, the likely outcome is **increased sales**, not decreased sales or charitable donations. Also, **digital technology** is the most plausible modern enabler of consumer campaigns, more than \u201cphysical technology\u201d or a specific \u201cblockchain technology\u201d in this general context.\n\nThus the best completion is:\n\n> In contrast to **boycotts**, **buycotts** aim to reward favourable behaviour by companies. The success of such campaigns has been heightened through the use of **digital technology**, which allow campaigns to facilitate the company in achieving **increased sales**.\n\nThat matches option **F**.\n\nFINAL=F", "content_chars": 1224, "content_excerpt": "\n\nThe sentence is describing **buycotts**, which are the positive counterpart to **boycotts**:\n\n- **Boycotts** punish or pressure companies by withholding purchases.\n- **Buycotts** reward companies fo", "failure_class": null, "finish_reason": "stop", "latency_ms": 15436.284299998078, "reasoning_chars": 4394, "reasoning_excerpt": "We need answer a multiple-choice fill-in-the-blank. Need reason through ten options, finish with FINAL=. Need determine correct option.\n\nQuestion: \"In contrast to _______, _______ aim to rewar", "reasoning_field": "reasoning_content", "reasoning_tokens": null, "status": "passed", "text_checks": [ { "name": "answer", "passed": true } ], "usage": { "completion_tokens": 1178, "prompt_tokens": 270, "prompt_tokens_details": null, "reasoning_tokens": 918, "total_tokens": 1448 } } ], "id": "mmlu-pro-15-business", "pass_count": 3, "pass_rate": 1.0, "required_pass_rate": 1.0, "status": "passed", "validator": "deterministic_text_checks" }, { "attempt_count": 3, "attempts": [ { "attempt": 1, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 2048, "reasoning_headroom_tokens": 1792, "visible_answer_tokens": 256 }, "content": "\n\nArthropods have an open circulatory system with a dorsal heart, so option B is accurate.\n\nFINAL=B", "content_chars": 99, "content_excerpt": "\n\nArthropods have an open circulatory system with a dorsal heart, so option B is accurate.\n\nFINAL=B", "failure_class": null, "finish_reason": "stop", "latency_ms": 3099.001299997326, "reasoning_chars": 847, "reasoning_excerpt": "We need answer user's question. Need reason through ten-option question, finish with FINAL=. Need choose accurate statement concerning arthropods. Options:\nA. exoskeleton composed primarily of", "reasoning_field": "reasoning_content", "reasoning_tokens": null, "status": "passed", "text_checks": [ { "name": "answer", "passed": true } ], "usage": { "completion_tokens": 230, "prompt_tokens": 178, "prompt_tokens_details": null, "reasoning_tokens": 205, "total_tokens": 408 } }, { "attempt": 2, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 2048, "reasoning_headroom_tokens": 1792, "visible_answer_tokens": 256 }, "content": "\n\nArthropods have an open circulatory system with a dorsal heart, so option B is accurate.\n\nFINAL=B", "content_chars": 99, "content_excerpt": "\n\nArthropods have an open circulatory system with a dorsal heart, so option B is accurate.\n\nFINAL=B", "failure_class": null, "finish_reason": "stop", "latency_ms": 3091.173599998001, "reasoning_chars": 847, "reasoning_excerpt": "We need answer user's question. Need reason through ten-option question, finish with FINAL=. Need choose accurate statement concerning arthropods. Options:\nA. exoskeleton composed primarily of", "reasoning_field": "reasoning_content", "reasoning_tokens": null, "status": "passed", "text_checks": [ { "name": "answer", "passed": true } ], "usage": { "completion_tokens": 230, "prompt_tokens": 178, "prompt_tokens_details": null, "reasoning_tokens": 205, "total_tokens": 408 } }, { "attempt": 3, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 2048, "reasoning_headroom_tokens": 1792, "visible_answer_tokens": 256 }, "content": "\n\nArthropods have an open circulatory system with a dorsal heart, so option B is accurate.\n\nFINAL=B", "content_chars": 99, "content_excerpt": "\n\nArthropods have an open circulatory system with a dorsal heart, so option B is accurate.\n\nFINAL=B", "failure_class": null, "finish_reason": "stop", "latency_ms": 3092.267100000754, "reasoning_chars": 847, "reasoning_excerpt": "We need answer user's question. Need reason through ten-option question, finish with FINAL=. Need choose accurate statement concerning arthropods. Options:\nA. exoskeleton composed primarily of", "reasoning_field": "reasoning_content", "reasoning_tokens": null, "status": "passed", "text_checks": [ { "name": "answer", "passed": true } ], "usage": { "completion_tokens": 230, "prompt_tokens": 178, "prompt_tokens_details": null, "reasoning_tokens": 205, "total_tokens": 408 } } ], "id": "mmlu-pro-20-biology", "pass_count": 3, "pass_rate": 1.0, "required_pass_rate": 1.0, "status": "passed", "validator": "deterministic_text_checks" }, { "attempt_count": 3, "attempts": [ { "attempt": 1, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 2048, "reasoning_headroom_tokens": 1792, "visible_answer_tokens": 256 }, "content": "\n\nThe methyl radical has:\n\n- one \\(^{13}\\text{C}\\) nucleus with \\(I = \\tfrac{1}{2}\\), giving \\(2I+1 = 2\\) lines;\n- three equivalent \\(^{1}\\text{H}\\) nuclei, each with \\(I = \\tfrac{1}{2}\\), giving \\(n+1 = 3+1 = 4\\) lines.\n\nWith no overlap, the total number of EPR lines is:\n\n\\[\n2 \\times 4 = 8\n\\]\n\nFINAL=B", "content_chars": 303, "content_excerpt": "\n\nThe methyl radical has:\n\n- one \\(^{13}\\text{C}\\) nucleus with \\(I = \\tfrac{1}{2}\\), giving \\(2I+1 = 2\\) lines;\n- three equivalent \\(^{1}\\text{H}\\) nuclei, each with \\(I = \\tfrac{1}{2}\\), giving \\(n+", "failure_class": null, "finish_reason": "stop", "latency_ms": 14531.2068000203, "reasoning_chars": 3037, "reasoning_excerpt": "We need answer question: Predict number of lines in EPR spectrum of solution of 13C-labelled methyl radical (13CH3 radical), assuming lines do not overlap. Options. Need reason. Need final = letter.\n\n", "reasoning_field": "reasoning_content", "reasoning_tokens": null, "status": "passed", "text_checks": [ { "name": "answer", "passed": true } ], "usage": { "completion_tokens": 1108, "prompt_tokens": 157, "prompt_tokens_details": null, "reasoning_tokens": 986, "total_tokens": 1265 } }, { "attempt": 2, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 2048, "reasoning_headroom_tokens": 1792, "visible_answer_tokens": 256 }, "content": "\n\nThe methyl radical has:\n\n- one \\(^{13}\\text{C}\\) nucleus with \\(I = \\tfrac{1}{2}\\), giving \\(2I+1 = 2\\) lines;\n- three equivalent \\(^{1}\\text{H}\\) nuclei, each with \\(I = \\tfrac{1}{2}\\), giving \\(n+1 = 3+1 = 4\\) lines.\n\nWith no overlap, the total number of EPR lines is:\n\n\\[\n2 \\times 4 = 8\n\\]\n\nFINAL=B", "content_chars": 303, "content_excerpt": "\n\nThe methyl radical has:\n\n- one \\(^{13}\\text{C}\\) nucleus with \\(I = \\tfrac{1}{2}\\), giving \\(2I+1 = 2\\) lines;\n- three equivalent \\(^{1}\\text{H}\\) nuclei, each with \\(I = \\tfrac{1}{2}\\), giving \\(n+", "failure_class": null, "finish_reason": "stop", "latency_ms": 14524.614399997517, "reasoning_chars": 3037, "reasoning_excerpt": "We need answer question: Predict number of lines in EPR spectrum of solution of 13C-labelled methyl radical (13CH3 radical), assuming lines do not overlap. Options. Need reason. Need final = letter.\n\n", "reasoning_field": "reasoning_content", "reasoning_tokens": null, "status": "passed", "text_checks": [ { "name": "answer", "passed": true } ], "usage": { "completion_tokens": 1108, "prompt_tokens": 157, "prompt_tokens_details": null, "reasoning_tokens": 986, "total_tokens": 1265 } }, { "attempt": 3, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 2048, "reasoning_headroom_tokens": 1792, "visible_answer_tokens": 256 }, "content": "\n\nThe methyl radical has:\n\n- one \\(^{13}\\text{C}\\) nucleus with \\(I = \\tfrac{1}{2}\\), giving \\(2I+1 = 2\\) lines;\n- three equivalent \\(^{1}\\text{H}\\) nuclei, each with \\(I = \\tfrac{1}{2}\\), giving \\(n+1 = 3+1 = 4\\) lines.\n\nWith no overlap, the total number of EPR lines is:\n\n\\[\n2 \\times 4 = 8\n\\]\n\nFINAL=B", "content_chars": 303, "content_excerpt": "\n\nThe methyl radical has:\n\n- one \\(^{13}\\text{C}\\) nucleus with \\(I = \\tfrac{1}{2}\\), giving \\(2I+1 = 2\\) lines;\n- three equivalent \\(^{1}\\text{H}\\) nuclei, each with \\(I = \\tfrac{1}{2}\\), giving \\(n+", "failure_class": null, "finish_reason": "stop", "latency_ms": 14522.14360004291, "reasoning_chars": 3037, "reasoning_excerpt": "We need answer question: Predict number of lines in EPR spectrum of solution of 13C-labelled methyl radical (13CH3 radical), assuming lines do not overlap. Options. Need reason. Need final = letter.\n\n", "reasoning_field": "reasoning_content", "reasoning_tokens": null, "status": "passed", "text_checks": [ { "name": "answer", "passed": true } ], "usage": { "completion_tokens": 1108, "prompt_tokens": 157, "prompt_tokens_details": null, "reasoning_tokens": 986, "total_tokens": 1265 } } ], "id": "mmlu-pro-25-chemistry", "pass_count": 3, "pass_rate": 1.0, "required_pass_rate": 1.0, "status": "passed", "validator": "deterministic_text_checks" }, { "attempt_count": 3, "attempts": [ { "attempt": 1, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 2048, "reasoning_headroom_tokens": 1792, "visible_answer_tokens": 256 }, "content": "", "content_chars": 0, "content_excerpt": "", "error": "answer", "failure_class": "reasoning_budget_exhausted", "finish_reason": "length", "latency_ms": 26782.584899978247, "reasoning_chars": 6644, "reasoning_excerpt": "We need answer multiple choice. Need reason carefully. Problem: pipelined RISC machine registers R0-R7, supports ADD Rs1, Rs2, Rd and MUL Rs1, Rs2, Rd. Operation normally takes one cycle; however, an ", "reasoning_field": "reasoning_content", "reasoning_tokens": null, "status": "failed", "text_checks": [ { "name": "answer", "passed": false } ], "usage": { "completion_tokens": 2048, "prompt_tokens": 220, "prompt_tokens_details": null, "reasoning_tokens": 2048, "total_tokens": 2268 } }, { "attempt": 2, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 2048, "reasoning_headroom_tokens": 1792, "visible_answer_tokens": 256 }, "content": "", "content_chars": 0, "content_excerpt": "", "error": "answer", "failure_class": "reasoning_budget_exhausted", "finish_reason": "length", "latency_ms": 26776.66390000377, "reasoning_chars": 6644, "reasoning_excerpt": "We need answer multiple choice. Need reason carefully. Problem: pipelined RISC machine registers R0-R7, supports ADD Rs1, Rs2, Rd and MUL Rs1, Rs2, Rd. Operation normally takes one cycle; however, an ", "reasoning_field": "reasoning_content", "reasoning_tokens": null, "status": "failed", "text_checks": [ { "name": "answer", "passed": false } ], "usage": { "completion_tokens": 2048, "prompt_tokens": 220, "prompt_tokens_details": null, "reasoning_tokens": 2048, "total_tokens": 2268 } }, { "attempt": 3, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 2048, "reasoning_headroom_tokens": 1792, "visible_answer_tokens": 256 }, "content": "", "content_chars": 0, "content_excerpt": "", "error": "answer", "failure_class": "reasoning_budget_exhausted", "finish_reason": "length", "latency_ms": 26783.28999999212, "reasoning_chars": 6644, "reasoning_excerpt": "We need answer multiple choice. Need reason carefully. Problem: pipelined RISC machine registers R0-R7, supports ADD Rs1, Rs2, Rd and MUL Rs1, Rs2, Rd. Operation normally takes one cycle; however, an ", "reasoning_field": "reasoning_content", "reasoning_tokens": null, "status": "failed", "text_checks": [ { "name": "answer", "passed": false } ], "usage": { "completion_tokens": 2048, "prompt_tokens": 220, "prompt_tokens_details": null, "reasoning_tokens": 2048, "total_tokens": 2268 } } ], "error": "answer", "id": "mmlu-pro-30-computer-science", "pass_count": 0, "pass_rate": 0.0, "required_pass_rate": 1.0, "status": "failed", "validator": "deterministic_text_checks" }, { "attempt_count": 3, "attempts": [ { "attempt": 1, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 2048, "reasoning_headroom_tokens": 1792, "visible_answer_tokens": 256 }, "content": "\n\nIn a stationary autoregressive process, the impact of a shock is temporary: the process mean-reverts, and the effect of the shock decays over time rather than persisting or exploding.\n\nFINAL=F", "content_chars": 194, "content_excerpt": "\n\nIn a stationary autoregressive process, the impact of a shock is temporary: the process mean-reverts, and the effect of the shock decays over time rather than persisting or exploding.\n\nFINAL=F", "failure_class": null, "finish_reason": "stop", "latency_ms": 3933.8318000081927, "reasoning_chars": 1136, "reasoning_excerpt": "We need answer a multiple-choice conceptual econometrics/time series question. Need reason through ten options, finish with FINAL=. Need be careful.\n\nQuestion: \"For a stationary autoregressive", "reasoning_field": "reasoning_content", "reasoning_tokens": null, "status": "passed", "text_checks": [ { "name": "answer", "passed": true } ], "usage": { "completion_tokens": 294, "prompt_tokens": 150, "prompt_tokens_details": null, "reasoning_tokens": 251, "total_tokens": 444 } }, { "attempt": 2, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 2048, "reasoning_headroom_tokens": 1792, "visible_answer_tokens": 256 }, "content": "\n\nIn a stationary autoregressive process, the impact of a shock is temporary: the process mean-reverts, and the effect of the shock decays over time rather than persisting or exploding.\n\nFINAL=F", "content_chars": 194, "content_excerpt": "\n\nIn a stationary autoregressive process, the impact of a shock is temporary: the process mean-reverts, and the effect of the shock decays over time rather than persisting or exploding.\n\nFINAL=F", "failure_class": null, "finish_reason": "stop", "latency_ms": 3934.042600041721, "reasoning_chars": 1136, "reasoning_excerpt": "We need answer a multiple-choice conceptual econometrics/time series question. Need reason through ten options, finish with FINAL=. Need be careful.\n\nQuestion: \"For a stationary autoregressive", "reasoning_field": "reasoning_content", "reasoning_tokens": null, "status": "passed", "text_checks": [ { "name": "answer", "passed": true } ], "usage": { "completion_tokens": 294, "prompt_tokens": 150, "prompt_tokens_details": null, "reasoning_tokens": 251, "total_tokens": 444 } }, { "attempt": 3, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 2048, "reasoning_headroom_tokens": 1792, "visible_answer_tokens": 256 }, "content": "\n\nIn a stationary autoregressive process, the impact of a shock is temporary: the process mean-reverts, and the effect of the shock decays over time rather than persisting or exploding.\n\nFINAL=F", "content_chars": 194, "content_excerpt": "\n\nIn a stationary autoregressive process, the impact of a shock is temporary: the process mean-reverts, and the effect of the shock decays over time rather than persisting or exploding.\n\nFINAL=F", "failure_class": null, "finish_reason": "stop", "latency_ms": 3926.0803000070155, "reasoning_chars": 1136, "reasoning_excerpt": "We need answer a multiple-choice conceptual econometrics/time series question. Need reason through ten options, finish with FINAL=. Need be careful.\n\nQuestion: \"For a stationary autoregressive", "reasoning_field": "reasoning_content", "reasoning_tokens": null, "status": "passed", "text_checks": [ { "name": "answer", "passed": true } ], "usage": { "completion_tokens": 294, "prompt_tokens": 150, "prompt_tokens_details": null, "reasoning_tokens": 251, "total_tokens": 444 } } ], "id": "mmlu-pro-35-economics", "pass_count": 3, "pass_rate": 1.0, "required_pass_rate": 1.0, "status": "passed", "validator": "deterministic_text_checks" }, { "attempt_count": 3, "attempts": [ { "attempt": 1, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 2048, "reasoning_headroom_tokens": 1792, "visible_answer_tokens": 256 }, "content": "", "content_chars": 0, "content_excerpt": "", "error": "answer", "failure_class": "reasoning_budget_exhausted", "finish_reason": "length", "latency_ms": 26789.449399977457, "reasoning_chars": 4275, "reasoning_excerpt": "We need answer physics multiple choice. Need reason. Problem: \"A point pole has a strength of 4 pi times 10^-4 weber. The force in newtons on a point pole of 4 pi times 1.5 times 10^-4 weber placed 10", "reasoning_field": "reasoning_content", "reasoning_tokens": null, "status": "failed", "text_checks": [ { "name": "answer", "passed": false } ], "usage": { "completion_tokens": 2048, "prompt_tokens": 191, "prompt_tokens_details": null, "reasoning_tokens": 2048, "total_tokens": 2239 } }, { "attempt": 2, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 2048, "reasoning_headroom_tokens": 1792, "visible_answer_tokens": 256 }, "content": "", "content_chars": 0, "content_excerpt": "", "error": "answer", "failure_class": "reasoning_budget_exhausted", "finish_reason": "length", "latency_ms": 26782.12160000112, "reasoning_chars": 4275, "reasoning_excerpt": "We need answer physics multiple choice. Need reason. Problem: \"A point pole has a strength of 4 pi times 10^-4 weber. The force in newtons on a point pole of 4 pi times 1.5 times 10^-4 weber placed 10", "reasoning_field": "reasoning_content", "reasoning_tokens": null, "status": "failed", "text_checks": [ { "name": "answer", "passed": false } ], "usage": { "completion_tokens": 2048, "prompt_tokens": 191, "prompt_tokens_details": null, "reasoning_tokens": 2048, "total_tokens": 2239 } }, { "attempt": 3, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 2048, "reasoning_headroom_tokens": 1792, "visible_answer_tokens": 256 }, "content": "", "content_chars": 0, "content_excerpt": "", "error": "answer", "failure_class": "reasoning_budget_exhausted", "finish_reason": "length", "latency_ms": 26786.550299962983, "reasoning_chars": 4275, "reasoning_excerpt": "We need answer physics multiple choice. Need reason. Problem: \"A point pole has a strength of 4 pi times 10^-4 weber. The force in newtons on a point pole of 4 pi times 1.5 times 10^-4 weber placed 10", "reasoning_field": "reasoning_content", "reasoning_tokens": null, "status": "failed", "text_checks": [ { "name": "answer", "passed": false } ], "usage": { "completion_tokens": 2048, "prompt_tokens": 191, "prompt_tokens_details": null, "reasoning_tokens": 2048, "total_tokens": 2239 } } ], "error": "answer", "id": "mmlu-pro-40-engineering", "pass_count": 0, "pass_rate": 0.0, "required_pass_rate": 1.0, "status": "failed", "validator": "deterministic_text_checks" }, { "attempt_count": 3, "attempts": [ { "attempt": 1, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 2048, "reasoning_headroom_tokens": 1792, "visible_answer_tokens": 256 }, "content": "\n\nThe statement says: \u201cTurtles live long lives and are happy creatures, unless they are injured.\u201d\n\nLet:\n- L = Turtles live long lives\n- H = Turtles are happy creatures\n- I = Turtles are injured\n\n\u201cP unless Q\u201d is usually symbolized as \u201cP or Q,\u201d because if Q is false, then P must be true.\n\nHere, P = \u201cTurtles live long lives and are happy creatures\u201d = L and H \nQ = \u201cthey are injured\u201d = I\n\nSo the best symbolization is:\n\n**(L and H) or I**\n\nFINAL=B", "content_chars": 446, "content_excerpt": "\n\nThe statement says: \u201cTurtles live long lives and are happy creatures, unless they are injured.\u201d\n\nLet:\n- L = Turtles live long lives\n- H = Turtles are happy creatures\n- I = Turtles are injured\n\n\u201cP un", "failure_class": null, "finish_reason": "stop", "latency_ms": 6120.97819999326, "reasoning_chars": 1174, "reasoning_excerpt": "We need answer user's question. Need reason through ten-option question, finish with FINAL=. Need parse: \"Which formula of propositional logic best symbolizes: Turtles live long lives and are ", "reasoning_field": "reasoning_content", "reasoning_tokens": null, "status": "passed", "text_checks": [ { "name": "answer", "passed": true } ], "usage": { "completion_tokens": 462, "prompt_tokens": 162, "prompt_tokens_details": null, "reasoning_tokens": 333, "total_tokens": 624 } }, { "attempt": 2, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 2048, "reasoning_headroom_tokens": 1792, "visible_answer_tokens": 256 }, "content": "\n\nThe statement says: \u201cTurtles live long lives and are happy creatures, unless they are injured.\u201d\n\nLet:\n- L = Turtles live long lives\n- H = Turtles are happy creatures\n- I = Turtles are injured\n\n\u201cP unless Q\u201d is usually symbolized as \u201cP or Q,\u201d because if Q is false, then P must be true.\n\nHere, P = \u201cTurtles live long lives and are happy creatures\u201d = L and H \nQ = \u201cthey are injured\u201d = I\n\nSo the best symbolization is:\n\n**(L and H) or I**\n\nFINAL=B", "content_chars": 446, "content_excerpt": "\n\nThe statement says: \u201cTurtles live long lives and are happy creatures, unless they are injured.\u201d\n\nLet:\n- L = Turtles live long lives\n- H = Turtles are happy creatures\n- I = Turtles are injured\n\n\u201cP un", "failure_class": null, "finish_reason": "stop", "latency_ms": 6120.7220000214875, "reasoning_chars": 1174, "reasoning_excerpt": "We need answer user's question. Need reason through ten-option question, finish with FINAL=. Need parse: \"Which formula of propositional logic best symbolizes: Turtles live long lives and are ", "reasoning_field": "reasoning_content", "reasoning_tokens": null, "status": "passed", "text_checks": [ { "name": "answer", "passed": true } ], "usage": { "completion_tokens": 462, "prompt_tokens": 162, "prompt_tokens_details": null, "reasoning_tokens": 333, "total_tokens": 624 } }, { "attempt": 3, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 2048, "reasoning_headroom_tokens": 1792, "visible_answer_tokens": 256 }, "content": "\n\nThe statement says: \u201cTurtles live long lives and are happy creatures, unless they are injured.\u201d\n\nLet:\n- L = Turtles live long lives\n- H = Turtles are happy creatures\n- I = Turtles are injured\n\n\u201cP unless Q\u201d is usually symbolized as \u201cP or Q,\u201d because if Q is false, then P must be true.\n\nHere, P = \u201cTurtles live long lives and are happy creatures\u201d = L and H \nQ = \u201cthey are injured\u201d = I\n\nSo the best symbolization is:\n\n**(L and H) or I**\n\nFINAL=B", "content_chars": 446, "content_excerpt": "\n\nThe statement says: \u201cTurtles live long lives and are happy creatures, unless they are injured.\u201d\n\nLet:\n- L = Turtles live long lives\n- H = Turtles are happy creatures\n- I = Turtles are injured\n\n\u201cP un", "failure_class": null, "finish_reason": "stop", "latency_ms": 6115.021200035699, "reasoning_chars": 1174, "reasoning_excerpt": "We need answer user's question. Need reason through ten-option question, finish with FINAL=. Need parse: \"Which formula of propositional logic best symbolizes: Turtles live long lives and are ", "reasoning_field": "reasoning_content", "reasoning_tokens": null, "status": "passed", "text_checks": [ { "name": "answer", "passed": true } ], "usage": { "completion_tokens": 462, "prompt_tokens": 162, "prompt_tokens_details": null, "reasoning_tokens": 333, "total_tokens": 624 } } ], "id": "mmlu-pro-45-philosophy", "pass_count": 3, "pass_rate": 1.0, "required_pass_rate": 1.0, "status": "passed", "validator": "deterministic_text_checks" }, { "attempt_count": 3, "attempts": [ { "attempt": 1, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 2048, "reasoning_headroom_tokens": 1792, "visible_answer_tokens": 256 }, "content": "\n\nA judge ad hoc at the ICJ is appointed for a particular case when a party does not have a judge of its nationality sitting on the Court. That person sits only in that case.\n\nFINAL=I", "content_chars": 183, "content_excerpt": "\n\nA judge ad hoc at the ICJ is appointed for a particular case when a party does not have a judge of its nationality sitting on the Court. That person sits only in that case.\n\nFINAL=I", "failure_class": null, "finish_reason": "stop", "latency_ms": 2998.1139000155963, "reasoning_chars": 801, "reasoning_excerpt": "We need answer a multiple-choice question: \"What is the judge ad hoc?\" Options A-J. Need reason through ten-option question, finish with FINAL=. Need likely correct option I: If a party before", "reasoning_field": "reasoning_content", "reasoning_tokens": null, "status": "passed", "text_checks": [ { "name": "answer", "passed": true } ], "usage": { "completion_tokens": 222, "prompt_tokens": 203, "prompt_tokens_details": null, "reasoning_tokens": 178, "total_tokens": 425 } }, { "attempt": 2, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 2048, "reasoning_headroom_tokens": 1792, "visible_answer_tokens": 256 }, "content": "\n\nA judge ad hoc at the ICJ is appointed for a particular case when a party does not have a judge of its nationality sitting on the Court. That person sits only in that case.\n\nFINAL=I", "content_chars": 183, "content_excerpt": "\n\nA judge ad hoc at the ICJ is appointed for a particular case when a party does not have a judge of its nationality sitting on the Court. That person sits only in that case.\n\nFINAL=I", "failure_class": null, "finish_reason": "stop", "latency_ms": 2988.9999000006355, "reasoning_chars": 801, "reasoning_excerpt": "We need answer a multiple-choice question: \"What is the judge ad hoc?\" Options A-J. Need reason through ten-option question, finish with FINAL=. Need likely correct option I: If a party before", "reasoning_field": "reasoning_content", "reasoning_tokens": null, "status": "passed", "text_checks": [ { "name": "answer", "passed": true } ], "usage": { "completion_tokens": 222, "prompt_tokens": 203, "prompt_tokens_details": null, "reasoning_tokens": 178, "total_tokens": 425 } }, { "attempt": 3, "budget": { "legacy_max_tokens_as_visible": false, "max_completion_tokens": 2048, "reasoning_headroom_tokens": 1792, "visible_answer_tokens": 256 }, "content": "\n\nA judge ad hoc at the ICJ is appointed for a particular case when a party does not have a judge of its nationality sitting on the Court. That person sits only in that case.\n\nFINAL=I", "content_chars": 183, "content_excerpt": "\n\nA judge ad hoc at the ICJ is appointed for a particular case when a party does not have a judge of its nationality sitting on the Court. That person sits only in that case.\n\nFINAL=I", "failure_class": null, "finish_reason": "stop", "latency_ms": 2991.4480000152253, "reasoning_chars": 801, "reasoning_excerpt": "We need answer a multiple-choice question: \"What is the judge ad hoc?\" Options A-J. Need reason through ten-option question, finish with FINAL=. Need likely correct option I: If a party before", "reasoning_field": "reasoning_content", "reasoning_tokens": null, "status": "passed", "text_checks": [ { "name": "answer", "passed": true } ], "usage": { "completion_tokens": 222, "prompt_tokens": 203, "prompt_tokens_details": null, "reasoning_tokens": 178, "total_tokens": 425 } } ], "id": "mmlu-pro-65-law", "pass_count": 3, "pass_rate": 1.0, "required_pass_rate": 1.0, "status": "passed", "validator": "deterministic_text_checks" } ], "date": "2026-07-12", "evidence_use": "diagnostic", "source": "tests\\fixtures\\eval-data\\hf-mmlu-pro-10-repeated.suite.json", "source_sha256": "f6112c6f592c5285f8dae86808285f328161d3b4ae30ef660440d173d47b0636", "status": "failed", "validator_strength": "deterministic_marker", "work_class": "independent-multidomain-reasoning-sanity" } }, "thinking": { "chat_template_kwargs": { "enable_thinking": true }, "control_evidence": null, "control_evidence_sha256": null, "control_mechanism": "chat_template_kwargs", "control_requested": { "enable_thinking": true }, "control_status": "requested_unverified", "mode": "enabled", "reasoning_effort": null, "unsupported": false }, "timing": { "chat": { "content_chunks": 0, "decode_tok_s_p50": null, "decode_tok_s_p95": null, "e2e_p50_ms": null, "e2e_p95_ms": null, "effective_prefill_tok_s_p50": null, "effective_prefill_tok_s_p95": null, "generation_p50_ms": null, "generation_p95_ms": null, "mean_inter_token_latency_ms_p50": null, "mean_inter_token_latency_ms_p95": null, "output_tokens": 0, "prompt_token_samples": 0, "prompt_tokens": null, "prompt_tokens_p50": null, "prompt_tokens_p95": null, "reasoning_chunks": 0, "time_to_first_output_p50_ms": null, "time_to_first_output_p95_ms": null, "ttft_p50_ms": null, "ttft_p95_ms": null, "ttft_samples": 0 }, "wall_ms": 317618.59089997597 }, "tool": { "checks": [], "status": "not_run" }, "voice": { "status": "not_run", "stt_latency_ms": null, "total_turn_latency_ms": null, "tts_latency_ms": null } }