{ "created_at": "2026-09-14T03:27:49.382485Z", "failure": null, "partial": false, "provenance": { "endpoint": { "auth_env": "", "base_url": "", "model": "llm.primary" }, "finished_at": "2026-09-14T03:27:49.382320Z", "started_at": "2026-09-14T03:16:33.044345Z", "submitted_at": "2026-09-14T03:16:32.913266+00:00", "worker": { "id": "" } }, "results": { "evidence": { "completeness": "completed", "created_at": "2026-09-14T03:27:49.373357Z", "evidence_kind": "measured", "failure": null, "identities": { "concurrency": { "requests": 1, "workers": 1 }, "context": { "configured_tokens": null, "output_headroom_tokens": 65536, "profile_buckets": [ 8192, 32768, 65536, 131072, 262144, 524288, 640000 ], "selected_buckets": [ 258000 ] }, "dataset": { "name": "deterministic-native", "profile_sha256": "6cc9e66696ae5fafed7af738cae898b59b7370574c829a9740300762dfbb92a7" }, "hardware": { "architecture": "x86_64", "container_capable": true, "free_disk_bytes": "", "harness_assets": [], "id": "", "owned_output_writable": true, "platform": "linux" }, "harnesses": { "native": { "schema": "anvil-serving.context/v1" } }, "image": { "status": "not-applicable" }, "model": { "quantization": "EXL3 uniform K4 routed experts; native non-routed weights; FP8 DS-MLA KV", "repo": "brandonmusic/GLM-5.3-Flash-tr3-4bpw", "revision": "a5fee929cf4888b1824323e33e8a19b60129e025" }, "runtime": { "configured_context": 327680, "dcp": 2, "engine": "vllm-glm53-exl3-b12x-v84-fp8-control", "ep": 2, "harness_commit": "f5d8fbc4308d27bdc964abac73d7801e738dcd2a", "image": "verdictai/glm53-flash-exl3-k4:r19-sm120-tp2-ep2-dcp2-v84-language-only@sha256:0f1cdcc8891f1cc3a444121eb61d366289a1cbba285f0892dcbb24bc94961692", "kv_cache": "fp8_ds_mla", "max_concurrency": 4, "mtp": 0, "reasoning_config": {}, "source_commit": "6dc2f516688fe6f84c6994dcd20fddf296853a6c", "source_recipe": "", "source_recipe_sha256": "030fb4775b70d891f4075f0f42e197eaa60f09b0d98817acc15e4eb3b0550064", "tp": 2 }, "served_model": { "configured_context": null, "id": "llm.primary" }, "topology": { "kind": "co-resident-benchmark-client", "worker_id": "" } }, "promotion": { "authorized": false, "message": "Benchmark evidence does not authorize model promotion; promotion is a separate human decision." }, "run": { "ownership_id": "intelligence-context-20260913", "profile": "deep", "run_id": "glm-r7-nospec-promoted-context258k-r1", "spec_sha256": "f727a43ab65c5ebd969741cfa3b5c0dccaa5f668614ffd000cf265b9a7f1bf23", "suite": "context" }, "schema": "anvil-serving.benchmark-evidence/v1", "stages": [ { "evidence": [ { "bytes": 569, "path": "evidence/0-assets.json", "sha256": "f54810ae97d1dad8c3b8784971edde8b18f39c6e0ad9fb517ada350972cbc1f9" } ], "name": "asset_preparation", "sequence": 0, "status": "completed" }, { "evidence": [ { "bytes": 2079, "path": "evidence/0-preflight.json", "sha256": "85716640faf7741799b95addfedc72f9ae33e1524ce2fa665e8fb0d64fee1423" } ], "name": "preflight", "sequence": 1, "status": "completed" }, { "evidence": [ { "bytes": 37806, "path": "evidence/0-context.json", "sha256": "54ad9f4626879440612a119cf3ede154de457481d8fa118b10ea8f641bd740f9" } ], "name": "context", "sequence": 2, "status": "completed" } ], "summary": { "advertised_context": 327680, "attempted_buckets": [ 258000 ], "buckets": [ { "completed_count": 9, "completion_rate": 1.0, "engine_telemetry": { "available": false, "observations": [] }, "failures": {}, "latency_ms": { "available": true, "mean": 74888.64516855554, "observations": [ 61501.18181598373, 63793.430492980406, 80184.67765901005, 118133.32542701392, 88020.22957801819, 66349.74837303162, 65311.766019964125, 64727.28689899668, 65976.16025200114 ] }, "meets_policy": true, "pass_rate": 1.0, "passed_count": 9, "relative_drop_from_baseline": 0.0, "requested_tokens": 258000, "sample_count": 9, "samples": [ { "answer_sha256": "ea77d2c40d5ae22b614345bf65faaf4a798b6926e2753eaa34d64261ffa8f4cb", "case_id": "native-order-258000-0.100-r0", "case_type": "native-order", "completed": true, "distractor_count": 0, "engine_telemetry": null, "failure": null, "finish_reason": "stop", "latency_ms": 61501.18181598373, "passed": true, "position": 0.1, "prompt_tokens": 255647, "raw_reasoning": "The user wants me to return the ALPHA, BETA, and GAMMA checkpoint values in that order, separated by ' | '.\n\nLooking at the reference document, I can find these lines:\n\n\"Checkpoint ALPHA stores K557508437185.\"\n\"Checkpoint BETA stores K434177192288.\"\n\"Checkpoint GAMMA stores K503557252183.\"\n\nSo the values are:\n- ALPHA: K557508437185\n- BETA: K434177192288\n- GAMMA: K503557252183\n\nThe user wants them separated by ' | '.", "raw_reasoning_field": "reasoning_content", "raw_reasoning_truncated": false, "requested_tokens": 258000, "schema": "anvil-serving.context-observation/v1", "target_count": 3, "throughput_tps": 2.5527964725906584, "token_measurement": "usage", "visible_answer": "", "visible_answer_truncated": false }, { "answer_sha256": "ea77d2c40d5ae22b614345bf65faaf4a798b6926e2753eaa34d64261ffa8f4cb", "case_id": "native-order-258000-0.500-r0", "case_type": "native-order", "completed": true, "distractor_count": 0, "engine_telemetry": null, "failure": null, "finish_reason": "stop", "latency_ms": 63793.430492980406, "passed": true, "position": 0.5, "prompt_tokens": 255647, "raw_reasoning": "The user is asking for the ALPHA, BETA, and GAMMA checkpoint values in a specific format. I need to look through the reference document to find these values.\n\nLooking through the document, I see there are many reference paragraphs labeled \"neutral cobalt ledger material for calibration.\" Then near the end, I see:\n\nCheckpoint ALPHA stores K557508437185.\nCheckpoint BETA stores K434177192288.\nCheckpoint GAMMA stores K503557252183.\n\nSo:\n- ALPHA = K557508437185\n- BETA = K434177192288\n- GAMMA = K503557252183\n\nThe user wants them in that order, separated by ' | '. So the answer should be:\nK557508437185 | K434177192288 | K503557252183\n\nWait, let me double-check if there are any other checkpoint values or if these are the only ones. Scanning through... yes, these appear to be the only checkpoint entries in the entire document. They appear after reference paragraph 0008093 and before reference paragraph 0008094.\n\nSo the final answer is:\nK557508437185 | K434177192288 | K503557252183", "raw_reasoning_field": "reasoning_content", "raw_reasoning_truncated": false, "requested_tokens": 258000, "schema": "anvil-serving.context-observation/v1", "target_count": 3, "throughput_tps": 4.7497053796682245, "token_measurement": "usage", "visible_answer": "", "visible_answer_truncated": false }, { "answer_sha256": "ea77d2c40d5ae22b614345bf65faaf4a798b6926e2753eaa34d64261ffa8f4cb", "case_id": "native-order-258000-0.900-r0", "case_type": "native-order", "completed": true, "distractor_count": 0, "engine_telemetry": null, "failure": null, "finish_reason": "stop", "latency_ms": 80184.67765901005, "passed": true, "position": 0.9, "prompt_tokens": 255647, "raw_reasoning": "The user is asking for the ALPHA, BETA, and GAMMA checkpoint values in a specific format. Looking at the reference document, I can see:\n\nCheckpoint ALPHA stores K557508437185.\nCheckpoint BETA stores K434177192288.\nCheckpoint GAMMA stores K503557252183.\n\nThe user wants these in the order ALPHA, BETA, GAMMA separated by \" | \".\n\nSo the answer should be:\nK557508437185 | K434177192288 | K503557252183\n\nLet me double-check the exact values from the document:\n- Checkpoint ALPHA stores K557508437185.\n- Checkpoint BETA stores K434177192288.\n- Checkpoint GAMMA stores K503557252183.\n\nYes, that's correct. The format requested is \"ALPHA_VALUE | BETA_VALUE | GAMMA_VALUE\" but the user says \"Return the ALPHA, BETA, and GAMMA checkpoint values in that order, separated by ' | '.\"\n\nSo it should be just the values, not the labels. Let me re-read: \"Return the ALPHA, BETA, and GAMMA checkpoint values in that order, separated by ' | '.\"\n\nThis means return the values (K557508437185, K434177192288, K503557252183) in that order separated by \" | \".\n\nSo: K557508437185 | K434177192288 | K503557252183\n\nWait, I should make sure I'm reading the values correctly. Let me look again carefully:\n\nCheckpoint ALPHA stores K557508437185.\nCheckpoint BETA stores K434177192288.\nCheckpoint GAMMA stores K503557252183.\n\nYes, those are the exact strings.\n\nThe answer is: K557508437185 | K434177192288 | K503557252183", "raw_reasoning_field": "reasoning_content", "raw_reasoning_truncated": false, "requested_tokens": 258000, "schema": "anvil-serving.context-observation/v1", "target_count": 3, "throughput_tps": 5.649458390621816, "token_measurement": "usage", "visible_answer": "", "visible_answer_truncated": false }, { "answer_sha256": "5912da657e0e0bd514e4d36a4b2e2c0ff194289d275e532bbc2a3d8ad9a57f6d", "case_id": "native-identifiers-258000-0.100-r0", "case_type": "native-identifiers", "completed": true, "distractor_count": 0, "engine_telemetry": null, "failure": null, "finish_reason": "stop", "latency_ms": 118133.32542701392, "passed": true, "position": 0.1, "prompt_tokens": 255672, "raw_reasoning": "The user is asking me to return the exact identifiers in a specific format: NAME= | UUID= | NUMBER= | IP= | SYMBOL=.\n\nLooking at the reference document, I can see there's one line that contains actual data:\n\n\"Operator Iris Valen-5745 owns ticket b447c0ce-b484-38b5-c41f-9dfd2cb85f3f, numeric account 24080090411, endpoint 198.51.100.235, and code symbol resolve_orchid_13530.\"\n\nAll other paragraphs are just \"neutral cobalt ledger material for calibration.\"\n\nSo from this single line, I can extract:\n- Name: Iris Valen-5745\n- UUID: b447c0ce-b484-38b5-c41f-9dfd2cb85f3f\n- Number: 24080090411\n- IP: 198.51.100.235\n- Symbol: resolve_orchid_13530\n\nThe format requested is: NAME= | UUID= | NUMBER= | IP= | SYMBOL=\n\nSo the answer should be:\nNAME=Iris Valen-5745 | UUID=b447c0ce-b484-38b5-c41f-9dfd2cb85f3f | NUMBER=24080090411 | IP=198.51.100.235 | SYMBOL=resolve_orchid_13530", "raw_reasoning_field": "reasoning_content", "raw_reasoning_truncated": false, "requested_tokens": 258000, "schema": "anvil-serving.context-observation/v1", "target_count": 1, "throughput_tps": 3.284424599049464, "token_measurement": "usage", "visible_answer": "", "visible_answer_truncated": false }, { "answer_sha256": "5912da657e0e0bd514e4d36a4b2e2c0ff194289d275e532bbc2a3d8ad9a57f6d", "case_id": "native-identifiers-258000-0.500-r0", "case_type": "native-identifiers", "completed": true, "distractor_count": 0, "engine_telemetry": null, "failure": null, "finish_reason": "stop", "latency_ms": 88020.22957801819, "passed": true, "position": 0.5, "prompt_tokens": 255672, "raw_reasoning": "The user is asking me to return exact identifiers in a specific format: NAME= | UUID= | NUMBER= | IP= | SYMBOL=.\n\nLooking at the reference document, I need to find the relevant information. The document contains thousands of paragraphs with \"neutral cobalt ledger material for calibration\" and one specific paragraph that contains actual identifiers.\n\nThe paragraph with actual data is:\n\"Operator Iris Valen-5745 owns ticket b447c0ce-b484-38b5-c41f-9dfd2cb85f3f, numeric account 24080090411, endpoint 198.51.100.235, and code symbol resolve_orchid_13530.\"\n\nSo the identifiers are:\n- NAME: Iris Valen-5745 (the operator's name)\n- UUID: b447c0ce-b484-38b5-c41f-9dfd2cb85f3f (the ticket)\n- NUMBER: 24080090411 (the numeric account)\n- IP: 198.51.100.235 (the endpoint)\n- SYMBOL: resolve_orchid_13530 (the code symbol)\n\nWait, let me re-read the question. The user asks to return the exact identifiers in the format NAME= | UUID= | NUMBER= | IP= | SYMBOL=.\n\nThe paragraph says \"Operator Iris Valen-5745 owns ticket b447c0ce-b484-38b5-c41f-9dfd2cb85f3f, numeric account 24080090411, endpoint 198.51.100.235, and code symbol resolve_orchid_13530.\"\n\nSo:\n- NAME = Iris Valen-5745\n- UUID = b447c0ce-b484-38b5-c41f-9dfd2cb85f3f\n- NUMBER = 24080090411\n- IP = 198.51.100.235\n- SYMBOL = resolve_orchid_13530\n\nThe answer should be formatted as requested.", "raw_reasoning_field": "reasoning_content", "raw_reasoning_truncated": false, "requested_tokens": 258000, "schema": "anvil-serving.context-observation/v1", "target_count": 1, "throughput_tps": 5.816844632814554, "token_measurement": "usage", "visible_answer": "", "visible_answer_truncated": false }, { "answer_sha256": "5912da657e0e0bd514e4d36a4b2e2c0ff194289d275e532bbc2a3d8ad9a57f6d", "case_id": "native-identifiers-258000-0.900-r0", "case_type": "native-identifiers", "completed": true, "distractor_count": 0, "engine_telemetry": null, "failure": null, "finish_reason": "stop", "latency_ms": 66349.74837303162, "passed": true, "position": 0.9, "prompt_tokens": 255672, "raw_reasoning": "The user is asking me to return exact identifiers in a specific format: NAME= | UUID= | NUMBER= | IP= | SYMBOL=.\n\nLooking at the reference document, I need to find where this information is contained. Let me scan through the document...\n\nI found it! In the middle of the document, there's a line:\n\n\"Operator Iris Valen-5745 owns ticket b447c0ce-b484-38b5-c41f-9dfd2cb85f3f, numeric account 24080090411, endpoint 198.51.100.235, and code symbol resolve_orchid_13530.\"\n\nSo the identifiers are:\n- NAME: Iris Valen-5745\n- UUID: b447c0ce-b484-38b5-c41f-9dfd2cb85f3f\n- NUMBER: 24080090411\n- IP: 198.51.100.235\n- SYMBOL: resolve_orchid_13530\n\nThe user wants this in the format: NAME= | UUID= | NUMBER= | IP= | SYMBOL=\n\nSo the answer should be:\nNAME=Iris Valen-5745 | UUID=b447c0ce-b484-38b5-c41f-9dfd2cb85f3f | NUMBER=24080090411 | IP=198.51.100.235 | SYMBOL=resolve_orchid_13530", "raw_reasoning_field": "reasoning_content", "raw_reasoning_truncated": false, "requested_tokens": 258000, "schema": "anvil-serving.context-observation/v1", "target_count": 1, "throughput_tps": 5.9080857066118355, "token_measurement": "usage", "visible_answer": "", "visible_answer_truncated": false }, { "answer_sha256": "f223b260b85326cac70f0d7b2aee301abc5624d3a9e80fb9e7104bb5ffaf0389", "case_id": "native-crosslink-258000-0.100-r0", "case_type": "native-crosslink", "completed": true, "distractor_count": 0, "engine_telemetry": null, "failure": null, "finish_reason": "stop", "latency_ms": 65311.766019964125, "passed": true, "position": 0.1, "prompt_tokens": 255654, "raw_reasoning": "The user has provided a reference document and asks me to \"Follow the relationships and return exactly: ASTER -> -> -> .\"\n\nLet me look through the document for the relevant facts:\n\n1. \"Project ASTER is owned by Mara Quill-5745.\" \u2014 so ASTER's owner is Mara Quill-5745.\n\n2. \"The deployment endpoint assigned to Mara Quill-5745 is 203.0.113.45.\" \u2014 so Mara Quill-5745's endpoint is 203.0.113.45.\n\n3. \"The service at 203.0.113.45 exports code symbol commit_aster_80803.\" \u2014 so the endpoint 203.0.113.45 exports code symbol commit_aster_80803.\n\nSo the chain is:\nASTER -> Mara Quill-5745 -> 203.0.113.45 -> commit_aster_80803\n\nThe user asks to return exactly: ASTER -> -> -> \n\nSo I should fill in the placeholders:\nASTER -> Mara Quill-5745 -> 203.0.113.45 -> commit_aster_80803\n\nThat's the answer. The rest of the document is just filler \"neutral cobalt ledger material for calibration.\"", "raw_reasoning_field": "reasoning_content", "raw_reasoning_truncated": false, "requested_tokens": 258000, "schema": "anvil-serving.context-observation/v1", "target_count": 3, "throughput_tps": 4.761776000742888, "token_measurement": "usage", "visible_answer": "", "visible_answer_truncated": false }, { "answer_sha256": "f223b260b85326cac70f0d7b2aee301abc5624d3a9e80fb9e7104bb5ffaf0389", "case_id": "native-crosslink-258000-0.500-r0", "case_type": "native-crosslink", "completed": true, "distractor_count": 0, "engine_telemetry": null, "failure": null, "finish_reason": "stop", "latency_ms": 64727.28689899668, "passed": true, "position": 0.5, "prompt_tokens": 255654, "raw_reasoning": "The user is asking me to follow relationships and return exactly: ASTER -> -> -> .\n\nLooking through the reference document, I need to find:\n1. What ASTER's owner is\n2. What the endpoint is\n3. What the code symbol is\n\nFrom the document:\n- \"Project ASTER is owned by Mara Quill-5745.\" (paragraph 0008092)\n- \"The deployment endpoint assigned to Mara Quill-5745 is 203.0.113.45.\" (paragraph 0014081)\n- \"The service at 203.0.113.45 exports code symbol commit_aster_80803.\" (paragraph 0003722)\n\nSo the chain is:\nASTER -> Mara Quill-5745 -> 203.0.113.45 -> commit_aster_80803\n\nThe user wants exactly: ASTER -> -> -> .\n\nSo the answer should be:\nASTER -> Mara Quill-5745 -> 203.0.113.45 -> commit_aster_80803", "raw_reasoning_field": "reasoning_content", "raw_reasoning_truncated": false, "requested_tokens": 258000, "schema": "anvil-serving.context-observation/v1", "target_count": 3, "throughput_tps": 4.109549662031689, "token_measurement": "usage", "visible_answer": "", "visible_answer_truncated": false }, { "answer_sha256": "f223b260b85326cac70f0d7b2aee301abc5624d3a9e80fb9e7104bb5ffaf0389", "case_id": "native-crosslink-258000-0.900-r0", "case_type": "native-crosslink", "completed": true, "distractor_count": 0, "engine_telemetry": null, "failure": null, "finish_reason": "stop", "latency_ms": 65976.16025200114, "passed": true, "position": 0.9, "prompt_tokens": 255654, "raw_reasoning": "The user is asking me to follow relationships and return exactly: ASTER -> -> -> .\n\nLet me look through the reference document for the relevant information:\n\n1. \"The deployment endpoint assigned to Mara Quill-5745 is 203.0.113.45.\" - This tells me Mara Quill-5745's endpoint is 203.0.113.45.\n\n2. \"The service at 203.0.113.45 exports code symbol commit_aster_80803.\" - This tells me the service at 203.0.113.45 exports the code symbol commit_aster_80803.\n\n3. \"Project ASTER is owned by Mara Quill-5745.\" - This tells me ASTER is owned by Mara Quill-5745.\n\nSo the chain is:\n- ASTER -> owner: Mara Quill-5745\n- Mara Quill-5745 -> endpoint: 203.0.113.45\n- 203.0.113.45 -> code symbol: commit_aster_80803\n\nSo the answer would be:\nASTER -> Mara Quill-5745 -> 203.0.113.45 -> commit_aster_80803\n\nThe user wants me to return exactly: ASTER -> -> -> , substituting the actual values.\n\nSo: ASTER -> Mara Quill-5745 -> 203.0.113.45 -> commit_aster_80803", "raw_reasoning_field": "reasoning_content", "raw_reasoning_truncated": false, "requested_tokens": 258000, "schema": "anvil-serving.context-observation/v1", "target_count": 3, "throughput_tps": 5.335260473715177, "token_measurement": "usage", "visible_answer": "", "visible_answer_truncated": false } ], "throughput_tps": { "available": true, "mean": 4.68532236864959, "observations": [ 2.5527964725906584, 4.7497053796682245, 5.649458390621816, 3.284424599049464, 5.816844632814554, 5.9080857066118355, 4.761776000742888, 4.109549662031689, 5.335260473715177 ] } } ], "effective_context": 258000, "first_material_degradation": null, "notes": [ "effective_context uses attempted buckets only", "missing engine telemetry is unavailable and is not inferred from latency" ], "schema": "anvil-serving.context-curve/v1", "threshold_policy": { "baseline_bucket": 258000, "baseline_pass_rate": 1.0, "max_relative_drop": 0.3, "pass_rate_floor": 0.7 } } } }, "run": { "ownership_id": "intelligence-context-20260913", "profile": "deep", "run_id": "glm-r7-nospec-promoted-context258k-r1", "spec_sha256": "f727a43ab65c5ebd969741cfa3b5c0dccaa5f668614ffd000cf265b9a7f1bf23", "suite": "context" }, "schema": "anvil-serving.benchmark-result/v1", "status": "completed" }