{
  "aggregates": {
    "case": {
      "eight-images-ordered": {
        "attempts": 2,
        "latency_p50_seconds": 7.734804499999999,
        "latency_p95_seconds": 8.975055,
        "pass_rate": 1.0,
        "passed": 2
      },
      "eight-images-reversed": {
        "attempts": 2,
        "latency_p50_seconds": 7.907716000000001,
        "latency_p95_seconds": 9.548457,
        "pass_rate": 1.0,
        "passed": 2
      }
    },
    "modality": {
      "image": {
        "attempts": 4,
        "latency_p50_seconds": 7.734804499999999,
        "latency_p95_seconds": 9.548457,
        "pass_rate": 1.0,
        "passed": 4
      }
    }
  },
  "attempts": [
    {
      "case_id": "eight-images-ordered",
      "failure": null,
      "finish_reason": "stop",
      "media": [
        {
          "bytes": 8679,
          "kind": "image",
          "mime": "image/png",
          "path": "card-A.png",
          "sha256": "e02b7e290b35c966e27a0a4eb5ca9928555155409700510e5caf669956a7a3ee"
        },
        {
          "bytes": 9861,
          "kind": "image",
          "mime": "image/png",
          "path": "card-B.png",
          "sha256": "fffe6b1b1f5819e0aacd015cb0bbbc1b43037c744238ea6ac3d8a0ea8ac2bff6"
        },
        {
          "bytes": 9214,
          "kind": "image",
          "mime": "image/png",
          "path": "card-C.png",
          "sha256": "6c8253ac2e97def51f996475ae4ce5436f5e69b4aaf8f96b410a9c0af9aeb799"
        },
        {
          "bytes": 9970,
          "kind": "image",
          "mime": "image/png",
          "path": "card-D.png",
          "sha256": "a7882977d87bd1e59d84029c34da4f41c2f16adfc4b65f16a08ad973c10e9154"
        },
        {
          "bytes": 9969,
          "kind": "image",
          "mime": "image/png",
          "path": "card-E.png",
          "sha256": "8e7ffb4c775589abcbc5e4507aed01a9d966fd635b1dc3cff87801155a73385f"
        },
        {
          "bytes": 8930,
          "kind": "image",
          "mime": "image/png",
          "path": "card-F.png",
          "sha256": "56b90ce7cd4edce9c9c3425135a2db924fcb9a822c4088ef6d4af692632b2de4"
        },
        {
          "bytes": 10211,
          "kind": "image",
          "mime": "image/png",
          "path": "card-G.png",
          "sha256": "87c4b85945a40b630f8d8e7548f6a55153f750904086e13aed1f99e2830de4e9"
        },
        {
          "bytes": 9622,
          "kind": "image",
          "mime": "image/png",
          "path": "card-H.png",
          "sha256": "6387a21bea8cb2f91a29f4139fcc79424b819b1b5c0e00f49a1c5cef6584bdd8"
        }
      ],
      "modality": "image",
      "output": "\n\nCARD A: VALUE 17\nCARD B: VALUE 42\nCARD C: VALUE 9\nCARD D: VALUE 63\nCARD E: VALUE 28\nCARD F: VALUE 51\nCARD G: VALUE 34\nCARD H: VALUE 76\nMAX H 76\nMIN C 9\nSUM 320",
      "passed": true,
      "reasoning": {
        "chars": 914,
        "field": "reasoning",
        "tokens": 453
      },
      "repetition": 1,
      "sampling": null,
      "seconds": 8.975055,
      "usage": {
        "completion_tokens": 535,
        "completion_tokens_details": {
          "reasoning_tokens": 453
        },
        "prompt_tokens": 3200,
        "prompt_tokens_details": null,
        "total_tokens": 3735
      },
      "validation": {
        "assertions": [
          {
            "passed": true,
            "type": "ordered_casefold",
            "values": [
              "CARD A",
              "17",
              "CARD B",
              "42",
              "CARD C",
              "9",
              "CARD D",
              "63",
              "CARD E",
              "28",
              "CARD F",
              "51",
              "CARD G",
              "34",
              "CARD H",
              "76"
            ]
          },
          {
            "passed": true,
            "type": "contains_casefold",
            "value": "320"
          },
          {
            "passed": true,
            "type": "contains_casefold",
            "value": "MAX H 76"
          },
          {
            "passed": true,
            "type": "contains_casefold",
            "value": "MIN C 9"
          }
        ],
        "finish_reason_allowed": true,
        "visible_answer": true
      }
    },
    {
      "case_id": "eight-images-ordered",
      "failure": null,
      "finish_reason": "stop",
      "media": [
        {
          "bytes": 8679,
          "kind": "image",
          "mime": "image/png",
          "path": "card-A.png",
          "sha256": "e02b7e290b35c966e27a0a4eb5ca9928555155409700510e5caf669956a7a3ee"
        },
        {
          "bytes": 9861,
          "kind": "image",
          "mime": "image/png",
          "path": "card-B.png",
          "sha256": "fffe6b1b1f5819e0aacd015cb0bbbc1b43037c744238ea6ac3d8a0ea8ac2bff6"
        },
        {
          "bytes": 9214,
          "kind": "image",
          "mime": "image/png",
          "path": "card-C.png",
          "sha256": "6c8253ac2e97def51f996475ae4ce5436f5e69b4aaf8f96b410a9c0af9aeb799"
        },
        {
          "bytes": 9970,
          "kind": "image",
          "mime": "image/png",
          "path": "card-D.png",
          "sha256": "a7882977d87bd1e59d84029c34da4f41c2f16adfc4b65f16a08ad973c10e9154"
        },
        {
          "bytes": 9969,
          "kind": "image",
          "mime": "image/png",
          "path": "card-E.png",
          "sha256": "8e7ffb4c775589abcbc5e4507aed01a9d966fd635b1dc3cff87801155a73385f"
        },
        {
          "bytes": 8930,
          "kind": "image",
          "mime": "image/png",
          "path": "card-F.png",
          "sha256": "56b90ce7cd4edce9c9c3425135a2db924fcb9a822c4088ef6d4af692632b2de4"
        },
        {
          "bytes": 10211,
          "kind": "image",
          "mime": "image/png",
          "path": "card-G.png",
          "sha256": "87c4b85945a40b630f8d8e7548f6a55153f750904086e13aed1f99e2830de4e9"
        },
        {
          "bytes": 9622,
          "kind": "image",
          "mime": "image/png",
          "path": "card-H.png",
          "sha256": "6387a21bea8cb2f91a29f4139fcc79424b819b1b5c0e00f49a1c5cef6584bdd8"
        }
      ],
      "modality": "image",
      "output": "\n\nCARD A: VALUE 17\nCARD B: VALUE 42\nCARD C: VALUE 9\nCARD D: VALUE 63\nCARD E: VALUE 28\nCARD F: VALUE 51\nCARD G: VALUE 34\nCARD H: VALUE 76\nMAX H 76\nMIN C 9\nSUM 320",
      "passed": true,
      "reasoning": {
        "chars": 723,
        "field": "reasoning",
        "tokens": 325
      },
      "repetition": 2,
      "sampling": null,
      "seconds": 6.494554,
      "usage": {
        "completion_tokens": 407,
        "completion_tokens_details": {
          "reasoning_tokens": 325
        },
        "prompt_tokens": 3200,
        "prompt_tokens_details": null,
        "total_tokens": 3607
      },
      "validation": {
        "assertions": [
          {
            "passed": true,
            "type": "ordered_casefold",
            "values": [
              "CARD A",
              "17",
              "CARD B",
              "42",
              "CARD C",
              "9",
              "CARD D",
              "63",
              "CARD E",
              "28",
              "CARD F",
              "51",
              "CARD G",
              "34",
              "CARD H",
              "76"
            ]
          },
          {
            "passed": true,
            "type": "contains_casefold",
            "value": "320"
          },
          {
            "passed": true,
            "type": "contains_casefold",
            "value": "MAX H 76"
          },
          {
            "passed": true,
            "type": "contains_casefold",
            "value": "MIN C 9"
          }
        ],
        "finish_reason_allowed": true,
        "visible_answer": true
      }
    },
    {
      "case_id": "eight-images-reversed",
      "failure": null,
      "finish_reason": "stop",
      "media": [
        {
          "bytes": 9622,
          "kind": "image",
          "mime": "image/png",
          "path": "card-H.png",
          "sha256": "6387a21bea8cb2f91a29f4139fcc79424b819b1b5c0e00f49a1c5cef6584bdd8"
        },
        {
          "bytes": 10211,
          "kind": "image",
          "mime": "image/png",
          "path": "card-G.png",
          "sha256": "87c4b85945a40b630f8d8e7548f6a55153f750904086e13aed1f99e2830de4e9"
        },
        {
          "bytes": 8930,
          "kind": "image",
          "mime": "image/png",
          "path": "card-F.png",
          "sha256": "56b90ce7cd4edce9c9c3425135a2db924fcb9a822c4088ef6d4af692632b2de4"
        },
        {
          "bytes": 9969,
          "kind": "image",
          "mime": "image/png",
          "path": "card-E.png",
          "sha256": "8e7ffb4c775589abcbc5e4507aed01a9d966fd635b1dc3cff87801155a73385f"
        },
        {
          "bytes": 9970,
          "kind": "image",
          "mime": "image/png",
          "path": "card-D.png",
          "sha256": "a7882977d87bd1e59d84029c34da4f41c2f16adfc4b65f16a08ad973c10e9154"
        },
        {
          "bytes": 9214,
          "kind": "image",
          "mime": "image/png",
          "path": "card-C.png",
          "sha256": "6c8253ac2e97def51f996475ae4ce5436f5e69b4aaf8f96b410a9c0af9aeb799"
        },
        {
          "bytes": 9861,
          "kind": "image",
          "mime": "image/png",
          "path": "card-B.png",
          "sha256": "fffe6b1b1f5819e0aacd015cb0bbbc1b43037c744238ea6ac3d8a0ea8ac2bff6"
        },
        {
          "bytes": 8679,
          "kind": "image",
          "mime": "image/png",
          "path": "card-A.png",
          "sha256": "e02b7e290b35c966e27a0a4eb5ca9928555155409700510e5caf669956a7a3ee"
        }
      ],
      "modality": "image",
      "output": "\n\nCARD H: VALUE 76\nCARD G: VALUE 34\nCARD F: VALUE 51\nCARD E: VALUE 28\nCARD D: VALUE 63\nCARD C: VALUE 9\nCARD B: VALUE 42\nCARD A: VALUE 17\nMAX H 76\nMIN C 9\nSUM 320",
      "passed": true,
      "reasoning": {
        "chars": 1124,
        "field": "reasoning",
        "tokens": 495
      },
      "repetition": 1,
      "sampling": null,
      "seconds": 9.548457,
      "usage": {
        "completion_tokens": 577,
        "completion_tokens_details": {
          "reasoning_tokens": 495
        },
        "prompt_tokens": 3200,
        "prompt_tokens_details": null,
        "total_tokens": 3777
      },
      "validation": {
        "assertions": [
          {
            "passed": true,
            "type": "ordered_casefold",
            "values": [
              "CARD H",
              "76",
              "CARD G",
              "34",
              "CARD F",
              "51",
              "CARD E",
              "28",
              "CARD D",
              "63",
              "CARD C",
              "9",
              "CARD B",
              "42",
              "CARD A",
              "17"
            ]
          },
          {
            "passed": true,
            "type": "contains_casefold",
            "value": "320"
          },
          {
            "passed": true,
            "type": "contains_casefold",
            "value": "MAX H 76"
          },
          {
            "passed": true,
            "type": "contains_casefold",
            "value": "MIN C 9"
          }
        ],
        "finish_reason_allowed": true,
        "visible_answer": true
      }
    },
    {
      "case_id": "eight-images-reversed",
      "failure": null,
      "finish_reason": "stop",
      "media": [
        {
          "bytes": 9622,
          "kind": "image",
          "mime": "image/png",
          "path": "card-H.png",
          "sha256": "6387a21bea8cb2f91a29f4139fcc79424b819b1b5c0e00f49a1c5cef6584bdd8"
        },
        {
          "bytes": 10211,
          "kind": "image",
          "mime": "image/png",
          "path": "card-G.png",
          "sha256": "87c4b85945a40b630f8d8e7548f6a55153f750904086e13aed1f99e2830de4e9"
        },
        {
          "bytes": 8930,
          "kind": "image",
          "mime": "image/png",
          "path": "card-F.png",
          "sha256": "56b90ce7cd4edce9c9c3425135a2db924fcb9a822c4088ef6d4af692632b2de4"
        },
        {
          "bytes": 9969,
          "kind": "image",
          "mime": "image/png",
          "path": "card-E.png",
          "sha256": "8e7ffb4c775589abcbc5e4507aed01a9d966fd635b1dc3cff87801155a73385f"
        },
        {
          "bytes": 9970,
          "kind": "image",
          "mime": "image/png",
          "path": "card-D.png",
          "sha256": "a7882977d87bd1e59d84029c34da4f41c2f16adfc4b65f16a08ad973c10e9154"
        },
        {
          "bytes": 9214,
          "kind": "image",
          "mime": "image/png",
          "path": "card-C.png",
          "sha256": "6c8253ac2e97def51f996475ae4ce5436f5e69b4aaf8f96b410a9c0af9aeb799"
        },
        {
          "bytes": 9861,
          "kind": "image",
          "mime": "image/png",
          "path": "card-B.png",
          "sha256": "fffe6b1b1f5819e0aacd015cb0bbbc1b43037c744238ea6ac3d8a0ea8ac2bff6"
        },
        {
          "bytes": 8679,
          "kind": "image",
          "mime": "image/png",
          "path": "card-A.png",
          "sha256": "e02b7e290b35c966e27a0a4eb5ca9928555155409700510e5caf669956a7a3ee"
        }
      ],
      "modality": "image",
      "output": "\n\nCARD H: VALUE 76\nCARD G: VALUE 34\nCARD F: VALUE 51\nCARD E: VALUE 28\nCARD D: VALUE 63\nCARD C: VALUE 9\nCARD B: VALUE 42\nCARD A: VALUE 17\nMAX H 76\nMIN C 9\nSUM 320",
      "passed": true,
      "reasoning": {
        "chars": 643,
        "field": "reasoning",
        "tokens": 309
      },
      "repetition": 2,
      "sampling": null,
      "seconds": 6.266975,
      "usage": {
        "completion_tokens": 391,
        "completion_tokens_details": {
          "reasoning_tokens": 309
        },
        "prompt_tokens": 3200,
        "prompt_tokens_details": null,
        "total_tokens": 3591
      },
      "validation": {
        "assertions": [
          {
            "passed": true,
            "type": "ordered_casefold",
            "values": [
              "CARD H",
              "76",
              "CARD G",
              "34",
              "CARD F",
              "51",
              "CARD E",
              "28",
              "CARD D",
              "63",
              "CARD C",
              "9",
              "CARD B",
              "42",
              "CARD A",
              "17"
            ]
          },
          {
            "passed": true,
            "type": "contains_casefold",
            "value": "320"
          },
          {
            "passed": true,
            "type": "contains_casefold",
            "value": "MAX H 76"
          },
          {
            "passed": true,
            "type": "contains_casefold",
            "value": "MIN C 9"
          }
        ],
        "finish_reason_allowed": true,
        "visible_answer": true
      }
    }
  ],
  "configuration": {
    "allowed_finish_reasons": [
      "stop"
    ],
    "chat_template_kwargs": null,
    "concurrency": 1,
    "max_images_per_request": 8,
    "max_tokens": 8192,
    "max_videos_per_request": 1,
    "reasoning_effort": null,
    "sampling": {
      "temperature": {
        "effective_request": 1.0,
        "requested": 1.0,
        "sent": true
      },
      "top_p": {
        "effective_request": 0.95,
        "requested": 0.95,
        "sent": true
      }
    },
    "thinking_mode": "default",
    "timeout_seconds": 300.0
  },
  "corpus": {
    "cases": [
      {
        "assertions": [
          {
            "type": "ordered_casefold",
            "values": [
              "CARD A",
              "17",
              "CARD B",
              "42",
              "CARD C",
              "9",
              "CARD D",
              "63",
              "CARD E",
              "28",
              "CARD F",
              "51",
              "CARD G",
              "34",
              "CARD H",
              "76"
            ]
          },
          {
            "type": "contains_casefold",
            "value": "320"
          },
          {
            "type": "contains_casefold",
            "value": "MAX H 76"
          },
          {
            "type": "contains_casefold",
            "value": "MIN C 9"
          }
        ],
        "id": "eight-images-ordered",
        "media": [
          {
            "bytes": 8679,
            "kind": "image",
            "mime": "image/png",
            "path": "card-A.png",
            "sha256": "e02b7e290b35c966e27a0a4eb5ca9928555155409700510e5caf669956a7a3ee"
          },
          {
            "bytes": 9861,
            "kind": "image",
            "mime": "image/png",
            "path": "card-B.png",
            "sha256": "fffe6b1b1f5819e0aacd015cb0bbbc1b43037c744238ea6ac3d8a0ea8ac2bff6"
          },
          {
            "bytes": 9214,
            "kind": "image",
            "mime": "image/png",
            "path": "card-C.png",
            "sha256": "6c8253ac2e97def51f996475ae4ce5436f5e69b4aaf8f96b410a9c0af9aeb799"
          },
          {
            "bytes": 9970,
            "kind": "image",
            "mime": "image/png",
            "path": "card-D.png",
            "sha256": "a7882977d87bd1e59d84029c34da4f41c2f16adfc4b65f16a08ad973c10e9154"
          },
          {
            "bytes": 9969,
            "kind": "image",
            "mime": "image/png",
            "path": "card-E.png",
            "sha256": "8e7ffb4c775589abcbc5e4507aed01a9d966fd635b1dc3cff87801155a73385f"
          },
          {
            "bytes": 8930,
            "kind": "image",
            "mime": "image/png",
            "path": "card-F.png",
            "sha256": "56b90ce7cd4edce9c9c3425135a2db924fcb9a822c4088ef6d4af692632b2de4"
          },
          {
            "bytes": 10211,
            "kind": "image",
            "mime": "image/png",
            "path": "card-G.png",
            "sha256": "87c4b85945a40b630f8d8e7548f6a55153f750904086e13aed1f99e2830de4e9"
          },
          {
            "bytes": 9622,
            "kind": "image",
            "mime": "image/png",
            "path": "card-H.png",
            "sha256": "6387a21bea8cb2f91a29f4139fcc79424b819b1b5c0e00f49a1c5cef6584bdd8"
          }
        ],
        "modality": "image",
        "prompt": "Read all eight images. Transcribe each in the supplied image order, exactly one line formatted CARD <letter>: VALUE <number>. Then add three lines: MAX <letter> <value>, MIN <letter> <value>, SUM <total>. Compute these across all eight images. No other text.",
        "repetitions": 2,
        "sampling": null
      },
      {
        "assertions": [
          {
            "type": "ordered_casefold",
            "values": [
              "CARD H",
              "76",
              "CARD G",
              "34",
              "CARD F",
              "51",
              "CARD E",
              "28",
              "CARD D",
              "63",
              "CARD C",
              "9",
              "CARD B",
              "42",
              "CARD A",
              "17"
            ]
          },
          {
            "type": "contains_casefold",
            "value": "320"
          },
          {
            "type": "contains_casefold",
            "value": "MAX H 76"
          },
          {
            "type": "contains_casefold",
            "value": "MIN C 9"
          }
        ],
        "id": "eight-images-reversed",
        "media": [
          {
            "bytes": 9622,
            "kind": "image",
            "mime": "image/png",
            "path": "card-H.png",
            "sha256": "6387a21bea8cb2f91a29f4139fcc79424b819b1b5c0e00f49a1c5cef6584bdd8"
          },
          {
            "bytes": 10211,
            "kind": "image",
            "mime": "image/png",
            "path": "card-G.png",
            "sha256": "87c4b85945a40b630f8d8e7548f6a55153f750904086e13aed1f99e2830de4e9"
          },
          {
            "bytes": 8930,
            "kind": "image",
            "mime": "image/png",
            "path": "card-F.png",
            "sha256": "56b90ce7cd4edce9c9c3425135a2db924fcb9a822c4088ef6d4af692632b2de4"
          },
          {
            "bytes": 9969,
            "kind": "image",
            "mime": "image/png",
            "path": "card-E.png",
            "sha256": "8e7ffb4c775589abcbc5e4507aed01a9d966fd635b1dc3cff87801155a73385f"
          },
          {
            "bytes": 9970,
            "kind": "image",
            "mime": "image/png",
            "path": "card-D.png",
            "sha256": "a7882977d87bd1e59d84029c34da4f41c2f16adfc4b65f16a08ad973c10e9154"
          },
          {
            "bytes": 9214,
            "kind": "image",
            "mime": "image/png",
            "path": "card-C.png",
            "sha256": "6c8253ac2e97def51f996475ae4ce5436f5e69b4aaf8f96b410a9c0af9aeb799"
          },
          {
            "bytes": 9861,
            "kind": "image",
            "mime": "image/png",
            "path": "card-B.png",
            "sha256": "fffe6b1b1f5819e0aacd015cb0bbbc1b43037c744238ea6ac3d8a0ea8ac2bff6"
          },
          {
            "bytes": 8679,
            "kind": "image",
            "mime": "image/png",
            "path": "card-A.png",
            "sha256": "e02b7e290b35c966e27a0a4eb5ca9928555155409700510e5caf669956a7a3ee"
          }
        ],
        "modality": "image",
        "prompt": "Read all eight images. Transcribe each in the supplied image order, exactly one line formatted CARD <letter>: VALUE <number>. Then add three lines: MAX <letter> <value>, MIN <letter> <value>, SUM <total>. Compute these across all eight images. No other text.",
        "repetitions": 2,
        "sampling": null
      }
    ],
    "path": "/source/docs/findings/2026-09-18-glm53-vision-evidence/eight-images/corpus.json",
    "provenance": {
      "scope": "Eight distinct cards, ordered and reversed, independent exact OCR ordering and sum assertions.",
      "synthetic": {
        "license": "CC0-1.0"
      }
    },
    "schema": "multimodal-corpus/v1",
    "sha256": "a3c84fbd456b1824bb2bc2383a8043efe6376b39d4ca2db831b8e02e09c096c3"
  },
  "failed_attempts": 0,
  "finished_at": "2026-09-26T14:30:30Z",
  "identity": {
    "base_url": "http://127.0.0.1:39144/v1",
    "engine_build_ref": null,
    "engine_revision": "98dff2a81d747d1dba01a47f939f48c3526d4206",
    "hardware": "1x RTX PRO 6000 Blackwell Max-Q; GPU0; co-resident other-GPU SWE; correctness only",
    "model": "thinkingcap27-awq-327k-apc-r1",
    "model_revision": "f8fe157f207a13f977bc3d620ce10a3e9ba5ab11",
    "runtime_image": "vllm/vllm-openai:v0.29.0@sha256:082ca6f035279109041ffd3fe0695cb568b29bc580b35c4f297a66a08b216c1b",
    "served_models": [
      "thinkingcap27-awq-327k-apc-r1"
    ]
  },
  "passed": true,
  "passed_attempts": 4,
  "requests": 4,
  "run_id": "multimodal-20260926T142959Z",
  "schema": "multimodal-benchmark-evidence/v1",
  "started_at": "2026-09-26T14:29:59Z"
}