{ "container_limits": { "mem_limit": 8589934592, "memswap_limit": 8589934592, "nano_cpus": 4000000000, "network_mode": "none", "pids_limit": 512 }, "created_at": "2026-10-13T22:12:12.125943Z", "dataset": "princeton-nlp/SWE-bench_Verified", "dataset_snapshot": { "files": { "test-00000-of-00001.parquet": "a45b1fe4e2f0c8390b2b2938ac83e92ed5979000856808f3679c07812e9e6dcd" }, "revision": "c104f840cc67f8b6eec6f759ebc8b2693d585d4a", "root": "[REDACTED_PATH]" }, "endpoint": { "base_url": "http://127.0.0.1:39170/v1", "model": "55a732e0c4c3d4614bc42b68493bb930d9b02c0a" }, "failure": null, "harnesses": { "agent": { "name": "mini-swe-agent", "revision": "a83fcae82d2a08f0ee0c688f9d137b3566c097f8" }, "grader": { "name": "swe-bench", "revision": "f7bbbb2ccdf479001d6467c9e34af59e44a840f9" }, "grader_containment": { "adapter_sha256": "23c56b27d62b79c39aca6dcfdd4c504030e241e3a6941620cdc6e947b9035a7b", "receipt": "[REDACTED_PATH]", "source_hashes": { "swebench/harness/docker_build.py": "5278842b60a7d38256f95f93c915dc84de2b8b4f286e9baae1b19280f768e484", "swebench/harness/run_evaluation.py": "6959f0b4e4eaf979771f529b88e3e9df1daa7fe86bc4291feec2e7d320bf7f2e" } }, "python_environment": { "architecture": "x86_64", "platform": "linux", "python": { "implementation": "CPython", "version": "3.14.4" }, "resolved_packages_sha256": "809336a2a74041fa07adad60eaca2208acd936f3728d2368d22f98657576a37a", "schema": "anvil-serving.swe-python-environment/v1" } }, "image_observations": [ { "images": [ { "image_id": "sha256:f8847eb2c89ef395c61e91c1f00166b274b2742aee6f310e3a315a10999f1e93", "instance_id": "sphinx-doc__sphinx-9658", "repo_digests": [ "swebench/sweb.eval.x86_64.sphinx-doc_1776_sphinx-9658@sha256:f8847eb2c89ef395c61e91c1f00166b274b2742aee6f310e3a315a10999f1e93" ], "tag": "swebench/sweb.eval.x86_64.sphinx-doc_1776_sphinx-9658:latest" }, { "image_id": "sha256:278dcceabbf7775dee7ee59434fd829e857d5476ac9e547fe9f9f204b2c3f4fe", "instance_id": "sympy__sympy-12096", "repo_digests": [ "swebench/sweb.eval.x86_64.sympy_1776_sympy-12096@sha256:278dcceabbf7775dee7ee59434fd829e857d5476ac9e547fe9f9f204b2c3f4fe" ], "tag": "swebench/sweb.eval.x86_64.sympy_1776_sympy-12096:latest" }, { "image_id": "sha256:730eab5b64c03ed8e8c94064da483db36fbd79d2465ffa98809acc6282bcfd82", "instance_id": "django__django-13512", "repo_digests": [ "swebench/sweb.eval.x86_64.django_1776_django-13512@sha256:730eab5b64c03ed8e8c94064da483db36fbd79d2465ffa98809acc6282bcfd82" ], "tag": "swebench/sweb.eval.x86_64.django_1776_django-13512:latest" }, { "image_id": "sha256:6783a6b3a574ea663b66d11da371b6fe00a18c6501e361e1384c8daeb978bf5c", "instance_id": "pydata__xarray-4356", "repo_digests": [ "swebench/sweb.eval.x86_64.pydata_1776_xarray-4356@sha256:6783a6b3a574ea663b66d11da371b6fe00a18c6501e361e1384c8daeb978bf5c" ], "tag": "swebench/sweb.eval.x86_64.pydata_1776_xarray-4356:latest" }, { "image_id": "sha256:a78e6034f627c1ae69c67b5a25ae5420192afb7279a4fd032a74095752e707d7", "instance_id": "pylint-dev__pylint-8898", "repo_digests": [ "swebench/sweb.eval.x86_64.pylint-dev_1776_pylint-8898@sha256:a78e6034f627c1ae69c67b5a25ae5420192afb7279a4fd032a74095752e707d7" ], "tag": "swebench/sweb.eval.x86_64.pylint-dev_1776_pylint-8898:latest" } ], "stage": "before_agent" }, { "images": [ { "image_id": "sha256:f8847eb2c89ef395c61e91c1f00166b274b2742aee6f310e3a315a10999f1e93", "instance_id": "sphinx-doc__sphinx-9658", "repo_digests": [ "swebench/sweb.eval.x86_64.sphinx-doc_1776_sphinx-9658@sha256:f8847eb2c89ef395c61e91c1f00166b274b2742aee6f310e3a315a10999f1e93" ], "tag": "swebench/sweb.eval.x86_64.sphinx-doc_1776_sphinx-9658:latest" }, { "image_id": "sha256:278dcceabbf7775dee7ee59434fd829e857d5476ac9e547fe9f9f204b2c3f4fe", "instance_id": "sympy__sympy-12096", "repo_digests": [ "swebench/sweb.eval.x86_64.sympy_1776_sympy-12096@sha256:278dcceabbf7775dee7ee59434fd829e857d5476ac9e547fe9f9f204b2c3f4fe" ], "tag": "swebench/sweb.eval.x86_64.sympy_1776_sympy-12096:latest" }, { "image_id": "sha256:730eab5b64c03ed8e8c94064da483db36fbd79d2465ffa98809acc6282bcfd82", "instance_id": "django__django-13512", "repo_digests": [ "swebench/sweb.eval.x86_64.django_1776_django-13512@sha256:730eab5b64c03ed8e8c94064da483db36fbd79d2465ffa98809acc6282bcfd82" ], "tag": "swebench/sweb.eval.x86_64.django_1776_django-13512:latest" }, { "image_id": "sha256:6783a6b3a574ea663b66d11da371b6fe00a18c6501e361e1384c8daeb978bf5c", "instance_id": "pydata__xarray-4356", "repo_digests": [ "swebench/sweb.eval.x86_64.pydata_1776_xarray-4356@sha256:6783a6b3a574ea663b66d11da371b6fe00a18c6501e361e1384c8daeb978bf5c" ], "tag": "swebench/sweb.eval.x86_64.pydata_1776_xarray-4356:latest" }, { "image_id": "sha256:a78e6034f627c1ae69c67b5a25ae5420192afb7279a4fd032a74095752e707d7", "instance_id": "pylint-dev__pylint-8898", "repo_digests": [ "swebench/sweb.eval.x86_64.pylint-dev_1776_pylint-8898@sha256:a78e6034f627c1ae69c67b5a25ae5420192afb7279a4fd032a74095752e707d7" ], "tag": "swebench/sweb.eval.x86_64.pylint-dev_1776_pylint-8898:latest" } ], "stage": "before_grader" }, { "images": [ { "image_id": "sha256:f8847eb2c89ef395c61e91c1f00166b274b2742aee6f310e3a315a10999f1e93", "instance_id": "sphinx-doc__sphinx-9658", "repo_digests": [ "swebench/sweb.eval.x86_64.sphinx-doc_1776_sphinx-9658@sha256:f8847eb2c89ef395c61e91c1f00166b274b2742aee6f310e3a315a10999f1e93" ], "tag": "swebench/sweb.eval.x86_64.sphinx-doc_1776_sphinx-9658:latest" }, { "image_id": "sha256:278dcceabbf7775dee7ee59434fd829e857d5476ac9e547fe9f9f204b2c3f4fe", "instance_id": "sympy__sympy-12096", "repo_digests": [ "swebench/sweb.eval.x86_64.sympy_1776_sympy-12096@sha256:278dcceabbf7775dee7ee59434fd829e857d5476ac9e547fe9f9f204b2c3f4fe" ], "tag": "swebench/sweb.eval.x86_64.sympy_1776_sympy-12096:latest" }, { "image_id": "sha256:730eab5b64c03ed8e8c94064da483db36fbd79d2465ffa98809acc6282bcfd82", "instance_id": "django__django-13512", "repo_digests": [ "swebench/sweb.eval.x86_64.django_1776_django-13512@sha256:730eab5b64c03ed8e8c94064da483db36fbd79d2465ffa98809acc6282bcfd82" ], "tag": "swebench/sweb.eval.x86_64.django_1776_django-13512:latest" }, { "image_id": "sha256:6783a6b3a574ea663b66d11da371b6fe00a18c6501e361e1384c8daeb978bf5c", "instance_id": "pydata__xarray-4356", "repo_digests": [ "swebench/sweb.eval.x86_64.pydata_1776_xarray-4356@sha256:6783a6b3a574ea663b66d11da371b6fe00a18c6501e361e1384c8daeb978bf5c" ], "tag": "swebench/sweb.eval.x86_64.pydata_1776_xarray-4356:latest" }, { "image_id": "sha256:a78e6034f627c1ae69c67b5a25ae5420192afb7279a4fd032a74095752e707d7", "instance_id": "pylint-dev__pylint-8898", "repo_digests": [ "swebench/sweb.eval.x86_64.pylint-dev_1776_pylint-8898@sha256:a78e6034f627c1ae69c67b5a25ae5420192afb7279a4fd032a74095752e707d7" ], "tag": "swebench/sweb.eval.x86_64.pylint-dev_1776_pylint-8898:latest" } ], "stage": "after_grader" } ], "instances": [ { "duration_s": null, "exit_status": "Submitted", "failure_class": null, "grader": { "completed": true, "name": "swe-bench", "report_sha256": "1c9ba6c446640cf2087cba5c90463f0bc304cec594567c0990df4d1684cc3451", "resolved": true, "revision": "f7bbbb2ccdf479001d6467c9e34af59e44a840f9" }, "instance_id": "sphinx-doc__sphinx-9658", "prediction_sha256": "828ec3438911e4c94acb692d497686ca900467a87f1c2d9fc92f27dcfc61148d", "prompt_identity": { "dataset": "princeton-nlp/SWE-bench_Verified", "split": "test" }, "request_ids": [ "public-request-011", "public-request-016", "public-request-033", "public-request-035", "public-request-055", "public-request-067", "public-request-069", "public-request-073", "public-request-102", "public-request-118", "public-request-122", "public-request-136", "public-request-144", "public-request-154", "public-request-157", "public-request-159", "public-request-167", "public-request-168", "public-request-177", "public-request-191", "public-request-195", "public-request-208", "public-request-217", "public-request-234", "public-request-240", "public-request-250", "public-request-252", "public-request-259", "public-request-263", "public-request-266", "public-request-269", "public-request-271", "public-request-283", "public-request-310", "public-request-313", "public-request-341" ], "tokens": { "completion_tokens": null, "prompt_tokens": null, "total_tokens": null }, "trajectory": { "path": "[REDACTED_PATH]", "sha256": "6ad149cac281a839dd554a0bce57ac6bcc37b3f93c19030cefc772d3c3fc0574" } }, { "duration_s": null, "exit_status": "Submitted", "failure_class": null, "grader": { "completed": true, "name": "swe-bench", "report_sha256": "1c9ba6c446640cf2087cba5c90463f0bc304cec594567c0990df4d1684cc3451", "resolved": true, "revision": "f7bbbb2ccdf479001d6467c9e34af59e44a840f9" }, "instance_id": "sympy__sympy-12096", "prediction_sha256": "fa0d3b57c9184ecbfc8c32ebf1c67b300d6fc6cd362997dd4d72f8760037376d", "prompt_identity": { "dataset": "princeton-nlp/SWE-bench_Verified", "split": "test" }, "request_ids": [ "public-request-101", "public-request-110", "public-request-116", "public-request-137", "public-request-184", "public-request-201", "public-request-230", "public-request-279", "public-request-290", "public-request-331" ], "tokens": { "completion_tokens": null, "prompt_tokens": null, "total_tokens": null }, "trajectory": { "path": "[REDACTED_PATH]", "sha256": "f190cf418a6c8a81affddbff70fe3f3fa81b46787f44bc152a6b07e7de53df52" } }, { "duration_s": null, "exit_status": "Submitted", "failure_class": null, "grader": { "completed": true, "name": "swe-bench", "report_sha256": "1c9ba6c446640cf2087cba5c90463f0bc304cec594567c0990df4d1684cc3451", "resolved": false, "revision": "f7bbbb2ccdf479001d6467c9e34af59e44a840f9" }, "instance_id": "django__django-13512", "prediction_sha256": "56293f34cabdd84bb662d630186316539dfef307584d4a82beca3dc638b7321c", "prompt_identity": { "dataset": "princeton-nlp/SWE-bench_Verified", "split": "test" }, "request_ids": [ "public-request-012", "public-request-020", "public-request-025", "public-request-040", "public-request-044", "public-request-048", "public-request-054", "public-request-056", "public-request-080", "public-request-095", "public-request-123", "public-request-128", "public-request-176", "public-request-186", "public-request-188", "public-request-192", "public-request-238", "public-request-270", "public-request-289", "public-request-335", "public-request-340" ], "tokens": { "completion_tokens": null, "prompt_tokens": null, "total_tokens": null }, "trajectory": { "path": "[REDACTED_PATH]", "sha256": "a403c2bd157f26d6ac46fb350d52de72172067d6dca085ce2be4963a890f9782" } }, { "duration_s": null, "exit_status": "Submitted", "failure_class": null, "grader": { "completed": true, "name": "swe-bench", "report_sha256": "1c9ba6c446640cf2087cba5c90463f0bc304cec594567c0990df4d1684cc3451", "resolved": true, "revision": "f7bbbb2ccdf479001d6467c9e34af59e44a840f9" }, "instance_id": "pydata__xarray-4356", "prediction_sha256": "b95b186b35305c32efbf8d09d9c6326b67c2e7697f9b1571c14652351fcf6f71", "prompt_identity": { "dataset": "princeton-nlp/SWE-bench_Verified", "split": "test" }, "request_ids": [ "public-request-017", "public-request-022", "public-request-028", "public-request-039", "public-request-047", "public-request-062", "public-request-082", "public-request-088", "public-request-094", "public-request-096", "public-request-098", "public-request-108", "public-request-114", "public-request-127", "public-request-134", "public-request-138", "public-request-155", "public-request-172", "public-request-189", "public-request-198", "public-request-233", "public-request-235", "public-request-262", "public-request-287", "public-request-294", "public-request-296", "public-request-300", "public-request-321" ], "tokens": { "completion_tokens": null, "prompt_tokens": null, "total_tokens": null }, "trajectory": { "path": "[REDACTED_PATH]", "sha256": "dbe5101aa89bdeb5687c28ff7b180d5be2b1af5465034b8ed7b06df2c243e488" } }, { "duration_s": null, "exit_status": "Submitted", "failure_class": null, "grader": { "completed": true, "name": "swe-bench", "report_sha256": "1c9ba6c446640cf2087cba5c90463f0bc304cec594567c0990df4d1684cc3451", "resolved": false, "revision": "f7bbbb2ccdf479001d6467c9e34af59e44a840f9" }, "instance_id": "pylint-dev__pylint-8898", "prediction_sha256": "4ff54e6250a24dd1a7550db180f45ee24cd42c3b743805559832c5b5a4c1d32c", "prompt_identity": { "dataset": "princeton-nlp/SWE-bench_Verified", "split": "test" }, "request_ids": [ "public-request-010", "public-request-038", "public-request-060", "public-request-066", "public-request-106", "public-request-160", "public-request-170", "public-request-183", "public-request-190", "public-request-204", "public-request-220", "public-request-225", "public-request-227", "public-request-245", "public-request-256", "public-request-276", "public-request-280", "public-request-285", "public-request-297", "public-request-314", "public-request-316", "public-request-323", "public-request-329", "public-request-334" ], "tokens": { "completion_tokens": null, "prompt_tokens": null, "total_tokens": null }, "trajectory": { "path": "[REDACTED_PATH]", "sha256": "b27cffd894af7b466870ecdfb51dab55896ff4984c38b01da1341ed52b05c6f2" } } ], "official_grader_complete": true, "ownership_id": "public-campaign-001", "paired_image_ids": { "django__django-13512": "sha256:730eab5b64c03ed8e8c94064da483db36fbd79d2465ffa98809acc6282bcfd82", "pydata__xarray-4356": "sha256:6783a6b3a574ea663b66d11da371b6fe00a18c6501e361e1384c8daeb978bf5c", "pylint-dev__pylint-8898": "sha256:a78e6034f627c1ae69c67b5a25ae5420192afb7279a4fd032a74095752e707d7", "sphinx-doc__sphinx-9658": "sha256:f8847eb2c89ef395c61e91c1f00166b274b2742aee6f310e3a315a10999f1e93", "sympy__sympy-12096": "sha256:278dcceabbf7775dee7ee59434fd829e857d5476ac9e547fe9f9f204b2c3f4fe" }, "plan_sha256": "209b98db8e979b38f3f7e63e35017d8be7451959f62f9c1cac3221340b49692f", "profile": "scout", "profile_sha256": "fce0e6e1e7497585bdfc6f76507d3baaff99a6b4f174d841d268de20c68de7f6", "promotion": { "authorized": false, "message": "Benchmark evidence does not authorize model promotion." }, "request_controls": { "reasoning_effort": "medium", "sampling": { "temperature": { "effective_request": 1.0, "requested": 1.0, "sent": true }, "top_p": { "effective_request": 0.95, "requested": 0.95, "sent": true } }, "thinking_mode": "default" }, "run_id": "public-run-002", "schema": "anvil-serving.swe-run/v1", "selection": { "instance_ids": [ "sphinx-doc__sphinx-9658", "sympy__sympy-12096", "django__django-13512", "pydata__xarray-4356", "pylint-dev__pylint-8898" ], "kind": "explicit_instance_ids", "sha256": "622934396fb97d950f803dfbb0791f6a398a5a6fe0fd94cc27b329f1a953cac4" }, "split": "test", "stages": [ { "command": [ "[REDACTED_PATH]", "[REDACTED_PATH]", "--output", "[REDACTED_PATH]", "--model", "openai/55a732e0c4c3d4614bc42b68493bb930d9b02c0a", "--subset", "[REDACTED_PATH]", "--split", "test", "--filter", "^(?:sphinx\\-doc__sphinx\\-9658|sympy__sympy\\-12096|django__django\\-13512|pydata__xarray\\-4356|pylint\\-dev__pylint\\-8898)$", "--workers", "1", "--environment-class", "docker", "--config", "[REDACTED_PATH]", "--config", "[REDACTED_PATH]" ], "duration_s": 862.434726, "name": "agent", "output_truncated": false, "returncode": 0, "status": "completed", "stderr": "", "stdout": "[REDACTED: private execution transcript]" }, { "command": [ "[REDACTED_PATH]", "[REDACTED_PATH]", "--grader-root", "[REDACTED_PATH]", "--adapter-sha256", "23c56b27d62b79c39aca6dcfdd4c504030e241e3a6941620cdc6e947b9035a7b", "--receipt", "[REDACTED_PATH]", "--image-identities", "[REDACTED_PATH]", "--image-identities-sha256", "e2880b3b24051b0658dd3095f7564ae592ab970e110fed76028f3a8a301f9b65", "--", "--dataset_name", "[REDACTED_PATH]", "--split", "test", "--predictions_path", "[REDACTED_PATH]", "--max_workers", "1", "--namespace", "swebench", "--run_id", "public-run-002", "--timeout", "1800", "--report_dir", "[REDACTED_PATH]", "--instance_ids", "sphinx-doc__sphinx-9658", "sympy__sympy-12096", "django__django-13512", "pydata__xarray-4356", "pylint-dev__pylint-8898" ], "duration_s": 120.8151, "name": "official_grader", "output_truncated": false, "returncode": 0, "status": "completed", "stderr": ":128: RuntimeWarning: 'swebench.harness.run_evaluation' found in sys.modules after import of package 'swebench.harness', but prior to execution of 'swebench.harness.run_evaluation'; this may result in unpredictable behaviour\n\rEvaluation: 0%| | 0/5 [00:00