feat(catalogue): offer GSM8K-Platinum as a second dataset of GSM8K

app/nullius/catalogue/benchmarks/gsm8k/benchmark.json

@@ -23,12 +23,25 @@
       "hub": "",
       "verifier": "The upstream's extract_answer and is_correct, applied to the agent's last message and the problem's reference answer.",
       "notes": "1,319 problems. MIT License."
+    },
+    {
+      "key": "platinum",
+      "title": "GSM8K-Platinum",
+      "tasks": 1209,
+      "official": "https://huggingface.co/datasets/madrylab/gsm8k-platinum",
+      "source": "Hugging Face",
+      "registry": "",
+      "hub": "",
+      "verifier": "The upstream's extract_answer and is_correct, applied to the agent's last message and the problem's reference answer.",
+      "notes": "The test split cleaned by Vendrow et al. (2025): 110 problems removed, 10 answers corrected. MIT License."
     }
   ],
-  "citation": "@article{cobbe2021gsm8k,\n  title={Training Verifiers to Solve Math Word Problems},\n  author={Cobbe, Karl and Kosaraju, Vineet and Bavarian, Mohammad and Chen, Mark and Jun, Heewoo and Kaiser, Lukasz and Plappert, Matthias and Tworek, Jerry and Hilton, Jacob and Nakano, Reiichiro and Hesse, Christopher and Schulman, John},\n  journal={arXiv preprint arXiv:2110.14168},\n  year={2021}\n}",
+  "task_count": 1319,
+  "citation": "@article{cobbe2021gsm8k,\n  title={Training Verifiers to Solve Math Word Problems},\n  author={Cobbe, Karl and Kosaraju, Vineet and Bavarian, Mohammad and Chen, Mark and Jun, Heewoo and Kaiser, Lukasz and Plappert, Matthias and Tworek, Jerry and Hilton, Jacob and Nakano, Reiichiro and Hesse, Christopher and Schulman, John},\n  journal={arXiv preprint arXiv:2110.14168},\n  year={2021}\n}\n\n@misc{vendrow2025largelanguagemodelbenchmarks,\n  title={Do Large Language Model Benchmarks Test Reliability?},\n  author={Joshua Vendrow and Edward Vendrow and Sara Beery and Aleksander Madry},\n  year={2025},\n  eprint={2502.03461},\n  archivePrefix={arXiv},\n  primaryClass={cs.LG},\n  url={https://arxiv.org/abs/2502.03461}\n}",
   "citation_source": "https://github.com/openai/grade-school-math",
   "good_to_know": [
     "Answers must end with a line '#### <number>'. The number is compared as text, so 18.00 does not match 18.",
-    "The test split is run. The Socratic variant has the same questions and answers, with different worked explanations."
+    "The test split is run. The Socratic variant has the same questions and answers, with different worked explanations.",
+    "GSM8K-Platinum is the test split with its errors cleaned. A problem has the same number in both, so the two runs compare problem by problem."
   ]
 }

app/nullius/catalogue/benchmarks/gsm8k/driver.py

@@ -1,12 +1,15 @@
 """GSM8K's driver for the Nullius standard (JSON lines on stdin and stdout).
 
-units  - one per problem of the test file, named by its line number ("0001")
+units  - one per problem of the test file, named by its line number ("0001");
+         a GSM8K-Platinum problem keeps the number of the test file's problem
+         it is (the same question, word for word), so a problem has one name
+         in both datasets and Platinum's numbers have gaps where it removed one
 setup  - the question goes in the prompt; the workspace stays empty
 score  - the upstream's own extract_answer and is_correct, imported from
          grade_school_math/dataset.py at the tested commit ($NULLIUS_UPSTREAM),
          applied to the agent's last message
-oracle - the reference control: the problem's own worked solution, the test
-         file's "answer" field, which ends in '#### <number>'
+oracle - the reference control: the problem's own worked solution, the
+         dataset file's "answer" field, which ends in '#### <number>'
 answer - the original setup: the problem asked of a model in one call through
          the unit's relay, in the format the paper's solutions end with
          ('#### <number>'), the reply scored as any agent's last message is.
@@ -16,8 +19,9 @@
          model that reasons refuses one)
 
 The upstream's module imports torch (for a training dataset class the score
-never uses), so the driver runs in the environment of its own that
-standard.json declares (driver.install), never in Nullius's.
+never uses), and Platinum is published as a parquet file, read with pyarrow, so
+the driver runs in the environment of its own that standard.json declares
+(driver.install), never in Nullius's.
 
 The reference solution is never in the workspace: it is read from the
 dataset only when the score is taken.
@@ -31,14 +35,25 @@
 import dataset as upstream                      # noqa: E402  grade_school_math/dataset.py, as published
 
 PROBLEMS = {}
+TEST = os.path.join(os.environ["NULLIUS_UPSTREAM"], "grade_school_math", "data", "test.jsonl")
+
+
+def numbered(path):
+    """The problems of a GSM8K file, each under its line number."""
+    with open(path) as f:
+        return {f"{n:04d}": json.loads(line) for n, line in enumerate(f, 1) if line.strip()}
 
 
 def load():
-    if not PROBLEMS:
-        with open(os.environ["NULLIUS_DATASET"]) as f:
-            for n, line in enumerate(f, 1):
-                if line.strip():
-                    PROBLEMS[f"{n:04d}"] = json.loads(line)
+    if PROBLEMS:
+        return PROBLEMS
+    if os.environ.get("NULLIUS_DATASET_ID") == "platinum":
+        import pyarrow.parquet as parquet       # Platinum's own format; the test file needs none
+        number = {p["question"]: n for n, p in numbered(TEST).items()}
+        for row in parquet.read_table(os.environ["NULLIUS_DATASET"]).to_pylist():
+            PROBLEMS[number[row["question"]]] = row     # a question the test file lacks is an error, never a new number
+    else:
+        PROBLEMS.update(numbered(os.environ["NULLIUS_DATASET"]))
     return PROBLEMS
 
 

app/nullius/catalogue/benchmarks/gsm8k/standard.json

@@ -20,6 +20,9 @@
     ],
     "install": {
       "python": "3.12",
+      "pip": [
+        "pyarrow==26.0.1"
+      ],
       "indexes": [
         {
           "url": "https://download.pytorch.org/whl/cpu",
@@ -37,6 +40,15 @@
       "path": "grade_school_math/data/test.jsonl",
       "sha256": "3730d312f6e3440559ace48831e51066acaca737f6eabec99bccb9e4b3c39d14",
       "units": 1319
+    },
+    {
+      "id": "platinum",
+      "label": "GSM8K-Platinum",
+      "summary": "The test split with its errors cleaned: 110 problems removed, 10 answers corrected.",
+      "url": "https://huggingface.co/datasets/madrylab/gsm8k-platinum/resolve/e762492455a1cf7967de89f05b6bef72fc713b66/main/test-00000-of-00001.parquet",
+      "sha256": "0c466762794c61ef364b7f72c3cb7e8bf9e5d8feca2c62e1bfce81eae7c7a1ea",
+      "bytes": 380973,
+      "units": 1209
     }
   ],
   "units": {
@@ -50,7 +62,7 @@
     },
     "oracle": {
       "source": "official",
-      "from": "The test file's answer field: the authors' worked solution, ending in '#### <number>'."
+      "from": "The dataset file's answer field: its authors' worked solution, ending in '#### <number>'."
     }
   },
   "score": {
@@ -93,6 +105,42 @@
         "source": "official",
         "on": "2026-10-13"
       }
+    },
+    "platinum": {
+      "units": {
+        "units": 1209,
+        "passed": "1209 ids, the same twice, unique",
+        "on": "2026-11-15"
+      },
+      "leak": {
+        "units": 1209,
+        "looked_for": 1209,
+        "short": 0,
+        "passed": "1209/1209",
+        "on": "2026-11-15"
+      },
+      "nop": {
+        "passed": "0/1209",
+        "value": 0.0,
+        "expect": {
+          "at_most": 0.0
+        },
+        "units": 1209,
+        "job": "2026-11-15__19-12-29__gsm8k__platinum__prove-nop",
+        "source": "nullius",
+        "on": "2026-11-15"
+      },
+      "oracle": {
+        "passed": "1209/1209",
+        "value": 1.0,
+        "expect": {
+          "at_least": 1.0
+        },
+        "units": 1209,
+        "job": "2026-11-15__19-12-29__gsm8k__platinum__prove-oracle",
+        "source": "official",
+        "on": "2026-11-15"
+      }
     }
   },
   "setups": {

app/nullius/catalogue/benchmarks/gsm8k/upstream-settings.json

@@ -62,6 +62,13 @@
   "default": "",
   "nullius": "not offered - not in the repository's code"
  },
+ {
+  "entry": "madrylab/gsm8k-platinum (Hugging Face)",
+  "setting": "cleaned test split",
+  "values": "main/test-00000-of-00001.parquet (1,209 problems)",
+  "default": "",
+  "nullius": "dataset:platinum"
+ },
  {
   "entry": "driver (ours)",
   "setting": "prompt",

app/tests/test_published_setups.py

@@ -37,6 +37,19 @@ def test_each_original_takes_any_model_and_is_not_the_papers(self):
             self.assertEqual(b["options"]["temperature"]["setups"], ["original"], key)
             self.assertEqual(b["options"]["temperature"]["default"], "default", key)
 
+    def test_gsm8k_offers_platinum_beside_its_test_split_pinned_and_proven(self):
+        b = definitions.get("benchmark", "gsm8k")
+        test, platinum = b["datasets"]
+        self.assertEqual((test["id"], test["units"], platinum["id"], platinum["units"]), ("test", 1319, "platinum", 1209))
+        self.assertRegex(platinum["url"], r"^https://huggingface\.co/datasets/madrylab/gsm8k-platinum/resolve/[0-9a-f]{40}/")
+        self.assertRegex(platinum["sha256"], r"^[0-9a-f]{64}$")
+        for ds in (test, platinum):
+            self.assertEqual(sorted(b["proven"][ds["id"]]), ["leak", "nop", "oracle", "units"], ds["id"])
+            self.assertEqual(b["proven"][ds["id"]]["oracle"]["units"], ds["units"])
+        cat = catalog.load()
+        self.assertEqual([(d.key, d.tasks) for d in cat.datasets_of(cat.benchmark("gsm8k"))], [("test", 1319), ("platinum", 1209)])
+        self.assertEqual(cat.tasks_of(cat.benchmark("gsm8k")), ("1,319", "tasks"))       # one set of problems, not their sum
+
     def test_a_setup_that_says_comparable_must_say_it_as_a_boolean(self):
         b = definitions.get("benchmark", "gsm8k")
         with open(os.path.join(b["_here"], "standard.json")) as f:

app/tests/test_standard_upstream_drivers.py

@@ -168,6 +168,37 @@ def test_the_verdict_is_the_upstreams_is_correct(self):
         self.assertEqual(said, theirs)
         self.assertTrue(said[0][0] and said[1][1] and not said[0][3])
 
+    def test_a_platinum_problem_keeps_its_number_in_the_test_file(self):
+        got = _installed("gsm8k")
+        if not got:
+            self.skipTest("GSM8K's upstream and driver environment are not installed (cli prove gsm8k)")
+        b, up, drv = got
+        tmp = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, tmp, True)
+        with open(os.path.join(up, "grade_school_math", "data", "test.jsonl")) as f:
+            test = [json.loads(line) for line in f]
+
+        def platinum(name, rows):
+            """A file in Platinum's format, written by the driver's own pyarrow."""
+            path = os.path.join(tmp, name)
+            _upstream_says(drv, up, "import json, sys, pyarrow as pa, pyarrow.parquet as pq\npath, rows = json.loads(sys.argv[1])\n"
+                                    "pq.write_table(pa.Table.from_pylist(rows), path)\nprint(1)", [path, rows])
+            return path
+
+        kept = [{"question": test[2]["question"], "answer": "revised\n#### 77", "cleaning_status": "revised"},
+                {"question": test[4]["question"], "answer": test[4]["answer"], "cleaning_status": "consensus"}]
+        p = _driver(self, b, up, platinum("kept.parquet", kept), NULLIUS_DATASET_ID="platinum")
+        self.assertEqual(p.request("units", dataset="platinum")["units"], ["0003", "0005"])      # the test file's numbers, gaps and all
+        self.assertIn(test[2]["question"], p.request("setup", unit="0003", dir=tmp)["prompt"])
+        self.assertTrue(p.request("score", unit="0003", dir=tmp, transcript=[], answer="#### 77")["passed"])     # Platinum's answer, not the test file's
+        self.assertEqual(p.request("oracle", unit="0003", dir=tmp)["answer"], "revised\n#### 77")
+        with self.assertRaises(jsonl.Failed):                                                    # a problem Platinum removed is not a unit
+            p.request("score", unit="0001", dir=tmp, transcript=[], answer="#### 18")
+        other = _driver(self, b, up, platinum("other.parquet", [dict(kept[0], question="a question the test file lacks")]),
+                        NULLIUS_DATASET_ID="platinum")
+        with self.assertRaises(jsonl.Failed):                                                    # never a number of its own
+            other.request("units", dataset="platinum")
+
 
 class HumanEvalNotTested(unittest.TestCase):
     """The workspace files solution.py imports are named beside the verdict, which stays the check's on

app/ui_tests/test_agents.py

@@ -543,7 +543,11 @@ def test_a_benchmark_page_shows_its_proofs_as_plain_chips_their_source_in_the_ti
         self.assertIn("<b>stable</b>", panel); self.assertIn("<b>no leak</b>", panel)
         self.assertIn('action="/benchmarks/gsm8k/prove"', panel)
         self.assertIn("No model calls or key required.", panel)
-        self.assertIn('type="hidden" name="dataset" value="test"', panel)
+        self.assertIn('<select name="dataset"><option value="">All datasets</option><option value="test">GSM8K test</option>'
+                      '<option value="platinum">GSM8K-Platinum</option></select>', panel)      # two datasets: a choice
+        html = self.get("/benchmarks/humaneval").get_data(as_text=True)
+        panel = html[html.index('id="standard"'):html.index('id="datasets"')]
+        self.assertIn('type="hidden" name="dataset" value="humaneval"', panel)                # a single one: a fixed value
         self.assertNotIn('<select name="dataset">', panel)
 
     def test_prove_it_starts_cli_prove_here_handed_no_key(self):

docs/benchmarks/supported/gsm8k.md

@@ -20,6 +20,7 @@ Grade-school math word problems that take two to eight steps of basic arithmetic
 | Dataset | Key | Tasks | From | Verifier |
 |---|---|---|---|---|
 | GSM8K test | `test` | 1319 | GitHub [openai/grade-school-math](https://github.com/openai/grade-school-math/tree/3101c7d5072418e28b9008a6636bde82a006892c) | The upstream's extract_answer and is_correct, applied to the agent's last message and the problem's reference answer. |
+| GSM8K-Platinum | `platinum` | 1209 | Hugging Face [madrylab/gsm8k-platinum](https://huggingface.co/datasets/madrylab/gsm8k-platinum/tree/e762492455a1cf7967de89f05b6bef72fc713b66) | The upstream's extract_answer and is_correct, applied to the agent's last message and the problem's reference answer. |
 
 ## On the standard
 
@@ -44,6 +45,7 @@ Grade-school math word problems that take two to eight steps of basic arithmetic
 
 - Answers must end with a line '#### <number>'. The number is compared as text, so 18.00 does not match 18.
 - The test split is run. The Socratic variant has the same questions and answers, with different worked explanations.
+- GSM8K-Platinum is the test split with its errors cleaned. A problem has the same number in both, so the two runs compare problem by problem.
 
 ## Nullius limitations
 
@@ -61,4 +63,14 @@ Grade-school math word problems that take two to eight steps of basic arithmetic
       journal={arXiv preprint arXiv:2110.14168},
       year={2021}
     }
+
+    @misc{vendrow2025largelanguagemodelbenchmarks,
+      title={Do Large Language Model Benchmarks Test Reliability?},
+      author={Joshua Vendrow and Edward Vendrow and Sara Beery and Aleksander Madry},
+      year={2025},
+      eprint={2502.03461},
+      archivePrefix={arXiv},
+      primaryClass={cs.LG},
+      url={https://arxiv.org/abs/2502.03461}
+    }
     ```