From ae06cc9717d6f2320fbbdb0b4af4135d75b56172 Mon Sep 17 00:00:00 2001 From: Etienne Filhol Date: Sun, 15 Nov 2026 19:37:58 +0100 Subject: [PATCH] feat(catalogue): offer GSM8K-Platinum as a second dataset of GSM8K GSM8K-Platinum is the test split with its errors cleaned (Vendrow et al., 2025): 110 problems removed, 10 answers corrected. It runs beside the original split so the same model can be scored on each. The file is pinned to a revision of madrylab/gsm8k-platinum and its sha256. A Platinum problem keeps the number of the test file's problem it is, found by its question, which is the same word for word in both, so the two datasets compare problem by problem. The check is unchanged: the upstream's extract_answer and is_correct. The driver's environment gains pyarrow, to read Platinum's parquet file. --- .../catalogue/benchmarks/gsm8k/benchmark.json | 17 ++++++- .../catalogue/benchmarks/gsm8k/driver.py | 35 +++++++++---- .../catalogue/benchmarks/gsm8k/standard.json | 50 ++++++++++++++++++- .../benchmarks/gsm8k/upstream-settings.json | 7 +++ app/tests/test_published_setups.py | 13 +++++ app/tests/test_standard_upstream_drivers.py | 31 ++++++++++++ app/ui_tests/test_agents.py | 6 ++- docs/benchmarks/supported/gsm8k.md | 12 +++++ 8 files changed, 157 insertions(+), 14 deletions(-) diff --git a/app/nullius/catalogue/benchmarks/gsm8k/benchmark.json b/app/nullius/catalogue/benchmarks/gsm8k/benchmark.json index d7ed83c..d85fd3e 100644 --- a/app/nullius/catalogue/benchmarks/gsm8k/benchmark.json +++ b/app/nullius/catalogue/benchmarks/gsm8k/benchmark.json @@ -23,12 +23,25 @@ "hub": "", "verifier": "The upstream's extract_answer and is_correct, applied to the agent's last message and the problem's reference answer.", "notes": "1,319 problems. MIT License." + }, + { + "key": "platinum", + "title": "GSM8K-Platinum", + "tasks": 1209, + "official": "https://huggingface.co/datasets/madrylab/gsm8k-platinum", + "source": "Hugging Face", + "registry": "", + "hub": "", + "verifier": "The upstream's extract_answer and is_correct, applied to the agent's last message and the problem's reference answer.", + "notes": "The test split cleaned by Vendrow et al. (2025): 110 problems removed, 10 answers corrected. MIT License." } ], - "citation": "@article{cobbe2021gsm8k,\n title={Training Verifiers to Solve Math Word Problems},\n author={Cobbe, Karl and Kosaraju, Vineet and Bavarian, Mohammad and Chen, Mark and Jun, Heewoo and Kaiser, Lukasz and Plappert, Matthias and Tworek, Jerry and Hilton, Jacob and Nakano, Reiichiro and Hesse, Christopher and Schulman, John},\n journal={arXiv preprint arXiv:2110.14168},\n year={2021}\n}", + "task_count": 1319, + "citation": "@article{cobbe2021gsm8k,\n title={Training Verifiers to Solve Math Word Problems},\n author={Cobbe, Karl and Kosaraju, Vineet and Bavarian, Mohammad and Chen, Mark and Jun, Heewoo and Kaiser, Lukasz and Plappert, Matthias and Tworek, Jerry and Hilton, Jacob and Nakano, Reiichiro and Hesse, Christopher and Schulman, John},\n journal={arXiv preprint arXiv:2110.14168},\n year={2021}\n}\n\n@misc{vendrow2025largelanguagemodelbenchmarks,\n title={Do Large Language Model Benchmarks Test Reliability?},\n author={Joshua Vendrow and Edward Vendrow and Sara Beery and Aleksander Madry},\n year={2025},\n eprint={2502.03461},\n archivePrefix={arXiv},\n primaryClass={cs.LG},\n url={https://arxiv.org/abs/2502.03461}\n}", "citation_source": "https://github.com/openai/grade-school-math", "good_to_know": [ "Answers must end with a line '#### '. The number is compared as text, so 18.00 does not match 18.", - "The test split is run. The Socratic variant has the same questions and answers, with different worked explanations." + "The test split is run. The Socratic variant has the same questions and answers, with different worked explanations.", + "GSM8K-Platinum is the test split with its errors cleaned. A problem has the same number in both, so the two runs compare problem by problem." ] } diff --git a/app/nullius/catalogue/benchmarks/gsm8k/driver.py b/app/nullius/catalogue/benchmarks/gsm8k/driver.py index 5e5d287..1296eb3 100644 --- a/app/nullius/catalogue/benchmarks/gsm8k/driver.py +++ b/app/nullius/catalogue/benchmarks/gsm8k/driver.py @@ -1,12 +1,15 @@ """GSM8K's driver for the Nullius standard (JSON lines on stdin and stdout). -units - one per problem of the test file, named by its line number ("0001") +units - one per problem of the test file, named by its line number ("0001"); + a GSM8K-Platinum problem keeps the number of the test file's problem + it is (the same question, word for word), so a problem has one name + in both datasets and Platinum's numbers have gaps where it removed one setup - the question goes in the prompt; the workspace stays empty score - the upstream's own extract_answer and is_correct, imported from grade_school_math/dataset.py at the tested commit ($NULLIUS_UPSTREAM), applied to the agent's last message -oracle - the reference control: the problem's own worked solution, the test - file's "answer" field, which ends in '#### ' +oracle - the reference control: the problem's own worked solution, the + dataset file's "answer" field, which ends in '#### ' answer - the original setup: the problem asked of a model in one call through the unit's relay, in the format the paper's solutions end with ('#### '), the reply scored as any agent's last message is. @@ -16,8 +19,9 @@ model that reasons refuses one) The upstream's module imports torch (for a training dataset class the score -never uses), so the driver runs in the environment of its own that -standard.json declares (driver.install), never in Nullius's. +never uses), and Platinum is published as a parquet file, read with pyarrow, so +the driver runs in the environment of its own that standard.json declares +(driver.install), never in Nullius's. The reference solution is never in the workspace: it is read from the dataset only when the score is taken. @@ -31,14 +35,25 @@ import dataset as upstream # noqa: E402 grade_school_math/dataset.py, as published PROBLEMS = {} +TEST = os.path.join(os.environ["NULLIUS_UPSTREAM"], "grade_school_math", "data", "test.jsonl") + + +def numbered(path): + """The problems of a GSM8K file, each under its line number.""" + with open(path) as f: + return {f"{n:04d}": json.loads(line) for n, line in enumerate(f, 1) if line.strip()} def load(): - if not PROBLEMS: - with open(os.environ["NULLIUS_DATASET"]) as f: - for n, line in enumerate(f, 1): - if line.strip(): - PROBLEMS[f"{n:04d}"] = json.loads(line) + if PROBLEMS: + return PROBLEMS + if os.environ.get("NULLIUS_DATASET_ID") == "platinum": + import pyarrow.parquet as parquet # Platinum's own format; the test file needs none + number = {p["question"]: n for n, p in numbered(TEST).items()} + for row in parquet.read_table(os.environ["NULLIUS_DATASET"]).to_pylist(): + PROBLEMS[number[row["question"]]] = row # a question the test file lacks is an error, never a new number + else: + PROBLEMS.update(numbered(os.environ["NULLIUS_DATASET"])) return PROBLEMS diff --git a/app/nullius/catalogue/benchmarks/gsm8k/standard.json b/app/nullius/catalogue/benchmarks/gsm8k/standard.json index d76a7ca..1525942 100644 --- a/app/nullius/catalogue/benchmarks/gsm8k/standard.json +++ b/app/nullius/catalogue/benchmarks/gsm8k/standard.json @@ -20,6 +20,9 @@ ], "install": { "python": "3.12", + "pip": [ + "pyarrow==26.0.1" + ], "indexes": [ { "url": "https://download.pytorch.org/whl/cpu", @@ -37,6 +40,15 @@ "path": "grade_school_math/data/test.jsonl", "sha256": "3730d312f6e3440559ace48831e51066acaca737f6eabec99bccb9e4b3c39d14", "units": 1319 + }, + { + "id": "platinum", + "label": "GSM8K-Platinum", + "summary": "The test split with its errors cleaned: 110 problems removed, 10 answers corrected.", + "url": "https://huggingface.co/datasets/madrylab/gsm8k-platinum/resolve/e762492455a1cf7967de89f05b6bef72fc713b66/main/test-00000-of-00001.parquet", + "sha256": "0c466762794c61ef364b7f72c3cb7e8bf9e5d8feca2c62e1bfce81eae7c7a1ea", + "bytes": 380973, + "units": 1209 } ], "units": { @@ -50,7 +62,7 @@ }, "oracle": { "source": "official", - "from": "The test file's answer field: the authors' worked solution, ending in '#### '." + "from": "The dataset file's answer field: its authors' worked solution, ending in '#### '." } }, "score": { @@ -93,6 +105,42 @@ "source": "official", "on": "2026-10-13" } + }, + "platinum": { + "units": { + "units": 1209, + "passed": "1209 ids, the same twice, unique", + "on": "2026-11-15" + }, + "leak": { + "units": 1209, + "looked_for": 1209, + "short": 0, + "passed": "1209/1209", + "on": "2026-11-15" + }, + "nop": { + "passed": "0/1209", + "value": 0.0, + "expect": { + "at_most": 0.0 + }, + "units": 1209, + "job": "2026-11-15__19-12-29__gsm8k__platinum__prove-nop", + "source": "nullius", + "on": "2026-11-15" + }, + "oracle": { + "passed": "1209/1209", + "value": 1.0, + "expect": { + "at_least": 1.0 + }, + "units": 1209, + "job": "2026-11-15__19-12-29__gsm8k__platinum__prove-oracle", + "source": "official", + "on": "2026-11-15" + } } }, "setups": { diff --git a/app/nullius/catalogue/benchmarks/gsm8k/upstream-settings.json b/app/nullius/catalogue/benchmarks/gsm8k/upstream-settings.json index cc0bdf2..3372485 100644 --- a/app/nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +++ b/app/nullius/catalogue/benchmarks/gsm8k/upstream-settings.json @@ -62,6 +62,13 @@ "default": "", "nullius": "not offered - not in the repository's code" }, + { + "entry": "madrylab/gsm8k-platinum (Hugging Face)", + "setting": "cleaned test split", + "values": "main/test-00000-of-00001.parquet (1,209 problems)", + "default": "", + "nullius": "dataset:platinum" + }, { "entry": "driver (ours)", "setting": "prompt", diff --git a/app/tests/test_published_setups.py b/app/tests/test_published_setups.py index af86d1d..e8abf44 100644 --- a/app/tests/test_published_setups.py +++ b/app/tests/test_published_setups.py @@ -37,6 +37,19 @@ def test_each_original_takes_any_model_and_is_not_the_papers(self): self.assertEqual(b["options"]["temperature"]["setups"], ["original"], key) self.assertEqual(b["options"]["temperature"]["default"], "default", key) + def test_gsm8k_offers_platinum_beside_its_test_split_pinned_and_proven(self): + b = definitions.get("benchmark", "gsm8k") + test, platinum = b["datasets"] + self.assertEqual((test["id"], test["units"], platinum["id"], platinum["units"]), ("test", 1319, "platinum", 1209)) + self.assertRegex(platinum["url"], r"^https://huggingface\.co/datasets/madrylab/gsm8k-platinum/resolve/[0-9a-f]{40}/") + self.assertRegex(platinum["sha256"], r"^[0-9a-f]{64}$") + for ds in (test, platinum): + self.assertEqual(sorted(b["proven"][ds["id"]]), ["leak", "nop", "oracle", "units"], ds["id"]) + self.assertEqual(b["proven"][ds["id"]]["oracle"]["units"], ds["units"]) + cat = catalog.load() + self.assertEqual([(d.key, d.tasks) for d in cat.datasets_of(cat.benchmark("gsm8k"))], [("test", 1319), ("platinum", 1209)]) + self.assertEqual(cat.tasks_of(cat.benchmark("gsm8k")), ("1,319", "tasks")) # one set of problems, not their sum + def test_a_setup_that_says_comparable_must_say_it_as_a_boolean(self): b = definitions.get("benchmark", "gsm8k") with open(os.path.join(b["_here"], "standard.json")) as f: diff --git a/app/tests/test_standard_upstream_drivers.py b/app/tests/test_standard_upstream_drivers.py index e33890a..79eabc1 100644 --- a/app/tests/test_standard_upstream_drivers.py +++ b/app/tests/test_standard_upstream_drivers.py @@ -168,6 +168,37 @@ def test_the_verdict_is_the_upstreams_is_correct(self): self.assertEqual(said, theirs) self.assertTrue(said[0][0] and said[1][1] and not said[0][3]) + def test_a_platinum_problem_keeps_its_number_in_the_test_file(self): + got = _installed("gsm8k") + if not got: + self.skipTest("GSM8K's upstream and driver environment are not installed (cli prove gsm8k)") + b, up, drv = got + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp, True) + with open(os.path.join(up, "grade_school_math", "data", "test.jsonl")) as f: + test = [json.loads(line) for line in f] + + def platinum(name, rows): + """A file in Platinum's format, written by the driver's own pyarrow.""" + path = os.path.join(tmp, name) + _upstream_says(drv, up, "import json, sys, pyarrow as pa, pyarrow.parquet as pq\npath, rows = json.loads(sys.argv[1])\n" + "pq.write_table(pa.Table.from_pylist(rows), path)\nprint(1)", [path, rows]) + return path + + kept = [{"question": test[2]["question"], "answer": "revised\n#### 77", "cleaning_status": "revised"}, + {"question": test[4]["question"], "answer": test[4]["answer"], "cleaning_status": "consensus"}] + p = _driver(self, b, up, platinum("kept.parquet", kept), NULLIUS_DATASET_ID="platinum") + self.assertEqual(p.request("units", dataset="platinum")["units"], ["0003", "0005"]) # the test file's numbers, gaps and all + self.assertIn(test[2]["question"], p.request("setup", unit="0003", dir=tmp)["prompt"]) + self.assertTrue(p.request("score", unit="0003", dir=tmp, transcript=[], answer="#### 77")["passed"]) # Platinum's answer, not the test file's + self.assertEqual(p.request("oracle", unit="0003", dir=tmp)["answer"], "revised\n#### 77") + with self.assertRaises(jsonl.Failed): # a problem Platinum removed is not a unit + p.request("score", unit="0001", dir=tmp, transcript=[], answer="#### 18") + other = _driver(self, b, up, platinum("other.parquet", [dict(kept[0], question="a question the test file lacks")]), + NULLIUS_DATASET_ID="platinum") + with self.assertRaises(jsonl.Failed): # never a number of its own + other.request("units", dataset="platinum") + class HumanEvalNotTested(unittest.TestCase): """The workspace files solution.py imports are named beside the verdict, which stays the check's on diff --git a/app/ui_tests/test_agents.py b/app/ui_tests/test_agents.py index 5dd9cf0..356b156 100644 --- a/app/ui_tests/test_agents.py +++ b/app/ui_tests/test_agents.py @@ -543,7 +543,11 @@ def test_a_benchmark_page_shows_its_proofs_as_plain_chips_their_source_in_the_ti self.assertIn("stable", panel); self.assertIn("no leak", panel) self.assertIn('action="/benchmarks/gsm8k/prove"', panel) self.assertIn("No model calls or key required.", panel) - self.assertIn('type="hidden" name="dataset" value="test"', panel) + self.assertIn('', panel) # two datasets: a choice + html = self.get("/benchmarks/humaneval").get_data(as_text=True) + panel = html[html.index('id="standard"'):html.index('id="datasets"')] + self.assertIn('type="hidden" name="dataset" value="humaneval"', panel) # a single one: a fixed value self.assertNotIn('