From 39a24dd5593f06a63e8cb7e11de43ac34003edd1 Mon Sep 17 00:00:00 2001 From: skishchampi <996985+skishchampi@users.noreply.github.com> Date: Mon, 17 Aug 2026 14:08:49 -0400 Subject: [PATCH 01/10] fix: doctor reads every pin form the org actually uses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found by running it against the seven live consumer files during a /pm recompute. It reported ONE pin where seven exist. Two assumptions were wrong. The requirement was anchored to the start of a line, so a `pyproject.toml` writing it quoted inside a dependency list read as no pin at all. And extras were not allowed, so `commoner-probe[http,pdf]==0.14.3` — the form four of the seven use — matched nothing. A file that names the package with no exact version now reports `unpinned` rather than nothing. The org pins with == or @vX.Y.Z, because a range moves under the consumer without anyone deciding to move it, and filing that beside the files that never mention the package hid it. The census this now produces: six files on 0.14.3 across five repos, one on v0.14.9, none on 0.15.0. Five tests, each written from a form measured in a real consumer file. --- commoner_probe/doctor.py | 28 ++++++++++++++++++++++++---- tests/test_doctor.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 4 deletions(-) diff --git a/commoner_probe/doctor.py b/commoner_probe/doctor.py index d6381c9..966d002 100644 --- a/commoner_probe/doctor.py +++ b/commoner_probe/doctor.py @@ -29,13 +29,24 @@ __all__ = ["VersionReport", "declared_pins", "installed_version", "source_version", "version_report"] -#: Requirement lines that pin this package, in the two forms the org uses: an -#: exact `==` pin from PyPI, and a git URL pinned to a tag. +#: Every form the org's consumers actually use, measured against the seven live +#: pin files on 2026-08-17. The requirement is NOT anchored to the start of a +#: line, because a `pyproject.toml` writes it quoted inside a dependency list, +#: and extras are optional, because four of the seven carry them +#: (`commoner-probe[http,pdf]==0.14.3`). The first reader required both and found +#: one pin where three existed. +_EXTRAS = r"(?:\[[^\]]*\])?" _PIN_PATTERNS = ( - re.compile(r"^\s*commoner[-_]probe\s*==\s*([0-9][^\s;#]*)", re.I | re.MULTILINE), - re.compile(r"commoner-probe(?:\.git)?@v?([0-9][^\s;#\"']*)", re.I), + re.compile(rf"commoner[-_]probe{_EXTRAS}\s*==\s*([0-9][^\s;#,\"']*)", re.I), + re.compile(rf"commoner-probe(?:\.git)?{_EXTRAS}@v?([0-9][^\s;#\"']*)", re.I), ) +#: The package named as a requirement with no exact version. The org requires an +#: exact pin, so this is a finding rather than an absence — reporting nothing +#: filed it beside the files that never mention the package at all. +_UNPINNED = re.compile( + rf"^\s*[\"']?commoner[-_]probe{_EXTRAS}\s*(?:[<>~!=]=|@|$)", re.I | re.MULTILINE) + class VersionReport: """The three versions, and whether they agree. @@ -62,6 +73,12 @@ def mismatches(self) -> list[str]: "this tree; one venv shared across worktrees reports whichever was " "installed last.") for where, pin in self.pins.items(): + if pin == "unpinned": + out.append( + f"{where} names this package with no exact version. The org pins " + "with == or @vX.Y.Z, because a range moves under the consumer " + "without anyone deciding to move it.") + continue if self.installed and pin != self.installed: out.append( f"{where} pins {pin} and the environment runs {self.installed}. A " @@ -136,6 +153,9 @@ def declared_pins(*paths: Path | str) -> dict[str, str]: if match: found[str(p)] = match.group(1) break + else: + if _UNPINNED.search(text): + found[str(p)] = "unpinned" return found diff --git a/tests/test_doctor.py b/tests/test_doctor.py index 8fd135a..7a17835 100644 --- a/tests/test_doctor.py +++ b/tests/test_doctor.py @@ -112,3 +112,39 @@ def test_this_checkout_and_this_environment_agree(self): assert report.source is not None assert report.installed is not None assert report.agrees, report.report + + +class TestEveryPinFormTheOrgActuallyUses: + """Measured against the seven live consumer files on 2026-08-17. The first + version of this reader found ONE pin where three existed, because it required + the requirement to start the line and carry no extras.""" + + def test_it_reads_a_pin_carrying_extras(self, tmp_path): + path = tmp_path / "requirements.txt" + path.write_text("commoner-probe[http,pdf]==0.14.3\n", encoding="utf-8") + assert declared_pins(path) == {str(path): "0.14.3"} + + def test_it_reads_a_pin_inside_a_pyproject_dependency_list(self, tmp_path): + path = tmp_path / "pyproject.toml" + path.write_text('[project]\ndependencies = [\n "commoner-probe==0.14.3",\n]\n', + encoding="utf-8") + assert declared_pins(path) == {str(path): "0.14.3"} + + def test_it_reads_a_pin_with_extras_inside_a_pyproject(self, tmp_path): + path = tmp_path / "pyproject.toml" + path.write_text('dependencies = ["commoner-probe[budget]==0.15.0"]\n', encoding="utf-8") + assert declared_pins(path) == {str(path): "0.15.0"} + + def test_a_file_that_names_the_package_and_pins_no_version_is_reported(self, tmp_path): + """Unpinned is not unmentioned. The org requires an exact pin, so a + consumer that depends on this package without one is a finding, and + returning nothing hid it among the files that never mention it.""" + path = tmp_path / "requirements.txt" + path.write_text("commoner-probe>=0.14\n", encoding="utf-8") + assert declared_pins(path) == {str(path): "unpinned"} + + def test_a_comment_naming_the_package_is_not_a_pin(self, tmp_path): + path = tmp_path / "requirements.txt" + path.write_text("# acquisition is delegated to commoner-probe\nrequests==2.32.0\n", + encoding="utf-8") + assert declared_pins(path) == {} From 1fdaa836fe11cc8b8446cb32f5de5d8fb57a03ac Mon Sep 17 00:00:00 2001 From: skishchampi <996985+skishchampi@users.noreply.github.com> Date: Mon, 17 Aug 2026 14:16:49 -0400 Subject: [PATCH 02/10] fix: a comment is not a pin, and an inline range is not a pin either MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex P2s on this PR, both caused by unanchoring the pattern. `search()` returns the first occurrence, so a commented old pin above an active one won. `doctor` then reported a mismatch that does not exist and exited 1. Comment tails are now removed before any pattern runs, treating `#` as a comment where it opens a line or follows whitespace — the form both requirements files and TOML use. The unpinned test kept its start-of-line anchor, so the valid compact form `dependencies = ["commoner-probe>=0.14"]` was unreachable and a violated exact-pin policy exited successfully. It is now unanchored like the pin patterns, which is safe because comments are already gone by then. Re-run against the seven live consumer files: still seven pins, unchanged. 1731 passed, ruff clean. Four tests, three of which failed first. --- commoner_probe/doctor.py | 20 +++++++++++++++++--- tests/test_doctor.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/commoner_probe/doctor.py b/commoner_probe/doctor.py index 966d002..e24c48d 100644 --- a/commoner_probe/doctor.py +++ b/commoner_probe/doctor.py @@ -43,9 +43,22 @@ #: The package named as a requirement with no exact version. The org requires an #: exact pin, so this is a finding rather than an absence — reporting nothing -#: filed it beside the files that never mention the package at all. -_UNPINNED = re.compile( - rf"^\s*[\"']?commoner[-_]probe{_EXTRAS}\s*(?:[<>~!=]=|@|$)", re.I | re.MULTILINE) +#: filed it beside the files that never mention the package at all. Unanchored, +#: like the pin patterns, because the compact TOML form +#: `dependencies = ["commoner-probe>=0.14"]` is valid and a start-of-line test +#: could not reach it: a violated pin policy then exited successfully. +_UNPINNED = re.compile(rf"commoner[-_]probe{_EXTRAS}\s*(?:[<>~!]=|<|>|@|[\"',\]\s]|$)", re.I) + + +def _uncommented(text: str) -> str: + """The text with comment tails removed. + + `search()` returns the FIRST occurrence, so a commented old pin above an + active one won: `doctor` reported a mismatch that did not exist and exited 1. + A `#` counts as a comment when it opens a line or follows whitespace, which + is how both requirements files and TOML write one. + """ + return re.sub(r"(?m)(?:^|(?<=\s))#.*$", "", text) class VersionReport: @@ -148,6 +161,7 @@ def declared_pins(*paths: Path | str) -> dict[str, str]: text = p.read_text(encoding="utf-8") except OSError: continue + text = _uncommented(text) for pattern in _PIN_PATTERNS: match = pattern.search(text) if match: diff --git a/tests/test_doctor.py b/tests/test_doctor.py index 7a17835..93483da 100644 --- a/tests/test_doctor.py +++ b/tests/test_doctor.py @@ -148,3 +148,34 @@ def test_a_comment_naming_the_package_is_not_a_pin(self, tmp_path): path.write_text("# acquisition is delegated to commoner-probe\nrequests==2.32.0\n", encoding="utf-8") assert declared_pins(path) == {} + + +class TestCommentsAndInlineRanges: + """Two ways the reader read the wrong thing, both from Codex on the fix that + made the pattern unanchored.""" + + def test_a_commented_pin_does_not_win_over_the_active_one(self, tmp_path): + """`search()` takes the first occurrence, so a commented old pin above an + active one made `doctor` report a mismatch that does not exist and exit 1.""" + path = tmp_path / "requirements.txt" + path.write_text("# commoner-probe==0.14.3\ncommoner-probe==0.15.0\n", + encoding="utf-8") + assert declared_pins(path) == {str(path): "0.15.0"} + + def test_a_file_holding_only_a_commented_pin_reports_nothing(self, tmp_path): + path = tmp_path / "requirements.txt" + path.write_text("# commoner-probe==0.14.3 (dropped)\nrequests==2.32.0\n", + encoding="utf-8") + assert declared_pins(path) == {} + + def test_an_inline_toml_range_is_reported_unpinned(self, tmp_path): + """The compact form is valid TOML, and the start-of-line test could not + reach it, so a violated exact-pin policy exited successfully.""" + path = tmp_path / "pyproject.toml" + path.write_text('dependencies = ["commoner-probe>=0.14"]\n', encoding="utf-8") + assert declared_pins(path) == {str(path): "unpinned"} + + def test_a_commented_range_is_not_reported_unpinned(self, tmp_path): + path = tmp_path / "requirements.txt" + path.write_text("# commoner-probe>=0.14 was the old floor\n", encoding="utf-8") + assert declared_pins(path) == {} From 37ff06839c208366c679cab8abf8d22ecded500a Mon Sep 17 00:00:00 2001 From: skishchampi <996985+skishchampi@users.noreply.github.com> Date: Mon, 17 Aug 2026 19:39:25 -0400 Subject: [PATCH 03/10] fix: the name must be this package, and prose is not a dependency my-commoner-probe==9.9.9 matched at the inner substring and was read as this package's pin. The patterns now require a name boundary. The unpinned pattern accepted any closing quote, so description = "built on commoner-probe" reported a file as unpinned. It now needs a range operator after the name, or the name as the whole requirement token. All seven live consumer pins still read. --- commoner_probe/doctor.py | 21 +++++++++++++++++---- tests/test_doctor.py | 25 +++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/commoner_probe/doctor.py b/commoner_probe/doctor.py index e24c48d..29a37bf 100644 --- a/commoner_probe/doctor.py +++ b/commoner_probe/doctor.py @@ -35,10 +35,15 @@ #: and extras are optional, because four of the seven carry them #: (`commoner-probe[http,pdf]==0.14.3`). The first reader required both and found #: one pin where three existed. +#: `my-commoner-probe==9.9.9` is a different package. Without a boundary the +#: pattern started matching at the inner substring and reported 9.9.9 as this +#: package's pin, so `doctor` failed a consumer that never depended on it. `/` +#: stays legal on the left, because a git URL carries one. _EXTRAS = r"(?:\[[^\]]*\])?" +_NAME = rf"(?=0.14"]` is valid and a start-of-line test #: could not reach it: a violated pin policy then exited successfully. -_UNPINNED = re.compile(rf"commoner[-_]probe{_EXTRAS}\s*(?:[<>~!]=|<|>|@|[\"',\]\s]|$)", re.I) +#: +#: Two shapes, and no third. A range operator follows the name, or the name IS +#: the whole requirement token. Accepting any closing quote made +#: `description = "built on commoner-probe"` a dependency, and prose is not a +#: declaration. +_UNPINNED_PATTERNS = ( + re.compile(rf"{_NAME}\s*(?:[<>~!]=|[<>@])", re.I), + re.compile(rf"(?:^|[\"'])\s*{_NAME}\s*(?:[\"']|$)", re.I | re.MULTILINE), +) def _uncommented(text: str) -> str: @@ -168,7 +181,7 @@ def declared_pins(*paths: Path | str) -> dict[str, str]: found[str(p)] = match.group(1) break else: - if _UNPINNED.search(text): + if any(p.search(text) for p in _UNPINNED_PATTERNS): found[str(p)] = "unpinned" return found diff --git a/tests/test_doctor.py b/tests/test_doctor.py index 93483da..67e7c64 100644 --- a/tests/test_doctor.py +++ b/tests/test_doctor.py @@ -179,3 +179,28 @@ def test_a_commented_range_is_not_reported_unpinned(self, tmp_path): path = tmp_path / "requirements.txt" path.write_text("# commoner-probe>=0.14 was the old floor\n", encoding="utf-8") assert declared_pins(path) == {} + + +class TestTheNameMustBeThisPackage: + """Both from Codex on the unanchored patterns. A false pin and a false + `unpinned` each make `doctor` exit 1 over a file that is correct, which + teaches a reader to stop believing it.""" + + def test_a_different_package_whose_name_ends_in_ours_is_not_our_pin(self, tmp_path): + path = tmp_path / "requirements.txt" + path.write_text("my-commoner-probe==9.9.9\n", encoding="utf-8") + assert declared_pins(path) == {} + + def test_prose_naming_the_package_is_not_a_dependency(self, tmp_path): + """`description = "built on commoner-probe"` declares nothing. The + closing quote satisfied the unpinned pattern.""" + path = tmp_path / "pyproject.toml" + path.write_text('[project]\ndescription = "built on commoner-probe"\n' + 'dependencies = ["requests==2.32.3"]\n', encoding="utf-8") + assert declared_pins(path) == {} + + def test_a_bare_dependency_with_no_version_is_still_unpinned(self, tmp_path): + """The narrowing must not lose the case it exists for.""" + path = tmp_path / "pyproject.toml" + path.write_text('dependencies = ["commoner-probe"]\n', encoding="utf-8") + assert declared_pins(path) == {str(path): "unpinned"} From e95d3ad8fe6a3c5efc47bc52740e22f4b90cd592 Mon Sep 17 00:00:00 2001 From: skishchampi <996985+skishchampi@users.noreply.github.com> Date: Mon, 17 Aug 2026 20:17:26 -0400 Subject: [PATCH 04/10] fix: a pin is a requirement token, not any text that reads like one The exact-pin pattern was left unanchored when the unpinned ones were narrowed, so `description = "built for commoner-probe==9.9.9"` reported 9.9.9 for a project that depends on nothing of ours. A reader sent to fix that pin finds no pin to fix. Both pin forms now have to open a line or a quoted element of a list. Running the reader over the eight live pin files found the second half. This package's own `pyproject.toml` read as unpinned: `name =` names the package, and so does the console-script key. Neither declares a dependency, so a bare name now counts only as a whole line or a whole quoted list element. The seven live consumer pins still read: 0.14.3 six times and 0.14.9 once. --- commoner_probe/doctor.py | 43 ++++++++++++++++++++++++-------------- tests/test_doctor.py | 45 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 16 deletions(-) diff --git a/commoner_probe/doctor.py b/commoner_probe/doctor.py index 29a37bf..9fd7243 100644 --- a/commoner_probe/doctor.py +++ b/commoner_probe/doctor.py @@ -30,20 +30,29 @@ "version_report"] #: Every form the org's consumers actually use, measured against the seven live -#: pin files on 2026-08-17. The requirement is NOT anchored to the start of a -#: line, because a `pyproject.toml` writes it quoted inside a dependency list, -#: and extras are optional, because four of the seven carry them +#: pin files on 2026-08-17. The requirement does not have to start the line, +#: because a `pyproject.toml` writes it quoted inside a dependency list, and +#: extras are optional, because four of the seven carry them #: (`commoner-probe[http,pdf]==0.14.3`). The first reader required both and found #: one pin where three existed. -#: `my-commoner-probe==9.9.9` is a different package. Without a boundary the -#: pattern started matching at the inner substring and reported 9.9.9 as this -#: package's pin, so `doctor` failed a consumer that never depended on it. `/` -#: stays legal on the left, because a git URL carries one. +#: It must still open a requirement token. `my-commoner-probe==9.9.9` is a +#: different package, and `description = "built for commoner-probe==9.9.9"` is +#: prose; each was read as this package's pin, and `doctor` then failed a +#: consumer that never depended on it. _EXTRAS = r"(?:\[[^\]]*\])?" -_NAME = rf"(?=0.14"]` is valid and a start-of-line test #: could not reach it: a violated pin policy then exited successfully. #: -#: Two shapes, and no third. A range operator follows the name, or the name IS -#: the whole requirement token. Accepting any closing quote made -#: `description = "built on commoner-probe"` a dependency, and prose is not a -#: declaration. +#: Three shapes, and no fourth. A range operator follows the name, or the name +#: is the whole line, or the name is a whole quoted element of a list. Accepting +#: any closing quote made `description = "built on commoner-probe"` a dependency +#: and `commoner-probe = "commoner_probe.cli:main"` a dependency, and neither +#: prose nor an entry-point key declares one. _UNPINNED_PATTERNS = ( - re.compile(rf"{_NAME}\s*(?:[<>~!]=|[<>@])", re.I), - re.compile(rf"(?:^|[\"'])\s*{_NAME}\s*(?:[\"']|$)", re.I | re.MULTILINE), + re.compile(rf"{_TOKEN}{_NAME}\s*(?:[<>~!]=|[<>@])", re.I | re.MULTILINE), + re.compile(rf"^\s*{_NAME}\s*$", re.I | re.MULTILINE), + re.compile(rf"[\[,]\s*[\"']\s*{_NAME}\s*[\"']", re.I | re.MULTILINE), ) diff --git a/tests/test_doctor.py b/tests/test_doctor.py index 67e7c64..c5f962a 100644 --- a/tests/test_doctor.py +++ b/tests/test_doctor.py @@ -204,3 +204,48 @@ def test_a_bare_dependency_with_no_version_is_still_unpinned(self, tmp_path): path = tmp_path / "pyproject.toml" path.write_text('dependencies = ["commoner-probe"]\n', encoding="utf-8") assert declared_pins(path) == {str(path): "unpinned"} + + +class TestProseIsNotAnExactPin: + """From Codex on `761c483`: only the unpinned patterns were narrowed to a + requirement token, so version-like prose was still read as a pin.""" + + def test_a_version_inside_prose_is_not_a_pin(self, tmp_path): + """`doctor` reported 9.9.9 for a project that depends on nothing of ours, + and a reader sent to fix that pin finds no pin to fix.""" + path = tmp_path / "pyproject.toml" + path.write_text('[project]\ndescription = "built for commoner-probe==9.9.9"\n' + 'dependencies = ["requests==2.32.3"]\n', encoding="utf-8") + assert declared_pins(path) == {} + + def test_the_pin_wins_over_prose_naming_another_version(self, tmp_path): + path = tmp_path / "pyproject.toml" + path.write_text('[project]\ndescription = "built for commoner-probe==9.9.9"\n' + 'dependencies = ["commoner-probe[http]==0.15.0"]\n', encoding="utf-8") + assert declared_pins(path) == {str(path): "0.15.0"} + + def test_a_git_url_pin_still_reads(self, tmp_path): + """The narrowing must not lose the second form the org uses: the name + sits inside a URL, so `/` stays legal on the left.""" + path = tmp_path / "requirements.txt" + path.write_text("commoner-probe @ git+https://github.com/CommonerLLP/" + "commoner-probe.git@v0.15.0\n", encoding="utf-8") + assert declared_pins(path) == {str(path): "0.15.0"} + + def test_the_package_s_own_metadata_is_not_a_dependency_on_itself(self, tmp_path): + """Found by running the reader over the eight live pin files. This + repo's own `pyproject.toml` was reported `unpinned`: `name =` and a + console-script key both name the package, and neither declares a + dependency.""" + path = tmp_path / "pyproject.toml" + path.write_text('[project]\nname = "commoner-probe"\ndependencies = ["requests==2.32.3"]\n' + '[project.scripts]\ncommoner-probe = "commoner_probe.cli:main"\n', + encoding="utf-8") + assert declared_pins(path) == {} + + def test_a_dependency_list_entry_is_still_reached(self, tmp_path): + """The narrowing must not lose either shape of the list.""" + path = tmp_path / "pyproject.toml" + path.write_text('[project]\nname = "consumer"\ndependencies = [\n' + ' "requests==2.32.3",\n "commoner-probe",\n]\n', encoding="utf-8") + assert declared_pins(path) == {str(path): "unpinned"} From d378b2a1330f55d978402ab441d58c9b81ce5c68 Mon Sep 17 00:00:00 2001 From: skishchampi <996985+skishchampi@users.noreply.github.com> Date: Mon, 17 Aug 2026 20:22:18 -0400 Subject: [PATCH 05/10] fix: only a dependency array declares a dependency, and a marker is not a pin Two findings from the review of the token narrowing. A quoted list element was read as a requirement wherever it sat, so `keywords = ["commoner-probe==9.9.9"]` reported a pin for a project that depends on nothing of ours. In a `.toml` file the search now runs inside dependency arrays only: `dependencies`, `requires`, and every key of an optional-dependency or dependency-group table. A requirements file is a list of requirements, so all of it still counts. `commoner-probe; python_version < "3.12"` matched nothing, so a file that violates the exact-pin policy passed the check. A marker now ends a bare requirement the way a line end does. The seven live consumer pins still read. --- commoner_probe/doctor.py | 54 +++++++++++++++++++++++++++++++++++++--- tests/test_doctor.py | 42 +++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 4 deletions(-) diff --git a/commoner_probe/doctor.py b/commoner_probe/doctor.py index 9fd7243..f667e49 100644 --- a/commoner_probe/doctor.py +++ b/commoner_probe/doctor.py @@ -69,10 +69,53 @@ #: prose nor an entry-point key declares one. _UNPINNED_PATTERNS = ( re.compile(rf"{_TOKEN}{_NAME}\s*(?:[<>~!]=|[<>@])", re.I | re.MULTILINE), - re.compile(rf"^\s*{_NAME}\s*$", re.I | re.MULTILINE), - re.compile(rf"[\[,]\s*[\"']\s*{_NAME}\s*[\"']", re.I | re.MULTILINE), + re.compile(rf"^\s*{_NAME}\s*(?:;|$)", re.I | re.MULTILINE), + re.compile(rf"[\[,]\s*[\"']\s*{_NAME}\s*(?:;[^\"']*)?[\"']", re.I | re.MULTILINE), ) +#: A `pyproject.toml` holds arrays that are not dependency lists — `keywords`, +#: `classifiers`, `packages` — and a requirement-shaped string in one of them +#: declares nothing. Only these keys carry requirements: `dependencies` and +#: `requires` by name, and every key of an optional-dependency or +#: dependency-group table. +_DEP_KEYS = frozenset({"dependencies", "requires", "requires-dist"}) +_DEP_TABLES = ("optional-dependencies", "dependency-groups") +_TABLE_HEADER = re.compile(r"^\s*\[([^\]]+)\]\s*$", re.MULTILINE) +_ARRAY_KEY = re.compile(r"^\s*([A-Za-z_][\w.-]*)\s*=\s*\[", re.MULTILINE) + + +def _dependency_arrays(text: str) -> list[str]: + """Every dependency array in a TOML text, brackets included. + + Bracket counting rather than a TOML parser: `tomllib` arrived in 3.11 and + this package supports 3.10, and an extras marker (`commoner-probe[http]`) + nests one balanced pair inside a string. + """ + tables = [(m.start(), m.group(1).strip()) for m in _TABLE_HEADER.finditer(text)] + out: list[str] = [] + for match in _ARRAY_KEY.finditer(text): + table = "" + for start, name in tables: + if start < match.start(): + table = name + else: + break + declares = (match.group(1) in _DEP_KEYS + or any(table.endswith(t) for t in _DEP_TABLES)) + if not declares: + continue + depth, i = 0, match.end() - 1 + while i < len(text): + if text[i] == "[": + depth += 1 + elif text[i] == "]": + depth -= 1 + if depth == 0: + out.append(text[match.end() - 1:i + 1]) + break + i += 1 + return out + def _uncommented(text: str) -> str: """The text with comment tails removed. @@ -186,13 +229,16 @@ def declared_pins(*paths: Path | str) -> dict[str, str]: except OSError: continue text = _uncommented(text) + # A requirements file is a list of requirements, so all of it counts. A + # `pyproject.toml` is mostly metadata, so only its dependency arrays do. + haystacks = _dependency_arrays(text) if p.suffix == ".toml" else [text] for pattern in _PIN_PATTERNS: - match = pattern.search(text) + match = next((m for m in map(pattern.search, haystacks) if m), None) if match: found[str(p)] = match.group(1) break else: - if any(p.search(text) for p in _UNPINNED_PATTERNS): + if any(pat.search(h) for h in haystacks for pat in _UNPINNED_PATTERNS): found[str(p)] = "unpinned" return found diff --git a/tests/test_doctor.py b/tests/test_doctor.py index c5f962a..d21e865 100644 --- a/tests/test_doctor.py +++ b/tests/test_doctor.py @@ -249,3 +249,45 @@ def test_a_dependency_list_entry_is_still_reached(self, tmp_path): path.write_text('[project]\nname = "consumer"\ndependencies = [\n' ' "requests==2.32.3",\n "commoner-probe",\n]\n', encoding="utf-8") assert declared_pins(path) == {str(path): "unpinned"} + + +class TestOnlyADependencyArrayDeclaresADependency: + """From Codex on `31f2381`. Narrowing to a quoted list element still read + every TOML array as a dependency list, and a marker with no version was read + as no declaration at all.""" + + def test_a_keyword_list_is_not_a_dependency_list(self, tmp_path): + path = tmp_path / "pyproject.toml" + path.write_text('[project]\nkeywords = ["commoner-probe==9.9.9"]\n' + 'dependencies = ["requests==2.32.3"]\n', encoding="utf-8") + assert declared_pins(path) == {} + + def test_an_optional_dependency_still_counts(self, tmp_path): + path = tmp_path / "pyproject.toml" + path.write_text('[project.optional-dependencies]\nprobe = ["commoner-probe==0.15.0"]\n', + encoding="utf-8") + assert declared_pins(path) == {str(path): "0.15.0"} + + def test_a_build_requirement_still_counts(self, tmp_path): + path = tmp_path / "pyproject.toml" + path.write_text('[build-system]\nrequires = ["commoner-probe==0.15.0"]\n', + encoding="utf-8") + assert declared_pins(path) == {str(path): "0.15.0"} + + def test_a_marker_with_no_version_is_unpinned(self, tmp_path): + """The org requires an exact pin. A marker is not a version, and + reporting nothing let the file pass the check it violates.""" + path = tmp_path / "requirements.txt" + path.write_text('commoner-probe; python_version < "3.12"\n', encoding="utf-8") + assert declared_pins(path) == {str(path): "unpinned"} + + def test_a_marker_with_no_version_in_toml_is_unpinned(self, tmp_path): + path = tmp_path / "pyproject.toml" + path.write_text("dependencies = [\"commoner-probe; python_version < '3.12'\"]\n", + encoding="utf-8") + assert declared_pins(path) == {str(path): "unpinned"} + + def test_a_pin_carrying_a_marker_is_still_a_pin(self, tmp_path): + path = tmp_path / "requirements.txt" + path.write_text('commoner-probe==0.15.0; python_version < "3.12"\n', encoding="utf-8") + assert declared_pins(path) == {str(path): "0.15.0"} From b2fb6c366b5be1fc1f4c2bb3389fb8d3390f7df0 Mon Sep 17 00:00:00 2001 From: skishchampi <996985+skishchampi@users.noreply.github.com> Date: Mon, 17 Aug 2026 20:42:02 -0400 Subject: [PATCH 06/10] fix: every declaration is classified, and the exit contract says so MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three findings from the review of the dependency-array narrowing. A file can name the package twice — an exact pin in `dependencies` and a range in an optional group. The reader stopped at the first pin, so the violation was never looked for, and a pin that matched the environment let `doctor` exit 0 over a file that breaks the policy it enforces. Classification is now per declaration: one line of a requirements file, one quoted element of a dependency array. A file-wide rule cannot work, because a git pin holds both shapes in one string — `@` opens an unpinned URL requirement and the tag closes an exact one. Bracket counting read a `]` inside a quoted marker as the end of the array, so every later entry fell outside the search. Strings are skipped. `doctor --help` and `docs/CLI.md` still said exit 1 meant two known numbers disagreeing. An unpinned dependency supplies no number and now exits 1 too. --- commoner_probe/cli.py | 3 +- commoner_probe/doctor.py | 74 +++++++++++++++++++++++++++++++++------- docs/CLI.md | 5 +++ tests/test_doctor.py | 28 +++++++++++++++ 4 files changed, 96 insertions(+), 14 deletions(-) diff --git a/commoner_probe/cli.py b/commoner_probe/cli.py index fb657dd..0b8c845 100644 --- a/commoner_probe/cli.py +++ b/commoner_probe/cli.py @@ -2199,7 +2199,8 @@ def build_parser() -> argparse.ArgumentParser: "doctor", help=( "Compare the source version, the installed metadata and any declared " - "pin. Exits 1 when two KNOWN numbers disagree." + "pin. Exits 1 when two KNOWN numbers disagree, and when a " + "--requirements file declares this package without an exact pin." ), epilog=( "Examples:\n" diff --git a/commoner_probe/doctor.py b/commoner_probe/doctor.py index f667e49..96814d6 100644 --- a/commoner_probe/doctor.py +++ b/commoner_probe/doctor.py @@ -104,11 +104,21 @@ def _dependency_arrays(text: str) -> list[str]: or any(table.endswith(t) for t in _DEP_TABLES)) if not declares: continue - depth, i = 0, match.end() - 1 + depth, i, quote = 0, match.end() - 1, "" while i < len(text): - if text[i] == "[": + char = text[i] + # A bracket inside a string is data, not structure. A marker may + # legally hold one — `"other; platform_version == \']\'"` — and + # counting it ended the array early, so every later dependency fell + # outside the search and an unpinned one was reported as absent. + if quote: + if char == quote: + quote = "" + elif char in "\"'": + quote = char + elif char == "[": depth += 1 - elif text[i] == "]": + elif char == "]": depth -= 1 if depth == 0: out.append(text[match.end() - 1:i + 1]) @@ -117,6 +127,30 @@ def _dependency_arrays(text: str) -> list[str]: return out +_TOML_STRING = re.compile(r"\"([^\"]*)\"|'([^']*)'") + + +def _declarations(text: str, *, toml: bool) -> list[str]: + """The requirement declarations in a file, one string each. + + A requirements file declares one per line. A `pyproject.toml` declares one + per quoted element of a dependency array, and the rest of the file is + metadata that must not be read as a requirement. + + One declaration at a time is what keeps the two pattern sets apart: a git + pin holds both an unpinned shape and an exact one, so classifying a whole + file at once cannot tell "pinned here, unpinned there" from "both in the + same requirement". + """ + if not toml: + return text.splitlines() + out: list[str] = [] + for array in _dependency_arrays(text): + for double, single in _TOML_STRING.findall(array): + out.append(double or single) + return out + + def _uncommented(text: str) -> str: """The text with comment tails removed. @@ -229,17 +263,31 @@ def declared_pins(*paths: Path | str) -> dict[str, str]: except OSError: continue text = _uncommented(text) - # A requirements file is a list of requirements, so all of it counts. A - # `pyproject.toml` is mostly metadata, so only its dependency arrays do. - haystacks = _dependency_arrays(text) if p.suffix == ".toml" else [text] - for pattern in _PIN_PATTERNS: - match = next((m for m in map(pattern.search, haystacks) if m), None) + haystacks = _declarations(text, toml=p.suffix == ".toml") + # Every declaration is classified, not just the first. One file can name + # the package twice — an exact pin in `dependencies` and a range in an + # optional group — and stopping at the first pin reported a compliant + # file; if that pin matched the environment, `doctor` exited 0 over a + # file that breaks the policy this check exists to enforce. + # + # Classification is per declaration, because the two pattern sets + # overlap inside one requirement. A git pin reads as `commoner-probe @ + # git+...@v0.15.0`: the `@` opens an unpinned URL requirement and the + # tag closes an exact one, in the same string. A file-wide "unpinned + # wins" rule therefore called every git tag pin unpinned. + pin: str | None = None + unpinned = False + for declaration in haystacks: + match = next( + (m for m in (pat.search(declaration) for pat in _PIN_PATTERNS) if m), None) if match: - found[str(p)] = match.group(1) - break - else: - if any(pat.search(h) for h in haystacks for pat in _UNPINNED_PATTERNS): - found[str(p)] = "unpinned" + pin = pin if pin is not None else match.group(1) + elif any(pat.search(declaration) for pat in _UNPINNED_PATTERNS): + unpinned = True + if unpinned: + found[str(p)] = "unpinned" + elif pin is not None: + found[str(p)] = pin return found diff --git a/docs/CLI.md b/docs/CLI.md index a5a7a60..86464d1 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -953,6 +953,11 @@ across worktrees reports whichever tree was installed last. Exits 1 when two KNOWN numbers disagree. A number that cannot be read is reported as unknown, never as agreement. +It also exits 1 when a `--requirements` file declares this package without an +exact pin. The org requires `==` or `@vX.Y.Z`, so a range, a bare name, or a +marker with no version is a finding rather than an absence. Reporting nothing +filed such a file beside the ones that never mention the package at all. + ### `commoner-probe stats` — corpus health ```bash diff --git a/tests/test_doctor.py b/tests/test_doctor.py index d21e865..2f90285 100644 --- a/tests/test_doctor.py +++ b/tests/test_doctor.py @@ -291,3 +291,31 @@ def test_a_pin_carrying_a_marker_is_still_a_pin(self, tmp_path): path = tmp_path / "requirements.txt" path.write_text('commoner-probe==0.15.0; python_version < "3.12"\n', encoding="utf-8") assert declared_pins(path) == {str(path): "0.15.0"} + + +class TestAViolationIsNotHiddenByAPin: + """From Codex on `17a6854`. Two of these let a file that breaks the exact-pin + policy pass the check that exists to enforce it.""" + + def test_an_unpinned_group_is_reported_even_beside_an_exact_pin(self, tmp_path): + """`search()` stopped at the first exact pin, so a range in another + group was never looked for. If that pin matched the environment, + `doctor` exited 0 over a file that violates the policy.""" + path = tmp_path / "pyproject.toml" + path.write_text('[project]\ndependencies = ["commoner-probe==0.15.0"]\n' + '[project.optional-dependencies]\nprobe = ["commoner-probe>=0.14"]\n', + encoding="utf-8") + assert declared_pins(path) == {str(path): "unpinned"} + + def test_a_bracket_inside_a_marker_does_not_end_the_array(self, tmp_path): + """Bracket counting read a `]` inside a quoted marker as the end of the + dependency list, so every later entry fell outside the search.""" + path = tmp_path / "pyproject.toml" + path.write_text('dependencies = [\n "other; platform_version == \']\'",\n' + ' "commoner-probe>=0.14",\n]\n', encoding="utf-8") + assert declared_pins(path) == {str(path): "unpinned"} + + def test_an_extras_bracket_inside_a_string_is_still_read(self, tmp_path): + path = tmp_path / "pyproject.toml" + path.write_text('dependencies = ["commoner-probe[http,pdf]==0.15.0"]\n', encoding="utf-8") + assert declared_pins(path) == {str(path): "0.15.0"} From 5bea361bf9ed4b420a6a3a000cd491ecd62c3672 Mon Sep 17 00:00:00 2001 From: skishchampi <996985+skishchampi@users.noreply.github.com> Date: Mon, 17 Aug 2026 20:48:29 -0400 Subject: [PATCH 07/10] fix: the TOML scanner reads escapes, quoted keys and the table it is in Three findings from the review of the per-declaration classifier. A basic TOML string processes escapes, so `\"` is a quote character. The scanner read it as the end of the string, went back into structure mode inside a marker, and let the marker's `]` close the array. Every later dependency then fell outside the search. A literal string has no escapes, so the rule is scoped to double quotes. A TOML key may be quoted, and quoting does not change its value. `"test" = [...]` names the same optional-dependency group as `test`. `dependencies` and `requires` now count only in the tables that give them meaning: the root, `[project]` and `[build-system]`. A tool's own `[tool.example] dependencies` array configures that tool and installs nothing, so a version in it was inventing a pin. The seven live consumer pins still read. --- commoner_probe/doctor.py | 26 +++++++++++++++++++++++--- tests/test_doctor.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 3 deletions(-) diff --git a/commoner_probe/doctor.py b/commoner_probe/doctor.py index 96814d6..0e93894 100644 --- a/commoner_probe/doctor.py +++ b/commoner_probe/doctor.py @@ -78,10 +78,20 @@ #: declares nothing. Only these keys carry requirements: `dependencies` and #: `requires` by name, and every key of an optional-dependency or #: dependency-group table. +#: And a key counts only in the table that gives it meaning. `[tool.example]` +#: may hold its own `dependencies` array; that is the tool's configuration and +#: it installs nothing, so a version in it is not a pin on this package. _DEP_KEYS = frozenset({"dependencies", "requires", "requires-dist"}) +#: The empty string is the root table: a bare `dependencies = [...]` with no +#: header above it is still a declaration, and only a NAMED foreign table is +#: someone else's configuration. +_DEP_KEY_TABLES = frozenset({"", "project", "build-system"}) _DEP_TABLES = ("optional-dependencies", "dependency-groups") _TABLE_HEADER = re.compile(r"^\s*\[([^\]]+)\]\s*$", re.MULTILINE) -_ARRAY_KEY = re.compile(r"^\s*([A-Za-z_][\w.-]*)\s*=\s*\[", re.MULTILINE) +#: A TOML key may be bare or quoted, and quoting does not change its value: +#: `"test" = [...]` names the same optional-dependency group as `test = [...]`. +_ARRAY_KEY = re.compile( + r"^\s*(?:([A-Za-z_][\w.-]*)|\"([^\"]+)\"|'([^']+)')\s*=\s*\[", re.MULTILINE) def _dependency_arrays(text: str) -> list[str]: @@ -100,8 +110,10 @@ def _dependency_arrays(text: str) -> list[str]: table = name else: break - declares = (match.group(1) in _DEP_KEYS - or any(table.endswith(t) for t in _DEP_TABLES)) + key = next(g for g in match.groups() if g is not None) + declares = ( + (key in _DEP_KEYS and table.strip('"\'') in _DEP_KEY_TABLES) + or any(table.endswith(t) for t in _DEP_TABLES)) if not declares: continue depth, i, quote = 0, match.end() - 1, "" @@ -112,6 +124,14 @@ def _dependency_arrays(text: str) -> list[str]: # counting it ended the array early, so every later dependency fell # outside the search and an unpinned one was reported as absent. if quote: + # A basic TOML string processes escapes, so `\\"` is a quote + # character and not the end of the string. Reading it as the end + # put the scanner back into structure mode inside a marker, and + # the marker's `]` then closed the array. A literal string + # (single quotes) has no escapes, which is why this is scoped. + if char == "\\" and quote == '"': + i += 2 + continue if char == quote: quote = "" elif char in "\"'": diff --git a/tests/test_doctor.py b/tests/test_doctor.py index 2f90285..bf6a850 100644 --- a/tests/test_doctor.py +++ b/tests/test_doctor.py @@ -319,3 +319,35 @@ def test_an_extras_bracket_inside_a_string_is_still_read(self, tmp_path): path = tmp_path / "pyproject.toml" path.write_text('dependencies = ["commoner-probe[http,pdf]==0.15.0"]\n', encoding="utf-8") assert declared_pins(path) == {str(path): "0.15.0"} + + +class TestTheTomlScannerIsNotFooled: + """From Codex on `2e63b1f`. Three ways a valid `pyproject.toml` defeated the + array scanner. Two hid a violation; one invented a pin.""" + + def test_an_escaped_quote_does_not_end_the_string(self, tmp_path): + path = tmp_path / "pyproject.toml" + path.write_text('dependencies = [\n "other; platform_version == \\"]\\"",\n' + ' "commoner-probe>=0.14",\n]\n', encoding="utf-8") + assert declared_pins(path) == {str(path): "unpinned"} + + def test_a_quoted_group_key_is_still_a_group(self, tmp_path): + path = tmp_path / "pyproject.toml" + path.write_text('[project.optional-dependencies]\n"test" = ["commoner-probe>=0.14"]\n', + encoding="utf-8") + assert declared_pins(path) == {str(path): "unpinned"} + + def test_a_tool_table_does_not_declare_project_dependencies(self, tmp_path): + """`[tool.example] dependencies = [...]` is that tool's own config. It + installs nothing, so a version in it is not a pin on this package.""" + path = tmp_path / "pyproject.toml" + path.write_text('[project]\ndependencies = ["requests==2.32.3"]\n' + '[tool.example]\ndependencies = ["commoner-probe==9.9.9"]\n', + encoding="utf-8") + assert declared_pins(path) == {} + + def test_the_project_and_build_tables_still_declare(self, tmp_path): + path = tmp_path / "pyproject.toml" + path.write_text('[build-system]\nrequires = ["commoner-probe==0.15.0"]\n' + '[project]\ndependencies = ["requests==2.32.3"]\n', encoding="utf-8") + assert declared_pins(path) == {str(path): "0.15.0"} From 71cee3eaba8c6ee86d4975dd7a6882ade1b6292b Mon Sep 17 00:00:00 2001 From: skishchampi <996985+skishchampi@users.noreply.github.com> Date: Mon, 17 Aug 2026 21:10:51 -0400 Subject: [PATCH 08/10] fix: a TOML reader reads TOML, and the floor moves to 3.11 Three more findings, and all three are the same defect as the four before them: a hand-written scanner meeting another thing real TOML does. A tool's own `[tool.x.optional-dependencies]` matched by suffix. A dotted `project.dependencies` key matched nothing. A `#` after punctuation was not read as a comment. So the scanner is deleted. `tomllib` reads the file, and it arrived in 3.11, so `requires-python` moves from `>=3.10` to `>=3.11`. Python 3.10 reaches end of life in October 2026 and SPEC 0 dropped it in 2024. Every consumer repo in the org runs 3.14; none is affected. `doctor.py` loses 97 lines and gains 64. `source_version()` uses the parser too: its regex took the first `version = "..."` in the file, which is the project's version only while no table above declares one. CI now runs 3.11, 3.12 and 3.13. The README, CONTRIBUTING and the classifiers say 3.11. --- .github/workflows/ci.yml | 2 +- CHANGELOG.md | 19 +++++ CONTRIBUTING.md | 2 +- README.md | 2 +- commoner_probe/doctor.py | 161 ++++++++++++++++----------------------- pyproject.toml | 4 +- tests/test_doctor.py | 26 +++++++ 7 files changed, 114 insertions(+), 102 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 52ebf46..8f880a2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,7 +10,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12"] + python-version: ["3.11", "3.12", "3.13"] steps: - name: Checkout uses: actions/checkout@v4 diff --git a/CHANGELOG.md b/CHANGELOG.md index eb25c29..5ee09f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,25 @@ ## Unreleased +### Changed — the minimum Python is now 3.11 + +**`requires-python` moves from `>=3.10` to `>=3.11`.** Install 0.15.0 on 3.10 +and pip will refuse it. Every consumer repo in the org runs 3.14, so nothing +here needs a change. Python 3.10 reaches end of life in October 2026, and +SPEC 0 dropped it in 2024. + +The floor bought a real deletion. `doctor` read `pyproject.toml` with a +hand-written scanner, because `tomllib` arrived in 3.11. Seven review findings +in three rounds were all the same defect: escaped quotes, quoted keys, dotted +keys, brackets inside markers, comments after punctuation, and a tool's own +tables read as project metadata. The scanner is gone and `tomllib` reads the +file. `doctor.py` loses 97 lines and gains 64. + +`source_version()` uses the parser too. The regex it replaced took the first +`version = "…"` in the file, which is the project's version only while no other +table declares one above it. + + **Take this one if you enumerate Lok Sabha sessions, or read any Sansad answer.** The degrading paginator that 0.14.8 and 0.14.9 built had no production caller, and an answer can be a different question's document. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d0a4088..ac125e1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -33,4 +33,4 @@ python -m twine check dist/* 2. Add or update tests for behavior changes. 3. Update docs (`README.md`, `docs/SCHEMAS.md`) if user-facing behavior changes. 4. Reference related issues in the PR description. -5. Ensure CI passes on Python 3.10, 3.11, and 3.12. +5. Ensure CI passes on Python 3.11, 3.12, and 3.13. diff --git a/README.md b/README.md index 48ff9ec..eae8520 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ provenance-rich, schema-validated JSONL and source files. ## Install -Requires Python 3.10+. Released on [PyPI](https://pypi.org/project/commoner-probe/). +Requires Python 3.11+. Released on [PyPI](https://pypi.org/project/commoner-probe/). ```bash pip install "commoner-probe[all]" # everything needed for acquisition + extraction diff --git a/commoner_probe/doctor.py b/commoner_probe/doctor.py index 0e93894..e3362c9 100644 --- a/commoner_probe/doctor.py +++ b/commoner_probe/doctor.py @@ -25,6 +25,7 @@ import re from pathlib import Path +from typing import Any __all__ = ["VersionReport", "declared_pins", "installed_version", "source_version", "version_report"] @@ -73,101 +74,57 @@ re.compile(rf"[\[,]\s*[\"']\s*{_NAME}\s*(?:;[^\"']*)?[\"']", re.I | re.MULTILINE), ) -#: A `pyproject.toml` holds arrays that are not dependency lists — `keywords`, -#: `classifiers`, `packages` — and a requirement-shaped string in one of them -#: declares nothing. Only these keys carry requirements: `dependencies` and -#: `requires` by name, and every key of an optional-dependency or -#: dependency-group table. -#: And a key counts only in the table that gives it meaning. `[tool.example]` -#: may hold its own `dependencies` array; that is the tool's configuration and -#: it installs nothing, so a version in it is not a pin on this package. -_DEP_KEYS = frozenset({"dependencies", "requires", "requires-dist"}) -#: The empty string is the root table: a bare `dependencies = [...]` with no -#: header above it is still a declaration, and only a NAMED foreign table is -#: someone else's configuration. -_DEP_KEY_TABLES = frozenset({"", "project", "build-system"}) -_DEP_TABLES = ("optional-dependencies", "dependency-groups") -_TABLE_HEADER = re.compile(r"^\s*\[([^\]]+)\]\s*$", re.MULTILINE) -#: A TOML key may be bare or quoted, and quoting does not change its value: -#: `"test" = [...]` names the same optional-dependency group as `test = [...]`. -_ARRAY_KEY = re.compile( - r"^\s*(?:([A-Za-z_][\w.-]*)|\"([^\"]+)\"|'([^']+)')\s*=\s*\[", re.MULTILINE) - - -def _dependency_arrays(text: str) -> list[str]: - """Every dependency array in a TOML text, brackets included. - - Bracket counting rather than a TOML parser: `tomllib` arrived in 3.11 and - this package supports 3.10, and an extras marker (`commoner-probe[http]`) - nests one balanced pair inside a string. - """ - tables = [(m.start(), m.group(1).strip()) for m in _TABLE_HEADER.finditer(text)] - out: list[str] = [] - for match in _ARRAY_KEY.finditer(text): - table = "" - for start, name in tables: - if start < match.start(): - table = name - else: - break - key = next(g for g in match.groups() if g is not None) - declares = ( - (key in _DEP_KEYS and table.strip('"\'') in _DEP_KEY_TABLES) - or any(table.endswith(t) for t in _DEP_TABLES)) - if not declares: - continue - depth, i, quote = 0, match.end() - 1, "" - while i < len(text): - char = text[i] - # A bracket inside a string is data, not structure. A marker may - # legally hold one — `"other; platform_version == \']\'"` — and - # counting it ended the array early, so every later dependency fell - # outside the search and an unpinned one was reported as absent. - if quote: - # A basic TOML string processes escapes, so `\\"` is a quote - # character and not the end of the string. Reading it as the end - # put the scanner back into structure mode inside a marker, and - # the marker's `]` then closed the array. A literal string - # (single quotes) has no escapes, which is why this is scoped. - if char == "\\" and quote == '"': - i += 2 - continue - if char == quote: - quote = "" - elif char in "\"'": - quote = char - elif char == "[": - depth += 1 - elif char == "]": - depth -= 1 - if depth == 0: - out.append(text[match.end() - 1:i + 1]) - break - i += 1 - return out - - -_TOML_STRING = re.compile(r"\"([^\"]*)\"|'([^']*)'") +#: The tables a `pyproject.toml` declares installable requirements in. A tool's +#: own `[tool.x]` table may hold a `dependencies` array; that configures the +#: tool and installs nothing, so a version in it is not a pin on this package. +#: The root entries are not PEP 621, but a file that writes `dependencies` with +#: no table above it is still declaring one, and reading it costs nothing. +_DEP_PATHS = ( + ("project", "dependencies"), + ("build-system", "requires"), + ("dependencies",), + ("requires",), +) +_DEP_GROUP_PATHS = ( + ("project", "optional-dependencies"), + ("dependency-groups",), + ("optional-dependencies",), +) -def _declarations(text: str, *, toml: bool) -> list[str]: - """The requirement declarations in a file, one string each. +def _toml_requirements(text: str) -> list[str]: + """Every requirement string a `pyproject.toml` declares, or [] if unreadable. - A requirements file declares one per line. A `pyproject.toml` declares one - per quoted element of a dependency array, and the rest of the file is - metadata that must not be read as a requirement. + Parsed by `tomllib`, not scanned. The scan this replaced grew a rule per + round of review — escaped quotes, quoted keys, dotted keys, brackets inside + markers, comments after punctuation — because each was another thing real + TOML does. Reading a format needs a reader for that format. - One declaration at a time is what keeps the two pattern sets apart: a git - pin holds both an unpinned shape and an exact one, so classifying a whole - file at once cannot tell "pinned here, unpinned there" from "both in the - same requirement". + A file that does not parse yields nothing rather than raising. `doctor` + reports what it can read and calls the rest unknown; a broken consumer file + is not this repo's version question. """ - if not toml: - return text.splitlines() + import tomllib + + try: + data = tomllib.loads(text) + except (tomllib.TOMLDecodeError, ValueError): + return [] out: list[str] = [] - for array in _dependency_arrays(text): - for double, single in _TOML_STRING.findall(array): - out.append(double or single) + for path in _DEP_PATHS: + node: Any = data + for key in path: + node = node.get(key) if isinstance(node, dict) else None + if isinstance(node, list): + out.extend(x for x in node if isinstance(x, str)) + for path in _DEP_GROUP_PATHS: + node: Any = data + for key in path: + node = node.get(key) if isinstance(node, dict) else None + if isinstance(node, dict): + for group in node.values(): + if isinstance(group, list): + out.extend(x for x in group if isinstance(x, str)) return out @@ -245,16 +202,22 @@ def report(self) -> str: def source_version(pyproject: Path | str) -> str | None: """The version in a ``pyproject.toml``, or None when it cannot be read. - Read with a regex rather than a TOML parser on purpose: ``tomllib`` arrived in - Python 3.11 and this package supports 3.10, so a parser import would make the - check unavailable on the oldest version it claims to run on. + Read by ``tomllib``. The regex this replaced took the first ``version = "…"`` + in the file, which is the project's version only while no other table + declares one above it. """ + import tomllib + try: text = Path(pyproject).read_text(encoding="utf-8") except OSError: return None - match = re.search(r'^version\s*=\s*"([^"]+)"', text, re.MULTILINE) - return match.group(1) if match else None + try: + data = tomllib.loads(text) + except (tomllib.TOMLDecodeError, ValueError): + return None + version = data.get("project", {}).get("version") + return version if isinstance(version, str) else None def installed_version(package: str = "commoner-probe") -> str | None: @@ -282,8 +245,12 @@ def declared_pins(*paths: Path | str) -> dict[str, str]: text = p.read_text(encoding="utf-8") except OSError: continue - text = _uncommented(text) - haystacks = _declarations(text, toml=p.suffix == ".toml") + if p.suffix == ".toml": + # `tomllib` handles TOML comments itself; stripping them first would + # corrupt a `#` that sits inside a string. + declarations = _toml_requirements(text) + else: + declarations = _uncommented(text).splitlines() # Every declaration is classified, not just the first. One file can name # the package twice — an exact pin in `dependencies` and a range in an # optional group — and stopping at the first pin reported a compliant @@ -297,7 +264,7 @@ def declared_pins(*paths: Path | str) -> dict[str, str]: # wins" rule therefore called every git tag pin unpinned. pin: str | None = None unpinned = False - for declaration in haystacks: + for declaration in declarations: match = next( (m for m in (pat.search(declaration) for pat in _PIN_PATTERNS) if m), None) if match: diff --git a/pyproject.toml b/pyproject.toml index 88ae64c..69fbede 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "commoner-probe" version = "0.15.0" description = "Sousveillance infrastructure for state mandatory-disclosure portals — parliamentary questions, committee reports, budget data, and state assembly records." readme = "README.md" -requires-python = ">=3.10" +requires-python = ">=3.11" authors = [{ name = "CommonerLLP" }] maintainers = [ { name = "Sreeram N R", email = "sreeram.nr@gmail.com" }, @@ -35,9 +35,9 @@ classifiers = [ "Intended Audience :: Science/Research", "Topic :: Sociology", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Operating System :: OS Independent", ] dependencies = [] diff --git a/tests/test_doctor.py b/tests/test_doctor.py index bf6a850..0a5e8dd 100644 --- a/tests/test_doctor.py +++ b/tests/test_doctor.py @@ -351,3 +351,29 @@ def test_the_project_and_build_tables_still_declare(self, tmp_path): path.write_text('[build-system]\nrequires = ["commoner-probe==0.15.0"]\n' '[project]\ndependencies = ["requests==2.32.3"]\n', encoding="utf-8") assert declared_pins(path) == {str(path): "0.15.0"} + + +class TestTheParserReadsWhatTheScannerCouldNot: + """Round 7 from Codex, and the reason the hand-written scanner went away. + Each of these is valid TOML that the scanner read wrongly. A real parser + answers all three without a rule per case.""" + + def test_a_tool_s_own_optional_dependencies_are_not_ours(self, tmp_path): + path = tmp_path / "pyproject.toml" + path.write_text('[project]\ndependencies = ["requests==2.32.3"]\n' + '[tool.example.optional-dependencies]\nx = ["commoner-probe==9.9.9"]\n', + encoding="utf-8") + assert declared_pins(path) == {} + + def test_a_dotted_key_declares_a_dependency(self, tmp_path): + path = tmp_path / "pyproject.toml" + path.write_text('project.dependencies = ["commoner-probe>=0.14"]\n', encoding="utf-8") + assert declared_pins(path) == {str(path): "unpinned"} + + def test_a_comment_after_punctuation_is_a_comment(self, tmp_path): + """TOML starts a comment at `#` outside a string, with or without a + space before it. The old remover required whitespace.""" + path = tmp_path / "pyproject.toml" + path.write_text('[project]\ndependencies = [#"commoner-probe==9.9.9"\n' + ' "requests==2.32.3",\n]\n', encoding="utf-8") + assert declared_pins(path) == {} From bd5993ec4189e8f8d4673c0debb7218255c6e3bc Mon Sep 17 00:00:00 2001 From: skishchampi <996985+skishchampi@users.noreply.github.com> Date: Mon, 17 Aug 2026 21:21:46 -0400 Subject: [PATCH 09/10] fix: a dot spells the same package, and two pins are a conflict Two findings from the review of the tomllib rewrite. PEP 503 normalises `-`, `_` and `.` to one distribution name, so `commoner.probe>=0.14` names this package and pip installs it. The reader saw neither an exact pin nor an unpinned one, and the file passed the check it breaks. One file can also carry two exact pins at different versions. Keeping the first hid the second, and a first that matched the environment exited 0. A file in that state is reported as a conflict, because which version installs depends on the resolver. --- commoner_probe/doctor.py | 28 +++++++++++++++++++++++----- tests/test_doctor.py | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 5 deletions(-) diff --git a/commoner_probe/doctor.py b/commoner_probe/doctor.py index e3362c9..5447001 100644 --- a/commoner_probe/doctor.py +++ b/commoner_probe/doctor.py @@ -41,7 +41,9 @@ #: prose; each was read as this package's pin, and `doctor` then failed a #: consumer that never depended on it. _EXTRAS = r"(?:\[[^\]]*\])?" -_NAME = rf"commoner[-_]probe{_EXTRAS}" +#: PEP 503 normalises `-`, `_` and `.` to one name, so `commoner.probe` is +#: this package and pip installs it as such. +_NAME = rf"commoner[-_.]probe{_EXTRAS}" #: A requirement token opens a line, or opens a quoted element of an array. #: Prose names the package mid-sentence, and TOML names it on both sides of a #: scalar assignment — `name = "commoner-probe"` and the console-script key — @@ -139,6 +141,10 @@ def _uncommented(text: str) -> str: return re.sub(r"(?m)(?:^|(?<=\s))#.*$", "", text) +#: Marks a file that pins this package at two versions at once. +_CONFLICT = "conflict" + + class VersionReport: """The three versions, and whether they agree. @@ -164,6 +170,12 @@ def mismatches(self) -> list[str]: "this tree; one venv shared across worktrees reports whichever was " "installed last.") for where, pin in self.pins.items(): + if pin.startswith(_CONFLICT): + out.append( + f"{where} pins this package at two different versions " + f"({pin.split(': ', 1)[-1]}). Which one installs depends on the " + "resolver, so the file does not state what the consumer runs.") + continue if pin == "unpinned": out.append( f"{where} names this package with no exact version. The org pins " @@ -262,19 +274,25 @@ def declared_pins(*paths: Path | str) -> dict[str, str]: # git+...@v0.15.0`: the `@` opens an unpinned URL requirement and the # tag closes an exact one, in the same string. A file-wide "unpinned # wins" rule therefore called every git tag pin unpinned. - pin: str | None = None + pins: list[str] = [] unpinned = False for declaration in declarations: match = next( (m for m in (pat.search(declaration) for pat in _PIN_PATTERNS) if m), None) if match: - pin = pin if pin is not None else match.group(1) + if match.group(1) not in pins: + pins.append(match.group(1)) elif any(pat.search(declaration) for pat in _UNPINNED_PATTERNS): unpinned = True if unpinned: found[str(p)] = "unpinned" - elif pin is not None: - found[str(p)] = pin + elif len(pins) > 1: + # Keeping the first hid the second. One file pinning two versions + # installs whichever resolver wins, and if the first matched the + # environment `doctor` exited 0 over it. + found[str(p)] = f"{_CONFLICT}: {', '.join(pins)}" + elif pins: + found[str(p)] = pins[0] return found diff --git a/tests/test_doctor.py b/tests/test_doctor.py index 0a5e8dd..6cc7174 100644 --- a/tests/test_doctor.py +++ b/tests/test_doctor.py @@ -377,3 +377,43 @@ def test_a_comment_after_punctuation_is_a_comment(self, tmp_path): path.write_text('[project]\ndependencies = [#"commoner-probe==9.9.9"\n' ' "requests==2.32.3",\n]\n', encoding="utf-8") assert declared_pins(path) == {} + + +class TestTwoMoreWaysAPolicyBreachHid: + """Round 8 from Codex, on the `tomllib` rewrite. Both let a file that + breaks the exact-pin policy exit 0.""" + + def test_a_dot_spelled_name_is_this_package(self, tmp_path): + """PEP 503 normalises `.`, `-` and `_` to one name, so + `commoner.probe` names this package and the reader must see it.""" + path = tmp_path / "pyproject.toml" + path.write_text('project.dependencies = ["commoner.probe>=0.14"]\n', encoding="utf-8") + assert declared_pins(path) == {str(path): "unpinned"} + + def test_a_dot_spelled_exact_pin_reads(self, tmp_path): + path = tmp_path / "requirements.txt" + path.write_text("commoner.probe==0.15.0\n", encoding="utf-8") + assert declared_pins(path) == {str(path): "0.15.0"} + + def test_two_exact_pins_that_disagree_are_a_conflict(self, tmp_path): + """Keeping the first hid the second. If the first matched the + environment, `doctor` exited 0 over a file pinning two versions.""" + path = tmp_path / "pyproject.toml" + path.write_text('[project]\ndependencies = ["commoner-probe==0.15.0"]\n' + '[project.optional-dependencies]\nx = ["commoner-probe==9.9.9"]\n', + encoding="utf-8") + assert declared_pins(path) == {str(path): "conflict: 0.15.0, 9.9.9"} + + def test_a_conflict_is_a_mismatch_even_when_one_pin_matches(self): + from commoner_probe.doctor import VersionReport + + report = VersionReport("0.15.0", "0.15.0", {"req.txt": "conflict: 0.15.0, 9.9.9"}) + assert not report.agrees + assert "two different versions" in report.report + + def test_one_version_repeated_is_not_a_conflict(self, tmp_path): + path = tmp_path / "pyproject.toml" + path.write_text('[project]\ndependencies = ["commoner-probe==0.15.0"]\n' + '[project.optional-dependencies]\nx = ["commoner-probe==0.15.0"]\n', + encoding="utf-8") + assert declared_pins(path) == {str(path): "0.15.0"} From 04c44433550c1686a64644e32df8a0984a725e2f Mon Sep 17 00:00:00 2001 From: skishchampi <996985+skishchampi@users.noreply.github.com> Date: Mon, 17 Aug 2026 21:29:45 -0400 Subject: [PATCH 10/10] fix: one declaration can carry two pins, and both are read `commoner-probe==0.15.0,==9.9.9` is one valid PEP 508 requirement and nothing satisfies it. The reader took the first specifier and reported a clean pin, so an environment on 0.15.0 exited 0 over a contradiction. Every `==` in the specifier set is collected. The marker after `;` is cut first, because a version inside it is a condition and not a pin. --- commoner_probe/doctor.py | 17 +++++++++++++++-- tests/test_doctor.py | 18 ++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/commoner_probe/doctor.py b/commoner_probe/doctor.py index 5447001..98eb519 100644 --- a/commoner_probe/doctor.py +++ b/commoner_probe/doctor.py @@ -141,6 +141,19 @@ def _uncommented(text: str) -> str: return re.sub(r"(?m)(?:^|(?<=\s))#.*$", "", text) +#: Every `==` version in one requirement's specifier set. PEP 508 allows a list +#: — `commoner-probe==0.15.0,==9.9.9` is one declaration and cannot be +#: satisfied — and reading only the first specifier reported it as a clean pin. +#: The marker after `;` is not a specifier and is cut before the scan. +_EXACT = re.compile(r"==\s*([0-9][^\s,;#\"']*)") + + +def _exact_versions(declaration: str, start: int) -> list[str]: + """The versions the specifier set after *start* pins, in order.""" + specifiers = declaration[start:].split(";", 1)[0] + return _EXACT.findall(specifiers) + + #: Marks a file that pins this package at two versions at once. _CONFLICT = "conflict" @@ -280,8 +293,8 @@ def declared_pins(*paths: Path | str) -> dict[str, str]: match = next( (m for m in (pat.search(declaration) for pat in _PIN_PATTERNS) if m), None) if match: - if match.group(1) not in pins: - pins.append(match.group(1)) + found_here = _exact_versions(declaration, match.start()) or [match.group(1)] + pins.extend(v for v in found_here if v not in pins) elif any(pat.search(declaration) for pat in _UNPINNED_PATTERNS): unpinned = True if unpinned: diff --git a/tests/test_doctor.py b/tests/test_doctor.py index 6cc7174..05eb9c9 100644 --- a/tests/test_doctor.py +++ b/tests/test_doctor.py @@ -417,3 +417,21 @@ def test_one_version_repeated_is_not_a_conflict(self, tmp_path): '[project.optional-dependencies]\nx = ["commoner-probe==0.15.0"]\n', encoding="utf-8") assert declared_pins(path) == {str(path): "0.15.0"} + + def test_one_declaration_carrying_two_pins_is_a_conflict(self, tmp_path): + """`commoner-probe==0.15.0,==9.9.9` is one valid PEP 508 declaration and + it cannot be satisfied. Reading only the first specifier reported a + clean pin.""" + path = tmp_path / "requirements.txt" + path.write_text("commoner-probe==0.15.0,==9.9.9\n", encoding="utf-8") + assert declared_pins(path) == {str(path): "conflict: 0.15.0, 9.9.9"} + + def test_a_marker_after_the_pin_is_not_a_second_version(self, tmp_path): + path = tmp_path / "requirements.txt" + path.write_text('commoner-probe==0.15.0; python_version >= "3.11"\n', encoding="utf-8") + assert declared_pins(path) == {str(path): "0.15.0"} + + def test_extras_and_one_pin_stay_one_pin(self, tmp_path): + path = tmp_path / "requirements.txt" + path.write_text("commoner-probe[http,pdf]==0.15.0\n", encoding="utf-8") + assert declared_pins(path) == {str(path): "0.15.0"}