diff --git a/CHANGELOG.md b/CHANGELOG.md index a1c9a5f..20ea73a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ All notable changes to this project are documented in this file. ### Breaking Changes - **Retrieval-source entries now use `resource_id` as their sole identifier.** Generated edges no longer duplicate each nested `sources` entry's provenance identifier into `sources[].id`; readers should use `sources[].resource_id`. Tablassert's validator supplies the inherited `id` only to its in-memory compatibility copy while the pinned Biolink model still requires it. +### Fixed +- **Negative-log p-value columns are un-logged instead of shipped verbatim.** `pvalue_target` already routed spellings like `negative log p value` / `-log10(p)` onto `p_value` / `adjusted_p_value`, but the score rode through untouched: a −log10(p)=8 enrichment column (the `mokg-v12` HOYER1 fixture ships exactly this shape) emitted `p_value` 8.0 — wildly out of range — and `sig` banded it `not_significant`, the exact inverse of the truth. The coercion now recognizes an explicit negation marker (`negative` / `negated` / `neg` / `-`) before a `log`/`log10` p-or-q token (`is_neglog10_column`) — requiring a complete p/q-value token (or a bare delimited `P`), so a word merely starting with p or q (`negative log protein`) no longer matches — and converts the score (`p = 10**-x`) when it lands on the numeric slot, in both `coerce_pvalue_columns` and `sig`. A raw p/q candidate now always beats a −log10 alias of the same statistic before fuzzy ranking runs, so a short raw name (`P`, `FDR`) can no longer lose to a long alias it would then be un-logged over. Float64 underflow floors extreme scores at `0.0` (indistinguishable from `p ~ 0`, and the band is identical); nulls stay null. A plain `log10` spelling without a negation marker is deliberately left verbatim — the sign convention is ambiguous there — and when a raw p-value column coexists with a −log10 alias the raw column still wins and the alias is left untouched. + ## 13.0.0 - 2026-08-24 ### Breaking Changes diff --git a/src/tablassert/coerce.py b/src/tablassert/coerce.py index b0abc91..f25e8a2 100644 --- a/src/tablassert/coerce.py +++ b/src/tablassert/coerce.py @@ -48,8 +48,6 @@ def sig(lf: pl.LazyFrame, col: str = "p_value", out: str = "statistical_signific null p-value → null qualifier. The qualifier may only be set when ``p_value``/``adjusted_p_value`` is populated (Biolink class rule). """ - from rapidfuzz import fuzz - names: list[str] = lf.collect_schema().names() # Same rigorous classification as ``coerce_pvalue_columns``: a column is a significance # source only when ``pvalue_target`` accepts it — NOT by naive substring — so non-p-value @@ -66,11 +64,14 @@ def sig(lf: pl.LazyFrame, col: str = "p_value", out: str = "statistical_signific # bucket is present. Raw p-value is the canonical significance source; adjusted is the fallback. preferred: str = col if col in buckets else next(iter(buckets)) candidates: list[str] = buckets[preferred] - reference: str = preferred.replace("_", " ") - # An existing canonical column always wins; fuzzy ranking only picks among aliases - # (same rule as ``coerce_pvalue_columns``). - chosen: str = preferred if preferred in candidates else max(candidates, key=lambda c: fuzz.ratio(c, reference)) + # Same selection rule as ``coerce_pvalue_columns``: canonical wins, then a + # raw candidate beats a -log10 alias, then fuzzy ranking among what is left. + chosen: str = _best_candidate(candidates, preferred, preferred) expr: pl.Expr = pl.col(chosen).cast(pl.Float64, strict=False) + # A -log10(p) score column must be un-logged before banding, or the bands + # invert (a score of 8 means p = 1e-8, not p = 8.0 -> not_significant). + if is_neglog10_column(chosen): + expr = _unlog10(expr) band: pl.Expr = ( pl.when(expr.is_null()) .then(pl.lit(None, dtype=pl.String)) @@ -173,6 +174,54 @@ def sig(lf: pl.LazyFrame, col: str = "p_value", out: str = "statistical_signific """, re.IGNORECASE | re.VERBOSE, ) +# A -log10(p/q) score column: an explicit negation marker (negative / negated / +# neg / -) before a log/log10 p-or-q token ("negative log p value" — the +# mokg-v12 HOYER1 spelling, "negative log10 p value", "-log10(p)", +# "neg log10 q value"). The trailing token must be a *complete* p/q-value +# token (or a bare delimited P), so the [pq] cannot swallow the first letter +# of an unrelated word ("negative log protein" stays out). A plain +# "log10 p value" without a negation marker does NOT match: the sign +# convention is ambiguous there, so those columns keep riding verbatim rather +# than being un-logged on a guess. +NEGLOG10_PVALUE_PATTERN: re.Pattern[str] = re.compile( + rf""" + (? bool: + """Return True when a column reports a -log10(p/q) score instead of the raw p/q value. + + Args: + name: Raw source column name. + + Returns: + True when the name carries an explicit negation marker before a + log/log10 p-or-q token, else False. Plain ``log``/``log10`` spellings + without a marker are deliberately excluded (sign convention + ambiguous), as is everything without a log token at all. + """ + return bool(NEGLOG10_PVALUE_PATTERN.search(name)) + + +def _unlog10(expr: pl.Expr) -> pl.Expr: + """Convert a -log10 score expression back to its original scale (10**-x). + + Float64 underflow floors extreme scores at ``0.0`` — indistinguishable + from ``p ~ 0`` in practice, and the significance band is identical either + way. Nulls stay null. + """ + return pl.lit(10.0) ** (-expr) def pvalue_target(name: str) -> str | None: @@ -215,10 +264,37 @@ def pvalue_target(name: str) -> str | None: return "adjusted_p_value" if is_adjusted else "p_value" +def _best_candidate(candidates: list[str], target: str, chosen: str) -> str: + """Pick the column a numeric p/q-value slot should read from. + + A raw (non-neglog) candidate is always preferred over a -log10 alias of + the same statistic: the raw column holds the p/q value itself, while the + alias needs un-logging. Fuzzy ranking only runs when no raw candidate + exists (or only neglog candidates do), so a short raw name ("P", "FDR") + can no longer lose to a long -log10 alias it would then be un-logged over. + + Args: + candidates: Column names bucketed onto ``target`` by ``pvalue_target``. + target: Canonical slot name (``p_value`` / ``adjusted_p_value``). + chosen: The canonical name when it is itself among the candidates + (the existing canonical-wins rule), else a sentinel that is not. + + Returns: + The winning column name. + """ + from rapidfuzz import fuzz + + pool: list[str] = [c for c in candidates if not is_neglog10_column(c)] or candidates + return chosen if chosen in pool else max(pool, key=lambda c: fuzz.ratio(c, target.replace("_", " "))) + + def coerce_pvalue_columns(lf: pl.LazyFrame) -> pl.LazyFrame: """Rename p-value-like columns to Biolink KGX-compliant ``p_value`` / ``adjusted_p_value``. Picks a single best fuzzy match per target when multiple candidates exist. + A chosen column that reports a -log10(p/q) score (see + :func:`is_neglog10_column`) is un-logged (``p = 10**-x``) as it is renamed, + so the slot receives the p-value the model types it as. Args: lf: Source LazyFrame. @@ -226,9 +302,6 @@ def coerce_pvalue_columns(lf: pl.LazyFrame) -> pl.LazyFrame: Returns: LazyFrame with the chosen columns renamed (no-op if no candidates). """ - # Picks a single best fuzzy match per target when multiple candidates exist. - from rapidfuzz import fuzz - names: list[str] = lf.collect_schema().names() buckets: dict[str, list[str]] = {} for n in names: @@ -237,14 +310,23 @@ def coerce_pvalue_columns(lf: pl.LazyFrame) -> pl.LazyFrame: buckets.setdefault(target, []).append(n) renames: dict[str, str] = {} + unlog_targets: list[str] = [] for target, candidates in buckets.items(): - reference: str = target.replace("_", " ") - # An existing canonical column always wins; fuzzy ranking only picks among aliases. - chosen: str = target if target in candidates else max(candidates, key=lambda c: fuzz.ratio(c, reference)) + # An existing canonical column always wins; a raw candidate beats a + # -log10 alias before fuzzy ranking has any say. + chosen: str = _best_candidate(candidates, target, target) if chosen != target: renames[chosen] = target + # A -log10(p) score must be un-logged when it lands on the numeric slot. + if is_neglog10_column(chosen): + unlog_targets.append(target) - return lf.rename(renames) if renames else lf + if not renames and not unlog_targets: + return lf + out: pl.LazyFrame = lf.rename(renames) if renames else lf + if unlog_targets: + out = out.with_columns([_unlog10(pl.col(t).cast(pl.Float64, strict=False)).alias(t) for t in unlog_targets]) + return out # --- Study-size fragments ---------------------------------------------------- diff --git a/src/tablassert/lib.py b/src/tablassert/lib.py index 9ea8e8d..d7f0d83 100644 --- a/src/tablassert/lib.py +++ b/src/tablassert/lib.py @@ -37,6 +37,7 @@ coerced_target, effect_size_target, effect_type_target, + is_neglog10_column, pvalue_target, sig, study_size_target, @@ -81,6 +82,7 @@ "effect_size_target", "effect_type_target", "infores", + "is_neglog10_column", "normalize_biolink_category", "predicate_options", "pvalue_target", diff --git a/tests/test_lib.py b/tests/test_lib.py index 78261a2..0d69a80 100644 --- a/tests/test_lib.py +++ b/tests/test_lib.py @@ -38,6 +38,7 @@ idx, idxname, infores, + is_neglog10_column, numeric_columns, parse_edge_name, publications, @@ -932,6 +933,105 @@ def test_sig_very_strongly_significant_band() -> None: assert list(result["statistical_significance_qualifier"]) == ["very_strongly_significant", "very_strongly_significant", "strongly_significant"] +@pytest.mark.parametrize( + "name", + [ + "negative log p value", # mokg-v12 HOYER1 spelling + "negative log10 p value", + "-log10(p)", + "neg log10 q value", + "Negated Log P Value", + "-LOG10 P VALUE", + ], +) +def test_is_neglog10_column_matches_negation_spellings(name: str) -> None: + assert is_neglog10_column(name) is True + + +@pytest.mark.parametrize( + "name", + [ + "p value", + "log p value", # no negation marker: sign convention ambiguous + "log10 p value", + "adjusted p value", + "negatively correlated", + "regulation", + "negative log protein", # [pq] must be a complete token, not a word prefix + "negative log qwerty", + ], +) +def test_is_neglog10_column_rejects_unmarked_or_unrelated(name: str) -> None: + assert is_neglog10_column(name) is False + + +def test_coerce_pvalue_columns_unlogs_negative_log_p_value() -> None: + """-log10(p)=8 means p=1e-8: the slot receives the recovered p-value, nulls stay null.""" + lf: pl.LazyFrame = pl.DataFrame({"negative log p value": ["8.0", "0.0522071", None]}).lazy() + out: pl.DataFrame = coerce_pvalue_columns(lf).collect() + assert out["p_value"][0] == pytest.approx(1e-8) + assert out["p_value"][1] == pytest.approx(10**-0.0522071) + assert out["p_value"][2] is None + + +def test_coerce_pvalue_columns_unlog_underflows_to_zero() -> None: + """Observed -log10 scores reach ~864; 10**-864 underflows float64 to 0.0.""" + lf: pl.LazyFrame = pl.DataFrame({"negative log10 p value": [864.066614351]}).lazy() + out: pl.DataFrame = coerce_pvalue_columns(lf).collect() + assert out["p_value"][0] == 0.0 + + +def test_coerce_pvalue_columns_prefers_raw_over_neglog10_alias() -> None: + """A raw p-value column beats a -log10 alias; the alias is left untouched.""" + lf: pl.LazyFrame = pl.DataFrame({"p value": [0.03], "negative log10 p value": [8.0]}).lazy() + out: pl.DataFrame = coerce_pvalue_columns(lf).collect() + assert out["p_value"][0] == pytest.approx(0.03) + assert out["negative log10 p value"][0] == pytest.approx(8.0) + + +def test_coerce_pvalue_columns_raw_beats_neglog10_despite_short_name() -> None: + """A short raw name ("P", fuzz-score 0 against "p value") must still beat a + long -log10 alias (score ~48) — fuzzy ranking never sees the alias.""" + lf: pl.LazyFrame = pl.DataFrame({"P": [0.03], "negative log10 p value": [8.0]}).lazy() + out: pl.DataFrame = coerce_pvalue_columns(lf).collect() + assert out["p_value"][0] == pytest.approx(0.03) + assert out["negative log10 p value"][0] == pytest.approx(8.0) + + +def test_coerce_pvalue_columns_raw_fdr_beats_neglog10_q_alias() -> None: + """ "FDR" also loses the raw/alias contest against "negative log10 q value".""" + lf: pl.LazyFrame = pl.DataFrame({"FDR": [0.02], "negative log10 q value": [3.0]}).lazy() + out: pl.DataFrame = coerce_pvalue_columns(lf).collect() + assert out["adjusted_p_value"][0] == pytest.approx(0.02) + assert out["negative log10 q value"][0] == pytest.approx(3.0) + + +def test_sig_raw_beats_neglog10_despite_short_name() -> None: + """sig applies the same raw-beats-alias rule: a bare "P" column wins over a + -log10 alias, so the band comes from the raw 0.03 (significant).""" + lf: pl.LazyFrame = pl.DataFrame({"P": [0.03], "negative log10 p value": [8.0]}).lazy() + result: pl.DataFrame = lib.sig(lf).collect() + assert list(result["statistical_significance_qualifier"]) == ["significant"] + + +def test_coerce_pvalue_columns_unlogs_neglog10_q_value_into_adjusted() -> None: + lf: pl.LazyFrame = pl.DataFrame({"negative log10 q value": [3.0]}).lazy() + out: pl.DataFrame = coerce_pvalue_columns(lf).collect() + assert out["adjusted_p_value"][0] == pytest.approx(1e-3) + + +def test_sig_unlogs_neglog10_source() -> None: + """Banding un-logs a -log10 score column: 8 -> very strongly significant, + 0 (p=1) -> not significant — the bands no longer invert.""" + lf: pl.LazyFrame = pl.DataFrame({"negative log p value": [8.0, 0.0, 2.0]}).lazy() + result: pl.DataFrame = lib.sig(lf).collect() + assert list(result["statistical_significance_qualifier"]) == [ + "very_strongly_significant", + "not_significant", + "strongly_significant", # 10**-2 == 0.01 boundary, inclusive + ] + + def test_sig_significant_band_boundary() -> None: """sig maps the 0.01 < p <= 0.05 band to significant (boundary included).""" lf: pl.LazyFrame = pl.DataFrame({"p_value": [0.05, 0.06]}).lazy()