From 8c5b2f72c76882bfdbef7ae0591a83467d576885 Mon Sep 17 00:00:00 2001 From: SkyeAv Date: Fri, 14 Aug 2026 11:05:37 -0700 Subject: [PATCH] fix(lib): don't fabricate a supporting study with no identity and no content `inline_supporting_study` emitted `has_supporting_studies` unconditionally, so a section declaring no `publications` got a Study keyed by its own config filename (`my_table.yaml`) whose single StudyResult was a row index into a file the pipeline regenerates. Biolink defines `has supporting studies` as "studies that produced information used as evidence", and nothing in NCATSTranslator/translator-ingests models evidence that way -- there a Study is a real cohort (ICEES), dataset (COHD), trial (CTKP), or text-mining group (SemMedDB/TMKP) carrying typed StudyResult slots, never a key/value dump keyed by a filename. The struct is now emitted only when it carries something. A section with a real publication keeps it unchanged (`PMID:123#Table_S7 row 12` is genuine provenance), as does any section with routed UNSATISFIABLE_EDGE_FIELDS statistics or class-pruned values to preserve -- the rescue path is why the filename fallback exists at all. The sheet-name and row-number columns are consumed either way; they never reach the edge. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_014gQKmpqbfNTVPfdzYw97L2 --- CHANGELOG.md | 1 + docs/configuration/table.md | 2 ++ src/tablassert/lib.py | 42 ++++++++++++++++++++++++--------- tests/test_lib.py | 47 +++++++++++++++++++++++++++++++++++++ 4 files changed, 81 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 33145a1..ee1290e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ All notable changes to this project are documented in this file. ### Fixed - **`species_context_qualifier` is no longer auto-derived or relocated into supporting studies.** The field was derived from resolved node taxon metadata, then stored as a `StudyResult.description` key/value when the selected association class could not accept it. It is now disabled for both qualifiers and annotations; node taxon metadata remains available on node records. +- **A section with no publication and nothing to preserve no longer fabricates a supporting study.** `inline_supporting_study` emitted `has_supporting_studies` unconditionally, so a table declaring no `publications` got a `Study` keyed by its own config filename (`my_table.yaml`) whose single `StudyResult` was a row index into a file the pipeline regenerates. Biolink defines `has supporting studies` as "studies that produced information used as evidence", and nothing in `NCATSTranslator/translator-ingests` models evidence that way — there a `Study` is a real cohort (ICEES), dataset (COHD), trial (CTKP), or text-mining group (SemMedDB/TMKP) with typed `StudyResult` slots. The struct is now emitted only when it carries something: a section with a real publication keeps it unchanged (`PMID:123#Table_S7 row 12` is genuine provenance), as does any section with routed `UNSATISFIABLE_EDGE_FIELDS` statistics or class-pruned values to preserve. The sheet-name and row-number columns are consumed either way — they are never meant to reach the edge. ## 11.0.0 - 2026-08-13 diff --git a/docs/configuration/table.md b/docs/configuration/table.md index 17ebff2..5fd75a7 100644 --- a/docs/configuration/table.md +++ b/docs/configuration/table.md @@ -497,6 +497,8 @@ This means nothing in your source data is silently dropped: context that doesn't In addition to user-declared annotations, every edge automatically carries `extracted_from_row_number`, a 1-based index into the original source table (matching Excel-style row numbering). It is not declared as an annotation — tablassert emits it internally so each edge always carries its source-row provenance. Together with the sheet name it identifies the edge's **inlined supporting study** (`has_supporting_studies`), where it is carried alongside any relocated unsatisfiable slots; neither is folded into `supporting_text`. +The supporting study is only emitted when it carries something. Biolink defines `has supporting studies` as "studies that produced information used as evidence", so a section that declares **no `publications`** and has **no** relocated slots or class-pruned values emits no `has_supporting_studies` at all: its `study_id` would fall back to the config filename, making every edge assert a `Study` named `my_table.yaml` whose only result is a row index. A section with a real publication always keeps the struct — `PMID:123#Table_S7 row 12` is genuine provenance — as does any section with values to preserve. The row and sheet columns are consumed either way. + #### Automatic column coercion Before the allow-list sweep runs, tablassert renames statistical columns to their canonical Biolink names so source headers do not have to match exactly. Recognition is delimiter-anchored (spaces, `_`, `-`, `.` are interchangeable) and the **best fuzzy match per target wins, with an existing canonical column always preferred** over a higher-scoring spaced alias. diff --git a/src/tablassert/lib.py b/src/tablassert/lib.py index c233594..dca44ab 100644 --- a/src/tablassert/lib.py +++ b/src/tablassert/lib.py @@ -377,7 +377,7 @@ def _retrieval_source(resource_id: str, resource_role: str, upstream: list[str] """ -def inline_supporting_study(lf: pl.LazyFrame, study_id: str, sheet: str | None) -> pl.LazyFrame: +def inline_supporting_study(lf: pl.LazyFrame, study_id: str, sheet: str | None, identified: bool = True) -> pl.LazyFrame: """Attach table provenance and homeless statistics as an inlined Biolink ``Study``. Follows the COHD/ICEES pattern in ``translator-ingests``: the edge carries @@ -396,21 +396,45 @@ def inline_supporting_study(lf: pl.LazyFrame, study_id: str, sheet: str | None) ``supporting_study_*`` slots become real ``Association`` fields and are left flat on the edge instead; nothing here is hardcoded to either state. + **Nothing to say, nothing emitted.** Biolink defines ``has supporting studies`` as + "studies that produced information used as evidence", so the struct has to earn its + place. An UNIDENTIFIED section -- one whose ``study_id`` fell back to the config + filename because it declares no publication -- with no routed statistics and nothing + pruned would otherwise emit a Study named ``my_table.yaml`` whose only StudyResult is a + row index into a file the pipeline regenerates. That is a fabricated study on every + edge, and no translator-ingests source models evidence that way. In that case the + struct is skipped; the routed/sheet/row columns are dropped either way. A section WITH + a publication keeps the full struct unchanged -- ``PMID:123#Table_S7 row 12`` is real + provenance -- as does any section that has statistics to carry. + ``study_id`` is a per-section constant, so it can key a static struct field. Args: lf: Edges LazyFrame after annotation and provenance ops. study_id: Stable study identifier (``"#"``). sheet: Worksheet name, when the source is a spreadsheet. + identified: Whether ``study_id`` names a real publication rather than falling back + to the config filename. ``False`` lets a contentless study be skipped. Returns: - LazyFrame with ``has_supporting_studies`` appended and the routed columns dropped. + LazyFrame with ``has_supporting_studies`` appended (when it carries anything) and + the routed columns dropped. """ names: list[str] = lf.collect_schema().names() routed: list[str] = sorted(c for c in names if c in UNSATISFIABLE_EDGE_FIELDS and c not in DISABLED_EDGE_FIELDS) disabled: list[str] = sorted(c for c in names if c in DISABLED_EDGE_FIELDS) row: str = "extracted_from_row_number" has_row: bool = row in names + pruned: bool = PRUNED_COLUMN in names + drop: list[str] = [ + *routed, + *disabled, + *([row] if has_row else []), + *(["sheet_name"] if "sheet_name" in names else []), + *([PRUNED_COLUMN] if pruned else []), + ] + if not identified and not routed and not pruned: + return lf.drop(drop) result_id: pl.Expr = pl.concat_str([pl.lit(f"{study_id}#row"), pl.col(row).cast(pl.String)]) if has_row else pl.lit(f"{study_id}#result") label: str = f"{sheet} row " if sheet else "row " @@ -420,7 +444,6 @@ def inline_supporting_study(lf: pl.LazyFrame, study_id: str, sheet: str | None) # than silently dropped; `StudyResult.has_attribute` is `list[str]` (not inlined), # so typed Attributes would require emitting Attribute rows into the nodes file. fields: list[pl.Expr] = [result_id.alias("id"), result_name.alias("name")] - pruned: bool = PRUNED_COLUMN in names if routed or pruned: parts: list[pl.Expr] = [] for col in routed: @@ -443,13 +466,6 @@ def inline_supporting_study(lf: pl.LazyFrame, study_id: str, sheet: str | None) pl.lit(study_id).alias("id"), pl.lit(sheet or study_id).alias("name"), pl.concat_list(pl.struct(fields)).alias("has_study_results") ) out: pl.LazyFrame = lf.with_columns(pl.struct(study.alias(study_id)).alias("has_supporting_studies")) - drop: list[str] = [ - *routed, - *disabled, - *([row] if has_row else []), - *(["sheet_name"] if "sheet_name" in names else []), - *([PRUNED_COLUMN] if pruned else []), - ] return out.drop(drop) @@ -1118,6 +1134,10 @@ def _provenance_ops(self: Self) -> list[Any]: # The study is the table itself: one publication, one worksheet. Both are # section constants, so the study id can key a static struct field. sheet: str | None = self.source.sheet if self.source.kind == Files.EXCEL else None # pyright: ignore + # No publication means no study to name: the fallback below is a filename, not an + # identifier, so `inline_supporting_study` skips the struct unless it has statistics + # to carry. + identified: bool = bool(publication_values) publication: str = publication_values[0] if publication_values else (self.config.name or "study") study_id: str = f"{publication}#{sheet}" if sheet else publication return [ @@ -1132,7 +1152,7 @@ def _provenance_ops(self: Self) -> list[Any]: (publications, (publication_values,)) if publication_values else None, # Prune first so class-rejected values are handed to the study rather than lost. (prune_to_class, ()), - (inline_supporting_study, (study_id, sheet)), + (inline_supporting_study, (study_id, sheet, identified)), (trim, ()), (format_numeric, ()), (to_store, (self.store, self.config.name)), diff --git a/tests/test_lib.py b/tests/test_lib.py index bc4a5d6..b8ea5ff 100644 --- a/tests/test_lib.py +++ b/tests/test_lib.py @@ -2948,3 +2948,50 @@ def test_prune_to_class_preserves_classless_approval_ids() -> None: ).lazy() out: pl.DataFrame = prune_to_class(lf).collect() assert out["approval_ids"].to_list() == ["011111|022222", "033333"] + + +def test_inline_supporting_study_skips_a_study_with_no_identity_and_nothing_to_carry() -> None: + """An unpublished section with nothing routed or pruned emits no ``has_supporting_studies``. + + ``study_id`` would be the config filename and the sole StudyResult a row index, so the + struct would assert a Study that never existed on every edge. The row and sheet columns + are still consumed -- they are never meant to reach the edge. + """ + from tablassert.lib import inline_supporting_study + + lf: pl.LazyFrame = pl.DataFrame({"subject": ["A"], "object": ["B"], "extracted_from_row_number": [6848], "sheet_name": ["Table_S7"]}).lazy() + out: pl.DataFrame = inline_supporting_study(lf, "my_table.yaml", None, False).collect() + assert "has_supporting_studies" not in out.columns + assert "extracted_from_row_number" not in out.columns + assert "sheet_name" not in out.columns + assert out["subject"].to_list() == ["A"] + + +def test_inline_supporting_study_keeps_an_unidentified_study_that_carries_statistics() -> None: + """Without a publication the struct still survives when it has values to preserve. + + The rescue path is the whole reason the fallback id exists: a class-refused value is real + evidence, and losing it would be worse than naming its carrier after a config file. + """ + from tablassert.lib import PRUNED_COLUMN, inline_supporting_study + + lf: pl.LazyFrame = pl.DataFrame( + {"subject": ["A"], "extracted_from_row_number": [12], PRUNED_COLUMN: [["species_context_qualifier=NCBITaxon:9606"]]} + ).lazy() + out: pl.DataFrame = inline_supporting_study(lf, "my_table.yaml", None, False).collect() + study: dict[str, Any] = out["has_supporting_studies"].to_list()[0]["my_table.yaml"] + assert study["has_study_results"][0]["description"] == "species_context_qualifier=NCBITaxon:9606" + assert study["has_study_results"][0]["id"] == "my_table.yaml#row12" + assert PRUNED_COLUMN not in out.columns + + +def test_inline_supporting_study_keeps_a_published_study_with_no_statistics() -> None: + """A real publication earns the struct on its own -- ``PMID:123 row 12`` is real provenance.""" + from tablassert.lib import inline_supporting_study + + lf: pl.LazyFrame = pl.DataFrame({"subject": ["A"], "extracted_from_row_number": [12]}).lazy() + out: pl.DataFrame = inline_supporting_study(lf, "PMID:123#Table_S7", "Table_S7", True).collect() + study: dict[str, Any] = out["has_supporting_studies"].to_list()[0]["PMID:123#Table_S7"] + assert study["name"] == "Table_S7" + assert study["has_study_results"][0]["name"] == "Table_S7 row 12" + assert "description" not in study["has_study_results"][0]