diff --git a/CHANGELOG.md b/CHANGELOG.md index a6ecd31..a1c9a5f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ All notable changes to this project are documented in this file. +## Unreleased + +### Breaking Changes +- **Retrieval-source entries now use `resource_id` as their sole identifier.** Generated edges no longer duplicate each nested `sources` entry's provenance identifier into `sources[].id`; readers should use `sources[].resource_id`. Tablassert's validator supplies the inherited `id` only to its in-memory compatibility copy while the pinned Biolink model still requires it. + ## 13.0.0 - 2026-08-24 ### Breaking Changes diff --git a/docs/configuration/table.md b/docs/configuration/table.md index 175a5e3..0c4a8bf 100644 --- a/docs/configuration/table.md +++ b/docs/configuration/table.md @@ -451,7 +451,7 @@ Override fields: | `knowledge_level` | String | No | Override-specific KL value. Defaults to `statistical_association`. | | `agent_type` | String | No | Override-specific AT value. Defaults to `data_analysis_pipeline`. | -Tablassert emits the graph-level infores (or `infores:` when unset) as the primary entry of the Biolink `sources` list on each edge, `{resource_id: "infores:multiomics-kg", resource_role: "primary_knowledge_source", upstream_resource_ids: [...], source_record_urls: [...]}`, with one additional `supporting_data_source` entry per upstream. When `override.upstream_source_record_urls` is set, the primary entry emits no `source_record_urls` and each mapped supporting entry carries its own instead. No flat `primary_knowledge_source` scalar is emitted: current translator-ingests practice carries retrieval provenance only in `sources`, and the Biolink `RetrievalSource` class is where `resource_id` / `upstream_resource_ids` / `source_record_urls` are defined. (Each entry also mirrors `resource_id` into `id` because the generated Biolink classes still require it; that mirror disappears once biolink-model [#1706](https://github.com/biolink/biolink-model/issues/1706) lands.) The override cannot set a per-section primary source; manual infores CURIEs belong in `upstream_resource_ids`. Older flat `resource_id` / `primary_knowledge_source` output has been removed so generated KGX matches the Biolink edge contract. +Tablassert emits the graph-level infores (or `infores:` when unset) as the primary entry of the Biolink `sources` list on each edge, `{resource_id: "infores:multiomics-kg", resource_role: "primary_knowledge_source", upstream_resource_ids: [...], source_record_urls: [...]}`, with one additional `supporting_data_source` entry per upstream. When `override.upstream_source_record_urls` is set, the primary entry emits no `source_record_urls` and each mapped supporting entry carries its own instead. No flat `primary_knowledge_source` scalar is emitted: current translator-ingests practice carries retrieval provenance only in `sources`, and the Biolink `RetrievalSource` class is where `resource_id` / `upstream_resource_ids` / `source_record_urls` are defined. Each retrieval-source entry uses `resource_id` as its sole identifier. The override cannot set a per-section primary source; manual infores CURIEs belong in `upstream_resource_ids`. Older flat `resource_id` / `primary_knowledge_source` output has been removed so generated KGX matches the Biolink edge contract. ### Annotations diff --git a/src/tablassert/biolink.py b/src/tablassert/biolink.py index 7dc6c1b..65aa2b5 100644 --- a/src/tablassert/biolink.py +++ b/src/tablassert/biolink.py @@ -643,6 +643,40 @@ def is_multivalued(cls: type[Any], field: str) -> bool: return any(get_origin(arg) is list for arg in get_args(annotation)) +@cache +def _retrieval_source_id_required() -> bool: + """Whether the installed model still requires the inherited source ``id`` field.""" + retrieval_source: Any = getattr(_bm, "RetrievalSource", None) + id_field: Any = getattr(retrieval_source, "model_fields", {}).get("id") + return id_field is not None and id_field.is_required() + + +def _validation_record(record: dict[str, Any], *, edge: bool) -> dict[str, Any]: + """Add only in-memory compatibility aliases needed by the installed Biolink model. + + ``RetrievalSource`` in the currently pinned model still requires the inherited + ``Entity.id`` even though ``resource_id`` is the canonical provenance identifier + emitted by Tablassert. The alias is used solely for Pydantic validation; it never + changes the decoded KGX record or the files written by the pipeline. + """ + if not edge: + return record + if not _retrieval_source_id_required(): + return record + sources: Any = record.get("sources") + if not isinstance(sources, list): + return record + normalized: list[Any] = [] + changed: bool = False + for source in sources: + if isinstance(source, dict) and "id" not in source and "resource_id" in source: + normalized.append({**source, "id": source["resource_id"]}) + changed = True + else: + normalized.append(source) + return {**record, "sources": normalized} if changed else record + + def validate_record(record: dict[str, Any], *, edge: bool) -> list[str]: """Validate one KGX record against the Biolink class named by its ``category``. @@ -659,7 +693,7 @@ def validate_record(record: dict[str, Any], *, edge: bool) -> list[str]: category: str = categories[0] if isinstance(categories, list) and categories else str(categories or "") cls: type[Any] = association_class(category) if edge else node_class(category) try: - cls(**record) + cls(**_validation_record(record, edge=edge)) except ValidationError as error: return [f"{'.'.join(str(part) for part in item['loc']) or '?'}: {item['type']}" for item in error.errors()] return [] diff --git a/src/tablassert/lib.py b/src/tablassert/lib.py index 0a6e5fc..9ea8e8d 100644 --- a/src/tablassert/lib.py +++ b/src/tablassert/lib.py @@ -354,16 +354,12 @@ def value(lf: pl.LazyFrame, col: str, x: object) -> pl.LazyFrame: def _retrieval_source(resource_id: str, resource_role: str, upstream: list[str] | None = None, urls: list[str] | None = None) -> pl.Expr: """Build one ``RetrievalSource`` struct expression. - Every entry declares the same five fields so that :func:`retrieval_sources` can + Every entry declares the same four fields so that :func:`retrieval_sources` can ``concat_list`` them into a single ``list[struct]`` column; absent list fields are typed nulls, which the Rust null-stripper removes from the emitted JSON. """ empty: pl.Expr = pl.lit(None, dtype=pl.List(pl.String)) return pl.struct( - # `id` duplicates `resource_id`, but RetrievalSource inherits `id` from - # `entity` and the generated Pydantic classes require it, so omitting it - # fails KGX validation. Stays until biolink-model #1706/#1731 land. - pl.lit(resource_id).alias("id"), pl.lit(resource_id).alias("resource_id"), pl.lit(resource_role).alias("resource_role"), (pl.concat_list([pl.lit(x) for x in upstream]) if upstream else empty).alias("upstream_resource_ids"), diff --git a/tests/test_biolink.py b/tests/test_biolink.py index 984770a..a1aed18 100644 --- a/tests/test_biolink.py +++ b/tests/test_biolink.py @@ -453,6 +453,29 @@ def test_validate_kgx_never_passes_a_missing_file(tmp_path: Path) -> None: assert report["edges"]["missing"] is True +def test_validate_kgx_accepts_resource_id_only_retrieval_sources(tmp_path: Path) -> None: + """Validation aliases ``resource_id`` to the pinned model's inherited ``id`` in memory only.""" + nodes: Path = tmp_path / "n.ndjson" + edges: Path = tmp_path / "e.ndjson" + nodes.write_text(json.dumps({"id": "HGNC:11998", "name": "TP53", "category": ["biolink:Gene"]}) + "\n") + edge: dict[str, Any] = { + "id": "e1", + "subject": "HGNC:11998", + "predicate": "biolink:associated_with", + "object": "MONDO:0008903", + "category": ["biolink:Association"], + "knowledge_level": "statistical_association", + "agent_type": "data_analysis_pipeline", + "sources": [{"resource_id": "infores:test", "resource_role": "primary_knowledge_source"}], + } + edges.write_text(json.dumps(edge) + "\n") + + report: dict[str, Any] = validate_kgx(nodes, edges) + assert "id" not in json.loads(edges.read_text())["sources"][0] + assert report["edges"]["valid"] == 1 + assert report["ok"] is True + + def test_validate_kgx_separates_pending_extras_from_real_failures(tmp_path: Path) -> None: """``valid_excluding_pending`` forgives a deliberate extra; ``valid`` stays strict.""" nodes: Path = tmp_path / "n.ndjson" diff --git a/tests/test_cover_lib.py b/tests/test_cover_lib.py index 5cf22b1..81102bd 100644 --- a/tests/test_cover_lib.py +++ b/tests/test_cover_lib.py @@ -205,14 +205,7 @@ def test_compile_graph_unlinks_stale_tmp_outputs(monkeypatch: Any, tmp_path: Pat "agent_type": ["manual_agent"], "primary_knowledge_source": ["infores:un-kg"], "sources": [ - [ - { - "id": "infores:un-kg", - "resource_id": "infores:un-kg", - "resource_role": "primary_knowledge_source", - "source_record_urls": ["https://example.org/un.tsv"], - } - ] + [{"resource_id": "infores:un-kg", "resource_role": "primary_knowledge_source", "source_record_urls": ["https://example.org/un.tsv"]}] ], } ).write_parquet(sub) diff --git a/tests/test_lib.py b/tests/test_lib.py index 50806b0..78261a2 100644 --- a/tests/test_lib.py +++ b/tests/test_lib.py @@ -577,10 +577,7 @@ def test_tcode_collect_nests_upstream_resource_ids_in_sources(fixtures_path: Pat primary: dict[str, Any] = next(s for s in sources if s["resource_role"] == "primary_knowledge_source") assert primary["upstream_resource_ids"] == ["infores:pubmed-central"] assert {s["resource_id"] for s in sources if s["resource_role"] == "supporting_data_source"} == {"infores:pubmed-central"} - # `id` mirrors `resource_id`: RetrievalSource inherits a required `id` from - # `entity` in the generated Biolink classes, so it stays until biolink-model - # #1706/#1731 land. - assert all(s["id"] == s["resource_id"] for s in sources) + assert all("id" not in s for s in sources) def test_normalize_category_list_with_biolink_prefix() -> None: @@ -740,7 +737,7 @@ def test_tcode_collect_nests_source_record_urls_in_sources(fixtures_path: Path) assert "source_record_urls" not in result.columns primary: dict[str, Any] = next(s for s in result["sources"].to_list()[0] if s["resource_role"] == "primary_knowledge_source") assert primary["source_record_urls"] == ["https://example.com/test.tsv"] - assert primary["id"] == primary["resource_id"] + assert "id" not in primary def test_tcode_collect_upstream_source_record_urls_rehome_urls(fixtures_path: Path) -> None: @@ -1195,13 +1192,12 @@ def test_compile_graph_emits_ndjson(monkeypatch: Any, tmp_path: Path, rig_factor monkeypatch.chdir(tmp_path) sub: Path = tmp_path / "sub.parquet" primary: dict[str, Any] = { - "id": "infores:smoke", "resource_id": "infores:smoke", "resource_role": "primary_knowledge_source", "upstream_resource_ids": ["infores:pubmed-central"], "source_record_urls": ["https://pmc.ncbi.nlm.nih.gov/bin/table1.xlsx"], } - supporting: dict[str, Any] = {"id": "infores:pubmed-central", "resource_id": "infores:pubmed-central", "resource_role": "supporting_data_source"} + supporting: dict[str, Any] = {"resource_id": "infores:pubmed-central", "resource_role": "supporting_data_source"} pl.DataFrame( { "subject": ["A", "B"], @@ -1316,7 +1312,6 @@ def spy_open( "sources": [ [ { - "id": "infores:utf8-kg", "resource_id": "infores:utf8-kg", "resource_role": "primary_knowledge_source", "source_record_urls": ["https://example.org/utf8.tsv"], @@ -1361,7 +1356,6 @@ def write_sub(p: Path, subj: str, obj: str) -> None: "sources": [ [ { - "id": "infores:cb-kg", "resource_id": "infores:cb-kg", "resource_role": "primary_knowledge_source", "source_record_urls": ["https://example.org/cb.tsv"], @@ -1422,7 +1416,6 @@ def test_compile_graph_keeps_qualifiers_and_publications_on_edges(monkeypatch: A "sources": [ [ { - "id": "infores:qual-kg", "resource_id": "infores:qual-kg", "resource_role": "primary_knowledge_source", "source_record_urls": ["https://example.org/qual.tsv"], @@ -2531,7 +2524,6 @@ def test_compile_graph_folds_unknown_annotations_into_supporting_text(monkeypatc "sources": [ [ { - "id": "infores:fold-kg", "resource_id": "infores:fold-kg", "resource_role": "primary_knowledge_source", "source_record_urls": ["https://example.org/fold.tsv"], @@ -2582,7 +2574,6 @@ def test_compile_graph_passes_approval_ids_through_verbatim(monkeypatch: Any, tm "sources": [ [ { - "id": "infores:approval-kg", "resource_id": "infores:approval-kg", "resource_role": "primary_knowledge_source", "source_record_urls": ["https://example.org/approval.tsv"], @@ -2940,9 +2931,8 @@ def test_build_pipeline_e2e_smoke_with_monkeypatched_fullmap(monkeypatch: Any, t primary_source: dict[str, Any] = next(x for x in edge_rows[0]["sources"] if x["resource_role"] == "primary_knowledge_source") assert primary_source["resource_id"] == "infores:pipeline-kg" assert primary_source["upstream_resource_ids"] == ["infores:pubmed-central"] - # `id` mirrors `resource_id` (required by the generated Biolink classes), and - # no flat scalar duplicates the primary source. - assert primary_source["id"] == primary_source["resource_id"] + # Source provenance uses `resource_id` without a duplicate Biolink `id`. + assert "id" not in primary_source assert "primary_knowledge_source" not in edge_rows[0] assert {row["id"] for row in node_rows} == {"HGNC:1100", "HGNC:11998"} assert rig["name"] == "PIPELINE_KG v0.1.0 Resource Ingest Guide" diff --git a/tests/test_rig.py b/tests/test_rig.py index 1db53d8..2f9b17b 100644 --- a/tests/test_rig.py +++ b/tests/test_rig.py @@ -22,7 +22,7 @@ def _source_entry(primary: str, url: str, upstream: list[str] | None = None) -> list[dict[str, Any]]: - entry: dict[str, Any] = {"id": primary, "resource_id": primary, "resource_role": "primary_knowledge_source", "source_record_urls": [url]} + entry: dict[str, Any] = {"resource_id": primary, "resource_role": "primary_knowledge_source", "source_record_urls": [url]} if upstream: entry["upstream_resource_ids"] = upstream return [entry] @@ -85,14 +85,13 @@ def test_rig_edge_type_info_separates_roles_properties_qualifiers_and_files(tmp_ "MONDO:1", sources=[ { - "id": "infores:test-kg", "resource_id": "infores:test-kg", "resource_role": "primary_knowledge_source", "upstream_resource_ids": ["infores:pubmed-central"], "source_record_urls": ["https://pmc.ncbi.nlm.nih.gov/bin/table1.xlsx"], }, - {"id": "infores:pubmed-central", "resource_id": "infores:pubmed-central", "resource_role": "supporting_data_source"}, - {"id": "infores:aggregator", "resource_id": "infores:aggregator", "resource_role": "aggregator_knowledge_source"}, + {"resource_id": "infores:pubmed-central", "resource_role": "supporting_data_source"}, + {"resource_id": "infores:aggregator", "resource_role": "aggregator_knowledge_source"}, ], p_value=0.01, publications=["PMID:1"],