Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,11 @@

All notable changes to this project are documented in this file.

## Unreleased

### Breaking Changes
- **Retrieval-source entries now use `resource_id` as their sole identifier.** Generated edges no longer duplicate each nested `sources` entry's provenance identifier into `sources[].id`; readers should use `sources[].resource_id`. Tablassert's validator supplies the inherited `id` only to its in-memory compatibility copy while the pinned Biolink model still requires it.

## 13.0.0 - 2026-08-24

### Breaking Changes
Expand Down
2 changes: 1 addition & 1 deletion docs/configuration/table.md
Original file line number Diff line number Diff line change
Expand Up @@ -451,7 +451,7 @@ Override fields:
| `knowledge_level` | String | No | Override-specific KL value. Defaults to `statistical_association`. |
| `agent_type` | String | No | Override-specific AT value. Defaults to `data_analysis_pipeline`. |

Tablassert emits the graph-level infores (or `infores:<graph-name>` when unset) as the primary entry of the Biolink `sources` list on each edge, `{resource_id: "infores:multiomics-kg", resource_role: "primary_knowledge_source", upstream_resource_ids: [...], source_record_urls: [...]}`, with one additional `supporting_data_source` entry per upstream. When `override.upstream_source_record_urls` is set, the primary entry emits no `source_record_urls` and each mapped supporting entry carries its own instead. No flat `primary_knowledge_source` scalar is emitted: current translator-ingests practice carries retrieval provenance only in `sources`, and the Biolink `RetrievalSource` class is where `resource_id` / `upstream_resource_ids` / `source_record_urls` are defined. (Each entry also mirrors `resource_id` into `id` because the generated Biolink classes still require it; that mirror disappears once biolink-model [#1706](https://github.com/biolink/biolink-model/issues/1706) lands.) The override cannot set a per-section primary source; manual infores CURIEs belong in `upstream_resource_ids`. Older flat `resource_id` / `primary_knowledge_source` output has been removed so generated KGX matches the Biolink edge contract.
Tablassert emits the graph-level infores (or `infores:<graph-name>` when unset) as the primary entry of the Biolink `sources` list on each edge, `{resource_id: "infores:multiomics-kg", resource_role: "primary_knowledge_source", upstream_resource_ids: [...], source_record_urls: [...]}`, with one additional `supporting_data_source` entry per upstream. When `override.upstream_source_record_urls` is set, the primary entry emits no `source_record_urls` and each mapped supporting entry carries its own instead. No flat `primary_knowledge_source` scalar is emitted: current translator-ingests practice carries retrieval provenance only in `sources`, and the Biolink `RetrievalSource` class is where `resource_id` / `upstream_resource_ids` / `source_record_urls` are defined. Each retrieval-source entry uses `resource_id` as its sole identifier. The override cannot set a per-section primary source; manual infores CURIEs belong in `upstream_resource_ids`. Older flat `resource_id` / `primary_knowledge_source` output has been removed so generated KGX matches the Biolink edge contract.

### Annotations

Expand Down
36 changes: 35 additions & 1 deletion src/tablassert/biolink.py
Original file line number Diff line number Diff line change
Expand Up @@ -643,6 +643,40 @@ def is_multivalued(cls: type[Any], field: str) -> bool:
return any(get_origin(arg) is list for arg in get_args(annotation))


@cache
def _retrieval_source_id_required() -> bool:
"""Whether the installed model still requires the inherited source ``id`` field."""
retrieval_source: Any = getattr(_bm, "RetrievalSource", None)
id_field: Any = getattr(retrieval_source, "model_fields", {}).get("id")
return id_field is not None and id_field.is_required()


def _validation_record(record: dict[str, Any], *, edge: bool) -> dict[str, Any]:
"""Add only in-memory compatibility aliases needed by the installed Biolink model.

``RetrievalSource`` in the currently pinned model still requires the inherited
``Entity.id`` even though ``resource_id`` is the canonical provenance identifier
emitted by Tablassert. The alias is used solely for Pydantic validation; it never
changes the decoded KGX record or the files written by the pipeline.
"""
if not edge:
return record
if not _retrieval_source_id_required():
return record
sources: Any = record.get("sources")
if not isinstance(sources, list):
return record
normalized: list[Any] = []
changed: bool = False
for source in sources:
if isinstance(source, dict) and "id" not in source and "resource_id" in source:
normalized.append({**source, "id": source["resource_id"]})
changed = True
else:
normalized.append(source)
return {**record, "sources": normalized} if changed else record


def validate_record(record: dict[str, Any], *, edge: bool) -> list[str]:
"""Validate one KGX record against the Biolink class named by its ``category``.

Expand All @@ -659,7 +693,7 @@ def validate_record(record: dict[str, Any], *, edge: bool) -> list[str]:
category: str = categories[0] if isinstance(categories, list) and categories else str(categories or "")
cls: type[Any] = association_class(category) if edge else node_class(category)
try:
cls(**record)
cls(**_validation_record(record, edge=edge))
except ValidationError as error:
return [f"{'.'.join(str(part) for part in item['loc']) or '?'}: {item['type']}" for item in error.errors()]
return []
Expand Down
6 changes: 1 addition & 5 deletions src/tablassert/lib.py
Original file line number Diff line number Diff line change
Expand Up @@ -354,16 +354,12 @@ def value(lf: pl.LazyFrame, col: str, x: object) -> pl.LazyFrame:
def _retrieval_source(resource_id: str, resource_role: str, upstream: list[str] | None = None, urls: list[str] | None = None) -> pl.Expr:
"""Build one ``RetrievalSource`` struct expression.

Every entry declares the same five fields so that :func:`retrieval_sources` can
Every entry declares the same four fields so that :func:`retrieval_sources` can
``concat_list`` them into a single ``list[struct]`` column; absent list fields are
typed nulls, which the Rust null-stripper removes from the emitted JSON.
"""
empty: pl.Expr = pl.lit(None, dtype=pl.List(pl.String))
return pl.struct(
# `id` duplicates `resource_id`, but RetrievalSource inherits `id` from
# `entity` and the generated Pydantic classes require it, so omitting it
# fails KGX validation. Stays until biolink-model #1706/#1731 land.
pl.lit(resource_id).alias("id"),
pl.lit(resource_id).alias("resource_id"),
pl.lit(resource_role).alias("resource_role"),
(pl.concat_list([pl.lit(x) for x in upstream]) if upstream else empty).alias("upstream_resource_ids"),
Expand Down
23 changes: 23 additions & 0 deletions tests/test_biolink.py
Original file line number Diff line number Diff line change
Expand Up @@ -453,6 +453,29 @@ def test_validate_kgx_never_passes_a_missing_file(tmp_path: Path) -> None:
assert report["edges"]["missing"] is True


def test_validate_kgx_accepts_resource_id_only_retrieval_sources(tmp_path: Path) -> None:
"""Validation aliases ``resource_id`` to the pinned model's inherited ``id`` in memory only."""
nodes: Path = tmp_path / "n.ndjson"
edges: Path = tmp_path / "e.ndjson"
nodes.write_text(json.dumps({"id": "HGNC:11998", "name": "TP53", "category": ["biolink:Gene"]}) + "\n")
edge: dict[str, Any] = {
"id": "e1",
"subject": "HGNC:11998",
"predicate": "biolink:associated_with",
"object": "MONDO:0008903",
"category": ["biolink:Association"],
"knowledge_level": "statistical_association",
"agent_type": "data_analysis_pipeline",
"sources": [{"resource_id": "infores:test", "resource_role": "primary_knowledge_source"}],
}
edges.write_text(json.dumps(edge) + "\n")

report: dict[str, Any] = validate_kgx(nodes, edges)
assert "id" not in json.loads(edges.read_text())["sources"][0]
assert report["edges"]["valid"] == 1
assert report["ok"] is True


def test_validate_kgx_separates_pending_extras_from_real_failures(tmp_path: Path) -> None:
"""``valid_excluding_pending`` forgives a deliberate extra; ``valid`` stays strict."""
nodes: Path = tmp_path / "n.ndjson"
Expand Down
9 changes: 1 addition & 8 deletions tests/test_cover_lib.py
Original file line number Diff line number Diff line change
Expand Up @@ -205,14 +205,7 @@ def test_compile_graph_unlinks_stale_tmp_outputs(monkeypatch: Any, tmp_path: Pat
"agent_type": ["manual_agent"],
"primary_knowledge_source": ["infores:un-kg"],
"sources": [
[
{
"id": "infores:un-kg",
"resource_id": "infores:un-kg",
"resource_role": "primary_knowledge_source",
"source_record_urls": ["https://example.org/un.tsv"],
}
]
[{"resource_id": "infores:un-kg", "resource_role": "primary_knowledge_source", "source_record_urls": ["https://example.org/un.tsv"]}]
],
}
).write_parquet(sub)
Expand Down
20 changes: 5 additions & 15 deletions tests/test_lib.py
Original file line number Diff line number Diff line change
Expand Up @@ -577,10 +577,7 @@ def test_tcode_collect_nests_upstream_resource_ids_in_sources(fixtures_path: Pat
primary: dict[str, Any] = next(s for s in sources if s["resource_role"] == "primary_knowledge_source")
assert primary["upstream_resource_ids"] == ["infores:pubmed-central"]
assert {s["resource_id"] for s in sources if s["resource_role"] == "supporting_data_source"} == {"infores:pubmed-central"}
# `id` mirrors `resource_id`: RetrievalSource inherits a required `id` from
# `entity` in the generated Biolink classes, so it stays until biolink-model
# #1706/#1731 land.
assert all(s["id"] == s["resource_id"] for s in sources)
assert all("id" not in s for s in sources)


def test_normalize_category_list_with_biolink_prefix() -> None:
Expand Down Expand Up @@ -740,7 +737,7 @@ def test_tcode_collect_nests_source_record_urls_in_sources(fixtures_path: Path)
assert "source_record_urls" not in result.columns
primary: dict[str, Any] = next(s for s in result["sources"].to_list()[0] if s["resource_role"] == "primary_knowledge_source")
assert primary["source_record_urls"] == ["https://example.com/test.tsv"]
assert primary["id"] == primary["resource_id"]
assert "id" not in primary


def test_tcode_collect_upstream_source_record_urls_rehome_urls(fixtures_path: Path) -> None:
Expand Down Expand Up @@ -1195,13 +1192,12 @@ def test_compile_graph_emits_ndjson(monkeypatch: Any, tmp_path: Path, rig_factor
monkeypatch.chdir(tmp_path)
sub: Path = tmp_path / "sub.parquet"
primary: dict[str, Any] = {
"id": "infores:smoke",
"resource_id": "infores:smoke",
"resource_role": "primary_knowledge_source",
"upstream_resource_ids": ["infores:pubmed-central"],
"source_record_urls": ["https://pmc.ncbi.nlm.nih.gov/bin/table1.xlsx"],
}
supporting: dict[str, Any] = {"id": "infores:pubmed-central", "resource_id": "infores:pubmed-central", "resource_role": "supporting_data_source"}
supporting: dict[str, Any] = {"resource_id": "infores:pubmed-central", "resource_role": "supporting_data_source"}
pl.DataFrame(
{
"subject": ["A", "B"],
Expand Down Expand Up @@ -1316,7 +1312,6 @@ def spy_open(
"sources": [
[
{
"id": "infores:utf8-kg",
"resource_id": "infores:utf8-kg",
"resource_role": "primary_knowledge_source",
"source_record_urls": ["https://example.org/utf8.tsv"],
Expand Down Expand Up @@ -1361,7 +1356,6 @@ def write_sub(p: Path, subj: str, obj: str) -> None:
"sources": [
[
{
"id": "infores:cb-kg",
"resource_id": "infores:cb-kg",
"resource_role": "primary_knowledge_source",
"source_record_urls": ["https://example.org/cb.tsv"],
Expand Down Expand Up @@ -1422,7 +1416,6 @@ def test_compile_graph_keeps_qualifiers_and_publications_on_edges(monkeypatch: A
"sources": [
[
{
"id": "infores:qual-kg",
"resource_id": "infores:qual-kg",
"resource_role": "primary_knowledge_source",
"source_record_urls": ["https://example.org/qual.tsv"],
Expand Down Expand Up @@ -2531,7 +2524,6 @@ def test_compile_graph_folds_unknown_annotations_into_supporting_text(monkeypatc
"sources": [
[
{
"id": "infores:fold-kg",
"resource_id": "infores:fold-kg",
"resource_role": "primary_knowledge_source",
"source_record_urls": ["https://example.org/fold.tsv"],
Expand Down Expand Up @@ -2582,7 +2574,6 @@ def test_compile_graph_passes_approval_ids_through_verbatim(monkeypatch: Any, tm
"sources": [
[
{
"id": "infores:approval-kg",
"resource_id": "infores:approval-kg",
"resource_role": "primary_knowledge_source",
"source_record_urls": ["https://example.org/approval.tsv"],
Expand Down Expand Up @@ -2940,9 +2931,8 @@ def test_build_pipeline_e2e_smoke_with_monkeypatched_fullmap(monkeypatch: Any, t
primary_source: dict[str, Any] = next(x for x in edge_rows[0]["sources"] if x["resource_role"] == "primary_knowledge_source")
assert primary_source["resource_id"] == "infores:pipeline-kg"
assert primary_source["upstream_resource_ids"] == ["infores:pubmed-central"]
# `id` mirrors `resource_id` (required by the generated Biolink classes), and
# no flat scalar duplicates the primary source.
assert primary_source["id"] == primary_source["resource_id"]
# Source provenance uses `resource_id` without a duplicate Biolink `id`.
assert "id" not in primary_source
assert "primary_knowledge_source" not in edge_rows[0]
assert {row["id"] for row in node_rows} == {"HGNC:1100", "HGNC:11998"}
assert rig["name"] == "PIPELINE_KG v0.1.0 Resource Ingest Guide"
Expand Down
7 changes: 3 additions & 4 deletions tests/test_rig.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@


def _source_entry(primary: str, url: str, upstream: list[str] | None = None) -> list[dict[str, Any]]:
entry: dict[str, Any] = {"id": primary, "resource_id": primary, "resource_role": "primary_knowledge_source", "source_record_urls": [url]}
entry: dict[str, Any] = {"resource_id": primary, "resource_role": "primary_knowledge_source", "source_record_urls": [url]}
if upstream:
entry["upstream_resource_ids"] = upstream
return [entry]
Expand Down Expand Up @@ -85,14 +85,13 @@ def test_rig_edge_type_info_separates_roles_properties_qualifiers_and_files(tmp_
"MONDO:1",
sources=[
{
"id": "infores:test-kg",
"resource_id": "infores:test-kg",
"resource_role": "primary_knowledge_source",
"upstream_resource_ids": ["infores:pubmed-central"],
"source_record_urls": ["https://pmc.ncbi.nlm.nih.gov/bin/table1.xlsx"],
},
{"id": "infores:pubmed-central", "resource_id": "infores:pubmed-central", "resource_role": "supporting_data_source"},
{"id": "infores:aggregator", "resource_id": "infores:aggregator", "resource_role": "aggregator_knowledge_source"},
{"resource_id": "infores:pubmed-central", "resource_role": "supporting_data_source"},
{"resource_id": "infores:aggregator", "resource_role": "aggregator_knowledge_source"},
],
p_value=0.01,
publications=["PMID:1"],
Expand Down
Loading