Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@ All notable changes to this project are documented in this file.
### Added
- **`build-kg --release` now drops edges whose `effect_size` is exactly zero.** When an `effect_size` column is present, release-mode builds filter out rows with a non-null zero effect size before fullmap resolution, matching the existing release-mode drop of `biolink:not_significant` edges. Rows with a null effect size are kept.

### Fixed
- **`species_context_qualifier` is no longer auto-derived or relocated into supporting studies.** The field was derived from resolved node taxon metadata, then stored as a `StudyResult.description` key/value when the selected association class could not accept it. It is now disabled for both qualifiers and annotations; node taxon metadata remains available on node records.

## 11.0.0 - 2026-08-13

### Breaking Changes
Expand Down
5 changes: 3 additions & 2 deletions docs/configuration/table.md
Original file line number Diff line number Diff line change
Expand Up @@ -370,8 +370,9 @@ Use the `"values"` token to reference column values in transformations.
### Qualifiers

Add context to edges (anatomical location, disease context, etc.).
`species_context_qualifier` is auto-derived from resolved subject/object taxon
metadata and should not be declared manually.
`species_context_qualifier` is intentionally disabled: it is neither derived from
node taxon metadata nor accepted as a qualifier or annotation. Taxon metadata is
still emitted on resolved nodes.

| Field | Type | Required | Description |
|-------|------|----------|-------------|
Expand Down
2 changes: 1 addition & 1 deletion src/tablassert/agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -2206,7 +2206,7 @@ def predicate_cheatsheet(pairs: Sequence[tuple[str, str]] = CHEATSHEET_PAIRS) ->
are EXEMPT from the validity score: a `biolink_valid_pct` below 1.0 is never caused by them.
- QUALIFIERS: enum-ranged qualifiers take a literal TOKEN, never a CURIE
(`object_direction_qualifier: increased`, not a UMLS id), and `species_context_qualifier` is
auto-derived from the resolved taxon — never author it.
disabled — never author it as a qualifier or annotation.

## ReAct workflow + planning
Reason in an explicit ReAct loop (Thought -> Action -> Observation) and re-plan every few steps:
Expand Down
22 changes: 18 additions & 4 deletions src/tablassert/biolink.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@
__all__ = [
"ALLOWED_EDGE_FIELDS",
"BIOLINK_VERSION",
"DISABLED_EDGE_FIELDS",
"EFFECT_TYPE_VALUES",
"ENUM_RANGED_QUALIFIERS",
"KNOWN_PENDING_EDGE_FIELDS",
Expand Down Expand Up @@ -420,7 +421,7 @@ def resolve_association_class(category: str, predicate: str) -> type[Any]:
# ``relationship_strength`` -- declared in the LinkML YAML but attached to zero
# Pydantic classes (see ``UNSATISFIABLE_EDGE_FIELDS``); they are routed onto the
# inlined ``Study`` / ``StudyResult`` instead.
# - ``taxon`` -- a node property; edges carry ``species_context_qualifier``.
# - ``taxon`` -- a node property; no species-context edge is synthesized from it.
TABLASERT_EDGE_EXTRAS: frozenset[str] = frozenset(
[
"broad_synonym",
Expand Down Expand Up @@ -523,6 +524,16 @@ class EffectTypes(str, Enum):
"""


DISABLED_EDGE_FIELDS: frozenset[str] = frozenset({"species_context_qualifier"})
"""Edge fields Tablassert intentionally never emits or accepts.

This policy is separate from :data:`UNSATISFIABLE_EDGE_FIELDS`: the latter tracks
Biolink Model attachment and may change with a dependency release, while this set is
a stable Tablassert product decision. Keeping it separate prevents a future Biolink
release from making ``species_context_qualifier`` silently emittable again.
"""


ENUM_RANGED_QUALIFIERS: dict[str, frozenset[str]] = {
qualifier.value: choices
for qualifier in Qualifiers
Expand Down Expand Up @@ -551,15 +562,18 @@ class EffectTypes(str, Enum):


ALLOWED_EDGE_FIELDS: frozenset[str] = (
frozenset(_association_model_fields()) | {q.value for q in Qualifiers} | TABLASERT_EDGE_EXTRAS
) - UNSATISFIABLE_EDGE_FIELDS
(frozenset(_association_model_fields()) | {q.value for q in Qualifiers} | TABLASERT_EDGE_EXTRAS)
- UNSATISFIABLE_EDGE_FIELDS
- DISABLED_EDGE_FIELDS
)
"""Authoritative biolink-compliant edge column allow-list.

Any column on an edge frame that is not in this set is folded into the
``supporting_text`` ``list[str]`` field by ``lib.fold_unknown_to_supporting_text()``
as a ``"column: value"`` string. Composed of the fields declared by *any* Biolink
association class, the derived qualifier slot names, and the curated
``TABLASERT_EDGE_EXTRAS`` -- less the slots that no Pydantic class can hold.
``TABLASERT_EDGE_EXTRAS`` -- less the slots that no Pydantic class can hold and the
fields disabled by Tablassert policy.

Note this is a per-*family* allow-list: a field being permitted here does not mean the
specific association class chosen for a given edge accepts it. Per-record pruning
Expand Down
2 changes: 1 addition & 1 deletion src/tablassert/errors.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
"annotation-split-by-empty",
"annotation-effect-size-without-type",
"annotation-effect-type-without-size",
"qualifier-auto-derived",
"field-disabled",
"qualifier-bad-value",
"qualifier-unsatisfiable",
"qualifier-nullable-literal",
Expand Down
46 changes: 21 additions & 25 deletions src/tablassert/lib.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
from tablassert._lazy import LazyModule
from tablassert.biolink import (
ALLOWED_EDGE_FIELDS,
DISABLED_EDGE_FIELDS,
ENUM_RANGED_QUALIFIERS,
UNSATISFIABLE_EDGE_FIELDS,
Categories,
Expand Down Expand Up @@ -260,8 +261,9 @@ def prune_to_class(lf: pl.LazyFrame) -> pl.LazyFrame:
``ALLOWED_EDGE_FIELDS`` is a per-*family* allow-list: it says a column is a slot
of *some* association class. Whether the specific class chosen for a given row
accepts it is a separate question, and getting it wrong is the single largest
source of ``extra_forbidden`` failures (``species_context_qualifier`` and friends
on a class that has no such slot).
source of ``extra_forbidden`` failures for qualifier fields on a class that has
no such slot. Tablassert-disabled fields are removed before this class-specific
masking so they cannot be rescued into study metadata.

Categories vary per row within a section, so this masks per row rather than
dropping columns: values are nulled where the row's class rejects them, and the
Expand All @@ -278,6 +280,10 @@ def prune_to_class(lf: pl.LazyFrame) -> pl.LazyFrame:
"""
schema: pl.Schema = lf.collect_schema()
names: list[str] = schema.names()
disabled: list[str] = [c for c in names if c in DISABLED_EDGE_FIELDS]
if disabled:
lf = lf.drop(disabled)
names = [c for c in names if c not in DISABLED_EDGE_FIELDS]
if "category" not in names:
return lf

Expand Down Expand Up @@ -343,22 +349,6 @@ def value(lf: pl.LazyFrame, col: str, x: object) -> pl.LazyFrame:
return lf.with_columns(pl.lit(x).alias(col))


def derive_species_context(lf: pl.LazyFrame) -> pl.LazyFrame:
"""Derive ``species_context_qualifier`` from resolved node taxon columns.

Uses ``subject_taxon`` first and falls back to ``object_taxon``. Null values
indicate no resolved taxon metadata and are later stripped from NDJSON
output by ``dedup_stream``.

Args:
lf: Source LazyFrame after subject/object fullmap resolution.

Returns:
LazyFrame with an auto-derived ``species_context_qualifier`` edge column.
"""
return lf.with_columns(pl.coalesce(pl.col("subject_taxon"), pl.col("object_taxon")).alias("species_context_qualifier"))


def _retrieval_source(resource_id: str, resource_role: str, upstream: list[str] | None = None, urls: list[str] | None = None) -> pl.Expr:
"""Build one ``RetrievalSource`` struct expression.

Expand Down Expand Up @@ -417,7 +407,8 @@ def inline_supporting_study(lf: pl.LazyFrame, study_id: str, sheet: str | None)
LazyFrame with ``has_supporting_studies`` appended and the routed columns dropped.
"""
names: list[str] = lf.collect_schema().names()
routed: list[str] = sorted(c for c in names if c in UNSATISFIABLE_EDGE_FIELDS)
routed: list[str] = sorted(c for c in names if c in UNSATISFIABLE_EDGE_FIELDS and c not in DISABLED_EDGE_FIELDS)
disabled: list[str] = sorted(c for c in names if c in DISABLED_EDGE_FIELDS)
row: str = "extracted_from_row_number"
has_row: bool = row in names

Expand Down Expand Up @@ -452,7 +443,13 @@ def inline_supporting_study(lf: pl.LazyFrame, study_id: str, sheet: str | None)
pl.lit(study_id).alias("id"), pl.lit(sheet or study_id).alias("name"), pl.concat_list(pl.struct(fields)).alias("has_study_results")
)
out: pl.LazyFrame = lf.with_columns(pl.struct(study.alias(study_id)).alias("has_supporting_studies"))
drop: list[str] = [*routed, *([row] if has_row else []), *(["sheet_name"] if "sheet_name" in names else []), *([PRUNED_COLUMN] if pruned else [])]
drop: list[str] = [
*routed,
*disabled,
*([row] if has_row else []),
*(["sheet_name"] if "sheet_name" in names else []),
*([PRUNED_COLUMN] if pruned else []),
]
return out.drop(drop)


Expand Down Expand Up @@ -1124,7 +1121,6 @@ def _provenance_ops(self: Self) -> list[Any]:
publication: str = publication_values[0] if publication_values else (self.config.name or "study")
study_id: str = f"{publication}#{sheet}" if sheet else publication
return [
(derive_species_context, ()),
(value, ("predicate", "biolink:" + self.statement.predicate)),
(edge_category, ("biolink:" + self.statement.predicate,)),
(value, ("knowledge_level", knowledge_level)),
Expand Down Expand Up @@ -1182,7 +1178,6 @@ def collect(self: Self, db: Path) -> list[tuple[Callable, tuple[Any]]] | Path:
resolve_batch: "resolve",
fullmap_audit: "qc",
column: "encode",
derive_species_context: "edge",
edge_category: "edge",
publications: "provenance",
retrieval_sources: "provenance",
Expand Down Expand Up @@ -1388,9 +1383,10 @@ def fold_unknown_to_supporting_text(lf: pl.LazyFrame) -> pl.LazyFrame:
"""
schema: pl.Schema = lf.collect_schema()
schema_names: list[str] = schema.names()
unknown: list[str] = sorted(c for c in schema_names if c not in ALLOWED_EDGE_FIELDS)
disabled: list[str] = sorted(c for c in schema_names if c in DISABLED_EDGE_FIELDS)
unknown: list[str] = sorted(c for c in schema_names if c not in ALLOWED_EDGE_FIELDS and c not in DISABLED_EDGE_FIELDS)
if not unknown:
return lf
return lf.drop(disabled) if disabled else lf

parts: list[pl.Expr] = []
for col in unknown:
Expand All @@ -1412,7 +1408,7 @@ def fold_unknown_to_supporting_text(lf: pl.LazyFrame) -> pl.LazyFrame:
else:
combined = derived

return lf.with_columns(combined.alias("supporting_text")).drop(unknown)
return lf.with_columns(combined.alias("supporting_text")).drop([*unknown, *disabled])


def _collect_subframes(
Expand Down
28 changes: 20 additions & 8 deletions src/tablassert/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
from tablassert.biolink import (
ALLOWED_EDGE_FIELDS,
BIOLINK_VERSION,
DISABLED_EDGE_FIELDS,
ENUM_RANGED_QUALIFIERS,
UNSATISFIABLE_EDGE_FIELDS,
AgentTypes,
Expand Down Expand Up @@ -339,17 +340,18 @@ def resolved(self: Self) -> bool:
return self.vocabulary is None

@model_validator(mode="after")
def reject_auto_derived_qualifiers(self: Self) -> Self:
"""Reject qualifiers that Tablassert derives from resolved node metadata.
def reject_disabled_qualifiers(self: Self) -> Self:
"""Reject qualifier fields that Tablassert has intentionally disabled.

``species_context_qualifier`` is populated automatically from resolved
subject/object taxon, so declaring it manually would make fullmap treat
it as an independently resolved query column and risk conflicting output.
The disabled policy is separate from Biolink's current slot attachment: a
dependency release must not make a field that Tablassert does not support
silently configurable again.
"""
if self.qualifier == "species_context_qualifier":
field: str = str(self.qualifier)
if field in DISABLED_EDGE_FIELDS:
raise TablassertValidationError(
"species_context_qualifier is auto-derived from resolved subject/object taxon; remove it from qualifiers.",
code="qualifier-auto-derived",
f"{field} is disabled in Tablassert; it is neither derived nor accepted as a configured qualifier or annotation.",
code="field-disabled",
)
return self

Expand Down Expand Up @@ -536,6 +538,16 @@ class Annotation(Encoding):
def clean_annotation(cls, annotation: str) -> str:
return annotation.lower().strip()

@model_validator(mode="after")
def reject_disabled_annotations(self: Self) -> Self:
"""Reject disabled edge fields even when they arrive through ``annotations``."""
if self.annotation in DISABLED_EDGE_FIELDS:
raise TablassertValidationError(
f"{self.annotation} is disabled in Tablassert; it is neither derived nor accepted as a configured qualifier or annotation.",
code="field-disabled",
)
return self

@model_validator(mode="after")
def split_by_requires_a_column(self) -> Self:
"""Enforce that ``split_by`` carries a real separator and a ``method: column`` encoding.
Expand Down
5 changes: 3 additions & 2 deletions tests/test_agent_assembly.py
Original file line number Diff line number Diff line change
Expand Up @@ -178,10 +178,11 @@ def test_instructions_carry_a_generated_predicate_cheatsheet() -> None:
assert "Gene ~ Disease -> GeneToDiseaseAssociation: affects, associated_with, contributes_to" in INSTRUCTIONS
assert "demoted_edge_pct" in INSTRUCTIONS

# And the two silent-relocation rules the pipeline enforces.
# And the relocation/disabled-field rules the pipeline enforces.
assert "supporting_study_size" in INSTRUCTIONS # named as a slot that does NOT reach the edge
assert "adjusted_p_value" in INSTRUCTIONS # the recommended alternative
assert "species_context_qualifier" in INSTRUCTIONS
assert "species_context_qualifier` is" in INSTRUCTIONS
assert "species_context_qualifier` is\n auto-derived" not in INSTRUCTIONS


def test_instructions_do_not_recommend_a_class_forbidden_predicate() -> None:
Expand Down
17 changes: 17 additions & 0 deletions tests/test_agent_derive.py
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,23 @@ def test_table_config_error_returns_the_actionable_message_the_gate_swallows() -
# Still False, still never raises -- only the reason is newly available.
assert validate_section(yaml.safe_dump(unsatisfiable, sort_keys=False)) is False

# A permanently disabled field is rejected through both supported config entry points.
disabled_qualifier: dict[str, Any] = copy.deepcopy(ALAMV6_TEMPLATE)
disabled_qualifier["template"]["statement"]["qualifiers"] = [
{"qualifier": "species_context_qualifier", "method": "value", "encoding": "NCBITaxon:9606"}
]
disabled_message: str | None = table_config_error(yaml.safe_dump(disabled_qualifier, sort_keys=False))
assert disabled_message is not None
assert "field-disabled" in disabled_message
assert validate_section(yaml.safe_dump(disabled_qualifier, sort_keys=False)) is False

disabled_annotation: dict[str, Any] = copy.deepcopy(ALAMV6_TEMPLATE)
disabled_annotation["template"]["annotations"] = [{"annotation": "species_context_qualifier", "method": "value", "encoding": "NCBITaxon:9606"}]
annotation_message: str | None = table_config_error(yaml.safe_dump(disabled_annotation, sort_keys=False))
assert annotation_message is not None
assert "field-disabled" in annotation_message
assert validate_section(yaml.safe_dump(disabled_annotation, sort_keys=False)) is False

# Never raises, whatever it is handed.
for nasty in ("", "[]", "{", "\x00", "a: [1, 2", "- - -"):
assert table_config_error(nasty) is None or isinstance(table_config_error(nasty), str)
9 changes: 8 additions & 1 deletion tests/test_biolink.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
from tablassert.biolink import (
ALLOWED_EDGE_FIELDS,
BIOLINK_VERSION,
DISABLED_EDGE_FIELDS,
EFFECT_TYPE_VALUES,
KNOWN_PENDING_EDGE_FIELDS,
TABLASERT_EDGE_EXTRAS,
Expand Down Expand Up @@ -331,6 +332,12 @@ def test_allowed_edge_fields_includes_subclass_only_slots() -> None:
assert slot in ALLOWED_EDGE_FIELDS, slot


def test_disabled_edge_fields_are_never_emittable() -> None:
"""Tablassert-disabled fields stay excluded even if a future Biolink model attaches them."""
assert "species_context_qualifier" in DISABLED_EDGE_FIELDS
assert DISABLED_EDGE_FIELDS.isdisjoint(ALLOWED_EDGE_FIELDS)


def test_allowed_edge_fields_excludes_unattached_qualifiers() -> None:
"""Qualifier slots attached to no Pydantic class are not emittable.

Expand All @@ -345,7 +352,7 @@ def test_allowed_edge_fields_excludes_unattached_qualifiers() -> None:

def test_allowed_edge_fields_covers_every_satisfiable_qualifier() -> None:
"""Every qualifier slot with a real home is an allowed edge column."""
satisfiable: set[str] = {q.value for q in Qualifiers} - set(UNSATISFIABLE_EDGE_FIELDS)
satisfiable: set[str] = {q.value for q in Qualifiers} - set(UNSATISFIABLE_EDGE_FIELDS) - set(DISABLED_EDGE_FIELDS)
assert satisfiable <= set(ALLOWED_EDGE_FIELDS)


Expand Down
Loading
Loading