diff --git a/.gitignore b/.gitignore index 4fb74c6..e839994 100644 --- a/.gitignore +++ b/.gitignore @@ -176,3 +176,6 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ + +# Local planning notes (not part of the shipped code) +PLAN.md diff --git a/shepherd_utils/statistical_significance_qualifier.py b/shepherd_utils/statistical_significance_qualifier.py new file mode 100644 index 0000000..ff9b350 --- /dev/null +++ b/shepherd_utils/statistical_significance_qualifier.py @@ -0,0 +1,46 @@ +"""Shared helpers for biolink:statistical_significance_qualifier (shepherd#134). + +The qualifier (enum StatisticalSignificanceQualifierEnum) is_a statement_qualifier, +a descendant of `qualifier` in biolink-model, so BMT/Retriever route it into TRAPI +edge.qualifiers[]. get_statistical_significance() therefore reads edge['qualifiers'] +ONLY. (ARAX is likewise qualifier-only as of RTXteam/RTX#2859 — its defensive +edge.attributes lookup was removed; add a fallback here only if a KP is found to +send the qualifier as an attribute.) Shared by aragorn_score + arax_rank (ranking). +""" + +from typing import Any, Dict, Optional + +SIGNIFICANCE_QUALIFIER_TYPE_ID = "biolink:statistical_significance_qualifier" + +# Conservative ranking scores per band. TODO: revisit once all KGs populate the +# qualifier (rollout asymmetry: qualifier-bearing edges scored against edges that +# lack it entirely). Mirrors the RTX ARAX_ranker change (RTXteam/RTX#2858). +SIGNIFICANCE_BAND_SCORES: Dict[str, float] = { + "very_strongly_significant": 0.70, + "strongly_significant": 0.55, + "significant": 0.40, + "suggestive": 0.15, + "not_significant": 0.0, +} + +# Source-agnostic trust weight applied to the band score in ranking (conservative; +# matches RTX trust=0.5). NOT routed through aragorn's per-source get_source_weight. +SIGNIFICANCE_SOURCE_WEIGHT: float = 0.5 + + +def _strip_biolink(value: Any) -> Optional[str]: + if isinstance(value, str) and value.startswith("biolink:"): + return value[len("biolink:") :] + return value + + +def get_statistical_significance(edge: Dict[str, Any]) -> Optional[str]: + """Return the bare significance band for a dict-based TRAPI edge, or None. + + Reads edge['qualifiers'] only (it is a biolink qualifier; BMT/Retriever route it + there). Strips any biolink: prefix from the value. + """ + for q in edge.get("qualifiers") or []: + if q.get("qualifier_type_id") == SIGNIFICANCE_QUALIFIER_TYPE_ID: + return _strip_biolink(q.get("qualifier_value")) + return None diff --git a/tests/unit/aragorn/test_aragorn_score_ranker.py b/tests/unit/aragorn/test_aragorn_score_ranker.py index 5bb28fa..625da45 100644 --- a/tests/unit/aragorn/test_aragorn_score_ranker.py +++ b/tests/unit/aragorn/test_aragorn_score_ranker.py @@ -673,3 +673,85 @@ def test_score_jaccard_like_returns_score_over_one_minus_score(): assert scored["analyses"][0]["score"] == pytest.approx( raw_score / (1 - raw_score) ) + + +# --- statistical significance qualifier (shepherd#134) -------------------- + + +def test_get_edge_values_extracts_significance_qualifier(): + """A qualifier-bearing edge gets a statistical_significance property.""" + edge = { + "subject": "A", + "object": "B", + "predicate": "biolink:related_to", + "sources": [ + {"resource_id": "infores:test", "resource_role": "primary_knowledge_source"} + ], + "qualifiers": [ + { + "qualifier_type_id": "biolink:statistical_significance_qualifier", + "qualifier_value": "very_strongly_significant", + } + ], + } + r = Ranker(_make_msg_with_edge(edge), logger) + vals = r.get_edge_values("e1") + assert "statistical_significance" in vals["infores:test"] + prop = vals["infores:test"]["statistical_significance"] + assert prop["value"] == "very_strongly_significant" + assert prop["weight"] > 0 + assert prop["weight"] == pytest.approx(0.70 * 0.5) + + +def test_get_edge_values_significance_strips_biolink_prefix(): + """biolink:-prefixed qualifier values are stripped.""" + edge = { + "subject": "A", + "object": "B", + "sources": [ + {"resource_id": "infores:test", "resource_role": "primary_knowledge_source"} + ], + "qualifiers": [ + { + "qualifier_type_id": "biolink:statistical_significance_qualifier", + "qualifier_value": "biolink:significant", + } + ], + } + r = Ranker(_make_msg_with_edge(edge), logger) + vals = r.get_edge_values("e1") + assert vals["infores:test"]["statistical_significance"]["value"] == "significant" + + +def test_not_significant_contributes_nothing(): + """Band score 0 -> property omitted -> no admittance contribution (no penalty).""" + edge = { + "subject": "A", + "object": "B", + "sources": [ + {"resource_id": "infores:test", "resource_role": "primary_knowledge_source"} + ], + "qualifiers": [ + { + "qualifier_type_id": "biolink:statistical_significance_qualifier", + "qualifier_value": "not_significant", + } + ], + } + r = Ranker(_make_msg_with_edge(edge), logger) + vals = r.get_edge_values("e1") + assert "statistical_significance" not in vals["infores:test"] + + +def test_qualifierless_edge_has_no_significance_property(): + """Edges without the qualifier get no statistical_significance property.""" + edge = { + "subject": "A", + "object": "B", + "sources": [ + {"resource_id": "infores:test", "resource_role": "primary_knowledge_source"} + ], + } + r = Ranker(_make_msg_with_edge(edge), logger) + vals = r.get_edge_values("e1") + assert "statistical_significance" not in vals["infores:test"] diff --git a/tests/unit/test_arax_rank_ranker.py b/tests/unit/test_arax_rank_ranker.py new file mode 100644 index 0000000..f2bca4d --- /dev/null +++ b/tests/unit/test_arax_rank_ranker.py @@ -0,0 +1,134 @@ +"""Tests for statistical significance qualifier scoring in arax_rank (shepherd#134).""" + +import logging + +from workers.arax_rank.ranker import ARAXRanker + +logger = logging.getLogger(__name__) + + +def _edge(**kw): + return {"subject": "A", "object": "B", "predicate": "biolink:related_to", **kw} + + +def test_significance_additive_boost(): + """A qualifier-bearing edge scores >= the same edge without the qualifier.""" + ranker = ARAXRanker(logger) + base = ranker._calculate_edge_confidence( + "infores:test--A--B", + _edge( + attributes=[ + { + "attribute_type_id": "biolink:pValue", + "original_attribute_name": "pValue", + "value": "0.001", + } + ] + ), + ) + boosted = ranker._calculate_edge_confidence( + "infores:test--A--B", + _edge( + attributes=[ + { + "attribute_type_id": "biolink:pValue", + "original_attribute_name": "pValue", + "value": "0.001", + } + ], + qualifiers=[ + { + "qualifier_type_id": "biolink:statistical_significance_qualifier", + "qualifier_value": "very_strongly_significant", + } + ], + ), + ) + assert boosted >= base # additive qualifier can only help or be neutral + + +def test_significance_score_mapping(): + """Each band maps to band_score * 0.5 trust, appended to the score list.""" + ranker = ARAXRanker(logger) + # Edge with no attributes and no qualifier -> base only + no_qual = ranker._calculate_edge_confidence("infores:test--A--B", _edge()) + # Edge with qualifier only (no attributes) -> base + qualifier boost + with_qual = ranker._calculate_edge_confidence( + "infores:test--A--B", + _edge( + qualifiers=[ + { + "qualifier_type_id": "biolink:statistical_significance_qualifier", + "qualifier_value": "significant", + } + ] + ), + ) + # significant = 0.40 * 0.5 = 0.20 additive boost + assert with_qual > no_qual + + +def test_not_significant_adds_nothing(): + """not_significant (score 0.0) adds no boost.""" + ranker = ARAXRanker(logger) + no_qual = ranker._calculate_edge_confidence("infores:test--A--B", _edge()) + not_sig = ranker._calculate_edge_confidence( + "infores:test--A--B", + _edge( + qualifiers=[ + { + "qualifier_type_id": "biolink:statistical_significance_qualifier", + "qualifier_value": "not_significant", + } + ] + ), + ) + assert not_sig == no_qual + + +def test_biolink_prefix_stripped(): + """biolink:-prefixed qualifier values are handled.""" + ranker = ARAXRanker(logger) + bare = ranker._calculate_edge_confidence( + "infores:test--A--B", + _edge( + qualifiers=[ + { + "qualifier_type_id": "biolink:statistical_significance_qualifier", + "qualifier_value": "strongly_significant", + } + ] + ), + ) + prefixed = ranker._calculate_edge_confidence( + "infores:test--A--B", + _edge( + qualifiers=[ + { + "qualifier_type_id": "biolink:statistical_significance_qualifier", + "qualifier_value": "biolink:strongly_significant", + } + ] + ), + ) + assert bare == prefixed + + +def test_qualifier_works_without_attributes(): + """Qualifier scoring works even for edges with no attributes at all.""" + ranker = ARAXRanker(logger) + # No attributes, no qualifier -> base only (0.5 for infores) + base_only = ranker._calculate_edge_confidence("infores:test--A--B", _edge()) + # No attributes, but has qualifier -> base + boost + with_qual = ranker._calculate_edge_confidence( + "infores:test--A--B", + _edge( + qualifiers=[ + { + "qualifier_type_id": "biolink:statistical_significance_qualifier", + "qualifier_value": "very_strongly_significant", + } + ] + ), + ) + assert with_qual > base_only diff --git a/tests/unit/test_statistical_significance_qualifier.py b/tests/unit/test_statistical_significance_qualifier.py new file mode 100644 index 0000000..3ac5cfc --- /dev/null +++ b/tests/unit/test_statistical_significance_qualifier.py @@ -0,0 +1,80 @@ +"""Tests for shepherd_utils.statistical_significance_qualifier (shepherd#134).""" + +from shepherd_utils.statistical_significance_qualifier import ( + SIGNIFICANCE_BAND_SCORES, + get_statistical_significance, +) + + +def test_band_scores_descending(): + bands = [ + "very_strongly_significant", + "strongly_significant", + "significant", + "suggestive", + "not_significant", + ] + scores = [SIGNIFICANCE_BAND_SCORES[b] for b in bands] + assert scores == sorted(scores, reverse=True) and scores[-1] == 0.0 + + +def test_lookup_in_qualifiers(): + edge = { + "qualifiers": [ + { + "qualifier_type_id": "biolink:statistical_significance_qualifier", + "qualifier_value": "significant", + } + ] + } + assert get_statistical_significance(edge) == "significant" + + +def test_attributes_are_ignored(): + # Qualifiers-only by design: an attributes-only qualifier is NOT read + # (matches ARAX, qualifier-only as of RTX#2859). + edge = { + "attributes": [ + { + "attribute_type_id": "biolink:statistical_significance_qualifier", + "value": "suggestive", + } + ] + } + assert get_statistical_significance(edge) is None + + +def test_lookup_strips_biolink_prefix(): + edge = { + "qualifiers": [ + { + "qualifier_type_id": "biolink:statistical_significance_qualifier", + "qualifier_value": "biolink:significant", + } + ] + } + assert get_statistical_significance(edge) == "significant" + + +def test_only_qualifiers_read(): + # The band comes from edge['qualifiers']; attributes are not consulted. + edge = { + "qualifiers": [ + { + "qualifier_type_id": "biolink:statistical_significance_qualifier", + "qualifier_value": "significant", + } + ], + "attributes": [ + { + "attribute_type_id": "biolink:statistical_significance_qualifier", + "value": "not_significant", + } + ], + } + assert get_statistical_significance(edge) == "significant" + + +def test_lookup_none_when_absent(): + assert get_statistical_significance({"attributes": []}) is None + assert get_statistical_significance({}) is None diff --git a/workers/aragorn_score/worker.py b/workers/aragorn_score/worker.py index c4f7c77..e8c83ef 100644 --- a/workers/aragorn_score/worker.py +++ b/workers/aragorn_score/worker.py @@ -17,6 +17,11 @@ from shepherd_utils.otel import setup_tracer from shepherd_utils.process_pool import ProcessPoolManager from shepherd_utils.shared import get_tasks, run_task_lifecycle +from shepherd_utils.statistical_significance_qualifier import ( + SIGNIFICANCE_BAND_SCORES, + SIGNIFICANCE_SOURCE_WEIGHT, + get_statistical_significance, +) # Queue name STREAM = "aragorn.score" @@ -890,6 +895,7 @@ def get_edge_values(self, edge_id): "literature_coocurrence": None, "p_value": None, "affinity": None, + "statistical_significance": None, } # Look through attributes and @@ -998,6 +1004,10 @@ def get_edge_values(self, edge_id): if orig_attr_name == "biolink:tmkp_confidence_score": usable_edge_attr["confidence_score"] = attribute.get("value", 0) + # Qualifier lives in edge["qualifiers"] (BMT/Retriever path); the + # attribute loop above won't see it. + usable_edge_attr["statistical_significance"] = get_statistical_significance(edge) + # At this point we have all of the information extracted from the edge # We have have looked through all attributes and updated # usable_edge_attr. Now we can construct the edge values using these @@ -1091,6 +1101,17 @@ def get_edge_values(self, edge_id): "weight": property_w * source_w, } + if usable_edge_attr["statistical_significance"] is not None: + band = usable_edge_attr["statistical_significance"] + property_w = SIGNIFICANCE_BAND_SCORES.get(band, 0.0) + if property_w > 0: + this_edge_vals[edge_source]["statistical_significance"] = { + "value": band, + "property_weight": property_w, + "source_weight": SIGNIFICANCE_SOURCE_WEIGHT, + "weight": property_w * SIGNIFICANCE_SOURCE_WEIGHT, + } + # Cache it self.edge_values[edge_id] = this_edge_vals return this_edge_vals diff --git a/workers/arax_rank/ranker.py b/workers/arax_rank/ranker.py index 3be4663..0042b16 100644 --- a/workers/arax_rank/ranker.py +++ b/workers/arax_rank/ranker.py @@ -24,6 +24,12 @@ import numpy.typing as npt import scipy.stats +from shepherd_utils.statistical_significance_qualifier import ( + SIGNIFICANCE_BAND_SCORES, + SIGNIFICANCE_SOURCE_WEIGHT, + get_statistical_significance, +) + # Default confidence for manual agent edges (matches ARAX_ranker.py line 24) EDGE_CONFIDENCE_MANUAL_AGENT = 0.90 @@ -294,6 +300,17 @@ def _calculate_edge_confidence(self, edge_key: str, edge: Dict) -> float: if normalized_score > 0: edge_attribute_score_list.append(normalized_score) + # Statistical significance qualifier is carried in edge["qualifiers"] + # (not attributes). Looked up separately (categorical bypass) so the + # enum string never hits the numeric attribute normalizer. Mirrors RTX + # _get_significance_qualifier_value + _significance_trust_weight + # (RTXteam/RTX#2859). + sig_value = get_statistical_significance(edge) + if sig_value is not None: + sig_score = SIGNIFICANCE_BAND_SCORES.get(sig_value, 0.0) + if sig_score > 0: + edge_attribute_score_list.append(sig_score * SIGNIFICANCE_SOURCE_WEIGHT) + # If no attributes scored, return base score (ARAX_ranker.py lines 379-384) if len(edge_attribute_score_list) == 0: return base