Improve term hierarchy construction and add hierarchical competency queries for anatomical nodes and paths

mapserver/competency/flatmap.py

@@ -36,6 +36,7 @@
 from ..utils import json_map_metadata
 
 from .knowledge import CompetencyKnowledge
+from ..knowledge.hierarchy import SparcHierarchy, NPO_ONTOLOGY, UBERON_ONTOLOGY
 
 #===============================================================================
 
@@ -121,6 +122,7 @@ def anatomical_map_knowledge(map_uuid: str, competency_db: CompetencyKnowledge)
         nerve_terms.update(term for node in knowledge_terms[path_id]['nerves'] for term in [node[0]] + node[1])
 
     # Non-path features with an anatomical term
+    hierarchy = SparcHierarchy(UBERON_ONTOLOGY, NPO_ONTOLOGY)
     for feature_id, properties in annotated_features.items():
         if feature_id not in knowledge_terms:
             label = properties.get('label', properties.get('name', feature_id))
@@ -129,6 +131,8 @@ def anatomical_map_knowledge(map_uuid: str, competency_db: CompetencyKnowledge)
                 'source': map_uuid,
                 'label': label,
                 'long-label': descriptions.get(feature_id, label),
+                'descendants': hierarchy.terminal_path_terms({feature_id}).intersection(knowledge_terms.keys())
+                                if hierarchy.has(feature_id) else []
             }
             if properties.get('type') == 'nerve' or feature_id in nerve_terms:
                 knowledge_terms[feature_id]['type'] = NERVE_TYPE

mapserver/competency/queries.d/query_29.yaml

@@ -0,0 +1,50 @@
+queries:
+  - id: 29
+    label: Neuron populations associated with a location and its children
+    sql: >
+      WITH term_data AS(
+        SELECT term_id, source_id
+        FROM feature_terms
+        WHERE %CONDITIONS%
+      ),
+      descendants AS (
+        SELECT DISTINCT fp.feature_0 AS child_id, fp.source_id, td.term_id AS parent_id
+        FROM feature_relationship fp JOIN term_data td
+          ON fp.source_id = td.source_id and fp.feature_1 = td.term_id
+        WHERE fp.relationship = 'descendant-ancestor'
+        UNION
+        SELECT term_id, source_id, term_id FROM term_data
+      ),
+      descendant_nodes AS (
+        SELECT DISTINCT pnf.path_id, pnf.node_id, pnf.source_id
+        FROM descendants d JOIN path_node_features pnf
+          ON d.child_id = pnf.feature_id AND d.source_id = pnf.source_id
+        GROUP BY pnf.node_id, pnf.source_id, pnf.path_id
+        HAVING COUNT(DISTINCT d.parent_id) = (
+          SELECT COUNT(*) FROM term_data
+        )
+      )
+      SELECT DISTINCT path_id, source_id FROM descendant_nodes
+    parameters:
+    - id: feature_id
+      column: term_id
+      label: Anatomical terms for locations
+      type: string
+      multiple: true
+    - id: source_id
+      column: source_id
+      label: Knowledge source
+      type: string
+      default_msg: the latest source is used
+      default_sql: >
+        select source_id from knowledge_sources where source_id like 'sckan%'
+        order by source_id desc limit 1
+    order: ''
+    results:
+    - key: source_id
+      label: Knowledge source
+      type: string
+    - key: path_id
+      label: Neuron population
+      type: string
+

mapserver/competency/queries.d/query_30.yaml

@@ -0,0 +1,50 @@
+queries:
+  - id: 30
+    label: Nodes as children
+    sql: >
+      WITH term_data AS(
+        SELECT term_id, source_id
+        FROM feature_terms
+        WHERE %CONDITIONS%
+      ),
+      descendants AS (
+        SELECT DISTINCT fp.feature_0 AS child_id, fp.source_id, td.term_id AS parent_id
+        FROM feature_relationship fp JOIN term_data td
+          ON fp.source_id = td.source_id and fp.feature_1 = td.term_id
+        WHERE fp.relationship = 'descendant-ancestor'
+        UNION
+        SELECT term_id, source_id, term_id FROM term_data
+      ),
+      descendant_nodes AS (
+        SELECT DISTINCT pnf.path_id, pnf.node_id, pnf.source_id
+        FROM descendants d JOIN path_node_features pnf
+          ON d.child_id = pnf.feature_id AND d.source_id = pnf.source_id
+        GROUP BY pnf.node_id, pnf.source_id, pnf.path_id
+        HAVING COUNT(DISTINCT d.parent_id) = (
+          SELECT COUNT(*) FROM term_data
+        )
+      )
+      SELECT DISTINCT node_id, source_id FROM descendant_nodes
+    parameters:
+    - id: feature_id
+      column: term_id
+      label: Anatomical terms for locations
+      type: string
+      multiple: true
+    - id: source_id
+      column: source_id
+      label: Knowledge source
+      type: string
+      default_msg: the latest source is used
+      default_sql: >
+        select source_id from knowledge_sources where source_id like 'sckan%'
+        order by source_id desc limit 1
+    order: ''
+    results:
+    - key: source_id
+      label: Knowledge source
+      type: string
+    - key: node_id
+      label: Anatomical node
+      type: string
+

mapserver/knowledge/hierarchy.py

@@ -44,10 +44,10 @@
 #===============================================================================
 
 # Bump this to automatically rebuild the SPARC term hierarchy
-SPARC_HIERARCHY_VERSION = '1.0'
+SPARC_HIERARCHY_VERSION = '1.1'
 
 # Bump this to automatically rebuild map term hierarchies
-MAP_TREE_VERSION = '1.3'
+MAP_TREE_VERSION = '1.4'
 
 #===============================================================================
 
@@ -246,7 +246,7 @@ def __init__(self, uberon_source: str, interlex_source: str):
             with open(self.__hierarchy_file) as fp:
                 graph_json = json.load(fp)
                 self.__graph = nx.node_link_graph(graph_json, edges='links', directed=True)  # type: ignore
-                if self.__graph.get('graph', {}).get('version', '') < SPARC_HIERARCHY_VERSION:
+                if graph_json.get('graph', {}).get('version', '') < SPARC_HIERARCHY_VERSION:
                     self.__create_sparc_hierarchy(uberon_source, interlex_source)
         except Exception:
             self.__create_sparc_hierarchy(uberon_source, interlex_source)
@@ -287,7 +287,7 @@ def __add_ilx_terms(self, interlex_source: str):
             else:
                 self.__add_ilx_child(ilx_term)
         depth = 0
-        while depth < 3 and len(have_ilx_parents):
+        while len(have_ilx_parents):
             new_parents = []
             for ilx_term in have_ilx_parents:
                 # Are all parents now in the graph?
@@ -296,10 +296,19 @@ def __add_ilx_terms(self, interlex_source: str):
                     self.__add_ilx_child(ilx_term)
                 else:
                     new_parents.append(ilx_term)
+            if len(new_parents) == len(have_ilx_parents):
+                # No progress — remaining terms form a cycle or reference unknown parents
+                for t in set(new_parents):
+                    missing_parents = [p.id for p in t.parents if p.id not in self.__graph]
+                    settings['LOGGER'].warning(
+                        f'Unresolved Interlex term: {t.uri.id}; missing parents: {missing_parents}'
+                    )
+                break
             have_ilx_parents = new_parents
             depth += 1
-        if len(have_ilx_parents):
-            raise ValueError('Some Interlex parts are too deeply nested')
+            if len(have_ilx_parents) and depth >= 3:
+                settings['LOGGER'].warning('Some Interlex parts are too deeply nested')
+                break
 
     def __add_ilx_child(self, ilx: IlxTerm):
     #=======================================