Improve SAB data

Makefile

@@ -9,6 +9,13 @@ source/sab.ttl: scripts/extract_sab_data_from_docx.py cache/esab-2015_1.docx
 	# TODO 2: In XL, add precomposed usages (extract from usage in records)? See:
 	# ../librisxl/marc_export/src/main/resources/se/kb/libris/export/sabrub.txt # precomposed
 
+source/sab/precoordinated.ttl: scripts/make_precoordinated_sab_terms.py source/sab.ttl cache/sab-usages.tsv.gz
+	python3 $^ > $@ 2>logs/sab-unknown.tsv
+
+cache/sab-usages.tsv.gz: scripts/sab-usages.rq
+	curl --fail-with-body https://libris.kb.se/sparql -HAccept:text/tab-separated-values --data-urlencode query@$^ | gzip - > /tmp/sab-usages.tsv.gz
+	cp /tmp/sab-usages.tsv.gz $@
+
 ## SSIF 2011 (Obsolete)
 #
 #cache/ssif.xlsx:

scripts/create_sab_skos_data.py

@@ -1,45 +1,48 @@
-# -*- coding: UTF-8 -*-
-from __future__ import unicode_literals
 import csv
 import json
 import re
 import sys
-import urllib
+from urllib.parse import quote
 
-
-SKOS = "http://www.w3.org/2004/02/skos/core#"
-SAB_BASE = "https://id.kb.se/def/scheme/sab/{0}"
-DDC_BASE = "http://dewey.info/class/{0}/"
+KBV = "https://id.kb.se/vocab/"
+# SKOS = "http://www.w3.org/2004/02/skos/core#"
+SAB_BASE = "https://id.kb.se/term/kssb/{0}"
+DDC_BASE = None  # "http://dewey.info/class/{0}/"
 LANG = 'sv'
 
 hint_link_map = {
     #'#H': SKOS+'closeMatch',
     #'#G': SKOS+'closeMatch',
     #'#M': inverseOf SKOS+'closeMatch',
-    '#1': SKOS+'exactMatch',
-    '#2': SKOS+'broadMatch',
-    '#3': SKOS+'narrowMatch',
-    '#4': SKOS+'closeMatch',
-    'DDK22': SKOS+'relatedMatch', # TODO
+    '#1': 'exactMatch',
+    '#2': 'broadMatch',
+    '#3': 'narrowMatch',
+    '#4': 'closeMatch',
+    'DDK22': 'relatedMatch',  # TODO
 }
 
-def create_data(sab_codes_fpath, ddc_map_fpath, limit):
+
+def create_data(ddc_map_fpath, sab_codes_fpath):
     rmap = {}
-    create_sab_skos_data(rmap, sab_codes_fpath, limit=limit)
+    if sab_codes_fpath:
+        create_sab_skos_data(rmap, sab_codes_fpath)
     create_sab_ddc_data(rmap, ddc_map_fpath)
     return {
         "@context": {
-            "@vocab": SKOS,
-            "prefLabel": {"@language": LANG}
+            "@vocab": KBV,
+            "@base": SAB_BASE.format(""),
+            "prefLabel": {"@language": LANG},
         },
-        "@graph": rmap.values()
+        "@graph": list(rmap.values()),
     }
 
+
 def to_uri(base, code):
-    slug = urllib.quote(code.encode('utf-8'), safe=b':(),')
+    slug = quote(code.encode('utf-8'), safe='')  # TODO: safe=':(),' (as in generated sab?)
     return base.format(slug)
 
-def create_sab_skos_data(rmap, fpath, limit=0):
+
+def create_sab_skos_data(rmap, fpath):
     label_map = {}
     pending_broader = []
 
@@ -49,53 +52,66 @@ def create_sab_skos_data(rmap, fpath, limit=0):
             "@id": uri,
             "@type": "Concept",
             "notation": code,
-            "prefLabel": label
+            "prefLabel": label,
         }
         label_map[label] = uri
         if ': ' in label:
             pending_broader.append((r, label.rsplit(': ', 1)[0]))
-        if limit and i > limit:
-            break
 
     for r, broader_label in pending_broader:
         broader_uri = label_map.get(broader_label)
         if broader_uri:
             r.setdefault("broader", []).append({"@id": broader_uri})
 
+
 def create_sab_ddc_data(rmap, fpath):
     for number, sab_code, ddc_code, hint in read_csv_items(
-            fpath, skip_comment=True, coding='utf-8', size=4):
+        fpath, skip_comment=True, size=4
+    ):
         hint = re.split(r'\s+|\w(?=#)', hint)[-1].strip()
         link = hint_link_map.get(hint)
         if not link:
-            print >> sys.stderr, "No link map for", hint.encode('utf-8')
+            print("No link map for", hint, file=sys.stderr)
             continue
-        uri = to_uri(SAB_BASE, sab_code)
+        uri = to_uri("{}", sab_code)
         rmap.setdefault(uri, {"@id": uri}).setdefault(link, []).append(
-                {"@id": to_uri(DDC_BASE, ddc_code)})
+            {
+                "@id": to_uri(DDC_BASE, ddc_code)
+            } if DDC_BASE else {
+                "@type": "ClassificationDdc",
+                "code": ddc_code
+            }
+        )
+
 
-def read_csv_items(fpath, skip_first=True, skip_comment=False,
-        csv_dialect='excel-tab', coding='latin-1', size=0):
-    with open(fpath, 'rb') as fp:
+def read_csv_items(
+    fpath,
+    skip_first=True,
+    skip_comment=False,
+    csv_dialect='excel-tab',
+    encoding='latin-1',
+    size=0,
+):
+    with open(fpath, 'rt', encoding=encoding) as fp:
         reader = csv.reader(fp, csv_dialect)
         if skip_first is True:
-            reader.next()
+            next(reader)
         for row in reader:
-            if not row or skip_comment and row[0].startswith(b'#'):
+            if not row or skip_comment and row[0].startswith('#'):
                 continue
-            cols = [col.strip().decode(coding) for col in row]
+            cols = [col.strip() for col in row]
             if size and len(cols) > size:
                 cols = cols[0:size]
             yield cols
 
 
 if __name__ == '__main__':
     args = sys.argv[1:]
-    sab_codes_fpath = args.pop(0)
     ddc_map_fpath = args.pop(0)
-    limit = int(args.pop(0)) if args else 0
+    sab_codes_fpath = args.pop(0) if args else None
 
-    data = create_data(sab_codes_fpath, ddc_map_fpath, limit)
-    print json.dumps(data,
-            indent=2, separators=(',', ': '), sort_keys=True, ensure_ascii=False
-            ).encode('utf-8')
+    data = create_data(ddc_map_fpath, sab_codes_fpath)
+    s = json.dumps(
+        data, indent=2, separators=(',', ': '), sort_keys=True, ensure_ascii=False
+    )
+    print(s)

scripts/extract_sab_data_from_docx.py

@@ -6,7 +6,7 @@
 from zipfile import ZipFile
 import json
 from lxml import etree
-from urllib.parse import quote
+from urllib.parse import quote, unquote
 
 
 NS = {'w': "http://schemas.openxmlformats.org/wordprocessingml/2006/main"}
@@ -61,32 +61,40 @@ def handle_row(self, section, level, parts):
 
     def handle_main_row(self, level, parts):
         current = self._stack[-1]
+
         node = self._make_node(parts, current=current)
         if node is None:
             return
 
         code = node['code']
 
+        dangling = False
+
         if node['@type'].endswith('Subdivision'):
             current.setdefault('element', []).append(node)
             return
         elif not level and not len(code) == 1:
-            return
+            # FIXME: we seem to lose parent 'B' from e.g. ':a' in HJÄLPTABELLER
+            dangling = True
 
         if node['@type'].endswith('Collection'):
             self._current_coll = [ node[ID] ] + code.split('--')
             # NOTE: we "incorrectly" link the collection to the parent
             # classification, instead of:
             #return
         elif self._current_coll:
-            # FIXME: _current_coll needs to be like _stack ...
+            # TODO 668fe2a3: _current_coll needs to be like _stack ...
             coll_id, coll_start, coll_end = self._current_coll
             if (len(code) == len(coll_start) and
                     code >= coll_start and code <= coll_end):
                 node['inCollection'] = {ID: coll_id}
 
-        if len(code) > len(current['code']):
-            # "Deeper" Collection nodes got lost otherwise (see FIXME though)
+        if dangling:
+            prev_parent = self._stack[-1]
+            if node['code'].startswith(prev_parent['code']):
+                prev_parent.setdefault('narrower', []).append(node)
+        elif len(code) > len(current['code']):
+            # "Deeper" Collection nodes got lost otherwise (but see 668fe2a3)
             if node['@type'].endswith('Collection'):
                 self.helptable.append(node)
             elif code.startswith(current['code']):
@@ -130,44 +138,55 @@ def _make_node(self, parts, element_type=None, current=None):
             #if debug: print(element_type, parts, file=sys.stderr)
             return None
 
-        node = None # TODO: USE self._index.get(code) OR merge in final step
+        node = self._index.get(code) # Obviates merge in final step?
         if not node:
             node = OrderedDict()
             if code[0].isalpha():
                 self._index[code] = node
         else:
-            assert node['prefLabel'] == label, "%r != %r" % (
-                    (node['code'], node['prefLabel']), (code, label))
+            if node['prefLabel'] != label:
+                if len(label) < len(node['prefLabel']):
+                    node['altLabel'] = node['prefLabel']
+                    node['prefLabel'] = label
+                else:
+                    node['altLabel'] = label
 
         if is_collection:
-            node['@type'] = 'Collection'
+            node['@type'] = 'TermCollection'
         elif element_type:
             node['@type'] = element_type
         else:
             node['@type'] = 'Classification'
 
         node['inScheme'] = {ID: f'/term/{SAB_CODE}'}
 
-        node_id = "%s" % quote(code.encode('utf-8'), safe=b'')
+        node_id = "%s" % quote(code, safe='')
 
+        prefixed_code = None
         # NOTE: Many local elements are similar to their top-level element, but
         # far from all (and special '.0' elements are always locally unique).
         if current and element_type:
             if code[0:2] != '.0':
                 node['broader'] = {ID: node_id}
-            node_id = current[ID] + code
+            prefixed_code = unquote(current[ID]) + code
+            node_id = quote(prefixed_code)
+            code = prefixed_code
 
         node[ID] = node_id
 
+        if 'code' in node:
+            assert node['code'] == code
         node['code'] = code
-        node['prefLabel'] = label
+
+        if 'prefLabel' not in node:
+            node['prefLabel'] = label
 
         if len(parts) > 2:
             node['comment'] = parts[2]
 
         # TODO: Really create additional Elements, or add alias ID and itemPortion?
         # (... Or only use in SAB-code parsing code?)
-        if not is_collection and len(code) > 1:
+        if not is_collection and not prefixed_code and len(code) > 1:
 
             if re.match(r'^F[åb-z]\w*$', code): # Fb--Få
                 if aux_elem := self._make_node(['=' + code[1:], parts[1]]):
@@ -271,6 +290,9 @@ def error_correct(parts):
 
     parts = [re.sub(r' +', ' ', p.strip()) for p in parts]
 
+    if parts[1].startswith('och'):
+        return
+
     if parts[1] == 'Sjukhusbibliot ek':
         parts[1] = 'Sjukhusbibliotek'
 
@@ -330,7 +352,7 @@ def extract_sab(doc, debug=False):
             assert not indent
             in_section = HEADING_MAP.get(parts[0], in_section)
             if debug:
-                print("#", parts[0])
+                print("#", parts[0], '=>', in_section)
         else:
             if len(parts) > 2:
                 parts[2] = ' '.join(parts[2:])
@@ -347,18 +369,30 @@ def extract_sab(doc, debug=False):
 
             if debug:
                 print(indent, sep='', end='')
-                print(('%s =' % parts[0]), *parts[1:], sep='\t')
+                print(in_section, '%s =' % parts[0], *parts[1:], sep='\t')
 
     return flatten(thandler.get_results())
 
 
-def flatten(data, results=None, broader=None):
+def flatten(data, results=None, broader_ref=None):
+    top_level = results is None
+
     results = {} if results is None else results
+    elements = {}
+
     for item in data:
-        if broader:
-            item['broader'] = broader
+        if broader_ref:
+            item['broader'] = broader_ref
+
         if 'narrower' in item:
-            flatten(item.pop('narrower'), results, broader={ID: item[ID]})
+            flatten(item.pop('narrower'), results, broader_ref={ID: item[ID]})
+
+        for element in item.get('element', []):
+            elements.setdefault(element[ID], []).append(element)  # there may be dups
+
+        for element in item.get('related', []):
+            elements.setdefault(element[ID], []).append(element)  # there may be dups?
+
         existing = results.get(item[ID])
         if existing:
             for k, v in item.items():
@@ -369,14 +403,43 @@ def flatten(data, results=None, broader=None):
         else:
             results[item[ID]] = item
 
+    # Merge element description(s) and add to result index
+    for elem_id, elem_dups in elements.items():
+        elem = dict()
+
+        for dup in elem_dups:
+            if 'prefLabel' in elem:
+                if len(dup.get('prefLabel', '')) > len(elem['prefLabel']):
+                    del dup['prefLabel']
+
+            elem.update(dup)
+
+            # Turn into references
+            dup.clear()
+            dup[ID] = elem_id
+
+        # Keep label from element accidentally interpreted as regular term
+        item = results.get(elem_id)
+        if item and elem['prefLabel'] != item['prefLabel']:
+            elem['altLabel'] = item['prefLabel']
+
+        # Add to flattended result index (and drop evntual said accidental item)
+        results[elem_id] = elem
+
+    if not top_level:
+        return None
+
+    items = sorted(results.values(), key=lambda x: x[ID])
+
     return {
         '@context': {
             '@vocab': 'https://id.kb.se/vocab/',
             '@base': f'https://id.kb.se/term/{SAB_CODE}/',
             'prefLabel': {'@language': 'sv'},
+            'altLabel': {'@language': 'sv'},
             'comment': {'@language': 'sv'},
         },
-        '@graph': list(results.values())
+        '@graph': items
     }