Wikipedia url

CHANGES

@@ -1,4 +1,10 @@
 CHANGES
+0.11.1 (June 12, 2026)
+- New field `book_wikipedia_url(s)` has been added to the DublinCore dispatcher.
+- Two functions `add_book_wikipedia_url` and `remove_book_wikipedia_url` have been added to the DublinCoreObject.
+- Wikipedia url dispatcher testing added to `test_load_from_json`.
+- Add/remove tesing can be found in `test_book_wikipedia_urls_add_and_remove` 
+
 0.11.0 (June *, 2026)
 - Now using SQLAlchemy 2.0
 - Upgraded all the packages inside of `Pipfile.lock` to most recent versions compatible with py3.9.

libgutenberg/DublinCore.py

@@ -21,6 +21,7 @@
 import textwrap
 import unicodedata
 from gettext import gettext as _
+from urllib.parse import unquote
 
 import six
 import lxml
@@ -523,6 +524,37 @@ def handle_dc_languages(dc, text):
                 pass
 
 
+WIKIMATCH = re.compile(
+    r"(?ix)https?://([a-z]{2,3})\.wikipedia\.org/wiki/([/!@i^*$a-z0-9_\(\)\-.:]+)")
+AVOID_WIKI = ["simple.", "File:", "/Category:", "Category:", "(disambiguation)"]
+UNTRUSTED_WIKI_LANGS = {'sco'}
+WIKIPEDIA_URL_PREFIX = 'Wikipedia page about this book: '
+
+
+def check_wikipedia_url(text):
+    """Return (lang, page_title) if text contains a valid wiki URL, else None."""
+    if not text:
+        return None
+    match = WIKIMATCH.search(text)
+    if not match:
+        return None
+    lang, page_title = match.group(1), match.group(2)
+    if lang in UNTRUSTED_WIKI_LANGS:
+        return None
+    if any(pattern in unquote(page_title) for pattern in AVOID_WIKI):
+        return None
+    return (lang, page_title)
+
+
+def wikipedia_url(lang, page_title):
+    return f"https://{lang}.wikipedia.org/wiki/{page_title}"
+
+
+def extract_wikipedia_url(text):
+    checked = check_wikipedia_url(text)
+    return wikipedia_url(*checked) if checked else None
+
+
 class GutenbergDublinCore(DublinCore):
     """ Parse from PG files. """
 
@@ -533,7 +565,23 @@ def __init__(self):
         self._project_gutenberg_id = None
         self.request_key = ''
         self.scan_urls = set()
+        self.wikipedia_urls = set()
+
+
+    def add_wikipedia_url(self, url):
+        url = (url or '').strip()
+        checked = check_wikipedia_url(url)
+        if not checked or url != wikipedia_url(*checked):
+            error('%s is not a valid wikipedia url', url)
+            return
+        self.wikipedia_urls.add(url)
+
 
+    def remove_wikipedia_url(self, url):
+        url = (url or '').strip()
+        checked = check_wikipedia_url(url)
+        if checked:
+            self.wikipedia_urls.discard(wikipedia_url(*checked))
 
 
     @property
@@ -794,6 +842,18 @@ def handle_scan_urls(self, key, value):
                 self.scan_urls.add(scan_url)
 
 
+        def handle_wikipedia_urls(self, key, value):
+            if isinstance(value, str):
+                value = [value]
+            elif isinstance(value, list):
+                pass
+            else:
+                error('%s is not a valid wikipedia url', value)
+                return
+            for url in value:
+                self.add_wikipedia_url(url)
+
+
         def handle_pubinfo(self, key, value):
             if key == 'publisher':
                 self.pubinfo.publisher = value
@@ -928,6 +988,7 @@ def dispatch(self, key, val):
             'alt_title':    store,
             'creator_role':  handle_creators,
             'scans_archive_url': handle_scan_urls,
+            'wikipedia_url': handle_wikipedia_urls,
             'credit':       store,
             'publisher':    handle_pubinfo,
             'publisher_country': handle_pubinfo,
@@ -951,6 +1012,9 @@ def dispatch(self, key, val):
             'created':                'source_publication_years',
             'produced by':            'credit',
             'publisher_place':        'place',
+            'wikipedia_urls':         'wikipedia_url',
+            'book_wikipedia_url':     'wikipedia_url',
+            'book_wikipedia_urls':    'wikipedia_url',
             }
 
         for role in list(self.inverse_role_map.keys()):

libgutenberg/DublinCoreMapping.py

@@ -23,6 +23,7 @@
 from sqlalchemy.exc import DBAPIError
 
 from . import DublinCore
+from .DublinCore import WIKIPEDIA_URL_PREFIX, check_wikipedia_url, wikipedia_url
 from . import GutenbergGlobals as gg
 from . import GutenbergDatabase
 from . import GutenbergFiles
@@ -179,7 +180,11 @@ def parse260(s):
             elif marc.code == '260':
                 (self.pubinfo.place, self.pubinfo.publisher, self.pubinfo.years) = parse260(marc.text) 
             elif marc.code == '500':
-                self.notes = marc.text
+                url = DublinCore.extract_wikipedia_url(marc.text)
+                if url:
+                    self.wikipedia_urls.add(url)
+                else:
+                    self.notes = marc.text
             elif marc.code == '505':
                 self.contents = marc.text
             elif marc.code == '508':
@@ -215,7 +220,7 @@ def parse260(s):
         # categories(text, audiobook, etc)
         if book.categories:
             self.dcmitypes = [struct(id=cat.dcmitype[0], description=cat.dcmitype[1])
-                              for cat in book.categories]
+                            for cat in book.categories]
         else:
             self.dcmitypes = [struct(id='Text', description='Text')]
 
@@ -293,7 +298,7 @@ def register_coverpage(self, id_, url, code=901):
         try:
             session.begin_nested()
             session.add(Attribute(fk_books=id_, fk_attriblist=code,
-                                  text=gg.archive2files(id_, url)))
+                                text=gg.archive2files(id_, url)))
             session.commit()
 
         except IntegrityError:  # Duplicate key
@@ -383,6 +388,8 @@ def save(self, updatemode=0):
         if self.request_key:
             self.add_attribute(self.book, self.request_key, marc=905)
 
+        self._update_wikipedia_urls()
+
         self.book.updatemode = 1 # prevent non-cataloguer changes
 
         session.commit()
@@ -480,6 +487,7 @@ def add_title(self, book, title, marc=245, subtitle=None):
         title = title.replace(' *_ *', '\n')
         self.add_attribute(book, title, nonfiling=nonfiling, marc=marc)
 
+
     def add_attribute(self, book, attr, nonfiling=0, marc=0):
         if not attr:
             return
@@ -505,6 +513,27 @@ def add_attribute(self, book, attr, nonfiling=0, marc=0):
                 book.attributes.append(Attribute(
                     fk_attriblist=marc, nonfiling=nonfiling, text=attr))
 
+
+    def _update_wikipedia_urls(self):
+        # Called from save(). Sync self.wikipedia_urls to MARC 500 wiki rows.
+        if not self.book:
+            return
+        remaining = set(self.wikipedia_urls)
+        for att in list(self.book.attributes):
+            if att.fk_attriblist != 500:
+                continue
+            wiki_key = check_wikipedia_url(att.text)
+            if not wiki_key:
+                continue
+            url = wikipedia_url(*wiki_key)
+            if url in remaining:
+                remaining.discard(url)
+            else:
+                self.book.attributes.remove(att)
+        for url in remaining:
+            self.book.attributes.append(
+                Attribute(fk_attriblist=500, text=f"{WIKIPEDIA_URL_PREFIX}{url}"))
+
     def delete(self):
         """ only delete the book! """
         session = self.get_my_session()

libgutenberg/tests/99999.json

@@ -26,6 +26,10 @@
             "https://archive.org/details/in.ernet.dli.2013.179137",
             "https://babel.hathitrust.org/cgi/pt?id=uieg.30152019845839"
         ],
+        "BOOK_WIKIPEDIA_URLS": [
+            "https://en.wikipedia.org/wiki/Test_Book",
+            "https://en.wikipedia.org/wiki/Another_Book"
+        ],
         "CREDIT": "Roger Frank and Sue Clark.",
         "LANGUAGE": "English",
         "PUBLISHER": "Frank A. Munsey Company",

libgutenberg/tests/test_dc.py

@@ -51,6 +51,11 @@ def test_orm_metadata(self):
         dc = DublinCoreMapping.DublinCoreObject()
         self.metadata_test2(dc)
 
+    def test_wikipedia_urls_load(self):
+        dc = DublinCoreMapping.DublinCoreObject()
+        dc.load_from_database(2701)  # Moby Dick
+        self.assertIn("https://en.wikipedia.org/wiki/Moby-Dick", dc.wikipedia_urls)
+
     def metadata_test1(self, dc):
         dc.load_from_database(self.ebook)
         self.assertEqual(dc.project_gutenberg_id, 20050)
@@ -224,7 +229,7 @@ def get_cover(ebook, dc):
         dc = GutenbergDatabaseDublinCore.GutenbergDatabaseDublinCore(self.dummypool)
         dc.register_coverpage(ebook, 'new_cover')
         # does nothing to avoid violates foreign key constraint
-        self.assertEqual(get_cover(ebook, dc), None) 
+        self.assertEqual(get_cover(ebook, dc), None)
 
     def tearDown(self):
         pass
@@ -297,6 +302,10 @@ def test_load_from_json(self):
         self.assertEqual(set_subtitle, "a true story : second line")
         self.assertEqual(len(dc.authors), 2)
         self.assertEqual(len(dc.scan_urls), 2)
+        self.assertEqual(
+            dc.wikipedia_urls,
+            {"https://en.wikipedia.org/wiki/Test_Book",
+             "https://en.wikipedia.org/wiki/Another_Book"})
         self.assertEqual(dc.pubinfo.first_year, '1920')
         self.assertEqual(dc.credit, 'Roger Frank and Sue Clark.')
         dc.add_credit('Sue Frank and Roger Clark.\n')
@@ -312,6 +321,10 @@ def test_load_from_json(self):
         dc.load_from_database(99999)
         self.assertEqual(set_title, dc.title)
         self.assertEqual(set_subtitle, dc.subtitle)
+        self.assertEqual(
+            dc.wikipedia_urls,
+            {"https://en.wikipedia.org/wiki/Test_Book",
+             "https://en.wikipedia.org/wiki/Another_Book"})
         marc260 = dc.session.query(Attribute).filter_by(book=dc.book, fk_attriblist=260).first().text
         self.assertTrue('1920' in marc260)
         self.assertEqual(
@@ -327,6 +340,11 @@ def test_load_from_json(self):
             len(dc.session.query(Attribute).filter_by(book=dc.book,
                 fk_attriblist=904).all()),
             2)
+        self.assertEqual(
+            len(dc.session.query(Attribute).filter_by(book=dc.book,
+                fk_attriblist=500).filter(
+                    Attribute.text.like('%wikipedia.org%')).all()),
+            2)
         self.assertEqual(
             dc.session.query(Attribute).filter_by(book=dc.book, fk_attriblist=905).first().text,
             '20210623194947brand')
@@ -337,8 +355,29 @@ def test_load_from_json(self):
         dc.session.flush()
         self.assertFalse(DBUtils.ebook_exists(99999))
 
+    def test_wikipedia_urls_add_and_remove(self):
+        dc = DublinCoreMapping.DublinCoreObject()
+        ebook = 99998
+        dc.load_or_create_book(ebook)
+        dc.rights = 'Public Domain in the USA.'
+        urls = ["https://en.wikipedia.org/wiki/Moby-Dick",
+                "https://en.wikipedia.org/wiki/Test_Book"]
+        dc.add_wikipedia_url(urls[0])
+        dc.add_wikipedia_url(f"See also: {urls[0]}")  # rejected
+        dc.add_wikipedia_url(urls[1])
+        dc.save(updatemode=0)
+
+        remaining = list(urls)
+        while remaining:
+            dc = DublinCoreMapping.DublinCoreObject()
+            dc.load_from_database(ebook)
+            self.assertEqual(dc.wikipedia_urls, set(remaining))
+            dc.remove_wikipedia_url(remaining.pop(0))
+            dc.save(updatemode=1)
+
     def tearDown(self):
         session = DBUtils.check_session(None)
         DBUtils.remove_author('Lorem Ipsum Jr.', session=session)
         session.query(Book).filter(Book.pk == 99999).delete()
-        session.commit()
+        session.query(Book).filter(Book.pk == 99998).delete()
+        session.commit()

setup.py

@@ -3,7 +3,8 @@
 # libgutenberg setup.py
 #
 
-__version__ = '0.11.0'
+__version__ = '0.11.1'
+
 
 from setuptools import setup