diff --git a/CHANGES b/CHANGES index b725de1..02bbd7a 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,10 @@ CHANGES +0.11.1 (June 12, 2026) +- New field `book_wikipedia_url(s)` has been added to the DublinCore dispatcher. +- Two functions `add_book_wikipedia_url` and `remove_book_wikipedia_url` have been added to the DublinCoreObject. +- Wikipedia url dispatcher testing added to `test_load_from_json`. +- Add/remove tesing can be found in `test_book_wikipedia_urls_add_and_remove` + 0.11.0 (June *, 2026) - Now using SQLAlchemy 2.0 - Upgraded all the packages inside of `Pipfile.lock` to most recent versions compatible with py3.9. diff --git a/libgutenberg/DublinCore.py b/libgutenberg/DublinCore.py index 984c441..1e6c882 100644 --- a/libgutenberg/DublinCore.py +++ b/libgutenberg/DublinCore.py @@ -21,6 +21,7 @@ import textwrap import unicodedata from gettext import gettext as _ +from urllib.parse import unquote import six import lxml @@ -523,6 +524,37 @@ def handle_dc_languages(dc, text): pass +WIKIMATCH = re.compile( + r"(?ix)https?://([a-z]{2,3})\.wikipedia\.org/wiki/([/!@i^*$a-z0-9_\(\)\-.:]+)") +AVOID_WIKI = ["simple.", "File:", "/Category:", "Category:", "(disambiguation)"] +UNTRUSTED_WIKI_LANGS = {'sco'} +WIKIPEDIA_URL_PREFIX = 'Wikipedia page about this book: ' + + +def check_wikipedia_url(text): + """Return (lang, page_title) if text contains a valid wiki URL, else None.""" + if not text: + return None + match = WIKIMATCH.search(text) + if not match: + return None + lang, page_title = match.group(1), match.group(2) + if lang in UNTRUSTED_WIKI_LANGS: + return None + if any(pattern in unquote(page_title) for pattern in AVOID_WIKI): + return None + return (lang, page_title) + + +def wikipedia_url(lang, page_title): + return f"https://{lang}.wikipedia.org/wiki/{page_title}" + + +def extract_wikipedia_url(text): + checked = check_wikipedia_url(text) + return wikipedia_url(*checked) if checked else None + + class GutenbergDublinCore(DublinCore): """ Parse from PG files. """ @@ -533,7 +565,23 @@ def __init__(self): self._project_gutenberg_id = None self.request_key = '' self.scan_urls = set() + self.wikipedia_urls = set() + + + def add_wikipedia_url(self, url): + url = (url or '').strip() + checked = check_wikipedia_url(url) + if not checked or url != wikipedia_url(*checked): + error('%s is not a valid wikipedia url', url) + return + self.wikipedia_urls.add(url) + + def remove_wikipedia_url(self, url): + url = (url or '').strip() + checked = check_wikipedia_url(url) + if checked: + self.wikipedia_urls.discard(wikipedia_url(*checked)) @property @@ -794,6 +842,18 @@ def handle_scan_urls(self, key, value): self.scan_urls.add(scan_url) + def handle_wikipedia_urls(self, key, value): + if isinstance(value, str): + value = [value] + elif isinstance(value, list): + pass + else: + error('%s is not a valid wikipedia url', value) + return + for url in value: + self.add_wikipedia_url(url) + + def handle_pubinfo(self, key, value): if key == 'publisher': self.pubinfo.publisher = value @@ -928,6 +988,7 @@ def dispatch(self, key, val): 'alt_title': store, 'creator_role': handle_creators, 'scans_archive_url': handle_scan_urls, + 'wikipedia_url': handle_wikipedia_urls, 'credit': store, 'publisher': handle_pubinfo, 'publisher_country': handle_pubinfo, @@ -951,6 +1012,9 @@ def dispatch(self, key, val): 'created': 'source_publication_years', 'produced by': 'credit', 'publisher_place': 'place', + 'wikipedia_urls': 'wikipedia_url', + 'book_wikipedia_url': 'wikipedia_url', + 'book_wikipedia_urls': 'wikipedia_url', } for role in list(self.inverse_role_map.keys()): diff --git a/libgutenberg/DublinCoreMapping.py b/libgutenberg/DublinCoreMapping.py index 86c5074..6fc0c02 100644 --- a/libgutenberg/DublinCoreMapping.py +++ b/libgutenberg/DublinCoreMapping.py @@ -23,6 +23,7 @@ from sqlalchemy.exc import DBAPIError from . import DublinCore +from .DublinCore import WIKIPEDIA_URL_PREFIX, check_wikipedia_url, wikipedia_url from . import GutenbergGlobals as gg from . import GutenbergDatabase from . import GutenbergFiles @@ -179,7 +180,11 @@ def parse260(s): elif marc.code == '260': (self.pubinfo.place, self.pubinfo.publisher, self.pubinfo.years) = parse260(marc.text) elif marc.code == '500': - self.notes = marc.text + url = DublinCore.extract_wikipedia_url(marc.text) + if url: + self.wikipedia_urls.add(url) + else: + self.notes = marc.text elif marc.code == '505': self.contents = marc.text elif marc.code == '508': @@ -215,7 +220,7 @@ def parse260(s): # categories(text, audiobook, etc) if book.categories: self.dcmitypes = [struct(id=cat.dcmitype[0], description=cat.dcmitype[1]) - for cat in book.categories] + for cat in book.categories] else: self.dcmitypes = [struct(id='Text', description='Text')] @@ -293,7 +298,7 @@ def register_coverpage(self, id_, url, code=901): try: session.begin_nested() session.add(Attribute(fk_books=id_, fk_attriblist=code, - text=gg.archive2files(id_, url))) + text=gg.archive2files(id_, url))) session.commit() except IntegrityError: # Duplicate key @@ -383,6 +388,8 @@ def save(self, updatemode=0): if self.request_key: self.add_attribute(self.book, self.request_key, marc=905) + self._update_wikipedia_urls() + self.book.updatemode = 1 # prevent non-cataloguer changes session.commit() @@ -480,6 +487,7 @@ def add_title(self, book, title, marc=245, subtitle=None): title = title.replace(' *_ *', '\n') self.add_attribute(book, title, nonfiling=nonfiling, marc=marc) + def add_attribute(self, book, attr, nonfiling=0, marc=0): if not attr: return @@ -505,6 +513,27 @@ def add_attribute(self, book, attr, nonfiling=0, marc=0): book.attributes.append(Attribute( fk_attriblist=marc, nonfiling=nonfiling, text=attr)) + + def _update_wikipedia_urls(self): + # Called from save(). Sync self.wikipedia_urls to MARC 500 wiki rows. + if not self.book: + return + remaining = set(self.wikipedia_urls) + for att in list(self.book.attributes): + if att.fk_attriblist != 500: + continue + wiki_key = check_wikipedia_url(att.text) + if not wiki_key: + continue + url = wikipedia_url(*wiki_key) + if url in remaining: + remaining.discard(url) + else: + self.book.attributes.remove(att) + for url in remaining: + self.book.attributes.append( + Attribute(fk_attriblist=500, text=f"{WIKIPEDIA_URL_PREFIX}{url}")) + def delete(self): """ only delete the book! """ session = self.get_my_session() diff --git a/libgutenberg/tests/99999.json b/libgutenberg/tests/99999.json index 291ebfb..8be6238 100644 --- a/libgutenberg/tests/99999.json +++ b/libgutenberg/tests/99999.json @@ -26,6 +26,10 @@ "https://archive.org/details/in.ernet.dli.2013.179137", "https://babel.hathitrust.org/cgi/pt?id=uieg.30152019845839" ], + "BOOK_WIKIPEDIA_URLS": [ + "https://en.wikipedia.org/wiki/Test_Book", + "https://en.wikipedia.org/wiki/Another_Book" + ], "CREDIT": "Roger Frank and Sue Clark.", "LANGUAGE": "English", "PUBLISHER": "Frank A. Munsey Company", diff --git a/libgutenberg/tests/test_dc.py b/libgutenberg/tests/test_dc.py index de91afa..726665c 100755 --- a/libgutenberg/tests/test_dc.py +++ b/libgutenberg/tests/test_dc.py @@ -51,6 +51,11 @@ def test_orm_metadata(self): dc = DublinCoreMapping.DublinCoreObject() self.metadata_test2(dc) + def test_wikipedia_urls_load(self): + dc = DublinCoreMapping.DublinCoreObject() + dc.load_from_database(2701) # Moby Dick + self.assertIn("https://en.wikipedia.org/wiki/Moby-Dick", dc.wikipedia_urls) + def metadata_test1(self, dc): dc.load_from_database(self.ebook) self.assertEqual(dc.project_gutenberg_id, 20050) @@ -224,7 +229,7 @@ def get_cover(ebook, dc): dc = GutenbergDatabaseDublinCore.GutenbergDatabaseDublinCore(self.dummypool) dc.register_coverpage(ebook, 'new_cover') # does nothing to avoid violates foreign key constraint - self.assertEqual(get_cover(ebook, dc), None) + self.assertEqual(get_cover(ebook, dc), None) def tearDown(self): pass @@ -297,6 +302,10 @@ def test_load_from_json(self): self.assertEqual(set_subtitle, "a true story : second line") self.assertEqual(len(dc.authors), 2) self.assertEqual(len(dc.scan_urls), 2) + self.assertEqual( + dc.wikipedia_urls, + {"https://en.wikipedia.org/wiki/Test_Book", + "https://en.wikipedia.org/wiki/Another_Book"}) self.assertEqual(dc.pubinfo.first_year, '1920') self.assertEqual(dc.credit, 'Roger Frank and Sue Clark.') dc.add_credit('Sue Frank and Roger Clark.\n') @@ -312,6 +321,10 @@ def test_load_from_json(self): dc.load_from_database(99999) self.assertEqual(set_title, dc.title) self.assertEqual(set_subtitle, dc.subtitle) + self.assertEqual( + dc.wikipedia_urls, + {"https://en.wikipedia.org/wiki/Test_Book", + "https://en.wikipedia.org/wiki/Another_Book"}) marc260 = dc.session.query(Attribute).filter_by(book=dc.book, fk_attriblist=260).first().text self.assertTrue('1920' in marc260) self.assertEqual( @@ -327,6 +340,11 @@ def test_load_from_json(self): len(dc.session.query(Attribute).filter_by(book=dc.book, fk_attriblist=904).all()), 2) + self.assertEqual( + len(dc.session.query(Attribute).filter_by(book=dc.book, + fk_attriblist=500).filter( + Attribute.text.like('%wikipedia.org%')).all()), + 2) self.assertEqual( dc.session.query(Attribute).filter_by(book=dc.book, fk_attriblist=905).first().text, '20210623194947brand') @@ -337,8 +355,29 @@ def test_load_from_json(self): dc.session.flush() self.assertFalse(DBUtils.ebook_exists(99999)) + def test_wikipedia_urls_add_and_remove(self): + dc = DublinCoreMapping.DublinCoreObject() + ebook = 99998 + dc.load_or_create_book(ebook) + dc.rights = 'Public Domain in the USA.' + urls = ["https://en.wikipedia.org/wiki/Moby-Dick", + "https://en.wikipedia.org/wiki/Test_Book"] + dc.add_wikipedia_url(urls[0]) + dc.add_wikipedia_url(f"See also: {urls[0]}") # rejected + dc.add_wikipedia_url(urls[1]) + dc.save(updatemode=0) + + remaining = list(urls) + while remaining: + dc = DublinCoreMapping.DublinCoreObject() + dc.load_from_database(ebook) + self.assertEqual(dc.wikipedia_urls, set(remaining)) + dc.remove_wikipedia_url(remaining.pop(0)) + dc.save(updatemode=1) + def tearDown(self): session = DBUtils.check_session(None) DBUtils.remove_author('Lorem Ipsum Jr.', session=session) session.query(Book).filter(Book.pk == 99999).delete() - session.commit() + session.query(Book).filter(Book.pk == 99998).delete() + session.commit() \ No newline at end of file diff --git a/setup.py b/setup.py index 50b4d2c..2a5d2af 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,8 @@ # libgutenberg setup.py # -__version__ = '0.11.0' +__version__ = '0.11.1' + from setuptools import setup