From 96b02705b04ab292372e29df994ed1a0248a810d Mon Sep 17 00:00:00 2001 From: Zachary Rosario Date: Thu, 11 Jun 2026 18:39:23 -0400 Subject: [PATCH 01/15] added functions to get and set wikipedia urls via DublinCoreMapping.py --- libgutenberg/DublinCoreMapping.py | 53 +++++++++++++++++++++++++++++-- libgutenberg/tests/test_dc.py | 35 +++++++++++++++++++- 2 files changed, 85 insertions(+), 3 deletions(-) diff --git a/libgutenberg/DublinCoreMapping.py b/libgutenberg/DublinCoreMapping.py index 86c5074..f40e1d5 100644 --- a/libgutenberg/DublinCoreMapping.py +++ b/libgutenberg/DublinCoreMapping.py @@ -215,7 +215,7 @@ def parse260(s): # categories(text, audiobook, etc) if book.categories: self.dcmitypes = [struct(id=cat.dcmitype[0], description=cat.dcmitype[1]) - for cat in book.categories] + for cat in book.categories] else: self.dcmitypes = [struct(id='Text', description='Text')] @@ -293,7 +293,7 @@ def register_coverpage(self, id_, url, code=901): try: session.begin_nested() session.add(Attribute(fk_books=id_, fk_attriblist=code, - text=gg.archive2files(id_, url))) + text=gg.archive2files(id_, url))) session.commit() except IntegrityError: # Duplicate key @@ -480,6 +480,7 @@ def add_title(self, book, title, marc=245, subtitle=None): title = title.replace(' *_ *', '\n') self.add_attribute(book, title, nonfiling=nonfiling, marc=marc) + def add_attribute(self, book, attr, nonfiling=0, marc=0): if not attr: return @@ -505,6 +506,54 @@ def add_attribute(self, book, attr, nonfiling=0, marc=0): book.attributes.append(Attribute( fk_attriblist=marc, nonfiling=nonfiling, text=attr)) + + def get_wikipedia_urls(self): + urls = set() + + for attrib in (e.text for e in self.marcs if e.code == '500'): + if not attrib: + continue + + match = re.search(r'https?://[^\s]*wikipedia\.org[^\s]*', attrib) + if match: + urls.add(match.group(0)) + + return urls + + def add_wikipedia_url(self, url): + """Add Wikipedia URL into MARC 500 attributes (deduplicated).""" + + if not url: + return + + session = self.get_my_session() + + if not self.book: + return + + url = url.strip() + + text = f"Wikipedia page about this book: {url}" + + # check duplicates using SQLAlchemy (consistent with your style) + exists = session.query(Attribute).where( + Attribute.book == self.book, + Attribute.fk_attriblist == 500, + Attribute.text == text + ).first() + + if exists: + return + + self.book.attributes.append( + Attribute( + fk_attriblist=500, + text=text + ) + ) + + session.commit() + def delete(self): """ only delete the book! """ session = self.get_my_session() diff --git a/libgutenberg/tests/test_dc.py b/libgutenberg/tests/test_dc.py index de91afa..92a4097 100755 --- a/libgutenberg/tests/test_dc.py +++ b/libgutenberg/tests/test_dc.py @@ -224,7 +224,7 @@ def get_cover(ebook, dc): dc = GutenbergDatabaseDublinCore.GutenbergDatabaseDublinCore(self.dummypool) dc.register_coverpage(ebook, 'new_cover') # does nothing to avoid violates foreign key constraint - self.assertEqual(get_cover(ebook, dc), None) + self.assertEqual(get_cover(ebook, dc), None) def tearDown(self): pass @@ -342,3 +342,36 @@ def tearDown(self): DBUtils.remove_author('Lorem Ipsum Jr.', session=session) session.query(Book).filter(Book.pk == 99999).delete() session.commit() + +@unittest.skipIf(not db_exists, 'database not configured') +class testDCFunctions(unittest.TestCase): + def test_get_wikipedia_urls(self): + dc = DublinCoreMapping.DublinCoreObject() + dc.load_from_database(2701) # Moby Dick + + urls = dc.get_wikipedia_urls() + + self.assertEqual( + urls, + {"https://en.wikipedia.org/wiki/Moby-Dick"} + ) + + def test_add_wikipedia_url(self): + dc = DublinCoreMapping.DublinCoreObject() + + ebook = 99998 # fake test id + + # ensure book exists + dc.load_or_create_book(ebook) + + url = "https://en.wikipedia.org/wiki/Moby-Dick" + + dc.add_wikipedia_url(url) + + # reload fresh object to ensure DB persistence + dc2 = DublinCoreMapping.DublinCoreObject() + dc2.load_from_database(ebook) + + urls = dc2.get_wikipedia_urls() + + self.assertIn(url, urls) \ No newline at end of file From 80f95c1dc14824c35e01107b630f976c3edf075f Mon Sep 17 00:00:00 2001 From: Zachary Rosario Date: Thu, 11 Jun 2026 18:43:07 -0400 Subject: [PATCH 02/15] updated version number --- CHANGES | 3 +++ setup.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 94e7164..1ca5c0b 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,7 @@ CHANGES +0.10.38 (June 11, 2026) +- Added two functions to `DublinCoreMapping.py` named `get_wikipedia_urls()` and `add_wikipedia_url`. The former returns a list of wikipedia urls for a given author, and the latter adds a wikipedia url to a books attributes in the db. + 0.10.37 (June 9, 2026) - Upgraded all the packages inside of `Pipfile.lock` to most recent versions. - Fixed syntax issues in `Models.py` preventing sqlalchemy upgrade. diff --git a/setup.py b/setup.py index b84d9ab..ef550d5 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ # libgutenberg setup.py # -__version__ = '0.10.37' +__version__ = '0.10.38' from setuptools import setup From 31656a3fab815bea028ec639281dd4e9f434af22 Mon Sep 17 00:00:00 2001 From: Zachary Rosario Date: Thu, 11 Jun 2026 20:04:03 -0400 Subject: [PATCH 03/15] remove comment --- libgutenberg/DublinCoreMapping.py | 1 - 1 file changed, 1 deletion(-) diff --git a/libgutenberg/DublinCoreMapping.py b/libgutenberg/DublinCoreMapping.py index f40e1d5..7170965 100644 --- a/libgutenberg/DublinCoreMapping.py +++ b/libgutenberg/DublinCoreMapping.py @@ -535,7 +535,6 @@ def add_wikipedia_url(self, url): text = f"Wikipedia page about this book: {url}" - # check duplicates using SQLAlchemy (consistent with your style) exists = session.query(Attribute).where( Attribute.book == self.book, Attribute.fk_attriblist == 500, From 3ce682dfe6d06f2647c5ef2bb741bc8e269af9f3 Mon Sep 17 00:00:00 2001 From: Zachary Rosario Date: Thu, 11 Jun 2026 20:08:40 -0400 Subject: [PATCH 04/15] added comment explain get wikipedia urls func --- libgutenberg/DublinCoreMapping.py | 1 + 1 file changed, 1 insertion(+) diff --git a/libgutenberg/DublinCoreMapping.py b/libgutenberg/DublinCoreMapping.py index 7170965..5759db1 100644 --- a/libgutenberg/DublinCoreMapping.py +++ b/libgutenberg/DublinCoreMapping.py @@ -508,6 +508,7 @@ def add_attribute(self, book, attr, nonfiling=0, marc=0): def get_wikipedia_urls(self): + """ Return a set of Wikipedia URLs from MARC 500 attributes. """ urls = set() for attrib in (e.text for e in self.marcs if e.code == '500'): From c7e33ea3cecfacc9a04ee84c64f7d539ed36e31d Mon Sep 17 00:00:00 2001 From: Zachary Rosario Date: Fri, 12 Jun 2026 19:43:10 -0400 Subject: [PATCH 05/15] Added wikipedia_urls field to dispatcher. Renanmed variable to book_wikipedia_urls. There is still a function to add urls and there is now one to remove them as well. Add url doesnt need a prefix, it will auto put one, but a custom one may be included and defualt will not be added (if wanted forever whatever reason). The remove function acccepts a inputs with prefix+url or just url. --- libgutenberg/DublinCore.py | 55 ++++++++++++++ libgutenberg/DublinCoreMapping.py | 69 ++++++----------- libgutenberg/tests/99999.json | 4 + libgutenberg/tests/test_dc.py | 119 ++++++++++++++++++++++-------- 4 files changed, 172 insertions(+), 75 deletions(-) diff --git a/libgutenberg/DublinCore.py b/libgutenberg/DublinCore.py index 984c441..3ae3aee 100644 --- a/libgutenberg/DublinCore.py +++ b/libgutenberg/DublinCore.py @@ -523,6 +523,26 @@ def handle_dc_languages(dc, text): pass +WIKI_URL_RE = re.compile(r'https?://[^\s]*wikipedia\.org[^\s]*') +WIKIPEDIA_URL_PREFIX = 'Wikipedia page about this book: ' + + +def extract_wikipedia_url(text): + match = WIKI_URL_RE.search(text or '') + return match.group(0) if match else None + + +def format_book_wikipedia_url(url_or_text): + """Bare URL gets default prefix; text with a URL already in it is kept as-is.""" + text = (url_or_text or '').strip() + if not text: + return '' + url = extract_wikipedia_url(text) + if url and text == url: + return f"{WIKIPEDIA_URL_PREFIX}{url}" + return text + + class GutenbergDublinCore(DublinCore): """ Parse from PG files. """ @@ -533,8 +553,27 @@ def __init__(self): self._project_gutenberg_id = None self.request_key = '' self.scan_urls = set() + self.book_wikipedia_urls = [] + def add_book_wikipedia_url(self, url_or_text): + text = format_book_wikipedia_url(url_or_text) + if not text: + return + url = extract_wikipedia_url(text) + if url and any(extract_wikipedia_url(t) == url for t in self.book_wikipedia_urls): + return + self.book_wikipedia_urls.append(text) + + + def remove_book_wikipedia_url(self, url_or_text): + needle = (url_or_text or '').strip() + key = extract_wikipedia_url(needle) + for i, text in enumerate(self.book_wikipedia_urls): + if text == needle or (key and extract_wikipedia_url(text) == key): + del self.book_wikipedia_urls[i] + return + @property def project_gutenberg_id(self): @@ -794,6 +833,18 @@ def handle_scan_urls(self, key, value): self.scan_urls.add(scan_url) + def handle_book_wikipedia_urls(self, key, value): + if isinstance(value, str): + value = [value] + elif isinstance(value, list): + pass + else: + error('%s is not a valid wikipedia url', value) + return + for url in value: + self.add_book_wikipedia_url(url) + + def handle_pubinfo(self, key, value): if key == 'publisher': self.pubinfo.publisher = value @@ -928,6 +979,7 @@ def dispatch(self, key, val): 'alt_title': store, 'creator_role': handle_creators, 'scans_archive_url': handle_scan_urls, + 'book_wikipedia_url': handle_book_wikipedia_urls, 'credit': store, 'publisher': handle_pubinfo, 'publisher_country': handle_pubinfo, @@ -951,6 +1003,9 @@ def dispatch(self, key, val): 'created': 'source_publication_years', 'produced by': 'credit', 'publisher_place': 'place', + 'book_wikipedia_urls': 'book_wikipedia_url', + 'wikipedia_urls': 'book_wikipedia_url', + 'wikipedia_url': 'book_wikipedia_url', } for role in list(self.inverse_role_map.keys()): diff --git a/libgutenberg/DublinCoreMapping.py b/libgutenberg/DublinCoreMapping.py index 5759db1..b4e618d 100644 --- a/libgutenberg/DublinCoreMapping.py +++ b/libgutenberg/DublinCoreMapping.py @@ -23,6 +23,7 @@ from sqlalchemy.exc import DBAPIError from . import DublinCore +from .DublinCore import extract_wikipedia_url from . import GutenbergGlobals as gg from . import GutenbergDatabase from . import GutenbergFiles @@ -179,7 +180,10 @@ def parse260(s): elif marc.code == '260': (self.pubinfo.place, self.pubinfo.publisher, self.pubinfo.years) = parse260(marc.text) elif marc.code == '500': - self.notes = marc.text + if extract_wikipedia_url(marc.text): + self.add_book_wikipedia_url(marc.text) + else: + self.notes = marc.text elif marc.code == '505': self.contents = marc.text elif marc.code == '508': @@ -383,6 +387,8 @@ def save(self, updatemode=0): if self.request_key: self.add_attribute(self.book, self.request_key, marc=905) + self._update_book_wikipedia_urls() + self.book.updatemode = 1 # prevent non-cataloguer changes session.commit() @@ -507,52 +513,25 @@ def add_attribute(self, book, attr, nonfiling=0, marc=0): fk_attriblist=marc, nonfiling=nonfiling, text=attr)) - def get_wikipedia_urls(self): - """ Return a set of Wikipedia URLs from MARC 500 attributes. """ - urls = set() - - for attrib in (e.text for e in self.marcs if e.code == '500'): - if not attrib: - continue - - match = re.search(r'https?://[^\s]*wikipedia\.org[^\s]*', attrib) - if match: - urls.add(match.group(0)) - - return urls - - def add_wikipedia_url(self, url): - """Add Wikipedia URL into MARC 500 attributes (deduplicated).""" - - if not url: - return - - session = self.get_my_session() - + def _update_book_wikipedia_urls(self): + """Sync MARC 500 wiki rows to book_wikipedia_urls (matched by URL).""" if not self.book: return - - url = url.strip() - - text = f"Wikipedia page about this book: {url}" - - exists = session.query(Attribute).where( - Attribute.book == self.book, - Attribute.fk_attriblist == 500, - Attribute.text == text - ).first() - - if exists: - return - - self.book.attributes.append( - Attribute( - fk_attriblist=500, - text=text - ) - ) - - session.commit() + wanted = {extract_wikipedia_url(text): text + for text in self.book_wikipedia_urls + if extract_wikipedia_url(text)} + for att in list(self.book.attributes): + if att.fk_attriblist != 500: + continue + url = extract_wikipedia_url(att.text) + if not url: + continue + if url in wanted: + del wanted[url] + else: + self.book.attributes.remove(att) + for text in wanted.values(): + self.book.attributes.append(Attribute(fk_attriblist=500, text=text)) def delete(self): """ only delete the book! """ diff --git a/libgutenberg/tests/99999.json b/libgutenberg/tests/99999.json index 291ebfb..8be6238 100644 --- a/libgutenberg/tests/99999.json +++ b/libgutenberg/tests/99999.json @@ -26,6 +26,10 @@ "https://archive.org/details/in.ernet.dli.2013.179137", "https://babel.hathitrust.org/cgi/pt?id=uieg.30152019845839" ], + "BOOK_WIKIPEDIA_URLS": [ + "https://en.wikipedia.org/wiki/Test_Book", + "https://en.wikipedia.org/wiki/Another_Book" + ], "CREDIT": "Roger Frank and Sue Clark.", "LANGUAGE": "English", "PUBLISHER": "Frank A. Munsey Company", diff --git a/libgutenberg/tests/test_dc.py b/libgutenberg/tests/test_dc.py index 92a4097..73fa25b 100755 --- a/libgutenberg/tests/test_dc.py +++ b/libgutenberg/tests/test_dc.py @@ -9,6 +9,7 @@ from libgutenberg.CommonOptions import Options from libgutenberg import GutenbergDatabase, GutenbergDatabaseDublinCore, DummyConnectionPool from libgutenberg import DBUtils, DublinCoreMapping +from libgutenberg.DublinCore import GutenbergDublinCore, WIKIPEDIA_URL_PREFIX, extract_wikipedia_url from libgutenberg.Logger import debug, warning from libgutenberg.Models import Attribute, Book @@ -51,6 +52,12 @@ def test_orm_metadata(self): dc = DublinCoreMapping.DublinCoreObject() self.metadata_test2(dc) + def test_book_wikipedia_urls_load(self): + dc = DublinCoreMapping.DublinCoreObject() + dc.load_from_database(2701) # Moby Dick + urls = [extract_wikipedia_url(text) for text in dc.book_wikipedia_urls] + self.assertIn("https://en.wikipedia.org/wiki/Moby-Dick", urls) + def metadata_test1(self, dc): dc.load_from_database(self.ebook) self.assertEqual(dc.project_gutenberg_id, 20050) @@ -297,6 +304,10 @@ def test_load_from_json(self): self.assertEqual(set_subtitle, "a true story : second line") self.assertEqual(len(dc.authors), 2) self.assertEqual(len(dc.scan_urls), 2) + self.assertEqual( + dc.book_wikipedia_urls, + [f"{WIKIPEDIA_URL_PREFIX}https://en.wikipedia.org/wiki/Test_Book", + f"{WIKIPEDIA_URL_PREFIX}https://en.wikipedia.org/wiki/Another_Book"]) self.assertEqual(dc.pubinfo.first_year, '1920') self.assertEqual(dc.credit, 'Roger Frank and Sue Clark.') dc.add_credit('Sue Frank and Roger Clark.\n') @@ -312,6 +323,10 @@ def test_load_from_json(self): dc.load_from_database(99999) self.assertEqual(set_title, dc.title) self.assertEqual(set_subtitle, dc.subtitle) + self.assertEqual( + dc.book_wikipedia_urls, + [f"{WIKIPEDIA_URL_PREFIX}https://en.wikipedia.org/wiki/Test_Book", + f"{WIKIPEDIA_URL_PREFIX}https://en.wikipedia.org/wiki/Another_Book"]) marc260 = dc.session.query(Attribute).filter_by(book=dc.book, fk_attriblist=260).first().text self.assertTrue('1920' in marc260) self.assertEqual( @@ -327,6 +342,11 @@ def test_load_from_json(self): len(dc.session.query(Attribute).filter_by(book=dc.book, fk_attriblist=904).all()), 2) + self.assertEqual( + len(dc.session.query(Attribute).filter_by(book=dc.book, + fk_attriblist=500).filter( + Attribute.text.like('%wikipedia.org%')).all()), + 2) self.assertEqual( dc.session.query(Attribute).filter_by(book=dc.book, fk_attriblist=905).first().text, '20210623194947brand') @@ -337,41 +357,80 @@ def test_load_from_json(self): dc.session.flush() self.assertFalse(DBUtils.ebook_exists(99999)) - def tearDown(self): - session = DBUtils.check_session(None) - DBUtils.remove_author('Lorem Ipsum Jr.', session=session) - session.query(Book).filter(Book.pk == 99999).delete() - session.commit() + def test_book_wikipedia_url_format(self): + bare = "https://en.wikipedia.org/wiki/Moby-Dick" + prefixed = f"{WIKIPEDIA_URL_PREFIX}{bare}" + custom = f"See also: {bare}" -@unittest.skipIf(not db_exists, 'database not configured') -class testDCFunctions(unittest.TestCase): - def test_get_wikipedia_urls(self): - dc = DublinCoreMapping.DublinCoreObject() - dc.load_from_database(2701) # Moby Dick - - urls = dc.get_wikipedia_urls() - - self.assertEqual( - urls, - {"https://en.wikipedia.org/wiki/Moby-Dick"} - ) + dc = GutenbergDublinCore() + dc.add_book_wikipedia_url(bare) + self.assertEqual(dc.book_wikipedia_urls, [prefixed]) + + dc.add_book_wikipedia_url(prefixed) + self.assertEqual(dc.book_wikipedia_urls, [prefixed]) + + dc2 = GutenbergDublinCore() + dc2.add_book_wikipedia_url(custom) + self.assertEqual(dc2.book_wikipedia_urls, [custom]) + + dc.add_book_wikipedia_url(bare) + dc.remove_book_wikipedia_url(bare) + self.assertEqual(dc.book_wikipedia_urls, []) - def test_add_wikipedia_url(self): + dc.add_book_wikipedia_url(bare) + dc.remove_book_wikipedia_url(prefixed) + self.assertEqual(dc.book_wikipedia_urls, []) + + def test_book_wikipedia_urls_add_and_remove(self): dc = DublinCoreMapping.DublinCoreObject() - + ebook = 99998 # fake test id - - # ensure book exists + dc.load_or_create_book(ebook) - + dc.rights = 'Public Domain in the USA.' + url = "https://en.wikipedia.org/wiki/Moby-Dick" - - dc.add_wikipedia_url(url) - - # reload fresh object to ensure DB persistence + url2 = "https://en.wikipedia.org/wiki/Test_Book" + url3 = "https://en.wikipedia.org/wiki/Herman_Melville" + text1 = f"{WIKIPEDIA_URL_PREFIX}{url}" + text2 = f"{WIKIPEDIA_URL_PREFIX}{url2}" + custom_text = f"Wikipedia page about this author: {url3}" + + dc.add_book_wikipedia_url(url) + dc.add_book_wikipedia_url(text2) + dc.add_book_wikipedia_url(custom_text) + dc.save(updatemode=0) + dc2 = DublinCoreMapping.DublinCoreObject() dc2.load_from_database(ebook) - - urls = dc2.get_wikipedia_urls() - - self.assertIn(url, urls) \ No newline at end of file + + self.assertEqual(dc2.book_wikipedia_urls, [text1, text2, custom_text]) + + dc2.remove_book_wikipedia_url(url) + dc2.save(updatemode=1) + + dc3 = DublinCoreMapping.DublinCoreObject() + dc3.load_from_database(ebook) + + self.assertEqual(dc3.book_wikipedia_urls, [text2, custom_text]) + + dc3.remove_book_wikipedia_url(custom_text) + dc3.save(updatemode=1) + + dc4 = DublinCoreMapping.DublinCoreObject() + dc4.load_from_database(ebook) + self.assertEqual(dc4.book_wikipedia_urls, [text2]) + + dc4.remove_book_wikipedia_url(text2) + dc4.save(updatemode=1) + + dc5 = DublinCoreMapping.DublinCoreObject() + dc5.load_from_database(ebook) + self.assertEqual(dc5.book_wikipedia_urls, []) + + def tearDown(self): + session = DBUtils.check_session(None) + DBUtils.remove_author('Lorem Ipsum Jr.', session=session) + session.query(Book).filter(Book.pk == 99999).delete() + session.query(Book).filter(Book.pk == 99998).delete() + session.commit() \ No newline at end of file From 869f37d78176cc8dceea37419c45d52a96f6e640 Mon Sep 17 00:00:00 2001 From: Zachary Rosario Date: Fri, 12 Jun 2026 22:18:37 -0400 Subject: [PATCH 06/15] strip whitespaces on inputs to add/remove wikipedia url funcs --- libgutenberg/DublinCore.py | 22 ++++++++++++---------- libgutenberg/tests/test_dc.py | 8 ++++++++ 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/libgutenberg/DublinCore.py b/libgutenberg/DublinCore.py index 3ae3aee..9014550 100644 --- a/libgutenberg/DublinCore.py +++ b/libgutenberg/DublinCore.py @@ -534,13 +534,12 @@ def extract_wikipedia_url(text): def format_book_wikipedia_url(url_or_text): """Bare URL gets default prefix; text with a URL already in it is kept as-is.""" - text = (url_or_text or '').strip() - if not text: + if not url_or_text: return '' - url = extract_wikipedia_url(text) - if url and text == url: + url = extract_wikipedia_url(url_or_text) + if url and url_or_text == url: return f"{WIKIPEDIA_URL_PREFIX}{url}" - return text + return url_or_text class GutenbergDublinCore(DublinCore): @@ -557,9 +556,10 @@ def __init__(self): def add_book_wikipedia_url(self, url_or_text): - text = format_book_wikipedia_url(url_or_text) - if not text: + url_or_text = (url_or_text or '').strip() + if not url_or_text: return + text = format_book_wikipedia_url(url_or_text) url = extract_wikipedia_url(text) if url and any(extract_wikipedia_url(t) == url for t in self.book_wikipedia_urls): return @@ -567,10 +567,12 @@ def add_book_wikipedia_url(self, url_or_text): def remove_book_wikipedia_url(self, url_or_text): - needle = (url_or_text or '').strip() - key = extract_wikipedia_url(needle) + url_or_text = (url_or_text or '').strip() + if not url_or_text: + return + key = extract_wikipedia_url(url_or_text) for i, text in enumerate(self.book_wikipedia_urls): - if text == needle or (key and extract_wikipedia_url(text) == key): + if text == url_or_text or (key and extract_wikipedia_url(text) == key): del self.book_wikipedia_urls[i] return diff --git a/libgutenberg/tests/test_dc.py b/libgutenberg/tests/test_dc.py index 73fa25b..81f2c28 100755 --- a/libgutenberg/tests/test_dc.py +++ b/libgutenberg/tests/test_dc.py @@ -381,6 +381,14 @@ def test_book_wikipedia_url_format(self): dc.remove_book_wikipedia_url(prefixed) self.assertEqual(dc.book_wikipedia_urls, []) + dc.add_book_wikipedia_url(f" {bare} ") + self.assertEqual(dc.book_wikipedia_urls, [prefixed]) + dc.remove_book_wikipedia_url(f" {bare} ") + self.assertEqual(dc.book_wikipedia_urls, []) + + dc.add_book_wikipedia_url(f"See also: {bare} ") + self.assertEqual(dc.book_wikipedia_urls, [f"See also: {bare}"]) + def test_book_wikipedia_urls_add_and_remove(self): dc = DublinCoreMapping.DublinCoreObject() From 1ce5c3dbe9d12a5f3db1a45532acd8675cf41a69 Mon Sep 17 00:00:00 2001 From: Zachary Rosario Date: Fri, 12 Jun 2026 23:02:09 -0400 Subject: [PATCH 07/15] updated changes --- CHANGES | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGES b/CHANGES index 1ca5c0b..5a76c07 100644 --- a/CHANGES +++ b/CHANGES @@ -1,7 +1,10 @@ CHANGES -0.10.38 (June 11, 2026) -- Added two functions to `DublinCoreMapping.py` named `get_wikipedia_urls()` and `add_wikipedia_url`. The former returns a list of wikipedia urls for a given author, and the latter adds a wikipedia url to a books attributes in the db. - +0.10.38 (June 12, 2026) +- New field `book_wikipedia_url(s)` has been added to the DublinCore dispatcher. +- Two functions `add_book_wikipedia_url` and `remove_book_wikipedia_url` have been added to the DublinCoreObject. +- Wikipedia url dispatcher testing added to `test_load_from_json`. +- Add/remove tesing can be found in `test_book_wikipedia_urls_add_and_remove` + - Plus `test_book_wikipedia_url_format` which covers striping white space from client input. 0.10.37 (June 9, 2026) - Upgraded all the packages inside of `Pipfile.lock` to most recent versions. - Fixed syntax issues in `Models.py` preventing sqlalchemy upgrade. From da02b0f429b43a35f123c2095cd55a3e550df82d Mon Sep 17 00:00:00 2001 From: Zach Rosario Date: Wed, 17 Jun 2026 11:29:24 -0400 Subject: [PATCH 08/15] Renamed book_wikipedia_url var to wikipedia_url. --- libgutenberg/DublinCore.py | 30 +++++++------- libgutenberg/DublinCoreMapping.py | 10 ++--- libgutenberg/tests/test_dc.py | 68 +++++++++++++++---------------- 3 files changed, 54 insertions(+), 54 deletions(-) diff --git a/libgutenberg/DublinCore.py b/libgutenberg/DublinCore.py index 9014550..18fb44f 100644 --- a/libgutenberg/DublinCore.py +++ b/libgutenberg/DublinCore.py @@ -532,7 +532,7 @@ def extract_wikipedia_url(text): return match.group(0) if match else None -def format_book_wikipedia_url(url_or_text): +def format_wikipedia_url(url_or_text): """Bare URL gets default prefix; text with a URL already in it is kept as-is.""" if not url_or_text: return '' @@ -552,28 +552,28 @@ def __init__(self): self._project_gutenberg_id = None self.request_key = '' self.scan_urls = set() - self.book_wikipedia_urls = [] + self.wikipedia_urls = [] - def add_book_wikipedia_url(self, url_or_text): + def add_wikipedia_url(self, url_or_text): url_or_text = (url_or_text or '').strip() if not url_or_text: return - text = format_book_wikipedia_url(url_or_text) + text = format_wikipedia_url(url_or_text) url = extract_wikipedia_url(text) - if url and any(extract_wikipedia_url(t) == url for t in self.book_wikipedia_urls): + if url and any(extract_wikipedia_url(t) == url for t in self.wikipedia_urls): return - self.book_wikipedia_urls.append(text) + self.wikipedia_urls.append(text) - def remove_book_wikipedia_url(self, url_or_text): + def remove_wikipedia_url(self, url_or_text): url_or_text = (url_or_text or '').strip() if not url_or_text: return key = extract_wikipedia_url(url_or_text) - for i, text in enumerate(self.book_wikipedia_urls): + for i, text in enumerate(self.wikipedia_urls): if text == url_or_text or (key and extract_wikipedia_url(text) == key): - del self.book_wikipedia_urls[i] + del self.wikipedia_urls[i] return @@ -835,7 +835,7 @@ def handle_scan_urls(self, key, value): self.scan_urls.add(scan_url) - def handle_book_wikipedia_urls(self, key, value): + def handle_wikipedia_urls(self, key, value): if isinstance(value, str): value = [value] elif isinstance(value, list): @@ -844,7 +844,7 @@ def handle_book_wikipedia_urls(self, key, value): error('%s is not a valid wikipedia url', value) return for url in value: - self.add_book_wikipedia_url(url) + self.add_wikipedia_url(url) def handle_pubinfo(self, key, value): @@ -981,7 +981,7 @@ def dispatch(self, key, val): 'alt_title': store, 'creator_role': handle_creators, 'scans_archive_url': handle_scan_urls, - 'book_wikipedia_url': handle_book_wikipedia_urls, + 'wikipedia_url': handle_wikipedia_urls, 'credit': store, 'publisher': handle_pubinfo, 'publisher_country': handle_pubinfo, @@ -1005,9 +1005,9 @@ def dispatch(self, key, val): 'created': 'source_publication_years', 'produced by': 'credit', 'publisher_place': 'place', - 'book_wikipedia_urls': 'book_wikipedia_url', - 'wikipedia_urls': 'book_wikipedia_url', - 'wikipedia_url': 'book_wikipedia_url', + 'wikipedia_urls': 'wikipedia_url', + 'book_wikipedia_url': 'wikipedia_url', + 'book_wikipedia_urls': 'wikipedia_url', } for role in list(self.inverse_role_map.keys()): diff --git a/libgutenberg/DublinCoreMapping.py b/libgutenberg/DublinCoreMapping.py index b4e618d..2586ea7 100644 --- a/libgutenberg/DublinCoreMapping.py +++ b/libgutenberg/DublinCoreMapping.py @@ -181,7 +181,7 @@ def parse260(s): (self.pubinfo.place, self.pubinfo.publisher, self.pubinfo.years) = parse260(marc.text) elif marc.code == '500': if extract_wikipedia_url(marc.text): - self.add_book_wikipedia_url(marc.text) + self.add_wikipedia_url(marc.text) else: self.notes = marc.text elif marc.code == '505': @@ -387,7 +387,7 @@ def save(self, updatemode=0): if self.request_key: self.add_attribute(self.book, self.request_key, marc=905) - self._update_book_wikipedia_urls() + self._update_wikipedia_urls() self.book.updatemode = 1 # prevent non-cataloguer changes @@ -513,12 +513,12 @@ def add_attribute(self, book, attr, nonfiling=0, marc=0): fk_attriblist=marc, nonfiling=nonfiling, text=attr)) - def _update_book_wikipedia_urls(self): - """Sync MARC 500 wiki rows to book_wikipedia_urls (matched by URL).""" + def _update_wikipedia_urls(self): + """Sync MARC 500 wiki rows to wikipedia_urls (matched by URL).""" if not self.book: return wanted = {extract_wikipedia_url(text): text - for text in self.book_wikipedia_urls + for text in self.wikipedia_urls if extract_wikipedia_url(text)} for att in list(self.book.attributes): if att.fk_attriblist != 500: diff --git a/libgutenberg/tests/test_dc.py b/libgutenberg/tests/test_dc.py index 81f2c28..31a7f37 100755 --- a/libgutenberg/tests/test_dc.py +++ b/libgutenberg/tests/test_dc.py @@ -52,10 +52,10 @@ def test_orm_metadata(self): dc = DublinCoreMapping.DublinCoreObject() self.metadata_test2(dc) - def test_book_wikipedia_urls_load(self): + def test_wikipedia_urls_load(self): dc = DublinCoreMapping.DublinCoreObject() dc.load_from_database(2701) # Moby Dick - urls = [extract_wikipedia_url(text) for text in dc.book_wikipedia_urls] + urls = [extract_wikipedia_url(text) for text in dc.wikipedia_urls] self.assertIn("https://en.wikipedia.org/wiki/Moby-Dick", urls) def metadata_test1(self, dc): @@ -305,7 +305,7 @@ def test_load_from_json(self): self.assertEqual(len(dc.authors), 2) self.assertEqual(len(dc.scan_urls), 2) self.assertEqual( - dc.book_wikipedia_urls, + dc.wikipedia_urls, [f"{WIKIPEDIA_URL_PREFIX}https://en.wikipedia.org/wiki/Test_Book", f"{WIKIPEDIA_URL_PREFIX}https://en.wikipedia.org/wiki/Another_Book"]) self.assertEqual(dc.pubinfo.first_year, '1920') @@ -324,7 +324,7 @@ def test_load_from_json(self): self.assertEqual(set_title, dc.title) self.assertEqual(set_subtitle, dc.subtitle) self.assertEqual( - dc.book_wikipedia_urls, + dc.wikipedia_urls, [f"{WIKIPEDIA_URL_PREFIX}https://en.wikipedia.org/wiki/Test_Book", f"{WIKIPEDIA_URL_PREFIX}https://en.wikipedia.org/wiki/Another_Book"]) marc260 = dc.session.query(Attribute).filter_by(book=dc.book, fk_attriblist=260).first().text @@ -357,39 +357,39 @@ def test_load_from_json(self): dc.session.flush() self.assertFalse(DBUtils.ebook_exists(99999)) - def test_book_wikipedia_url_format(self): + def test_wikipedia_url_format(self): bare = "https://en.wikipedia.org/wiki/Moby-Dick" prefixed = f"{WIKIPEDIA_URL_PREFIX}{bare}" custom = f"See also: {bare}" dc = GutenbergDublinCore() - dc.add_book_wikipedia_url(bare) - self.assertEqual(dc.book_wikipedia_urls, [prefixed]) + dc.add_wikipedia_url(bare) + self.assertEqual(dc.wikipedia_urls, [prefixed]) - dc.add_book_wikipedia_url(prefixed) - self.assertEqual(dc.book_wikipedia_urls, [prefixed]) + dc.add_wikipedia_url(prefixed) + self.assertEqual(dc.wikipedia_urls, [prefixed]) dc2 = GutenbergDublinCore() - dc2.add_book_wikipedia_url(custom) - self.assertEqual(dc2.book_wikipedia_urls, [custom]) + dc2.add_wikipedia_url(custom) + self.assertEqual(dc2.wikipedia_urls, [custom]) - dc.add_book_wikipedia_url(bare) - dc.remove_book_wikipedia_url(bare) - self.assertEqual(dc.book_wikipedia_urls, []) + dc.add_wikipedia_url(bare) + dc.remove_wikipedia_url(bare) + self.assertEqual(dc.wikipedia_urls, []) - dc.add_book_wikipedia_url(bare) - dc.remove_book_wikipedia_url(prefixed) - self.assertEqual(dc.book_wikipedia_urls, []) + dc.add_wikipedia_url(bare) + dc.remove_wikipedia_url(prefixed) + self.assertEqual(dc.wikipedia_urls, []) - dc.add_book_wikipedia_url(f" {bare} ") - self.assertEqual(dc.book_wikipedia_urls, [prefixed]) - dc.remove_book_wikipedia_url(f" {bare} ") - self.assertEqual(dc.book_wikipedia_urls, []) + dc.add_wikipedia_url(f" {bare} ") + self.assertEqual(dc.wikipedia_urls, [prefixed]) + dc.remove_wikipedia_url(f" {bare} ") + self.assertEqual(dc.wikipedia_urls, []) - dc.add_book_wikipedia_url(f"See also: {bare} ") - self.assertEqual(dc.book_wikipedia_urls, [f"See also: {bare}"]) + dc.add_wikipedia_url(f"See also: {bare} ") + self.assertEqual(dc.wikipedia_urls, [f"See also: {bare}"]) - def test_book_wikipedia_urls_add_and_remove(self): + def test_wikipedia_urls_add_and_remove(self): dc = DublinCoreMapping.DublinCoreObject() ebook = 99998 # fake test id @@ -404,37 +404,37 @@ def test_book_wikipedia_urls_add_and_remove(self): text2 = f"{WIKIPEDIA_URL_PREFIX}{url2}" custom_text = f"Wikipedia page about this author: {url3}" - dc.add_book_wikipedia_url(url) - dc.add_book_wikipedia_url(text2) - dc.add_book_wikipedia_url(custom_text) + dc.add_wikipedia_url(url) + dc.add_wikipedia_url(text2) + dc.add_wikipedia_url(custom_text) dc.save(updatemode=0) dc2 = DublinCoreMapping.DublinCoreObject() dc2.load_from_database(ebook) - self.assertEqual(dc2.book_wikipedia_urls, [text1, text2, custom_text]) + self.assertEqual(dc2.wikipedia_urls, [text1, text2, custom_text]) - dc2.remove_book_wikipedia_url(url) + dc2.remove_wikipedia_url(url) dc2.save(updatemode=1) dc3 = DublinCoreMapping.DublinCoreObject() dc3.load_from_database(ebook) - self.assertEqual(dc3.book_wikipedia_urls, [text2, custom_text]) + self.assertEqual(dc3.wikipedia_urls, [text2, custom_text]) - dc3.remove_book_wikipedia_url(custom_text) + dc3.remove_wikipedia_url(custom_text) dc3.save(updatemode=1) dc4 = DublinCoreMapping.DublinCoreObject() dc4.load_from_database(ebook) - self.assertEqual(dc4.book_wikipedia_urls, [text2]) + self.assertEqual(dc4.wikipedia_urls, [text2]) - dc4.remove_book_wikipedia_url(text2) + dc4.remove_wikipedia_url(text2) dc4.save(updatemode=1) dc5 = DublinCoreMapping.DublinCoreObject() dc5.load_from_database(ebook) - self.assertEqual(dc5.book_wikipedia_urls, []) + self.assertEqual(dc5.wikipedia_urls, []) def tearDown(self): session = DBUtils.check_session(None) From e6619910453689fb19f92037e3406e4866297c61 Mon Sep 17 00:00:00 2001 From: Zach Rosario Date: Wed, 17 Jun 2026 11:48:18 -0400 Subject: [PATCH 09/15] Updated wikipedia url handling to match summary writers implemntation. --- libgutenberg/DublinCore.py | 56 +++++++++++++++++++++++++------ libgutenberg/DublinCoreMapping.py | 18 +++++----- 2 files changed, 55 insertions(+), 19 deletions(-) diff --git a/libgutenberg/DublinCore.py b/libgutenberg/DublinCore.py index 18fb44f..b129e07 100644 --- a/libgutenberg/DublinCore.py +++ b/libgutenberg/DublinCore.py @@ -21,6 +21,7 @@ import textwrap import unicodedata from gettext import gettext as _ +from urllib.parse import unquote import six import lxml @@ -523,21 +524,46 @@ def handle_dc_languages(dc, text): pass -WIKI_URL_RE = re.compile(r'https?://[^\s]*wikipedia\.org[^\s]*') +WIKIMATCH = re.compile( + r"(?ix)https?://([a-z]{2,3})\.wikipedia\.org/wiki/([/!@i^*$a-z0-9_\(\)\-.:]+)") +AVOID_WIKI = ["simple.", "File:", "/Category:", "Category:", "(disambiguation)"] +UNTRUSTED_WIKI_LANGS = {'sco'} WIKIPEDIA_URL_PREFIX = 'Wikipedia page about this book: ' +def check_wikipedia_url(text): + """Return (lang, page_title) if text contains a valid wiki URL, else None.""" + if not text: + return None + match = WIKIMATCH.search(text) + if not match: + return None + lang, page_title = match.group(1), match.group(2) + if lang in UNTRUSTED_WIKI_LANGS: + return None + if any(pattern in unquote(page_title) for pattern in AVOID_WIKI): + return None + return (lang, page_title) + + +def wikipedia_url(lang, page_title): + return f"https://{lang}.wikipedia.org/wiki/{page_title}" + + def extract_wikipedia_url(text): - match = WIKI_URL_RE.search(text or '') - return match.group(0) if match else None + checked = check_wikipedia_url(text) + return wikipedia_url(*checked) if checked else None def format_wikipedia_url(url_or_text): """Bare URL gets default prefix; text with a URL already in it is kept as-is.""" if not url_or_text: return '' - url = extract_wikipedia_url(url_or_text) - if url and url_or_text == url: + checked = check_wikipedia_url(url_or_text) + if not checked: + return url_or_text + url = wikipedia_url(*checked) + if url_or_text.strip() == url: return f"{WIKIPEDIA_URL_PREFIX}{url}" return url_or_text @@ -559,10 +585,17 @@ def add_wikipedia_url(self, url_or_text): url_or_text = (url_or_text or '').strip() if not url_or_text: return - text = format_wikipedia_url(url_or_text) - url = extract_wikipedia_url(text) - if url and any(extract_wikipedia_url(t) == url for t in self.wikipedia_urls): + checked = check_wikipedia_url(url_or_text) + if not checked: + warning('%s is not a valid wikipedia url', url_or_text) return + if any(check_wikipedia_url(text) == checked for text in self.wikipedia_urls): + return + url = wikipedia_url(*checked) + if url_or_text == url: + text = f"{WIKIPEDIA_URL_PREFIX}{url}" + else: + text = url_or_text self.wikipedia_urls.append(text) @@ -570,9 +603,9 @@ def remove_wikipedia_url(self, url_or_text): url_or_text = (url_or_text or '').strip() if not url_or_text: return - key = extract_wikipedia_url(url_or_text) + checked = check_wikipedia_url(url_or_text) for i, text in enumerate(self.wikipedia_urls): - if text == url_or_text or (key and extract_wikipedia_url(text) == key): + if text == url_or_text or (checked and check_wikipedia_url(text) == checked): del self.wikipedia_urls[i] return @@ -844,6 +877,9 @@ def handle_wikipedia_urls(self, key, value): error('%s is not a valid wikipedia url', value) return for url in value: + if not check_wikipedia_url(url): + error('%s is not a valid wikipedia url', url) + continue self.add_wikipedia_url(url) diff --git a/libgutenberg/DublinCoreMapping.py b/libgutenberg/DublinCoreMapping.py index 2586ea7..f250873 100644 --- a/libgutenberg/DublinCoreMapping.py +++ b/libgutenberg/DublinCoreMapping.py @@ -23,7 +23,7 @@ from sqlalchemy.exc import DBAPIError from . import DublinCore -from .DublinCore import extract_wikipedia_url +from .DublinCore import check_wikipedia_url from . import GutenbergGlobals as gg from . import GutenbergDatabase from . import GutenbergFiles @@ -180,7 +180,7 @@ def parse260(s): elif marc.code == '260': (self.pubinfo.place, self.pubinfo.publisher, self.pubinfo.years) = parse260(marc.text) elif marc.code == '500': - if extract_wikipedia_url(marc.text): + if check_wikipedia_url(marc.text): self.add_wikipedia_url(marc.text) else: self.notes = marc.text @@ -514,20 +514,20 @@ def add_attribute(self, book, attr, nonfiling=0, marc=0): def _update_wikipedia_urls(self): - """Sync MARC 500 wiki rows to wikipedia_urls (matched by URL).""" + """Sync MARC 500 wiki rows to wikipedia_urls (matched by lang and title).""" if not self.book: return - wanted = {extract_wikipedia_url(text): text + wanted = {check_wikipedia_url(text): text for text in self.wikipedia_urls - if extract_wikipedia_url(text)} + if check_wikipedia_url(text)} for att in list(self.book.attributes): if att.fk_attriblist != 500: continue - url = extract_wikipedia_url(att.text) - if not url: + checked = check_wikipedia_url(att.text) + if not checked: continue - if url in wanted: - del wanted[url] + if checked in wanted: + del wanted[checked] else: self.book.attributes.remove(att) for text in wanted.values(): From 47e8bae88126d615bbe4319cf1b37ccf66ca92b3 Mon Sep 17 00:00:00 2001 From: Zach Rosario Date: Wed, 17 Jun 2026 11:53:11 -0400 Subject: [PATCH 10/15] upgraded from warning to error --- libgutenberg/DublinCore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libgutenberg/DublinCore.py b/libgutenberg/DublinCore.py index b129e07..5b60415 100644 --- a/libgutenberg/DublinCore.py +++ b/libgutenberg/DublinCore.py @@ -587,7 +587,7 @@ def add_wikipedia_url(self, url_or_text): return checked = check_wikipedia_url(url_or_text) if not checked: - warning('%s is not a valid wikipedia url', url_or_text) + error('%s is not a valid wikipedia url', url_or_text) return if any(check_wikipedia_url(text) == checked for text in self.wikipedia_urls): return From 04cd42f8c28e0c1d48a4bff30521765ba53a0d93 Mon Sep 17 00:00:00 2001 From: Zach Rosario Date: Thu, 2 Jul 2026 15:35:09 -0400 Subject: [PATCH 11/15] refactor(dublincore): store wikipedia_urls as bare URL set --- libgutenberg/DublinCore.py | 53 +++++++++----------------------------- 1 file changed, 12 insertions(+), 41 deletions(-) diff --git a/libgutenberg/DublinCore.py b/libgutenberg/DublinCore.py index 5b60415..1e6c882 100644 --- a/libgutenberg/DublinCore.py +++ b/libgutenberg/DublinCore.py @@ -555,19 +555,6 @@ def extract_wikipedia_url(text): return wikipedia_url(*checked) if checked else None -def format_wikipedia_url(url_or_text): - """Bare URL gets default prefix; text with a URL already in it is kept as-is.""" - if not url_or_text: - return '' - checked = check_wikipedia_url(url_or_text) - if not checked: - return url_or_text - url = wikipedia_url(*checked) - if url_or_text.strip() == url: - return f"{WIKIPEDIA_URL_PREFIX}{url}" - return url_or_text - - class GutenbergDublinCore(DublinCore): """ Parse from PG files. """ @@ -578,36 +565,23 @@ def __init__(self): self._project_gutenberg_id = None self.request_key = '' self.scan_urls = set() - self.wikipedia_urls = [] + self.wikipedia_urls = set() - def add_wikipedia_url(self, url_or_text): - url_or_text = (url_or_text or '').strip() - if not url_or_text: - return - checked = check_wikipedia_url(url_or_text) - if not checked: - error('%s is not a valid wikipedia url', url_or_text) - return - if any(check_wikipedia_url(text) == checked for text in self.wikipedia_urls): + def add_wikipedia_url(self, url): + url = (url or '').strip() + checked = check_wikipedia_url(url) + if not checked or url != wikipedia_url(*checked): + error('%s is not a valid wikipedia url', url) return - url = wikipedia_url(*checked) - if url_or_text == url: - text = f"{WIKIPEDIA_URL_PREFIX}{url}" - else: - text = url_or_text - self.wikipedia_urls.append(text) + self.wikipedia_urls.add(url) - def remove_wikipedia_url(self, url_or_text): - url_or_text = (url_or_text or '').strip() - if not url_or_text: - return - checked = check_wikipedia_url(url_or_text) - for i, text in enumerate(self.wikipedia_urls): - if text == url_or_text or (checked and check_wikipedia_url(text) == checked): - del self.wikipedia_urls[i] - return + def remove_wikipedia_url(self, url): + url = (url or '').strip() + checked = check_wikipedia_url(url) + if checked: + self.wikipedia_urls.discard(wikipedia_url(*checked)) @property @@ -877,9 +851,6 @@ def handle_wikipedia_urls(self, key, value): error('%s is not a valid wikipedia url', value) return for url in value: - if not check_wikipedia_url(url): - error('%s is not a valid wikipedia url', url) - continue self.add_wikipedia_url(url) From 7905c1ea7ef53c229835abe9eeeec62237408cc2 Mon Sep 17 00:00:00 2001 From: Zach Rosario Date: Thu, 2 Jul 2026 15:36:22 -0400 Subject: [PATCH 12/15] refactor(dublincoremapping): load bare wiki URLs, prefix MARC 500 only on save --- libgutenberg/DublinCoreMapping.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/libgutenberg/DublinCoreMapping.py b/libgutenberg/DublinCoreMapping.py index f250873..6fc0c02 100644 --- a/libgutenberg/DublinCoreMapping.py +++ b/libgutenberg/DublinCoreMapping.py @@ -23,7 +23,7 @@ from sqlalchemy.exc import DBAPIError from . import DublinCore -from .DublinCore import check_wikipedia_url +from .DublinCore import WIKIPEDIA_URL_PREFIX, check_wikipedia_url, wikipedia_url from . import GutenbergGlobals as gg from . import GutenbergDatabase from . import GutenbergFiles @@ -180,8 +180,9 @@ def parse260(s): elif marc.code == '260': (self.pubinfo.place, self.pubinfo.publisher, self.pubinfo.years) = parse260(marc.text) elif marc.code == '500': - if check_wikipedia_url(marc.text): - self.add_wikipedia_url(marc.text) + url = DublinCore.extract_wikipedia_url(marc.text) + if url: + self.wikipedia_urls.add(url) else: self.notes = marc.text elif marc.code == '505': @@ -514,24 +515,24 @@ def add_attribute(self, book, attr, nonfiling=0, marc=0): def _update_wikipedia_urls(self): - """Sync MARC 500 wiki rows to wikipedia_urls (matched by lang and title).""" + # Called from save(). Sync self.wikipedia_urls to MARC 500 wiki rows. if not self.book: return - wanted = {check_wikipedia_url(text): text - for text in self.wikipedia_urls - if check_wikipedia_url(text)} + remaining = set(self.wikipedia_urls) for att in list(self.book.attributes): if att.fk_attriblist != 500: continue - checked = check_wikipedia_url(att.text) - if not checked: + wiki_key = check_wikipedia_url(att.text) + if not wiki_key: continue - if checked in wanted: - del wanted[checked] + url = wikipedia_url(*wiki_key) + if url in remaining: + remaining.discard(url) else: self.book.attributes.remove(att) - for text in wanted.values(): - self.book.attributes.append(Attribute(fk_attriblist=500, text=text)) + for url in remaining: + self.book.attributes.append( + Attribute(fk_attriblist=500, text=f"{WIKIPEDIA_URL_PREFIX}{url}")) def delete(self): """ only delete the book! """ From b9409bbc7fe4aefe099a1d0f2b94f8ea3662ff2f Mon Sep 17 00:00:00 2001 From: Zach Rosario Date: Thu, 2 Jul 2026 15:36:32 -0400 Subject: [PATCH 13/15] test(dc): update wikipedia URL assertions for bare URL set --- libgutenberg/tests/test_dc.py | 63 ++++++++++++----------------------- 1 file changed, 21 insertions(+), 42 deletions(-) diff --git a/libgutenberg/tests/test_dc.py b/libgutenberg/tests/test_dc.py index 31a7f37..ff681bb 100755 --- a/libgutenberg/tests/test_dc.py +++ b/libgutenberg/tests/test_dc.py @@ -9,7 +9,7 @@ from libgutenberg.CommonOptions import Options from libgutenberg import GutenbergDatabase, GutenbergDatabaseDublinCore, DummyConnectionPool from libgutenberg import DBUtils, DublinCoreMapping -from libgutenberg.DublinCore import GutenbergDublinCore, WIKIPEDIA_URL_PREFIX, extract_wikipedia_url +from libgutenberg.DublinCore import GutenbergDublinCore from libgutenberg.Logger import debug, warning from libgutenberg.Models import Attribute, Book @@ -55,8 +55,7 @@ def test_orm_metadata(self): def test_wikipedia_urls_load(self): dc = DublinCoreMapping.DublinCoreObject() dc.load_from_database(2701) # Moby Dick - urls = [extract_wikipedia_url(text) for text in dc.wikipedia_urls] - self.assertIn("https://en.wikipedia.org/wiki/Moby-Dick", urls) + self.assertIn("https://en.wikipedia.org/wiki/Moby-Dick", dc.wikipedia_urls) def metadata_test1(self, dc): dc.load_from_database(self.ebook) @@ -306,8 +305,8 @@ def test_load_from_json(self): self.assertEqual(len(dc.scan_urls), 2) self.assertEqual( dc.wikipedia_urls, - [f"{WIKIPEDIA_URL_PREFIX}https://en.wikipedia.org/wiki/Test_Book", - f"{WIKIPEDIA_URL_PREFIX}https://en.wikipedia.org/wiki/Another_Book"]) + {"https://en.wikipedia.org/wiki/Test_Book", + "https://en.wikipedia.org/wiki/Another_Book"}) self.assertEqual(dc.pubinfo.first_year, '1920') self.assertEqual(dc.credit, 'Roger Frank and Sue Clark.') dc.add_credit('Sue Frank and Roger Clark.\n') @@ -325,8 +324,8 @@ def test_load_from_json(self): self.assertEqual(set_subtitle, dc.subtitle) self.assertEqual( dc.wikipedia_urls, - [f"{WIKIPEDIA_URL_PREFIX}https://en.wikipedia.org/wiki/Test_Book", - f"{WIKIPEDIA_URL_PREFIX}https://en.wikipedia.org/wiki/Another_Book"]) + {"https://en.wikipedia.org/wiki/Test_Book", + "https://en.wikipedia.org/wiki/Another_Book"}) marc260 = dc.session.query(Attribute).filter_by(book=dc.book, fk_attriblist=260).first().text self.assertTrue('1920' in marc260) self.assertEqual( @@ -359,35 +358,27 @@ def test_load_from_json(self): def test_wikipedia_url_format(self): bare = "https://en.wikipedia.org/wiki/Moby-Dick" - prefixed = f"{WIKIPEDIA_URL_PREFIX}{bare}" - custom = f"See also: {bare}" dc = GutenbergDublinCore() dc.add_wikipedia_url(bare) - self.assertEqual(dc.wikipedia_urls, [prefixed]) + self.assertEqual(dc.wikipedia_urls, {bare}) - dc.add_wikipedia_url(prefixed) - self.assertEqual(dc.wikipedia_urls, [prefixed]) + dc.add_wikipedia_url(bare) + self.assertEqual(dc.wikipedia_urls, {bare}) - dc2 = GutenbergDublinCore() - dc2.add_wikipedia_url(custom) - self.assertEqual(dc2.wikipedia_urls, [custom]) + dc.add_wikipedia_url(f" {bare} ") + self.assertEqual(dc.wikipedia_urls, {bare}) - dc.add_wikipedia_url(bare) dc.remove_wikipedia_url(bare) - self.assertEqual(dc.wikipedia_urls, []) + self.assertEqual(dc.wikipedia_urls, set()) dc.add_wikipedia_url(bare) - dc.remove_wikipedia_url(prefixed) - self.assertEqual(dc.wikipedia_urls, []) - - dc.add_wikipedia_url(f" {bare} ") - self.assertEqual(dc.wikipedia_urls, [prefixed]) dc.remove_wikipedia_url(f" {bare} ") - self.assertEqual(dc.wikipedia_urls, []) + self.assertEqual(dc.wikipedia_urls, set()) - dc.add_wikipedia_url(f"See also: {bare} ") - self.assertEqual(dc.wikipedia_urls, [f"See also: {bare}"]) + dc2 = GutenbergDublinCore() + dc2.add_wikipedia_url(f"See also: {bare}") + self.assertEqual(dc2.wikipedia_urls, set()) def test_wikipedia_urls_add_and_remove(self): dc = DublinCoreMapping.DublinCoreObject() @@ -399,20 +390,15 @@ def test_wikipedia_urls_add_and_remove(self): url = "https://en.wikipedia.org/wiki/Moby-Dick" url2 = "https://en.wikipedia.org/wiki/Test_Book" - url3 = "https://en.wikipedia.org/wiki/Herman_Melville" - text1 = f"{WIKIPEDIA_URL_PREFIX}{url}" - text2 = f"{WIKIPEDIA_URL_PREFIX}{url2}" - custom_text = f"Wikipedia page about this author: {url3}" dc.add_wikipedia_url(url) - dc.add_wikipedia_url(text2) - dc.add_wikipedia_url(custom_text) + dc.add_wikipedia_url(url2) dc.save(updatemode=0) dc2 = DublinCoreMapping.DublinCoreObject() dc2.load_from_database(ebook) - self.assertEqual(dc2.wikipedia_urls, [text1, text2, custom_text]) + self.assertEqual(dc2.wikipedia_urls, {url, url2}) dc2.remove_wikipedia_url(url) dc2.save(updatemode=1) @@ -420,21 +406,14 @@ def test_wikipedia_urls_add_and_remove(self): dc3 = DublinCoreMapping.DublinCoreObject() dc3.load_from_database(ebook) - self.assertEqual(dc3.wikipedia_urls, [text2, custom_text]) + self.assertEqual(dc3.wikipedia_urls, {url2}) - dc3.remove_wikipedia_url(custom_text) + dc3.remove_wikipedia_url(url2) dc3.save(updatemode=1) dc4 = DublinCoreMapping.DublinCoreObject() dc4.load_from_database(ebook) - self.assertEqual(dc4.wikipedia_urls, [text2]) - - dc4.remove_wikipedia_url(text2) - dc4.save(updatemode=1) - - dc5 = DublinCoreMapping.DublinCoreObject() - dc5.load_from_database(ebook) - self.assertEqual(dc5.wikipedia_urls, []) + self.assertEqual(dc4.wikipedia_urls, set()) def tearDown(self): session = DBUtils.check_session(None) From 7312801b392ab3860e17d2306129c24290cd5b54 Mon Sep 17 00:00:00 2001 From: Zach Rosario Date: Thu, 2 Jul 2026 15:45:03 -0400 Subject: [PATCH 14/15] test(dc): get rid of redundant wikipedia URL tests --- libgutenberg/tests/test_dc.py | 66 +++++++---------------------------- 1 file changed, 13 insertions(+), 53 deletions(-) diff --git a/libgutenberg/tests/test_dc.py b/libgutenberg/tests/test_dc.py index ff681bb..726665c 100755 --- a/libgutenberg/tests/test_dc.py +++ b/libgutenberg/tests/test_dc.py @@ -9,7 +9,6 @@ from libgutenberg.CommonOptions import Options from libgutenberg import GutenbergDatabase, GutenbergDatabaseDublinCore, DummyConnectionPool from libgutenberg import DBUtils, DublinCoreMapping -from libgutenberg.DublinCore import GutenbergDublinCore from libgutenberg.Logger import debug, warning from libgutenberg.Models import Attribute, Book @@ -356,64 +355,25 @@ def test_load_from_json(self): dc.session.flush() self.assertFalse(DBUtils.ebook_exists(99999)) - def test_wikipedia_url_format(self): - bare = "https://en.wikipedia.org/wiki/Moby-Dick" - - dc = GutenbergDublinCore() - dc.add_wikipedia_url(bare) - self.assertEqual(dc.wikipedia_urls, {bare}) - - dc.add_wikipedia_url(bare) - self.assertEqual(dc.wikipedia_urls, {bare}) - - dc.add_wikipedia_url(f" {bare} ") - self.assertEqual(dc.wikipedia_urls, {bare}) - - dc.remove_wikipedia_url(bare) - self.assertEqual(dc.wikipedia_urls, set()) - - dc.add_wikipedia_url(bare) - dc.remove_wikipedia_url(f" {bare} ") - self.assertEqual(dc.wikipedia_urls, set()) - - dc2 = GutenbergDublinCore() - dc2.add_wikipedia_url(f"See also: {bare}") - self.assertEqual(dc2.wikipedia_urls, set()) - def test_wikipedia_urls_add_and_remove(self): dc = DublinCoreMapping.DublinCoreObject() - - ebook = 99998 # fake test id - + ebook = 99998 dc.load_or_create_book(ebook) dc.rights = 'Public Domain in the USA.' - - url = "https://en.wikipedia.org/wiki/Moby-Dick" - url2 = "https://en.wikipedia.org/wiki/Test_Book" - - dc.add_wikipedia_url(url) - dc.add_wikipedia_url(url2) + urls = ["https://en.wikipedia.org/wiki/Moby-Dick", + "https://en.wikipedia.org/wiki/Test_Book"] + dc.add_wikipedia_url(urls[0]) + dc.add_wikipedia_url(f"See also: {urls[0]}") # rejected + dc.add_wikipedia_url(urls[1]) dc.save(updatemode=0) - dc2 = DublinCoreMapping.DublinCoreObject() - dc2.load_from_database(ebook) - - self.assertEqual(dc2.wikipedia_urls, {url, url2}) - - dc2.remove_wikipedia_url(url) - dc2.save(updatemode=1) - - dc3 = DublinCoreMapping.DublinCoreObject() - dc3.load_from_database(ebook) - - self.assertEqual(dc3.wikipedia_urls, {url2}) - - dc3.remove_wikipedia_url(url2) - dc3.save(updatemode=1) - - dc4 = DublinCoreMapping.DublinCoreObject() - dc4.load_from_database(ebook) - self.assertEqual(dc4.wikipedia_urls, set()) + remaining = list(urls) + while remaining: + dc = DublinCoreMapping.DublinCoreObject() + dc.load_from_database(ebook) + self.assertEqual(dc.wikipedia_urls, set(remaining)) + dc.remove_wikipedia_url(remaining.pop(0)) + dc.save(updatemode=1) def tearDown(self): session = DBUtils.check_session(None) From d31dc729fecc9264ffa27d2e60e8e25c245f8dba Mon Sep 17 00:00:00 2001 From: Zach Rosario Date: Thu, 2 Jul 2026 15:48:38 -0400 Subject: [PATCH 15/15] docs(changes): remove dropped wikipedia format test from release notes --- CHANGES | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGES b/CHANGES index 80f6511..02bbd7a 100644 --- a/CHANGES +++ b/CHANGES @@ -4,7 +4,6 @@ CHANGES - Two functions `add_book_wikipedia_url` and `remove_book_wikipedia_url` have been added to the DublinCoreObject. - Wikipedia url dispatcher testing added to `test_load_from_json`. - Add/remove tesing can be found in `test_book_wikipedia_urls_add_and_remove` - - Plus `test_book_wikipedia_url_format` which covers striping white space from client input. 0.11.0 (June *, 2026) - Now using SQLAlchemy 2.0