my.hackernews: use hpikit, kloggy and koncurrent

pyproject.toml

@@ -457,6 +457,7 @@ commands = [
         "my.pdfs",
         "my.github.ghexport",
         "my.hackernews.harmonic",
+        "my.hackernews.materialistic",
         "my.reddit.rexport",
         "my.twitter.archive",
     ],
@@ -512,6 +513,7 @@ hpi_modules = [
     "my.google.maps.android",
     "my.google.takeout.parser",
     "my.hackernews.harmonic",
+    "my.hackernews.materialistic",
     "my.hypothesis",
     "my.instapaper",
     "my.ip.all",

src/my/hackernews/harmonic.py

@@ -4,7 +4,7 @@
 
 from __future__ import annotations
 
-REQUIRES = ['lxml', 'orjson', 'kloggy', 'koncurrent']
+REQUIRES = ['lxml', 'hpikit', 'msgspec', 'kloggy', 'koncurrent']
 
 from abc import abstractmethod
 from collections.abc import Iterator, Sequence
@@ -14,20 +14,16 @@
 from typing import Any, Protocol, TypedDict, cast
 
 import koncurrent
-import orjson
+import msgspec
+from hpikit.datetime import AwareDateTime, aware
+from hpikit.json import loads
+from hpikit.paths import Paths, get_files
 from kloggy import get_logger
 from kloggy.progress import progress
 from lxml import etree
 from more_itertools import one, only
 
-from my.core import (
-    Paths,
-    Res,
-    Stats,
-    datetime_aware,
-    get_files,
-    stat,
-)
+from my.core import Res, Stats, stat
 from my.core.common import unique_everseen
 
 from .common import SavedBase, hackernews_link
@@ -71,13 +67,13 @@ class Cached(TypedDict):
 class Saved(SavedBase):
     raw: Cached
 
-    # datetime_aware evidence: Harmonic's cached Algolia records pair created_at_i with created_at ending in Z.
+    # Timestamp evidence: Harmonic's cached Algolia records pair created_at_i with created_at ending in Z.
     # Algolia documents created_at_i filters as seconds.
     # https://hn.algolia.com/api
     @property
-    def when(self) -> datetime_aware:
+    def when(self) -> AwareDateTime:
         ts = self.raw['created_at_i']
-        return datetime.fromtimestamp(ts, tz=UTC)
+        return aware(datetime.fromtimestamp(ts, tz=UTC))
 
     @property
     def uid(self) -> str:
@@ -97,7 +93,7 @@ def hackernews_link(self) -> str:
 
     def __hash__(self) -> int:
         # meh. but seems like the easiest and fastest way to hash a dict?
-        return hash(orjson.dumps(self.raw))
+        return hash(msgspec.json.encode(self.raw))
 
 
 _PREFIX = 'com.simon.harmonichackernews.KEY_SHARED_PREFERENCES'
@@ -116,7 +112,7 @@ def _process_path(path: Path) -> tuple[set[str], list[tuple[str, Cached | None]]
         # The corresponding CACHED_STORY{id} XML entries can therefore be absent.
         if res is None:
             continue
-        j = orjson.loads(res.text)
+        j = loads(res.text)
         cached[sid] = j
 
     res = one(cast(list[Any], tr.xpath(f'//*[@name="{_PREFIX}_BOOKMARKS"]')))

src/my/hackernews/materialistic.py

@@ -2,22 +2,28 @@
 [[https://play.google.com/store/apps/details?id=io.github.hidroh.materialistic][Materialistic]] app for Hackernews
 """
 
+REQUIRES = ['hpikit', 'kloggy', 'koncurrent']
+
 from collections.abc import Iterator, Sequence
 from datetime import UTC, datetime
 from pathlib import Path
 from typing import Any, NamedTuple
 
+import koncurrent
+from hpikit.datetime import AwareDateTime, aware
+from hpikit.paths import get_files
+from kloggy import get_logger
+from kloggy.progress import progress
 from more_itertools import unique_everseen
 
-from my.core import datetime_aware, get_files, make_logger
 from my.core.sqlite import sqlite_connection
 
 from .common import hackernews_link
 
 # todo migrate config to my.hackernews.materialistic
 from my.config import materialistic as config  # isort: skip
 
-logger = make_logger(__name__)
+logger = get_logger(__name__)
 
 
 def inputs() -> Sequence[Path]:
@@ -30,16 +36,16 @@ def inputs() -> Sequence[Path]:
 class Saved(NamedTuple):
     row: Row
 
-    # datetime_aware evidence: the pinned upstream SavedStory.from writes System.currentTimeMillis() to time.
+    # Timestamp evidence: the pinned upstream SavedStory.from writes System.currentTimeMillis() to time.
     # https://github.com/hidroh/materialistic/blob/b631d5111b7487d2328f463bd95e8507c74c3566/app/src/main/java/io/github/hidroh/materialistic/data/MaterialisticDatabase.java#L210-L230
     # https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/System.html#currentTimeMillis()
     @property
-    def when(self) -> datetime_aware:
+    def when(self) -> AwareDateTime:
         """
         NOTE: this is the time the item was saved, not the Hacker News item's creation time.
         """
         ts = int(self.row['time']) / 1000
-        return datetime.fromtimestamp(ts, tz=UTC)
+        return aware(datetime.fromtimestamp(ts, tz=UTC))
 
     @property
     def uid(self) -> str:
@@ -58,14 +64,20 @@ def hackernews_link(self) -> str:
         return hackernews_link(self.uid)
 
 
+def _process_path(path: Path) -> list[Row]:
+    with sqlite_connection(path, immutable=True, row_factory='dict') as conn:
+        return list(conn.execute('SELECT * FROM saved ORDER BY time'))
+
+
 def _all_raw() -> Iterator[Row]:
     paths = inputs()
-    total = len(paths)
-    width = len(str(total))
-    for idx, path in enumerate(paths):
-        logger.info(f'processing [{idx:>{width}}/{total:>{width}}] {path}')
-        with sqlite_connection(path, immutable=True, row_factory='dict') as conn:
-            yield from conn.execute('SELECT * FROM saved ORDER BY time')
+    with koncurrent.map_results(_process_path, paths) as results:
+        files = progress(paths, logger=logger, unit='files')
+        for path, result in zip(files, results, strict=True):
+            if isinstance(result, Exception):
+                result.add_note(f'^ while processing {path}')
+                raise result
+            yield from result
 
 
 def raw() -> Iterator[Row]:

src/my/tests/hackernews/dogsheep.py

@@ -0,0 +1,49 @@
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import assert_type, get_type_hints
+
+import pytest
+
+from my.core.datetime import AwareDateTime, aware
+from my.core.sqlite import sqlite_connection
+from my.hackernews import dogsheep
+
+
+def test_dogsheep_items(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    db = tmp_path / 'hackernews.sqlite'
+    with sqlite_connection(db) as conn:
+        conn.execute('CREATE TABLE items (id TEXT, type TEXT, time INTEGER, title TEXT, text TEXT, url TEXT)')
+        conn.executemany(
+            'INSERT INTO items VALUES (?, ?, ?, ?, ?, ?)',
+            [
+                ('2', 'story', 1704067200, 'A story', None, 'https://example.com/story'),
+                ('1', 'comment', 0, None, '<p>A comment</p>', None),
+            ],
+        )
+
+    monkeypatch.setattr(dogsheep.config, 'export_path', db)
+    results = list(dogsheep.items())
+    assert results == [
+        dogsheep.Item(
+            id='1',
+            type='comment',
+            created=aware(datetime(1970, 1, 1, tzinfo=UTC)),
+            title=None,
+            text_html='<p>A comment</p>',
+            url=None,
+        ),
+        dogsheep.Item(
+            id='2',
+            type='story',
+            created=aware(datetime(2024, 1, 1, tzinfo=UTC)),
+            title='A story',
+            text_html=None,
+            url='https://example.com/story',
+        ),
+    ]
+    for item in results:
+        assert not isinstance(item, Exception)
+        assert_type(item.created, AwareDateTime)
+        assert item.created.tzinfo is UTC
+        assert item.permalink == f'https://news.ycombinator.com/item?id={item.id}'
+    assert get_type_hints(dogsheep.Item)['created'] is datetime

src/my/tests/hackernews.py → src/my/tests/hackernews/harmonic.py

@@ -1,52 +1,9 @@
 from datetime import UTC, datetime
 from pathlib import Path
-from typing import assert_type, get_type_hints
+from typing import assert_type
 
 import pytest
-
-from my.core.datetime import AwareDateTime, aware
-from my.core.sqlite import sqlite_connection
-from my.hackernews import dogsheep
-
-
-def test_dogsheep_items(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
-    db = tmp_path / 'hackernews.sqlite'
-    with sqlite_connection(db) as conn:
-        conn.execute('CREATE TABLE items (id TEXT, type TEXT, time INTEGER, title TEXT, text TEXT, url TEXT)')
-        conn.executemany(
-            'INSERT INTO items VALUES (?, ?, ?, ?, ?, ?)',
-            [
-                ('2', 'story', 1704067200, 'A story', None, 'https://example.com/story'),
-                ('1', 'comment', 0, None, '<p>A comment</p>', None),
-            ],
-        )
-
-    monkeypatch.setattr(dogsheep.config, 'export_path', db)
-    results = list(dogsheep.items())
-    assert results == [
-        dogsheep.Item(
-            id='1',
-            type='comment',
-            created=aware(datetime(1970, 1, 1, tzinfo=UTC)),
-            title=None,
-            text_html='<p>A comment</p>',
-            url=None,
-        ),
-        dogsheep.Item(
-            id='2',
-            type='story',
-            created=aware(datetime(2024, 1, 1, tzinfo=UTC)),
-            title='A story',
-            text_html=None,
-            url='https://example.com/story',
-        ),
-    ]
-    for item in results:
-        assert not isinstance(item, Exception)
-        assert_type(item.created, AwareDateTime)
-        assert item.created.tzinfo is UTC
-        assert item.permalink == f'https://news.ycombinator.com/item?id={item.id}'
-    assert get_type_hints(dogsheep.Item)['created'] is datetime
+from hpikit.datetime import AwareDateTime
 
 
 @pytest.mark.parametrize('with_cached_story', [False, True])
@@ -95,6 +52,7 @@ def test_harmonic_saved_yields_missing_cache_errors(
         assert first.uid == '1'
         assert first.hackernews_link == 'https://news.ycombinator.com/item?id=1'
         assert first.when == datetime(1970, 1, 1, tzinfo=UTC)
+        assert_type(first.when, AwareDateTime)
         assert first.title == 'A story'
         assert first.url == 'https://example.com/story'
         assert isinstance(results[0], RuntimeError)
@@ -119,7 +77,6 @@ def test_harmonic_saved_continues_after_bad_file(
     *,
     malformed: str,
 ) -> None:
-    import orjson
     from lxml import etree
 
     from my.hackernews import harmonic
@@ -150,7 +107,7 @@ def test_harmonic_saved_continues_after_bad_file(
     assert [one.uid, two.uid] == ['1', '2']
     assert isinstance(error, Exception)
     if malformed == 'json':
-        assert isinstance(error, orjson.JSONDecodeError)
+        assert isinstance(error, ValueError)
     else:
         assert isinstance(error, etree.XMLSyntaxError)
     assert str(broken) in '\n'.join(error.__notes__)

src/my/tests/hackernews/materialistic.py

@@ -0,0 +1,68 @@
+from datetime import UTC, datetime
+from pathlib import Path
+from sqlite3 import OperationalError
+from typing import assert_type
+
+import pytest
+from hpikit.datetime import AwareDateTime
+
+from my.core.sqlite import sqlite_connection
+
+
+def test_materialistic_saves(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    from my.hackernews import materialistic
+
+    first = tmp_path / 'z.db'
+    second = tmp_path / 'a.db'
+    for path, rows in [
+        (
+            first,
+            [
+                ('1', 2000, 'Original title', 'https://example.com/one'),
+                ('2', 0, 'Earlier save', 'https://example.com/two'),
+            ],
+        ),
+        (
+            second,
+            [
+                ('1', 1000, 'Updated title', 'https://example.com/updated'),
+                ('3', 500, 'New save', 'https://example.com/three'),
+            ],
+        ),
+    ]:
+        with sqlite_connection(path) as conn:
+            conn.execute('CREATE TABLE saved (itemid TEXT, time INTEGER, title TEXT, url TEXT)')
+            conn.executemany('INSERT INTO saved VALUES (?, ?, ?, ?)', rows)
+
+    # Explicit input order wins over filenames, with timestamp ordering only within each database.
+    monkeypatch.setattr(materialistic.config, 'export_path', [first, second], raising=False)
+    results = list(materialistic.saves())
+    assert [item.uid for item in results] == ['2', '1', '3']
+    assert [item.title for item in results] == ['Earlier save', 'Original title', 'New save']
+    assert [item.url for item in results] == [
+        'https://example.com/two',
+        'https://example.com/one',
+        'https://example.com/three',
+    ]
+    assert [item.when for item in results] == [
+        datetime(1970, 1, 1, tzinfo=UTC),
+        datetime(1970, 1, 1, second=2, tzinfo=UTC),
+        datetime(1970, 1, 1, microsecond=500_000, tzinfo=UTC),
+    ]
+    for item in results:
+        assert_type(item.when, AwareDateTime)
+        assert item.when.tzinfo is UTC
+        assert item.hackernews_link == f'https://news.ycombinator.com/item?id={item.uid}'
+
+
+def test_materialistic_saves_raises_database_error(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    from my.hackernews import materialistic
+
+    broken = tmp_path / 'broken.db'
+    with sqlite_connection(broken) as conn:
+        conn.execute('CREATE TABLE unrelated (id TEXT)')
+    monkeypatch.setattr(materialistic.config, 'export_path', broken, raising=False)
+
+    with pytest.raises(OperationalError, match='no such table: saved') as raised:
+        list(materialistic.saves())
+    assert f'^ while processing {broken}' in raised.value.__notes__