From b865bcdf3076580eb6c6e35010ee211fae66caed Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sat, 31 Oct 2026 19:22:23 +0000 Subject: [PATCH] my.hackernews: use `hpikit`, `kloggy` and `koncurrent` Use `hpikit` for paths and checked datetimes in Harmonic and Materialistic. Move Harmonic JSON parsing to `hpikit.json` and hashing to `msgspec`. Replace Materialistic's manual progress logging with `kloggy.progress` and read snapshots through `koncurrent.map_results`. Split `my.tests.hackernews` by provider and declare Materialistic's dependencies in the full test and typecheck environments. --- pyproject.toml | 2 + src/my/hackernews/harmonic.py | 26 +++---- src/my/hackernews/materialistic.py | 34 +++++++--- src/my/tests/hackernews/dogsheep.py | 49 +++++++++++++ .../{hackernews.py => hackernews/harmonic.py} | 51 ++------------ src/my/tests/hackernews/materialistic.py | 68 +++++++++++++++++++ 6 files changed, 157 insertions(+), 73 deletions(-) create mode 100644 src/my/tests/hackernews/dogsheep.py rename src/my/tests/{hackernews.py => hackernews/harmonic.py} (70%) create mode 100644 src/my/tests/hackernews/materialistic.py diff --git a/pyproject.toml b/pyproject.toml index 71d6ea2e..2d81bc67 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -457,6 +457,7 @@ commands = [ "my.pdfs", "my.github.ghexport", "my.hackernews.harmonic", + "my.hackernews.materialistic", "my.reddit.rexport", "my.twitter.archive", ], @@ -512,6 +513,7 @@ hpi_modules = [ "my.google.maps.android", "my.google.takeout.parser", "my.hackernews.harmonic", + "my.hackernews.materialistic", "my.hypothesis", "my.instapaper", "my.ip.all", diff --git a/src/my/hackernews/harmonic.py b/src/my/hackernews/harmonic.py index 4b623e56..461fcd0d 100644 --- a/src/my/hackernews/harmonic.py +++ b/src/my/hackernews/harmonic.py @@ -4,7 +4,7 @@ from __future__ import annotations -REQUIRES = ['lxml', 'orjson', 'kloggy', 'koncurrent'] +REQUIRES = ['lxml', 'hpikit', 'msgspec', 'kloggy', 'koncurrent'] from abc import abstractmethod from collections.abc import Iterator, Sequence @@ -14,20 +14,16 @@ from typing import Any, Protocol, TypedDict, cast import koncurrent -import orjson +import msgspec +from hpikit.datetime import AwareDateTime, aware +from hpikit.json import loads +from hpikit.paths import Paths, get_files from kloggy import get_logger from kloggy.progress import progress from lxml import etree from more_itertools import one, only -from my.core import ( - Paths, - Res, - Stats, - datetime_aware, - get_files, - stat, -) +from my.core import Res, Stats, stat from my.core.common import unique_everseen from .common import SavedBase, hackernews_link @@ -71,13 +67,13 @@ class Cached(TypedDict): class Saved(SavedBase): raw: Cached - # datetime_aware evidence: Harmonic's cached Algolia records pair created_at_i with created_at ending in Z. + # Timestamp evidence: Harmonic's cached Algolia records pair created_at_i with created_at ending in Z. # Algolia documents created_at_i filters as seconds. # https://hn.algolia.com/api @property - def when(self) -> datetime_aware: + def when(self) -> AwareDateTime: ts = self.raw['created_at_i'] - return datetime.fromtimestamp(ts, tz=UTC) + return aware(datetime.fromtimestamp(ts, tz=UTC)) @property def uid(self) -> str: @@ -97,7 +93,7 @@ def hackernews_link(self) -> str: def __hash__(self) -> int: # meh. but seems like the easiest and fastest way to hash a dict? - return hash(orjson.dumps(self.raw)) + return hash(msgspec.json.encode(self.raw)) _PREFIX = 'com.simon.harmonichackernews.KEY_SHARED_PREFERENCES' @@ -116,7 +112,7 @@ def _process_path(path: Path) -> tuple[set[str], list[tuple[str, Cached | None]] # The corresponding CACHED_STORY{id} XML entries can therefore be absent. if res is None: continue - j = orjson.loads(res.text) + j = loads(res.text) cached[sid] = j res = one(cast(list[Any], tr.xpath(f'//*[@name="{_PREFIX}_BOOKMARKS"]'))) diff --git a/src/my/hackernews/materialistic.py b/src/my/hackernews/materialistic.py index 2aa4b9d5..1958ebb9 100644 --- a/src/my/hackernews/materialistic.py +++ b/src/my/hackernews/materialistic.py @@ -2,14 +2,20 @@ [[https://play.google.com/store/apps/details?id=io.github.hidroh.materialistic][Materialistic]] app for Hackernews """ +REQUIRES = ['hpikit', 'kloggy', 'koncurrent'] + from collections.abc import Iterator, Sequence from datetime import UTC, datetime from pathlib import Path from typing import Any, NamedTuple +import koncurrent +from hpikit.datetime import AwareDateTime, aware +from hpikit.paths import get_files +from kloggy import get_logger +from kloggy.progress import progress from more_itertools import unique_everseen -from my.core import datetime_aware, get_files, make_logger from my.core.sqlite import sqlite_connection from .common import hackernews_link @@ -17,7 +23,7 @@ # todo migrate config to my.hackernews.materialistic from my.config import materialistic as config # isort: skip -logger = make_logger(__name__) +logger = get_logger(__name__) def inputs() -> Sequence[Path]: @@ -30,16 +36,16 @@ def inputs() -> Sequence[Path]: class Saved(NamedTuple): row: Row - # datetime_aware evidence: the pinned upstream SavedStory.from writes System.currentTimeMillis() to time. + # Timestamp evidence: the pinned upstream SavedStory.from writes System.currentTimeMillis() to time. # https://github.com/hidroh/materialistic/blob/b631d5111b7487d2328f463bd95e8507c74c3566/app/src/main/java/io/github/hidroh/materialistic/data/MaterialisticDatabase.java#L210-L230 # https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/System.html#currentTimeMillis() @property - def when(self) -> datetime_aware: + def when(self) -> AwareDateTime: """ NOTE: this is the time the item was saved, not the Hacker News item's creation time. """ ts = int(self.row['time']) / 1000 - return datetime.fromtimestamp(ts, tz=UTC) + return aware(datetime.fromtimestamp(ts, tz=UTC)) @property def uid(self) -> str: @@ -58,14 +64,20 @@ def hackernews_link(self) -> str: return hackernews_link(self.uid) +def _process_path(path: Path) -> list[Row]: + with sqlite_connection(path, immutable=True, row_factory='dict') as conn: + return list(conn.execute('SELECT * FROM saved ORDER BY time')) + + def _all_raw() -> Iterator[Row]: paths = inputs() - total = len(paths) - width = len(str(total)) - for idx, path in enumerate(paths): - logger.info(f'processing [{idx:>{width}}/{total:>{width}}] {path}') - with sqlite_connection(path, immutable=True, row_factory='dict') as conn: - yield from conn.execute('SELECT * FROM saved ORDER BY time') + with koncurrent.map_results(_process_path, paths) as results: + files = progress(paths, logger=logger, unit='files') + for path, result in zip(files, results, strict=True): + if isinstance(result, Exception): + result.add_note(f'^ while processing {path}') + raise result + yield from result def raw() -> Iterator[Row]: diff --git a/src/my/tests/hackernews/dogsheep.py b/src/my/tests/hackernews/dogsheep.py new file mode 100644 index 00000000..2990e13d --- /dev/null +++ b/src/my/tests/hackernews/dogsheep.py @@ -0,0 +1,49 @@ +from datetime import UTC, datetime +from pathlib import Path +from typing import assert_type, get_type_hints + +import pytest + +from my.core.datetime import AwareDateTime, aware +from my.core.sqlite import sqlite_connection +from my.hackernews import dogsheep + + +def test_dogsheep_items(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + db = tmp_path / 'hackernews.sqlite' + with sqlite_connection(db) as conn: + conn.execute('CREATE TABLE items (id TEXT, type TEXT, time INTEGER, title TEXT, text TEXT, url TEXT)') + conn.executemany( + 'INSERT INTO items VALUES (?, ?, ?, ?, ?, ?)', + [ + ('2', 'story', 1704067200, 'A story', None, 'https://example.com/story'), + ('1', 'comment', 0, None, '

A comment

', None), + ], + ) + + monkeypatch.setattr(dogsheep.config, 'export_path', db) + results = list(dogsheep.items()) + assert results == [ + dogsheep.Item( + id='1', + type='comment', + created=aware(datetime(1970, 1, 1, tzinfo=UTC)), + title=None, + text_html='

A comment

', + url=None, + ), + dogsheep.Item( + id='2', + type='story', + created=aware(datetime(2024, 1, 1, tzinfo=UTC)), + title='A story', + text_html=None, + url='https://example.com/story', + ), + ] + for item in results: + assert not isinstance(item, Exception) + assert_type(item.created, AwareDateTime) + assert item.created.tzinfo is UTC + assert item.permalink == f'https://news.ycombinator.com/item?id={item.id}' + assert get_type_hints(dogsheep.Item)['created'] is datetime diff --git a/src/my/tests/hackernews.py b/src/my/tests/hackernews/harmonic.py similarity index 70% rename from src/my/tests/hackernews.py rename to src/my/tests/hackernews/harmonic.py index ee2cae84..88711965 100644 --- a/src/my/tests/hackernews.py +++ b/src/my/tests/hackernews/harmonic.py @@ -1,52 +1,9 @@ from datetime import UTC, datetime from pathlib import Path -from typing import assert_type, get_type_hints +from typing import assert_type import pytest - -from my.core.datetime import AwareDateTime, aware -from my.core.sqlite import sqlite_connection -from my.hackernews import dogsheep - - -def test_dogsheep_items(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - db = tmp_path / 'hackernews.sqlite' - with sqlite_connection(db) as conn: - conn.execute('CREATE TABLE items (id TEXT, type TEXT, time INTEGER, title TEXT, text TEXT, url TEXT)') - conn.executemany( - 'INSERT INTO items VALUES (?, ?, ?, ?, ?, ?)', - [ - ('2', 'story', 1704067200, 'A story', None, 'https://example.com/story'), - ('1', 'comment', 0, None, '

A comment

', None), - ], - ) - - monkeypatch.setattr(dogsheep.config, 'export_path', db) - results = list(dogsheep.items()) - assert results == [ - dogsheep.Item( - id='1', - type='comment', - created=aware(datetime(1970, 1, 1, tzinfo=UTC)), - title=None, - text_html='

A comment

', - url=None, - ), - dogsheep.Item( - id='2', - type='story', - created=aware(datetime(2024, 1, 1, tzinfo=UTC)), - title='A story', - text_html=None, - url='https://example.com/story', - ), - ] - for item in results: - assert not isinstance(item, Exception) - assert_type(item.created, AwareDateTime) - assert item.created.tzinfo is UTC - assert item.permalink == f'https://news.ycombinator.com/item?id={item.id}' - assert get_type_hints(dogsheep.Item)['created'] is datetime +from hpikit.datetime import AwareDateTime @pytest.mark.parametrize('with_cached_story', [False, True]) @@ -95,6 +52,7 @@ def test_harmonic_saved_yields_missing_cache_errors( assert first.uid == '1' assert first.hackernews_link == 'https://news.ycombinator.com/item?id=1' assert first.when == datetime(1970, 1, 1, tzinfo=UTC) + assert_type(first.when, AwareDateTime) assert first.title == 'A story' assert first.url == 'https://example.com/story' assert isinstance(results[0], RuntimeError) @@ -119,7 +77,6 @@ def test_harmonic_saved_continues_after_bad_file( *, malformed: str, ) -> None: - import orjson from lxml import etree from my.hackernews import harmonic @@ -150,7 +107,7 @@ def test_harmonic_saved_continues_after_bad_file( assert [one.uid, two.uid] == ['1', '2'] assert isinstance(error, Exception) if malformed == 'json': - assert isinstance(error, orjson.JSONDecodeError) + assert isinstance(error, ValueError) else: assert isinstance(error, etree.XMLSyntaxError) assert str(broken) in '\n'.join(error.__notes__) diff --git a/src/my/tests/hackernews/materialistic.py b/src/my/tests/hackernews/materialistic.py new file mode 100644 index 00000000..d8032c70 --- /dev/null +++ b/src/my/tests/hackernews/materialistic.py @@ -0,0 +1,68 @@ +from datetime import UTC, datetime +from pathlib import Path +from sqlite3 import OperationalError +from typing import assert_type + +import pytest +from hpikit.datetime import AwareDateTime + +from my.core.sqlite import sqlite_connection + + +def test_materialistic_saves(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + from my.hackernews import materialistic + + first = tmp_path / 'z.db' + second = tmp_path / 'a.db' + for path, rows in [ + ( + first, + [ + ('1', 2000, 'Original title', 'https://example.com/one'), + ('2', 0, 'Earlier save', 'https://example.com/two'), + ], + ), + ( + second, + [ + ('1', 1000, 'Updated title', 'https://example.com/updated'), + ('3', 500, 'New save', 'https://example.com/three'), + ], + ), + ]: + with sqlite_connection(path) as conn: + conn.execute('CREATE TABLE saved (itemid TEXT, time INTEGER, title TEXT, url TEXT)') + conn.executemany('INSERT INTO saved VALUES (?, ?, ?, ?)', rows) + + # Explicit input order wins over filenames, with timestamp ordering only within each database. + monkeypatch.setattr(materialistic.config, 'export_path', [first, second], raising=False) + results = list(materialistic.saves()) + assert [item.uid for item in results] == ['2', '1', '3'] + assert [item.title for item in results] == ['Earlier save', 'Original title', 'New save'] + assert [item.url for item in results] == [ + 'https://example.com/two', + 'https://example.com/one', + 'https://example.com/three', + ] + assert [item.when for item in results] == [ + datetime(1970, 1, 1, tzinfo=UTC), + datetime(1970, 1, 1, second=2, tzinfo=UTC), + datetime(1970, 1, 1, microsecond=500_000, tzinfo=UTC), + ] + for item in results: + assert_type(item.when, AwareDateTime) + assert item.when.tzinfo is UTC + assert item.hackernews_link == f'https://news.ycombinator.com/item?id={item.uid}' + + +def test_materialistic_saves_raises_database_error(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + from my.hackernews import materialistic + + broken = tmp_path / 'broken.db' + with sqlite_connection(broken) as conn: + conn.execute('CREATE TABLE unrelated (id TEXT)') + monkeypatch.setattr(materialistic.config, 'export_path', broken, raising=False) + + with pytest.raises(OperationalError, match='no such table: saved') as raised: + list(materialistic.saves()) + assert f'^ while processing {broken}' in raised.value.__notes__