test(python): speed-of-light e2e tests for the article's use cases

packages/python/tests/e2e/speed_of_light.py

@@ -0,0 +1,124 @@
+"""Harness for the use-case speed-of-light tests.
+
+Each use-case test asks one question of a generated corpus twice: once through
+the fastest native pipeline (find, sort, join, awk, a single parser process)
+and once through the real `dirsql` launcher. The rows must be identical and
+dirsql's wall time, after subtracting the launcher's measured startup, must be
+within ten percent of native. The bar is a ratio, never a millisecond number,
+so it holds on any machine.
+
+Startup is the mean of ten `dirsql --help` runs; `--help` and `SELECT 1` cost
+the same, so it stands in for the fixed per-invocation cost. A corpus too
+small to time is grown by doubling until the native pipeline takes at least a
+second, so the comparison is above the noise floor. Each side is run three
+times and the fastest run counts.
+"""
+
+from __future__ import annotations
+
+import json
+import shutil
+import subprocess
+import time
+
+ONE_SECOND = 1.0
+TOLERANCE = 1.1
+STARTUP_PROBES = 10
+BEST_OF = 3
+
+
+def cli() -> str:
+    dirsql = shutil.which("dirsql")
+    assert dirsql is not None, (
+        "`dirsql` console script not on PATH -- run `uv run maturin develop`"
+    )
+    return dirsql
+
+
+def timed(argv, cwd, timeout, env=None):
+    started = time.perf_counter()
+    proc = subprocess.run(
+        argv,
+        cwd=str(cwd),
+        stdin=subprocess.DEVNULL,
+        capture_output=True,
+        text=True,
+        timeout=timeout,
+        env=env,
+        check=False,
+    )
+    return proc, time.perf_counter() - started
+
+
+def startup_seconds(argv=None, cwd=None) -> float:
+    argv = [cli(), "--help"] if argv is None else argv
+    samples = []
+    for _ in range(STARTUP_PROBES):
+        proc, seconds = timed(argv, cwd or ".", timeout=60)
+        assert proc.returncode == 0, proc.stderr
+        samples.append(seconds)
+    return sum(samples) / len(samples)
+
+
+def dirsql_rows(proc, columns):
+    assert proc.returncode == 0, (
+        f"dirsql failed: stdout={proc.stdout!r} stderr={proc.stderr!r}"
+    )
+    return [tuple(row[column] for column in columns) for row in json.loads(proc.stdout)]
+
+
+def grow_until_native_takes_a_second(build, native, start, ceiling):
+    """Grow the corpus by doubling until `native` runs for a second.
+
+    `build(lo, hi)` adds items `lo` through `hi - 1` to the corpus already
+    holding `lo` items, so each doubling writes only the new half.
+    """
+    built, size = 0, start
+    while True:
+        build(built, size)
+        built = size
+        _, seconds = native()
+        if seconds >= ONE_SECOND or size >= ceiling:
+            return size
+        size *= 2
+
+
+def best_of(run, name, hopeless=None):
+    """Run `run` three times; return the rows and wall time of the fastest.
+
+    `run` returns `(rows, seconds)`. A run that raises `TimeoutExpired` fails
+    the test immediately with the timeout it blew through. A run slower than
+    `hopeless` seconds ends the series early: noise does not account for an
+    order of magnitude, so no retry could bring the best under the bar.
+    """
+    fastest = None
+    for _ in range(BEST_OF):
+        try:
+            rows, seconds = run()
+        except subprocess.TimeoutExpired as expired:
+            raise AssertionError(
+                f"{name} did not finish within {expired.timeout:.0f}s"
+            ) from None
+        if fastest is None or seconds < fastest[1]:
+            fastest = (rows, seconds)
+        if hopeless is not None and seconds > hopeless:
+            break
+    return fastest
+
+
+def hopeless_seconds(native_seconds: float, startup: float) -> float:
+    return startup + 10 * TOLERANCE * native_seconds
+
+
+def dirsql_timeout(native_seconds: float) -> float:
+    return max(60.0, 30 * native_seconds)
+
+
+def assert_speed_of_light(name, native_seconds, dirsql_seconds, startup):
+    measured = dirsql_seconds - startup
+    budget = TOLERANCE * native_seconds
+    assert measured <= budget, (
+        f"{name}: dirsql {dirsql_seconds:.3f}s minus startup {startup:.3f}s"
+        f" = {measured:.3f}s, over {TOLERANCE}x native {native_seconds:.3f}s"
+        f" = {budget:.3f}s"
+    )

packages/python/tests/e2e/use_case_dossier_test.py

@@ -0,0 +1,153 @@
+"""Use case 1, the dossier: three selections over a note tree, UNIONed.
+
+A standing set of background files, the five newest dailies by name, and the
+ten most recently modified markdown files outside the dailies, in one ranked
+list. Native is three `find` runs with `-printf`, each sorted and cut, then
+one final sort. dirsql is the article's query through the real launcher.
+No mocks: real console script, real process, real filesystem.
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+import time
+
+import pytest
+
+from .speed_of_light import (
+    assert_speed_of_light,
+    best_of,
+    cli,
+    dirsql_rows,
+    dirsql_timeout,
+    grow_until_native_takes_a_second,
+    hopeless_seconds,
+    startup_seconds,
+    timed,
+)
+
+DAY = 86400
+WINDOW_DAYS = 14
+STANDING = ("situation.md", "projects.md", "profile.md")
+COLUMNS = ("rank", "kind", "path", "mtime")
+
+QUERY = """
+WITH standing AS (
+  SELECT path, mtime FROM './'
+    WHERE dir = 'planner/background'
+      AND basename IN ('situation.md', 'projects.md', 'profile.md')
+),
+dailies AS (
+  SELECT path, mtime FROM './'
+    WHERE path LIKE 'vault/Dailies/Daily %'
+    ORDER BY basename DESC LIMIT 5
+),
+recent AS (
+  SELECT path, mtime FROM './'
+    WHERE ext = 'md'
+      AND path NOT LIKE 'vault/Dailies/%'
+      AND mtime > strftime('%s', 'now') - 14 * 86400
+    ORDER BY mtime DESC LIMIT 10
+)
+SELECT 1 AS rank, 'standing' AS kind, path, mtime FROM standing
+UNION ALL SELECT 2, 'daily', path, mtime FROM dailies
+UNION ALL SELECT 3, 'recent', path, mtime FROM recent
+ORDER BY rank, mtime DESC, path
+"""
+
+NATIVE = r"""
+set -e
+export LC_ALL=C
+cutoff=$(( $(date +%s) - 14 * 86400 ))
+{
+  find planner/background -maxdepth 1 \
+    \( -name situation.md -o -name projects.md -o -name profile.md \) \
+    -printf '1\tstanding\t%p\t%T@\n'
+  find vault/Dailies -name 'Daily *' -printf '%f\t%p\t%T@\n' \
+    | sort -t "$(printf '\t')" -k1,1r | head -5 \
+    | awk -F '\t' -v OFS='\t' '{ print 2, "daily", $2, $3 }'
+  find . -path ./vault/Dailies -prune -o -name '*.md' -newermt "@$cutoff" \
+    -printf '%T@\t%P\n' \
+    | sort -rn | head -10 \
+    | awk -F '\t' -v OFS='\t' '{ print 3, "recent", $2, $1 }'
+} | sort -t "$(printf '\t')" -k1,1n -k4,4rn -k3,3
+"""
+
+
+def _write(path, mtime, body="# note\nbody\n"):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(body)
+    os.utime(path, (mtime, mtime))
+
+
+def build_tree(root, lo, hi):
+    """Bulk notes `lo` through `hi - 1`: the first thousand inside the recency
+    window, the rest older, every tenth one a `.txt` the ext filter must
+    drop. Mtimes are distinct integers nowhere near the window's edge."""
+    now = int(time.time())
+    if lo == 0:
+        for name in STANDING:
+            _write(root / "planner" / "background" / name, now - 70 * DAY)
+        for day in range(1, 41):
+            stamp = now - day * DAY
+            date = time.strftime("%Y.%m.%d", time.gmtime(stamp))
+            _write(root / "vault" / "Dailies" / f"Daily {date}.md", stamp)
+    for i in range(lo, hi):
+        ext = "txt" if i % 10 == 0 else "md"
+        mtime = now - 3600 - i * 600 if i < 1000 else now - 15 * DAY - i * 7
+        _write(
+            root / "planner" / "internal" / "notes" / str(i % 1000) / f"note-{i}.{ext}",
+            mtime,
+        )
+
+
+def native_rows(proc):
+    assert proc.returncode == 0, proc.stderr
+    rows = []
+    for line in proc.stdout.splitlines():
+        rank, kind, path, mtime = line.split("\t")
+        rows.append((int(rank), kind, path, int(float(mtime))))
+    return rows
+
+
+def describe_dossier_speed_of_light():
+    @pytest.fixture
+    def root(tmp_path):
+        tree = tmp_path / "notes"
+        try:
+            yield tree
+        finally:
+            shutil.rmtree(tree, ignore_errors=True)
+
+    @pytest.mark.xfail(
+        strict=True,
+        reason="each CTE's path-table reference walks the tree again, and one walk is slower than find",
+    )
+    def it_matches_native_rows_within_the_bar(root):
+        startup = startup_seconds()
+
+        def native():
+            proc, seconds = timed(["bash", "-c", NATIVE], root, timeout=600)
+            return native_rows(proc), seconds
+
+        grow_until_native_takes_a_second(
+            lambda lo, hi: build_tree(root, lo, hi), native, start=50_000, ceiling=2**21
+        )
+        expected, native_seconds = best_of(native, "native")
+        assert len(expected) == 18, expected
+
+        def dirsql():
+            proc, seconds = timed(
+                [cli(), "query", QUERY],
+                root,
+                timeout=dirsql_timeout(native_seconds),
+            )
+            return dirsql_rows(proc, COLUMNS), seconds
+
+        actual, dirsql_seconds = best_of(
+            dirsql, "dirsql", hopeless_seconds(native_seconds, startup)
+        )
+
+        assert actual == expected
+        assert_speed_of_light("dossier", native_seconds, dirsql_seconds, startup)

packages/python/tests/e2e/use_case_join_test.py

@@ -0,0 +1,117 @@
+"""Use case 2, the arXiv sidecar layout: a join of two path tables.
+
+One folder per paper holding `abstract.md` and `title.md`; the question is
+every paper that has both, with its title and the abstract's size. Native is
+two `find` listings, one through `awk` for the title text, sorted and joined
+with `join(1)`. dirsql joins the two path tables on `dir` through the real
+launcher. No mocks: real console script, real process, real filesystem.
+"""
+
+from __future__ import annotations
+
+import shutil
+
+import pytest
+
+from .speed_of_light import (
+    assert_speed_of_light,
+    best_of,
+    cli,
+    dirsql_rows,
+    dirsql_timeout,
+    grow_until_native_takes_a_second,
+    hopeless_seconds,
+    startup_seconds,
+    timed,
+)
+
+COLUMNS = ("dir", "title", "abstract_bytes")
+QUERY = """
+SELECT a.dir, t.content AS title, a.size AS abstract_bytes
+FROM './papers/*/abstract.md' a
+JOIN './papers/*/title.md' t ON t.dir = a.dir
+ORDER BY a.dir
+"""
+
+NATIVE = r"""
+set -e
+export LC_ALL=C
+tab="$(printf '\t')"
+join -t "$tab" \
+  <(find papers -name title.md \
+      -exec awk -v OFS='\t' '{ split(FILENAME, p, "/"); print p[1] "/" p[2], $0 }' {} + \
+      | sort) \
+  <(find papers -name abstract.md -printf '%h\t%s\n' | sort)
+"""
+
+SENTENCE = (
+    "We study the problem from first principles and report a method that "
+    "improves on the baseline across every setting we tried. "
+)
+
+
+def build_papers(root, lo, hi):
+    """Paper folders `lo` through `hi - 1`. Every seventh lacks a title and
+    every eleventh an abstract, so the join has to drop rows rather than
+    pair everything."""
+    papers = root / "papers"
+    for i in range(lo, hi):
+        folder = papers / f"2609.{i:06d}"
+        folder.mkdir(parents=True)
+        if i % 7:
+            (folder / "title.md").write_text(f"Paper {i}: on the {i % 13}th method")
+        if i % 11:
+            (folder / "abstract.md").write_text(SENTENCE * (1 + i % 5))
+
+
+def native_rows(proc):
+    assert proc.returncode == 0, proc.stderr
+    rows = []
+    for line in proc.stdout.splitlines():
+        folder, title, size = line.split("\t")
+        rows.append((folder, title, int(size)))
+    return rows
+
+
+def describe_join_speed_of_light():
+    @pytest.fixture
+    def root(tmp_path):
+        tree = tmp_path / "arxiv"
+        try:
+            yield tree
+        finally:
+            shutil.rmtree(tree, ignore_errors=True)
+
+    @pytest.mark.xfail(
+        strict=True, reason="path-table join is an unindexed nested loop"
+    )
+    def it_matches_native_rows_within_the_bar(root):
+        startup = startup_seconds()
+
+        def native():
+            proc, seconds = timed(["bash", "-c", NATIVE], root, timeout=600)
+            return native_rows(proc), seconds
+
+        n = grow_until_native_takes_a_second(
+            lambda lo, hi: build_papers(root, lo, hi),
+            native,
+            start=10_000,
+            ceiling=2**20,
+        )
+        expected, native_seconds = best_of(native, "native")
+        assert len(expected) == sum(1 for i in range(n) if i % 7 and i % 11)
+
+        def dirsql():
+            proc, seconds = timed(
+                [cli(), "query", QUERY],
+                root,
+                timeout=dirsql_timeout(native_seconds),
+            )
+            return dirsql_rows(proc, COLUMNS), seconds
+
+        actual, dirsql_seconds = best_of(
+            dirsql, "dirsql", hopeless_seconds(native_seconds, startup)
+        )
+
+        assert actual == expected
+        assert_speed_of_light("join", native_seconds, dirsql_seconds, startup)