Test recursive and prefixed globs against find at speed of light

packages/python/tests/e2e/glob_speed_recursive_test.py

@@ -0,0 +1,118 @@
+"""Glob speed: recursive `'./**/*.md'` and the literal prefix `'./a/b/**/*.md'`.
+
+A deep, wide tree of markdown files mixed with other files, dotfiles and
+dot-named directories that globstar with dotglob off must not return. A
+quarter of it sits under `a/b`, the rest beside it, so a prefixed glob that
+walked the whole tree would read four times what it needs. Native is `find`
+pruning dot-named entries, faster here than bash globstar. No mocks: real
+console script, real process, real filesystem.
+"""
+
+from __future__ import annotations
+
+import shutil
+
+import pytest
+
+from .speed_of_light import (
+    assert_speed_of_light,
+    best_of,
+    cli,
+    dirsql_rows,
+    dirsql_timeout,
+    grow_until_native_takes_a_second,
+    hopeless_seconds,
+    startup_seconds,
+    timed,
+)
+
+ALL_NATIVE = (
+    "find . -mindepth 1 -name '.*' -prune -o -type f -name '*.md' -printf '%P\\n'"
+)
+PREFIX_NATIVE = "find a/b -mindepth 1 -name '.*' -prune -o -type f -name '*.md' -print"
+FANOUT = 8
+
+
+def home(i):
+    """The top of item `i`'s subtree: a quarter under `a/b`, the rest beside it."""
+    return ("a/b", "a/c", "t1", "t2")[i % 4]
+
+
+def build_tree(root, lo, hi):
+    """Items `lo` through `hi - 1`, each a file three directories below its
+    home. Every ninth is a `.txt`, every fiftieth a dotfile, and every
+    twenty-fifth sits in a dot-named directory."""
+    for i in range(lo, hi):
+        j = i // 4
+        folder = (
+            root
+            / home(i)
+            / f"x{j % FANOUT}"
+            / f"y{j // FANOUT % FANOUT}"
+            / f"z{j // FANOUT**2 % FANOUT}"
+        )
+        if i % 25 == 0:
+            folder = folder / ".cache"
+        folder.mkdir(parents=True, exist_ok=True)
+        if i % 9 == 0:
+            name = f"n{i}.txt"
+        elif i % 50 == 0:
+            name = f".h{i}.md"
+        else:
+            name = f"n{i}.md"
+        (folder / name).touch()
+
+
+def matched(n, prefix):
+    return sum(
+        1
+        for i in range(n)
+        if home(i).startswith(prefix) and i % 25 and i % 9 and i % 50
+    )
+
+
+def native(root, script):
+    proc, seconds = timed(["bash", "-c", script], root, timeout=600)
+    assert proc.returncode == 0, proc.stderr
+    return sorted(proc.stdout.splitlines()), seconds
+
+
+def describe_recursive_glob_speed_of_light():
+    @pytest.fixture
+    def root(tmp_path):
+        tree = tmp_path / "recursive"
+        tree.mkdir()
+        try:
+            yield tree
+        finally:
+            shutil.rmtree(tree, ignore_errors=True)
+
+    @pytest.mark.parametrize(
+        ("glob", "script", "prefix"),
+        [("./**/*.md", ALL_NATIVE, ""), ("./a/b/**/*.md", PREFIX_NATIVE, "a/b")],
+    )
+    def it_matches_native_files_within_the_bar(root, glob, script, prefix):
+        startup = startup_seconds()
+        n = grow_until_native_takes_a_second(
+            lambda lo, hi: build_tree(root, lo, hi),
+            lambda: native(root, script),
+            start=100_000,
+            ceiling=2**22,
+        )
+        expected, native_seconds = best_of(lambda: native(root, script), "native")
+        assert len(expected) == matched(n, prefix)
+
+        def dirsql():
+            proc, seconds = timed(
+                [cli(), "query", f"SELECT path FROM '{glob}'"],
+                root,
+                timeout=dirsql_timeout(native_seconds),
+            )
+            return sorted(row for (row,) in dirsql_rows(proc, ("path",))), seconds
+
+        actual, dirsql_seconds = best_of(
+            dirsql, "dirsql", hopeless_seconds(native_seconds, startup)
+        )
+
+        assert actual == expected
+        assert_speed_of_light(glob, native_seconds, dirsql_seconds, startup)