diff --git a/packages/python/tests/e2e/glob_speed_recursive_test.py b/packages/python/tests/e2e/glob_speed_recursive_test.py new file mode 100644 index 000000000..b82bf6b8f --- /dev/null +++ b/packages/python/tests/e2e/glob_speed_recursive_test.py @@ -0,0 +1,118 @@ +"""Glob speed: recursive `'./**/*.md'` and the literal prefix `'./a/b/**/*.md'`. + +A deep, wide tree of markdown files mixed with other files, dotfiles and +dot-named directories that globstar with dotglob off must not return. A +quarter of it sits under `a/b`, the rest beside it, so a prefixed glob that +walked the whole tree would read four times what it needs. Native is `find` +pruning dot-named entries, faster here than bash globstar. No mocks: real +console script, real process, real filesystem. +""" + +from __future__ import annotations + +import shutil + +import pytest + +from .speed_of_light import ( + assert_speed_of_light, + best_of, + cli, + dirsql_rows, + dirsql_timeout, + grow_until_native_takes_a_second, + hopeless_seconds, + startup_seconds, + timed, +) + +ALL_NATIVE = ( + "find . -mindepth 1 -name '.*' -prune -o -type f -name '*.md' -printf '%P\\n'" +) +PREFIX_NATIVE = "find a/b -mindepth 1 -name '.*' -prune -o -type f -name '*.md' -print" +FANOUT = 8 + + +def home(i): + """The top of item `i`'s subtree: a quarter under `a/b`, the rest beside it.""" + return ("a/b", "a/c", "t1", "t2")[i % 4] + + +def build_tree(root, lo, hi): + """Items `lo` through `hi - 1`, each a file three directories below its + home. Every ninth is a `.txt`, every fiftieth a dotfile, and every + twenty-fifth sits in a dot-named directory.""" + for i in range(lo, hi): + j = i // 4 + folder = ( + root + / home(i) + / f"x{j % FANOUT}" + / f"y{j // FANOUT % FANOUT}" + / f"z{j // FANOUT**2 % FANOUT}" + ) + if i % 25 == 0: + folder = folder / ".cache" + folder.mkdir(parents=True, exist_ok=True) + if i % 9 == 0: + name = f"n{i}.txt" + elif i % 50 == 0: + name = f".h{i}.md" + else: + name = f"n{i}.md" + (folder / name).touch() + + +def matched(n, prefix): + return sum( + 1 + for i in range(n) + if home(i).startswith(prefix) and i % 25 and i % 9 and i % 50 + ) + + +def native(root, script): + proc, seconds = timed(["bash", "-c", script], root, timeout=600) + assert proc.returncode == 0, proc.stderr + return sorted(proc.stdout.splitlines()), seconds + + +def describe_recursive_glob_speed_of_light(): + @pytest.fixture + def root(tmp_path): + tree = tmp_path / "recursive" + tree.mkdir() + try: + yield tree + finally: + shutil.rmtree(tree, ignore_errors=True) + + @pytest.mark.parametrize( + ("glob", "script", "prefix"), + [("./**/*.md", ALL_NATIVE, ""), ("./a/b/**/*.md", PREFIX_NATIVE, "a/b")], + ) + def it_matches_native_files_within_the_bar(root, glob, script, prefix): + startup = startup_seconds() + n = grow_until_native_takes_a_second( + lambda lo, hi: build_tree(root, lo, hi), + lambda: native(root, script), + start=100_000, + ceiling=2**22, + ) + expected, native_seconds = best_of(lambda: native(root, script), "native") + assert len(expected) == matched(n, prefix) + + def dirsql(): + proc, seconds = timed( + [cli(), "query", f"SELECT path FROM '{glob}'"], + root, + timeout=dirsql_timeout(native_seconds), + ) + return sorted(row for (row,) in dirsql_rows(proc, ("path",))), seconds + + actual, dirsql_seconds = best_of( + dirsql, "dirsql", hopeless_seconds(native_seconds, startup) + ) + + assert actual == expected + assert_speed_of_light(glob, native_seconds, dirsql_seconds, startup)