diff --git a/2026-06-14-010342-this-session-is-being-continued-from-a-previous-c.txt b/2026-06-14-010342-this-session-is-being-continued-from-a-previous-c.txt deleted file mode 100644 index b1e7104..0000000 --- a/2026-06-14-010342-this-session-is-being-continued-from-a-previous-c.txt +++ /dev/null @@ -1,19207 +0,0 @@ - ▐▛███▜▌ Claude Code v2.1.153 -▝▜█████▛▘ claude-fable-5 · Claude Max - ▘▘ ▝▝ ~/src/millpond - - -✻ Conversation compacted (ctrl+o for history) - - -⏺ PR up: https://github.com/PostHog/viaduck/pull/24 — two commits: the power-law - demo + UI legibility work (519f0a6), and the duck deck restyle (673f6e5). - Once merged and promoted, tomorrow's demo gets 20 skewed tenants, sortable - columns, honest lag, and a dashboard that looks like it belongs next to the - Duckgres node deck. - -✻ Brewed for 55s - -※ recap: Prepping tomorrow's viaduck demo: PR #24 (20 power-law demo destinations, sortable UI, lag in seconds, duck-deck restyle) is pushed and awaiting your merge, then promote-to-dev and a rollout restart. Tombstone work already merged earlier. - -❯ /compact - ⎿ Compacted (ctrl+o to see full summary) - ⎿  Referenced file ../../../../tmp/peepernetes.html - ⎿  Read ../../.claude/projects/-Users-jakob-src/memory/project_viaduck_rowid_reuse.md (21 lines) - ⎿  Referenced file viaduck/main.py - ⎿  Referenced file viaduck/apply.py - ⎿  Read tla/Viaduck.cfg (20 lines) - ⎿  Plan file referenced (~/.claude/plans/groovy-noodling-sun.md) - ⎿  Skills restored (plannotator-last, check-last-screenshot, update-config, pr-publish, render-appset) - -※ recap: Prepping tomorrow's viaduck demo: PR #24 (20 power-law demo destinations, sortable UI, lag in seconds, duck-deck restyle) is pushed and awaiting your merge. Next: merge it, promote to dev, and rollout-restart the dev pod. - -❯ Let's hop over to millpond and fix this: "2026-06-12T15:55:52 INFO - [viaduck-0][viaduck.main] Source snapshot 13628517: 12507192 data files - (1154426.66 GiB), 0.00 MiB of delete data - " Let's get that GiB number better formatted (to MB, GB, TB, etc) - automatically. - -⏺ I'll find where that log line is produced — the user said millpond, but the - logger is viaduck.main, so let me check both. - - Searched for 1 pattern (ctrl+o to expand) - ⎿  Interrupted · What should Claude do instead? - -❯ Yeah, ViaDuck. Sorry. - -⏺ Bash(sed -n 385,420p /Users/jakob/src/viaduck/viaduck/main.py; grep -n - "GiB\|MiB" /Users/jakob/src/viaduck/viaduck/main.py) - ⎿   len(assigned_ids), - ) - return - … +34 lines (ctrl+o to expand) - - Searched for 1 pattern (ctrl+o to expand) - -⏺ No Python-side helper exists. Let me check the JS fmtBytes in server.py to - match its convention. - - Searched for 2 patterns, read 1 file (ctrl+o to expand) - -⏺ Update(~/src/viaduck/viaduck/main.py) - ⎿  Added 8 lines - 162 elapsed += chunk - 163 - 164 - 165 +def _fmt_bytes(n: float) -> str: - 166 + for unit in ("B", "KiB", "MiB", "GiB", "TiB", "PiB"): - 167 + if abs(n) < 1024 or unit == "PiB": - 168 + return f"{n:.2f} {unit}" if unit != "B" else f"{in - +t(n)} B" - 169 + n /= 1024 - 170 + return f"{n:.2f} PiB" - 171 + - 172 + - 173 def _group_by_cursor( - 174 cursors: dict[str, int], - 175 all_dest_ids: list[str], - -⏺ Update(~/src/viaduck/viaduck/main.py) - ⎿  Added 2 lines, removed 2 lines - 163 - 164 - 165 def _fmt_bytes(n: float) -> str: - 166 - for unit in ("B", "KiB", "MiB", "GiB", "TiB", "PiB"): - 167 - if abs(n) < 1024 or unit == "PiB": - 166 + for unit in ("B", "KiB", "MiB", "GiB", "TiB"): - 167 + if abs(n) < 1024: - 168 return f"{n:.2f} {unit}" if unit != "B" else f"{in - t(n)} B" - 169 n /= 1024 - 170 return f"{n:.2f} PiB" - -⏺ Update(~/src/viaduck/viaduck/main.py) - ⎿  Added 3 lines, removed 3 lines - 409 data_bytes = pc.sum(files.column("data_file_size_bytes - ")).as_py() or 0 - 410 delete_bytes = pc.sum(files.column("delete_file_size_b - ytes")).as_py() or 0 - 411 log.info( - 412 - "Source snapshot %d: %d data files (%.2f GiB), %.2 - -f MiB of delete data", - 412 + "Source snapshot %d: %d data files (%s), %s of del - +ete data", - 413 current_id, - 414 file_count, - 415 - data_bytes / (1024**3), - 416 - delete_bytes / (1024**2), - 415 + _fmt_bytes(data_bytes), - 416 + _fmt_bytes(delete_bytes), - 417 ) - 418 except Exception: - 419 log.exception("Could not read snapshot file inventory; - continuing without it") - - Searched for 1 pattern (ctrl+o to expand) - -⏺ Bash(uv run python -c " - from viaduck.main import _fmt_bytes…) ⎿  0 -> 0 B - 512 -> 512 B - 1239556187582627.8 -> 1.10 PiB - … +3 lines (ctrl+o to expand) - -⏺ Done. viaduck/main.py:165 now has _fmt_bytes (auto-scales B → PiB, binary - units to match the UI's fmtBytes), and the seed-scan log line uses it for both - data and delete sizes. Your example now renders as 1.10 PiB instead of - 1154426.66 GiB. Lint clean; uncommitted on jakob/demo-power-law. - -✻ Worked for 1m 7s - -❯ I already merged that branch. Let's rebase off a new branch from main. - -⏺ Bash(git -C /Users/jakob/src/viaduck fetch origin && git -C - /Users/jakob/src/viaduck status --short && git -C /Users/jakob/src/viaduck - branch --show-current) - ⎿  From github.com:PostHog/viaduck - 3de9f04..191e560 main -> origin/main - * [new tag] v0.0.24 -> v0.0.24 - … +5 lines (ctrl+o to expand) - -⏺ Bash(git -C /Users/jakob/src/viaduck checkout main && git -C - /Users/jakob/src/viaduck pull --ff-only && git -C /Users/jakob/src/viaduck - checkout -b jakob/fmt-bytes &…) - ⎿  Error: Exit code 1 - error: Your local changes to the following files would be overwritten by - checkout: - viaduck/main.py - Please commit your changes or stash them before you switch branches. - Aborting - -⏺ Bash(git -C /Users/jakob/src/viaduck stash push viaduck/main.py -m "fmt-bytes - wip" && git -C /Users/jakob/src/viaduck checkout main && git -C - /Users/jakob/src/viaduc…) - ⎿  Saved working directory and index state On jakob/demo-power-law: fmt-b - ytes wip - Switched to branch 'main' - … +28 lines (ctrl+o to expand) - -⏺ Bash(git diff --stat && uv run ruff check viaduck/main.py && uv run python -c - "from viaduck.main import _fmt_bytes; - print(_fmt_bytes(1154426.66*1024**3))") - ⎿   viaduck/main.py | 14 +++++++++++--- - 1 file changed, 11 insertions(+), 3 deletions(-) - All checks passed! - 1.10 PiB - -⏺ On jakob/fmt-bytes off updated main (191e560, includes the merged demo - branch), with the _fmt_bytes change applied cleanly and verified. Uncommitted, - awaiting your call. - -✻ Brewed for 39s - -❯ Go for it. - -⏺ Bash(git log --oneline -5) - ⎿  191e560 Merge pull request #24 from PostHog/jakob/demo-power-law - 673f6e5 Duck deck: restyle the status UI in the peepernetes design lan - guage - … +5 lines (ctrl+o to expand) - -⏺ Bash(git add viaduck/main.py && git commit -m "Auto-scale byte sizes in the - seed-scan log line" && git status --short) - ⎿  [jakob/fmt-bytes 7693551] Auto-scale byte sizes in the seed-scan log l - ine - 1 file changed, 11 insertions(+), 3 deletions(-) - … +3 lines (ctrl+o to expand) - -⏺ Committed 7693551 on jakob/fmt-bytes. Push and open the PR too? - -✻ Crunched for 25s - -❯ Yeah - -⏺ Bash(git push -u origin jakob/fmt-bytes && gh pr create --title "Auto-scale - byte sizes in the seed-scan log line" --body "$(cat <<'EOF' - ## Summary…) - ⎿  pre-push: checking uv.lock consistency... - remote: - remote: Create a pull request for 'jakob/fmt-bytes' on GitHub by visit - … +9 lines (ctrl+o to expand) - -⏺ PR #25: https://github.com/PostHog/viaduck/pull/25 - -✻ Crunched for 18s - -❯ I have a sneaking feeling we didn't run any tests... - -⏺ Guilty — only ruff and a REPL spot-check. Running the suite now. - -⏺ Bash(just --list 2>/dev/null | head -20) - ⎿  Available recipes: - default # Default recipe: list all available recipes - … +22 lines (ctrl+o to expand) - -⏺ Bash(just --list 2>/dev/null | sed -n 20,40p) - ⎿   down # Stop the dev/demo environment (wipes volumes — - next start is a fresh source) - minio # Open the MinIO console (requires `just up` firs - … +20 lines (ctrl+o to expand) - -⏺ Bash(just test 2>&1 | tail -5) - ⎿  INFO viaduck.main:main.py:544 Seeded destination dest-1: 1 rows in - 1 batches (total=0.0s, write=0.0s) - =========================== short test summary info ================== - … +6 lines (ctrl+o to expand) -⏺ One failure, exactly where you'd expect — a test asserting the old log format. - - Searched for 1 pattern (ctrl+o to expand) - -⏺ Bash(sed -n 2007,2040p tests/unit/test_main.py) - ⎿   mock_table = MagicMock() - mock_table.scan.return_value.count.return_value = 0 # empty dest: - no truncate - … +31 lines (ctrl+o to expand) - -⏺ The 3 GiB assertion still passes; 768 KiB now renders as 768.00 KiB instead of - 0.75 MiB. Updating the test to match the auto-scaled output. - -⏺ Bash(python - <<'EOF' - import re…) - ⎿  (eval):1: command not found: python - - Read 1 file (ctrl+o to expand) - -⏺ Update(~/src/viaduck/tests/unit/test_main.py) - ⎿  Added 1 line, removed 1 line - 2020 line = prescan[0] - 2021 assert "2 data files" in line - 2022 assert "3.00 GiB" in line - 2023 - assert "0.75 MiB" in line - 2023 + assert "768.00 KiB" in line - 2024 - 2025 - 2026 def test_seed_new_destinations_continues_when_prescan_stats_f - ail(caplog): - -⏺ Bash(just test 2>&1 | tail -2 && just test-integration 2>&1 | tail -2) - ⎿  ============================= 357 passed in 4.37s ==================== - ========== - -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.htm - … +3 lines (ctrl+o to expand) - -⏺ All green: 357 unit + 53 integration. The fix is a one-line test update (0.75 - MiB → 768.00 KiB in test_seed_new_destinations_logs_prescan_stats), - uncommitted. Want me to commit and push it onto PR #25? - -✻ Sautéed for 1m 17s - -❯ yes - -⏺ Bash(git add tests/unit/test_main.py && git commit -m "Update prescan-stats - test for auto-scaled byte units" && git push) - ⎿  [jakob/fmt-bytes eaaa483] Update prescan-stats test for auto-scaled by - te units - 1 file changed, 1 insertion(+), 1 deletion(-) - … +3 lines (ctrl+o to expand) - -⏺ Pushed eaaa483 to PR #25 — full suite green (357 unit + 53 integration). - -✻ Cogitated for 9s - -※ recap: PR #25 (jakob/fmt-bytes) auto-scales the seed-scan log's byte units; tests fixed and green, both commits pushed. Next action: you merge PR #25 when ready. - -❯ Looks like we've got a scale issue with backfills: "2026-06-12T16:02:52 INFO - [viaduck-0][viaduck.main] Seed scan for destination team-2: DuckDB - pre-execution, 420s elapsed - Traceback (most recent call last): - File "/app/.venv/bin/viaduck", line 10, in - sys.exit(main()) - ^^^^^^ - File "/app/viaduck/main.py", line 839, in main - run(cfg) - File "/app/viaduck/main.py", line 601, in run - _seed_new_destinations(src_table, state_mgr, dest_pool, cfg, - assigned_ids) - File "/app/viaduck/main.py", line 497, in _seed_new_destinations - for batch in reader: - ^^^^^^ - File "pyarrow/ipc.pxi", line 757, in pyarrow.lib.RecordBatchReader.__next__ - File "pyarrow/ipc.pxi", line 791, in - pyarrow.lib.RecordBatchReader.read_next_batch - File "pyarrow/error.pxi", line 92, in pyarrow.lib.check_status - OSError: Invalid Input Error: Arrow Appender: The maximum total string size - for regular string buffers is 2147483647 but the offset of 2147548897 exceeds - this. - * SET arrow_large_buffer_size=true to use large string buffers" - -⏺ That's DuckDB's Arrow bridge overflowing the 32-bit string-offset limit on a - big seed batch — DuckDB itself tells us the fix: SET - arrow_large_buffer_size=true on the source connection so it emits large_string - buffers. Let me find where we set up the source connection. - - Read 1 file, listed 1 directory (ctrl+o to expand) - -⏺ Bash(python3 -c "import pyducklake, inspect, os; - print(os.path.dirname(inspect.getfile(pyducklake)))" 2>/dev/null || uv run - python -c "import pyducklake, inspect, os…) - ⎿  /Users/jakob/src/viaduck/.venv/lib/python3.12/site-packages/pyducklake - - Searched for 1 pattern, read 1 file (ctrl+o to expand) - -⏺ Bash(uv run python -c " - import duckdb…) - ⎿  [(True,)] - 1.5.2 - -⏺ Works. Now adding it to the source connection defaults in source.py. - - Searched for 1 pattern (ctrl+o to expand) - -⏺ Bash(sed -n 180,225p viaduck/destination.py) - ⎿   self._pins.clear() - metrics.pool_open_connections.set(0) - … +43 lines (ctrl+o to expand) -⏺ All catalogs (source + destinations) go through with_pool_defaults, so adding - the setting there covers every DuckDB→Arrow export path. - -⏺ Update(~/src/viaduck/viaduck/source.py) - ⎿  Added 21 lines, removed 12 lines - 20 - 21 META_COLUMNS = ("change_type", "snapshot_id", "rowid") - 22 - 23 -# DuckDB's postgres extension sized `pg_pool_max_connections` t - -o the thread - 24 -# count (e.g. 12) in older builds. A parallel CDC scan on the s - -ource catalog - 25 -# can claim every slot, leaving a concurrent state-table write - -to wait the - 26 -# full 30s pool timeout and fail (observed: production containe - -r exits with - 27 -# the loop's fatal-error path). Pin an explicit floor regardles - -s of upstream - 28 -# defaults drifting. `pg_connection_limit` is the legacy alias - -for - 29 -# `pg_pool_max_connections`; the new name is registered but not - - SET-able in - 30 -# duckdb 1.5.2's postgres extension. - 31 -_PG_POOL_DEFAULTS = { - 23 +# pg_connection_limit: DuckDB's postgres extension sized - 24 +# `pg_pool_max_connections` to the thread count (e.g. 12) in ol - +der builds. A - 25 +# parallel CDC scan on the source catalog can claim every slot, - + leaving a - 26 +# concurrent state-table write to wait the full 30s pool timeou - +t and fail - 27 +# (observed: production container exits with the loop's fatal-e - +rror path). - 28 +# Pin an explicit floor regardless of upstream defaults driftin - +g. - 29 +# `pg_connection_limit` is the legacy alias for `pg_pool_max_co - +nnections`; - 30 +# the new name is registered but not SET-able in duckdb 1.5.2's - + postgres - 31 +# extension. - 32 +# - 33 +# arrow_large_buffer_size: DuckDB's Arrow export defaults to 32 - +-bit string - 34 +# offsets, capping any one exported buffer at 2 GiB of string d - +ata — large - 35 +# seed scans blow past it (observed: backfill of a 12.5M-file s - +ource died - 36 +# mid-read with "maximum total string size ... exceeds this"). - +Large buffers - 37 +# use 64-bit offsets (pyarrow large_string), which Arrow comput - +e and the - 38 +# postgres writer handle transparently. - 39 +_CONNECTION_DEFAULTS = { - 40 "pg_connection_limit": "64", - 41 + "arrow_large_buffer_size": "true", - 42 } - 43 - 44 - 36 -def with_pool_defaults(props: dict[str, str]) -> dict[str, str] - -: - 37 - """Merge pg pool defaults under user-supplied properties (u - -ser wins).""" - 38 - merged = dict(_PG_POOL_DEFAULTS) - 45 +def with_connection_defaults(props: dict[str, str]) -> dict[str - +, str]: - 46 + """Merge DuckDB connection defaults under user-supplied pro - +perties (user wins).""" - 47 + merged = dict(_CONNECTION_DEFAULTS) - 48 merged.update(props) - 49 return merged - 50 - -⏺ Bash(perl -pi -e 's/with_pool_defaults/with_connection_defaults/g' - viaduck/source.py viaduck/destination.py && grep -rn "with_pool_defaults" - viaduck/ tests/ --includ…) - ⎿  --- - viaduck/source.py:45:def with_connection_defaults(props: dict[str, str - ]) -> dict[str, str]: - … +8 lines (ctrl+o to expand) - -⏺ Bash(just test 2>&1 | tail -2 && just test-integration 2>&1 | tail -2) - ⎿  ======================== 1 failed, 356 passed in 4.44s =============== - ========== - error: recipe `test` failed on line 49 with exit code 1 - … +4 lines (ctrl+o to expand) - -⏺ Bash(just test 2>&1 | grep -B2 -A15 "FAILED\|assert" | head -40) - ⎿   connect(cfg) - > MockCatalog.assert_called_once_with( - "src", - … +36 lines (ctrl+o to expand) - - Searched for 1 pattern, read 1 file (ctrl+o to expand) - -⏺ Update(~/src/viaduck/tests/unit/test_source.py) - ⎿  Added 1 line, removed 1 line - 90 "src", - 91 "postgres:host=localhost dbname=test", - 92 data_path="/tmp/data", - 93 - properties={"pg_connection_limit": "64"}, - 93 + properties={"pg_connection_limit": "64", "arrow - +_large_buffer_size": "true"}, - 94 ) - 95 - 96 - -⏺ Bash(just test 2>&1 | tail -2 && uv run ruff check viaduck/ tests/) - ⎿  ============================= 357 passed in 4.32s ==================== - ========== - All checks passed! - -⏺ Fix in place — all green (357 unit, 53 integration; the integration suite runs - real DuckDB connections, so large-buffer mode is exercised end-to-end). - - What changed: - - viaduck/source.py: _PG_POOL_DEFAULTS → _CONNECTION_DEFAULTS, adding - arrow_large_buffer_size: true (64-bit string offsets on DuckDB→Arrow export; - the 2 GiB offset cap is what killed the team-2 seed scan). Comment documents - both settings. - - Helper renamed with_pool_defaults → with_connection_defaults; covers source, - dest, and schema-fetch catalogs. User-supplied properties still override. - - tests/unit/test_source.py: expected properties updated. - - This is sitting uncommitted on jakob/fmt-bytes. Separate commit on this - branch, or a fresh branch off main? - -✻ Brewed for 2m 34s - -❯ fresh branch, I already merged. - -⏺ Bash(git fetch origin && git stash push viaduck/source.py - viaduck/destination.py tests/unit/test_source.py -m "arrow large buffers - wip" && git checkout main && git p…) - ⎿  From github.com:PostHog/viaduck - 191e560..e8de0d0 main -> origin/main - * [new tag] v0.0.25 -> v0.0.25 - … +32 lines (ctrl+o to expand) - -⏺ Bash(just test 2>&1 | tail -1) - ⎿  ============================= 357 passed in 4.36s ==================== - ========== - -⏺ On jakob/arrow-large-buffers off updated main (e8de0d0, includes merged #25), - change applied cleanly, tests green. Ready to commit/push/PR on your word. - -✻ Sautéed for 21s - -❯ Go for it. - -⏺ Bash(git add viaduck/source.py viaduck/destination.py - tests/unit/test_source.py && git commit -m "Enable DuckDB large Arrow - buffers to survive >2 GiB seed scans" &&…) - ⎿  [jakob/arrow-large-buffers 56b9507] Enable DuckDB large Arrow buffers - to survive >2 GiB seed scans - 3 files changed, 26 insertions(+), 17 deletions(-) - … +14 lines (ctrl+o to expand) - -⏺ PR #26: https://github.com/PostHog/viaduck/pull/26 - -✻ Worked for 18s - -※ recap: Fixing backfill scale issues: PR #26 (large Arrow buffers, fixes the 2 GiB seed-scan crash) is pushed with tests green, awaiting your merge. PR #25 (byte formatting) merged earlier. Next: merge #26 and re-run the backfill. - -❯ "This XML file does not appear to have any style information associated with - it. The document tree is shown below. - - InvalidBucketName - The specified bucket is not valid. - /ui - 18B8625746E6F361 - dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8 - " - -⏺ That's MinIO's S3 API answering — whatever URL you hit (/ui) landed on the S3 - port, not the viaduck web server. Let me check the port wiring. - - Searched for 1 pattern, listed 1 directory (ctrl+o to expand) - -⏺ Bash(sed -n 100,145p justfile; find . -name "*compose*" -not -path "./.git/*" - -not -path "./.venv/*" 2>/dev/null) - ⎿  # Stop the dev/demo environment (wipes volumes — next start is a fresh - source) - [group('docker')] - … +44 lines (ctrl+o to expand) - Searched for 1 pattern (ctrl+o to expand) - ⎿  Interrupted · What should Claude do instead? - -❯ This is what I'm getting back from dev, it's not the demo. - - Listed 2 directories (ctrl+o to expand) - -⏺ Dev has no MinIO in the viaduck values — both source and destination are real - S3 buckets, and the chart serves /ui on the same port 8000 as metrics/health. - That error (the HostId is MinIO's hardcoded default) means your request landed - on an actual MinIO server somewhere, not viaduck. - - How are you reaching dev — exact URL and the port-forward command (if any)? If - it's kubectl port-forward, check what it's targeting: - - kubectl -n viaduck get svc,pods -o wide - - My bet: the forward is pointed at a different service/namespace (something in - the dev cluster runs MinIO), or a stale forward from a previous session is - still holding the local port and proxying to the wrong pod. - -✻ Cooked for 54s - -※ recap: Goal: get the dev viaduck web UI loading; PRs 25/26 (byte formatting, large Arrow buffers) are merged/open. Your /ui request is hitting a MinIO server, not viaduck. Next: share your dev URL and port-forward command so we can retarget it. - -❯ Let's put that aside for now. Is there any reasonable way to get a percentage - progress on the seed scan? Right now it's hard to tell how long one can - expect it to take: "2026-06-12T16:28:33 INFO [viaduck-0][viaduck.main] - viaduck 0.0.26 starting - 2026-06-12T16:28:33 INFO [viaduck-0][viaduck.server] HTTP server listening - on port 8000 (/metrics, /healthz, /readyz, /status, /ui) - 2026-06-12T16:28:34 INFO [viaduck-0][viaduck.state] State table - 'viaduck_state' ready - 2026-06-12T16:28:34 INFO [viaduck-0][viaduck.state] All 1 assigned - destinations already have state rows; nothing to initialize - 2026-06-12T16:30:41 INFO [viaduck-0][viaduck.main] Seed scan: 1 destinations - need initial seed (source snapshot=13634405) - 2026-06-12T16:31:27 INFO [viaduck-0][viaduck.main] Source snapshot 13634405: - 12512628 data files (1.10 PiB), 0 B of delete data - 2026-06-12T16:31:27 INFO [viaduck-0][viaduck.main] Seeding destination - team-2: scanning source for routing_value=2, snapshot=13634405 - 2026-06-12T16:31:32 INFO [viaduck-0][viaduck.main] Seed scan for destination - team-2: DuckDB pre-execution, 5s elapsed - 2026-06-12T16:31:37 INFO [viaduck-0][viaduck.main] Seed scan for destination - team-2: DuckDB pre-execution, 10s elapsed - 2026-06-12T16:31:42 INFO [viaduck-0][viaduck.main] Seed scan for destination - team-2: DuckDB pre-execution, 15s elapsed - 2026-06-12T16:31:46 INFO [viaduck-0][viaduck.destination] Connected to - destination team-2 (catalog=team-2, table=posthog.events_via_viaduck) - 2026-06-12T16:31:47 INFO [viaduck-0][viaduck.main] Seed scan for destination - team-2: DuckDB pre-execution, 20s elapsed - 2026-06-12T16:31:52 INFO [viaduck-0][viaduck.main] Seed scan for destination - team-2: DuckDB pre-execution, 25s elapsed - 2026-06-12T16:31:57 INFO [viaduck-0][viaduck.main] Seed scan for destination - team-2: DuckDB pre-execution, 30s elapsed - 2026-06-12T16:32:02 INFO [viaduck-0][viaduck.main] Seed scan for destination - team-2: DuckDB pre-execution, 35s elapsed - 2026-06-12T16:32:07 INFO [viaduck-0][viaduck.main] Seed scan for destination - team-2: DuckDB pre-execution, 40s elapsed - 2026-06-12T16:32:12 INFO [viaduck-0][viaduck.main] Seed scan for destination - team-2: DuckDB pre-execution, 45s elapsed - 2026-06-12T16:32:17 INFO [viaduck-0][viaduck.main] Seed scan for destination - team-2: DuckDB pre-execution, 50s elapsed - 2026-06-12T16:32:22 INFO [viaduck-0][viaduck.main] Seed scan for destination - team-2: DuckDB pre-execution, 55s elapsed - 2026-06-12T16:32:27 INFO [viaduck-0][viaduck.main] Seed s - ──── (48 lines hidden) ────────────────────────────────────────────────────── - uck-0][viaduck.main] Seed scan for destination team-2: DuckDB pre-execution, - 20s elapsed - 2026-06-12T16:31:52 INFO [viaduck-0][viaduck.main] Seed scan for destination - team-2: DuckDB pre-execution, 25s elapsed - 2026-06-12T16:31:57 INFO [viaduck-0][viaduck.main] Seed scan for destination - team-2: DuckDB pre-execution, 30s elapsed - 2026-06-12T16:32:02 INFO [viaduck-0][viaduck.main] Seed scan for destination - team-2: DuckDB pre-execution, 35s elapsed - 2026-06-12T16:32:07 INFO [viaduck-0][viaduck.main] Seed scan for destination - team-2: DuckDB pre-execution, 40s elapsed - 2026-06-12T16:32:12 INFO [viaduck-0][viaduck.main] Seed scan for destination - team-2: DuckDB pre-execution, 45s elapsed - 2026-06-12T16:32:17 INFO [viaduck-0][viaduck.main] Seed scan for destination - team-2: DuckDB pre-execution, 50s elapsed - 2026-06-12T16:32:22 INFO [viaduck-0][viaduck.main] Seed scan for destination - team-2: DuckDB pre-execution, 55s elapsed - 2026-06-12T16:32:27 INFO [viaduck-0][viaduck.main] Seed scan for destination - team-2: DuckDB pre-execution, 60s elapsed - 2026-06-12T16:32:57 INFO [viaduck-0][viaduck.main] Seed scan for destination - team-2: DuckDB pre-execution, 90s elapsed - 2026-06-12T16:33:27 INFO [viaduck-0][viaduck.main] Seed scan for destination - team-2: DuckDB pre-execution, 120s elapsed - 2026-06-12T16:33:57 INFO [viaduck-0][viaduck.main] Seed scan for destination - team-2: DuckDB pre-execution, 150s elapsed - 2026-06-12T16:34:27 INFO [viaduck-0][viaduck.main] Seed scan for destination - team-2: DuckDB pre-execution, 180s elapsed - 2026-06-12T16:34:57 INFO [viaduck-0][viaduck.main] Seed scan for destination - team-2: DuckDB pre-execution, 210s elapsed - 2026-06-12T16:35:27 INFO [viaduck-0][viaduck.main] Seed scan for destination - team-2: DuckDB pre-execution, 240s elapsed - 2026-06-12T16:35:57 INFO [viaduck-0][viaduck.main] Seed scan for destination - team-2: DuckDB pre-execution, 270s elapsed - 2026-06-12T16:36:27 INFO [viaduck-0][viaduck.main] Seed scan for destination - team-2: DuckDB pre-execution, 300s elapsed - 2026-06-12T16:36:57 INFO [viaduck-0][viaduck.main] Seed scan for destination - team-2: DuckDB pre-execution, 330s elapsed - 2026-06-12T16:37:27 INFO [viaduck-0][viaduck.main] Seed scan for destination - team-2: DuckDB pre-execution, 360s elapsed - 2026-06-12T16:37:57 INFO [viaduck-0][viaduck.main] Seed scan for destination - team-2: DuckDB pre-execution, 390s elapsed - 2026-06-12T16:38:27 INFO [viaduck-0][viaduck.main] Seed scan for destination - team-2: DuckDB pre-execution, 420s elapsed" - -⏺ The heartbeat already exists — let me look at what it reports and what hooks - pyducklake/DuckDB give us for actual progress. - - Read 1 file (ctrl+o to expand) - -⏺ Bash(uv run python -c " - import duckdb…) - ⎿  ['query_progress'] - 1.5.2 - -⏺ Bash(uv run python <<'EOF' - import duckdb, threading, time…) - ⎿  [-1.0] - - -⏺ -1.0 — query likely finished too fast or progress wasn't sampled mid-flight. - Trying a heavier query. - -⏺ Bash(uv run python <<'EOF' - import duckdb, threading, time…) - ⎿  query took 3.7s, progress samples: [-1.0, 5.7, 9.8, 14.7, 24.6, 29.1, - 36.9, 40.6, 45.1, 49.6, 52.8, 56.5, 61.0, 65.9, 70.5, 74.5, 78.6, 82.7 - , 86.0, 90.5, 95.4, 100.0] - -⏺ Cross-thread query_progress() works: 0–100 float while the query runs, needs - enable_progress_bar=true + enable_progress_bar_print=false. Now checking - whether pyducklake exposes the underlying connection. - - Searched for 1 pattern (ctrl+o to expand) - -⏺ Yes — and cleanly. DuckDB tracks per-query progress internally, and - duckdb-python 1.5.2 exposes conn.query_progress() as a cross-thread call. - Verified locally: polling from a second thread while a query streams returns a - 0–100 float (5.7 → 9.8 → ... → 100.0), -1.0 when nothing's running. - pyducklake exposes the connection publicly (table.catalog.connection). - - The wiring would be: - - 1. Connection defaults (source.py): add enable_progress_bar: "true" + - enable_progress_bar_print: "false" — progress is only computed when the - progress bar is enabled; print-off keeps it silent. Negligible overhead. - 2. Heartbeat (_start_progress_heartbeat): accept the source connection; each - tick call query_progress() and, when ≥ 0, append ~N% scanned, est. Xm - remaining (ETA = elapsed × (100−pct)/pct). Covers exactly the blind spot — the - 7-minute "pre-execution" phase is the query executing before the first batch - surfaces, and progress ticks through it. - 3. Call site (_seed_new_destinations): pass src_table.catalog.connection. - - Caveats: - - The percentage is DuckDB's input-scanned estimate (cardinality-based), so - the ETA is honest-ish, not exact — fine for "minutes vs hours". - - query_progress() reads whatever query is live on that connection; during - seed that's unambiguous since the poll loop hasn't started. - - Want me to build it? - -✻ Cooked for 1m 39s - -❯ What would happen if we were seeding multiple destinations concurrently? - -⏺ Today it can't happen — _seed_new_destinations seeds sequentially in a loop on - the main thread, so there's exactly one scan live on the source connection - and query_progress() is unambiguous. - - If we ever parallelized seeding, the progress poll is the least of the - problems: DuckDB invalidates an open streaming result when a new query starts - on the same connection, so concurrent seeds would clobber each other's batch - readers before progress reporting ever got confused. Concurrent seeding would - force per-seed connections (e.g. conn.cursor() or separate catalogs) — and - since query_progress() is per-connection, each heartbeat would then poll its - own connection and stay accurate for free. - - So: the design holds as long as "one scan per connection" holds, which DuckDB - already forces on us. Worth a one-line comment at the call site noting that - invariant. - -✻ Sautéed for 13s - -❯ OK, let's prototype it up. - -⏺ Bash(git fetch origin && gh pr view 26 --json state,mergeCommit -q '.state' && - git status --short | head -5 && git branch --show-current) - ⎿  From github.com:PostHog/viaduck - e8de0d0..f495aaa main -> origin/main - * [new tag] v0.0.26 -> v0.0.26 - … +5 lines (ctrl+o to expand) -⏺ Bash(git checkout main && git pull --ff-only && git checkout -b - jakob/seed-scan-progress) - ⎿  Switched to branch 'main' - Your branch is behind 'origin/main' by 2 commits, and can be fast-forw - arded. - … +8 lines (ctrl+o to expand) - -⏺ Update(~/src/viaduck/viaduck/source.py) - ⎿  Added 6 lines - 36 # mid-read with "maximum total string size ... exceeds this"). - Large buffers - 37 # use 64-bit offsets (pyarrow large_string), which Arrow comput - e and the - 38 # postgres writer handle transparently. - 39 +# enable_progress_bar(+print=false): DuckDB only computes query - + progress - 40 +# when the progress bar is enabled; print-off keeps it out of t - +he terminal. - 41 +# This is what lets `conn.query_progress()` (polled cross-threa - +d by the - 42 +# seed-scan heartbeat) return a real percentage instead of -1. - 43 _CONNECTION_DEFAULTS = { - 44 "pg_connection_limit": "64", - 45 "arrow_large_buffer_size": "true", - 46 + "enable_progress_bar": "true", - 47 + "enable_progress_bar_print": "false", - 48 } - 49 - 50 - -⏺ Update(~/src/viaduck/viaduck/main.py) - ⎿  Added 1 line - 75 early_interval_s: float | None = None, - 76 early_duration_s: float = 60.0, - 77 pre_progress_label: str = "no progress yet", - 78 + progress_conn=None, - 79 ) -> threading.Event: - 80 """Start a background heartbeat for a long-running blocking - operation. - 81 - -⏺ Update(~/src/viaduck/viaduck/main.py) - ⎿  Added 14 lines - 99 operators faster confirmation the pod is alive during cold - -start without - 100 spamming the log forever. - 101 - 102 + `progress_conn` (a duckdb connection, e.g. `table.catalog. - +connection`) - 103 + enables percentage reporting: each tick polls `query_progr - +ess()` — - 104 + a lightweight cross-thread read of DuckDB's executor state - + — and, when - 105 + a query is live (>= 0), appends "~N% scanned, est. M remai - +ning" to the - 106 + line (ETA extrapolated as elapsed * (100-pct)/pct). Requir - +es - 107 + `enable_progress_bar=true` on the connection (set in - 108 + source._CONNECTION_DEFAULTS) or the poll returns -1 foreve - +r. The - 109 + percentage belongs to whatever query is currently running - +on that - 110 + connection, so this is only meaningful while the caller ho - +lds the - 111 + connection's single query slot (true for the seed scan: Du - +ckDB - 112 + invalidates an open streaming result if another query star - +ts on the - 113 + same connection, so one-scan-per-connection is already str - +ucturally - 114 + enforced). - 115 + - 116 The rate format only kicks in once `state["rows"] > 0`. Wh - ile the - 117 counter is still at 0 — i.e. no batch has arrived yet — th - e tick - 118 instead logs `