fix: sweep a stale persist cache on a connection that carries its extensions

packages/python/e2e-attestations/cc-keen-knuth-f11gyq.json

@@ -0,0 +1,7 @@
+{
+  "command": "uv run python -m pytest tests/e2e/ -x -q",
+  "ran_at": 1787281790,
+  "exit_code": 0,
+  "commit": "40ea3cfe1a19eb7a83afeaf46d168ec9ce368423",
+  "branch": "cc/keen-knuth-f11gyq"
+}

packages/rust/changelog.d/2026-08-21-sweep-loads-extensions.md

@@ -0,0 +1,3 @@
+**Fixed**
+
+- **Editing the `ddl` of a `--persist` cache holding an extension-provided virtual table no longer wedges it.** The sweep that precedes a rebuild ran on the cache connection at the moment it was opened, before anything loaded the configured `[[dirsql.extension]]` entries — so `DROP TABLE` on a virtual table whose module came from an extension answered `SQLite error: no such module: vec0`, and every subsequent run repeated the failure until `.dirsql` was deleted by hand. The sweep now runs once the connection carries the same extensions and `[[dirsql.function]]` registrations the batch that created those tables had, so it can drop whatever that batch was able to create. FTS5 masked this: its module is compiled into SQLite and registered on every connection, loadable ones are not. (#1008)

packages/rust/src/lib.rs

@@ -787,6 +787,7 @@ impl DirSQL {
                 deleted,
                 meta: ctx.expected_meta,
                 meta_current: ctx.meta_current,
+                needs_sweep: ctx.needs_sweep,
             }),
             poll_interval,
             path_table_parser,
@@ -841,13 +842,13 @@ impl DirSQL {
             no_ignore,
         } = prepared;
 
-        let (mut db, persist_ready) = match persist {
+        let (mut db, persist_ready, needs_sweep) = match persist {
             Some(p) => {
                 let mut db = p.db;
                 db.set_path_table_cache(p.path);
-                (db, Some((p.deleted, p.meta, p.meta_current)))
+                (db, Some((p.deleted, p.meta, p.meta_current)), p.needs_sweep)
             }
-            None => (Db::new()?, None),
+            None => (Db::new()?, None, false),
         };
         db.set_path_table_root(root.clone());
         db.set_hint_legacy_files_table(hint_legacy_files_table);
@@ -873,6 +874,17 @@ impl DirSQL {
         // query's first call to the function.
         functions::register_all(db.conn(), &functions).map_err(DirSqlError::sqlite)?;
 
+        // Sweep a cache the reconcile rejected. It happens *here*, after the
+        // extensions and functions are on the connection, rather than where
+        // the cache was opened: the drop has to see the same modules the batch
+        // that created these tables saw. A virtual table from a
+        // `[[dirsql.extension]]` is undroppable otherwise -- `DROP TABLE`
+        // answers `no such module`, and the cache stays wedged on every later
+        // run until it is deleted by hand (#1008).
+        if needs_sweep {
+            drop_user_tables(db.conn()).map_err(DirSqlError::sqlite)?;
+        }
+
         let mut on_file_map: HashMap<String, Arc<OnFileFn>> = HashMap::new();
         let mut strict_map: HashMap<String, bool> = HashMap::new();
         let mut ddl_map: HashMap<String, String> = HashMap::new();
@@ -1423,6 +1435,10 @@ pub struct PreparedPersist {
     /// write is skipped, so a run that changed nothing leaves the cache file
     /// byte-for-byte alone.
     meta_current: bool,
+    /// Whether the stale cache still has to be swept. Carried across the split
+    /// so the drop runs on a fully configured connection — see
+    /// [`PersistContext::needs_sweep`].
+    needs_sweep: bool,
 }
 
 /// A file the reconcile decided to trust: its cached rows are kept as-is and
@@ -1444,6 +1460,13 @@ struct PersistContext {
     expected_meta: HashMap<String, String>,
     /// Whether the on-disk meta already equals `expected_meta`.
     meta_current: bool,
+    /// Whether the cache holds tables from an incompatible prior run that must
+    /// be dropped before the DDL batch re-creates them. The drop itself is
+    /// deferred to `finish_build`, which runs it once the connection carries
+    /// the configured extensions and functions: dropping a virtual table needs
+    /// its module registered, and an extension-provided module exists only on
+    /// a connection that loaded that extension (#1008).
+    needs_sweep: bool,
 }
 
 fn compile_matcher(
@@ -1476,9 +1499,9 @@ fn compile_matcher(
 }
 
 /// Open (or create) the persistent SQLite cache and read its meta. If the
-/// meta is missing or incompatible with the current build, the cache is
-/// wiped and the resulting [`PersistContext`] carries an empty file index,
-/// so the rest of the pipeline treats every file as new.
+/// meta is missing or incompatible with the current build, the resulting
+/// [`PersistContext`] carries an empty file index and asks for a sweep, so
+/// the rest of the pipeline treats every file as new.
 fn prepare_persist(
     root: &Path,
     tables: &[Table],
@@ -1501,7 +1524,6 @@ fn prepare_persist(
     let cached = if compatible {
         read_cached_files(db.conn()).map_err(DirSqlError::sqlite)?
     } else {
-        drop_user_tables(db.conn()).map_err(DirSqlError::sqlite)?;
         HashMap::new()
     };
 
@@ -1513,6 +1535,7 @@ fn prepare_persist(
         // equality, which is what lets the write be skipped outright.
         meta_current: compatible && cached_meta.len() == expected_meta.len(),
         expected_meta,
+        needs_sweep: !compatible,
     })
 }
 
@@ -2629,6 +2652,7 @@ mod internal_tests {
             cached,
             expected_meta: HashMap::new(),
             meta_current: false,
+            needs_sweep: false,
         };
         let scanned = vec![(abs.clone(), "t".to_string())];
         let (to_parse, trusted, deleted) =
@@ -2666,6 +2690,7 @@ mod internal_tests {
             cached,
             expected_meta: HashMap::new(),
             meta_current: false,
+            needs_sweep: false,
         };
         let scanned = vec![(abs.clone(), "t".to_string())];
         let (to_parse, trusted, _deleted) =
@@ -2683,6 +2708,7 @@ mod internal_tests {
             cached: HashMap::new(),
             expected_meta: HashMap::new(),
             meta_current: false,
+            needs_sweep: false,
         };
         let missing = dir.path().join("ghost.txt");
         let scanned = vec![(missing, "t".to_string())];
@@ -3350,6 +3376,57 @@ mod internal_tests {
         let ctx = prepare_persist(dir.path(), &tables, &[], None).unwrap();
         assert!(ctx.cached.is_empty());
         assert!(!ctx.expected_meta.is_empty());
+        assert!(
+            ctx.needs_sweep,
+            "an incompatible cache must be handed on for sweeping"
+        );
+    }
+
+    /// The counterpart: a cache whose meta matches keeps its tables, so no
+    /// sweep is requested and the warm rows survive.
+    #[test]
+    fn prepare_persist_warm_cache_asks_for_no_sweep() {
+        let dir = TempDir::new().unwrap();
+        let cache = dir.path().join("cache.db");
+        let tables = vec![Table::new(
+            "t",
+            "CREATE TABLE t (x TEXT)",
+            "*.txt",
+            |_| vec![],
+        )];
+        drop(
+            DirSQL::builder()
+                .root(dir.path())
+                .tables(tables.clone())
+                .persist(Some(&cache))
+                .build()
+                .unwrap(),
+        );
+        let ctx = prepare_persist(dir.path(), &tables, &[], Some(&cache)).unwrap();
+        assert!(!ctx.needs_sweep, "a compatible cache must not be swept");
+    }
+
+    /// The sweep itself, from the build side: an edited `ddl` invalidates the
+    /// cache, and only a sweep lets the batch re-run — an un-dropped `t` reads
+    /// as a warm table and the new column never appears.
+    #[test]
+    fn an_edited_ddl_sweeps_the_cache_before_rebuilding() {
+        let dir = TempDir::new().unwrap();
+        let cache = dir.path().join("cache.db");
+        let build = |ddl: &'static str| {
+            DirSQL::builder()
+                .root(dir.path())
+                .tables(vec![Table::new("t", ddl, "*.txt", |_| vec![])])
+                .persist(Some(&cache))
+                .build()
+        };
+        drop(build("CREATE TABLE t (x TEXT)").unwrap());
+
+        let second = build("CREATE TABLE t (x TEXT, y TEXT)").unwrap();
+        assert!(
+            second.query("SELECT y FROM t").is_ok(),
+            "the rebuilt table must carry the edited batch's new column"
+        );
     }
 
     /// `snapshot_ns > mtime_ns`: the file is outside the racy window, so a
@@ -3377,6 +3454,7 @@ mod internal_tests {
             cached,
             expected_meta: HashMap::new(),
             meta_current: false,
+            needs_sweep: false,
         };
         let scanned = vec![(abs, "t".to_string())];
         let (to_parse, trusted, deleted) =
@@ -3411,6 +3489,7 @@ mod internal_tests {
             cached,
             expected_meta: HashMap::new(),
             meta_current: false,
+            needs_sweep: false,
         };
         let scanned = vec![(abs, "t".to_string())];
         let (to_parse, trusted, _deleted) =
@@ -3439,6 +3518,7 @@ mod internal_tests {
             cached,
             expected_meta: HashMap::new(),
             meta_current: false,
+            needs_sweep: false,
         };
         let fake = FakeFs::default();
         let (to_parse, trusted, deleted) =

packages/rust/tests/common/mod.rs

@@ -0,0 +1,55 @@
+//! Helpers shared across integration and e2e test binaries.
+
+use std::path::PathBuf;
+use std::process::Command;
+
+/// Build the test-only loadable extension fixture
+/// (`tests/fixtures/testext`) and return the path to its compiled shared
+/// library. Built into `CARGO_TARGET_TMPDIR` with coverage/instrumentation
+/// env removed so the nested build is independent of an outer
+/// `cargo llvm-cov` invocation.
+pub fn build_fixture_extension() -> PathBuf {
+    let manifest = concat!(
+        env!("CARGO_MANIFEST_DIR"),
+        "/tests/fixtures/testext/Cargo.toml"
+    );
+    let output = Command::new(env!("CARGO"))
+        .args([
+            "build",
+            "--manifest-path",
+            manifest,
+            "--target-dir",
+            env!("CARGO_TARGET_TMPDIR"),
+            "--message-format=json",
+        ])
+        .env_remove("RUSTFLAGS")
+        .env_remove("RUSTDOCFLAGS")
+        .env_remove("LLVM_PROFILE_FILE")
+        .output()
+        .expect("failed to spawn cargo build for the extension fixture");
+    assert!(
+        output.status.success(),
+        "fixture build failed:\n{}",
+        String::from_utf8_lossy(&output.stderr),
+    );
+
+    // Parse cargo's JSON output for the cdylib artifact path (platform-
+    // independent: .so / .dylib / .dll).
+    let stdout = String::from_utf8(output.stdout).unwrap();
+    let mut artifact: Option<PathBuf> = None;
+    for line in stdout.lines() {
+        let Ok(msg) = serde_json::from_str::<serde_json::Value>(line) else {
+            continue;
+        };
+        if msg["reason"] == "compiler-artifact" {
+            if let Some(files) = msg["filenames"].as_array() {
+                for f in files.iter().filter_map(|f| f.as_str()) {
+                    if f.ends_with(".so") || f.ends_with(".dylib") || f.ends_with(".dll") {
+                        artifact = Some(PathBuf::from(f));
+                    }
+                }
+            }
+        }
+    }
+    artifact.expect("no cdylib artifact in cargo build output for the fixture")
+}

packages/rust/tests/ddl_batch.rs

@@ -5,7 +5,10 @@
 //! settled afterwards against SQLite's own catalog. That is what makes
 //! indexes, FTS5 virtual tables and triggers expressible in config.
 
-use dirsql::{DirSQL, Row, Table, Value};
+mod common;
+
+use common::build_fixture_extension;
+use dirsql::{DirSQL, Extension, Row, Table, Value};
 use std::collections::HashMap;
 use std::fs;
 use std::path::Path;
@@ -25,6 +28,12 @@ CREATE TRIGGER notes_ad AFTER DELETE ON notes BEGIN
   INSERT INTO notes_fts(notes_fts, rowid, body) VALUES ('delete', old.rowid, old.body);
 END;";
 
+/// The same shape as [`FTS_DDL`], but the virtual table's module arrives with
+/// a loaded extension instead of being compiled into SQLite.
+const EXT_VTAB_DDL: &str = "\
+CREATE TABLE notes (path TEXT, body TEXT);
+CREATE VIRTUAL TABLE notes_ext USING dirsql_testext_vtab();";
+
 /// A `notes` table whose rows are each markdown file's name and contents.
 fn notes_table(ddl: &str) -> Table {
     Table::new("notes", ddl, "*.md", |path| {
@@ -61,6 +70,18 @@ fn build_persist(root: &Path, ddl: &str) -> dirsql::Result<DirSQL> {
         .build()
 }
 
+fn build_persist_with_extension(root: &Path, ddl: &str, ext: &Path) -> dirsql::Result<DirSQL> {
+    DirSQL::builder()
+        .root(root)
+        .table(notes_table(ddl))
+        .extension(Extension {
+            path: ext.to_path_buf(),
+            entrypoint: Some("sqlite3_extension_init".into()),
+        })
+        .persist(None::<&Path>)
+        .build()
+}
+
 /// The error message from a build that must fail. `DirSQL` is not `Debug`,
 /// so `expect_err` is unavailable.
 fn build_error(result: dirsql::Result<DirSQL>, what: &str) -> String {
@@ -253,3 +274,32 @@ fn a_rebuild_sweeps_a_cache_holding_virtual_tables() {
         "the rebuilt index must hold the re-ingested row, got {hits:?}"
     );
 }
+
+/// The sibling [`a_rebuild_sweeps_a_cache_holding_virtual_tables`] cannot be:
+/// FTS5's module is compiled into SQLite, so it is registered on every
+/// connection whether or not dirsql configured anything. A module that arrives
+/// with a `[[dirsql.extension]]` is registered only where that extension was
+/// loaded — so the sweep must run somewhere that loaded it, or `DROP TABLE`
+/// answers `no such module` and the cache is wedged for every later run.
+#[test]
+fn a_rebuild_sweeps_a_cache_holding_an_extension_backed_virtual_table() {
+    let ext = build_fixture_extension();
+    let root = TempDir::new().unwrap();
+    seed(root.path());
+    drop(build_persist_with_extension(root.path(), EXT_VTAB_DDL, &ext).expect("first build"));
+
+    let db = build_persist_with_extension(
+        root.path(),
+        &format!("{EXT_VTAB_DDL}\nCREATE INDEX notes_path ON notes(path);"),
+        &ext,
+    )
+    .expect("sweeping a cache holding an extension-provided virtual table must succeed");
+
+    let rows = db.query("SELECT n FROM notes_ext").unwrap();
+    assert_eq!(
+        rows.len(),
+        1,
+        "the rebuilt batch must have re-created the extension's virtual table, got {rows:?}"
+    );
+    assert_eq!(rows[0]["n"], Value::Integer(42));
+}